Ticket #1083: make-ftp-use-encrypted-temporary-file.dpatch

Sun Jul 11 22:37:21 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083).

Sat Jul 17 06:46:47 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile.

New patches:

[Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083).
david-sarah@jacaranda.org**20100711213721
 Ignore-this: e452e8ca66391aa2a1a49afe0114f317
] {
hunk ./src/allmydata/frontends/ftpd.py 2
 
-import tempfile
 from zope.interface import implements
 from twisted.application import service, strports
 from twisted.internet import defer
hunk ./src/allmydata/frontends/ftpd.py 12
 from allmydata.interfaces import IDirectoryNode, ExistingChildError, \
      NoSuchChildError
 from allmydata.immutable.upload import FileHandle
+from allmydata.util.fileutil import EncryptedTemporaryFile
 
 class ReadFile:
     implements(ftp.IReadFile)
hunk ./src/allmydata/frontends/ftpd.py 30
             raise NotImplementedError("Non-streaming producer not supported.")
         # we write the data to a temporary file, since Tahoe can't do
         # streaming upload yet.
-        self.f = tempfile.TemporaryFile()
+        self.f = EncryptedTemporaryFile()
         return None
 
     def unregisterProducer(self):
hunk ./src/allmydata/frontends/sftpd.py 2
 
-import os, tempfile, heapq, binascii, traceback, array, stat, struct
+import heapq, traceback, array, stat, struct
 from types import NoneType
 from stat import S_IFREG, S_IFDIR
 from time import time, strftime, localtime
hunk ./src/allmydata/frontends/sftpd.py 35
 from allmydata.mutable.common import NotWriteableError
 from allmydata.immutable.upload import FileHandle
 from allmydata.dirnode import update_metadata
-
-from pycryptopp.cipher.aes import AES
+from allmydata.util.fileutil import EncryptedTemporaryFile
 
 noisy = True
 use_foolscap_logging = True
hunk ./src/allmydata/frontends/sftpd.py 290
     return None
 
 
-class EncryptedTemporaryFile(PrefixingLogMixin):
-    # not implemented: next, readline, readlines, xreadlines, writelines
-
-    def __init__(self):
-        PrefixingLogMixin.__init__(self, facility="tahoe.sftp")
-        self.file = tempfile.TemporaryFile()
-        self.key = os.urandom(16)  # AES-128
-
-    def _crypt(self, offset, data):
-        # TODO: use random-access AES (pycryptopp ticket #18)
-        offset_big = offset // 16
-        offset_small = offset % 16
-        iv = binascii.unhexlify("%032x" % offset_big)
-        cipher = AES(self.key, iv=iv)
-        cipher.process("\x00"*offset_small)
-        return cipher.process(data)
-
-    def close(self):
-        self.file.close()
-
-    def flush(self):
-        self.file.flush()
-
-    def seek(self, offset, whence=0):  # 0 = SEEK_SET
-        if noisy: self.log(".seek(%r, %r)" % (offset, whence), level=NOISY)
-        self.file.seek(offset, whence)
-
-    def tell(self):
-        offset = self.file.tell()
-        if noisy: self.log(".tell() = %r" % (offset,), level=NOISY)
-        return offset
-
-    def read(self, size=-1):
-        if noisy: self.log(".read(%r)" % (size,), level=NOISY)
-        index = self.file.tell()
-        ciphertext = self.file.read(size)
-        plaintext = self._crypt(index, ciphertext)
-        return plaintext
-
-    def write(self, plaintext):
-        if noisy: self.log(".write(<data of length %r>)" % (len(plaintext),), level=NOISY)
-        index = self.file.tell()
-        ciphertext = self._crypt(index, plaintext)
-        self.file.write(ciphertext)
-
-    def truncate(self, newsize):
-        if noisy: self.log(".truncate(%r)" % (newsize,), level=NOISY)
-        self.file.truncate(newsize)
-
-
 class OverwriteableFileConsumer(PrefixingLogMixin):
     implements(IFinishableConsumer)
     """I act both as a consumer for the download of the original file contents, and as a
hunk ./src/allmydata/util/fileutil.py 5
 Futz with files like a pro.
 """
 
-import sys, exceptions, os, stat, tempfile, time
+import sys, exceptions, os, stat, tempfile, time, binascii
 
 from twisted.python import log
 
hunk ./src/allmydata/util/fileutil.py 9
+from pycryptopp.cipher.aes import AES
+
+
 def rename(src, dst, tries=4, basedelay=0.1):
     """ Here is a superkludge to workaround the fact that occasionally on
     Windows some other process (e.g. an anti-virus scanner, a local search
hunk ./src/allmydata/util/fileutil.py 118
         if self.cleanup and hasattr(self, 'name'):
             rm_dir(self.name)
 
+class EncryptedTemporaryFile:
+    # not implemented: next, readline, readlines, xreadlines, writelines
+
+    def __init__(self):
+        self.file = tempfile.TemporaryFile()
+        self.key = os.urandom(16)  # AES-128
+
+    def _crypt(self, offset, data):
+        offset_big = offset // 16
+        offset_small = offset % 16
+        iv = binascii.unhexlify("%032x" % offset_big)
+        cipher = AES(self.key, iv=iv)
+        cipher.process("\x00"*offset_small)
+        return cipher.process(data)
+
+    def close(self):
+        self.file.close()
+
+    def flush(self):
+        self.file.flush()
+
+    def seek(self, offset, whence=0):  # 0 = SEEK_SET
+        self.file.seek(offset, whence)
+
+    def tell(self):
+        offset = self.file.tell()
+        return offset
+
+    def read(self, size=-1):
+        index = self.file.tell()
+        ciphertext = self.file.read(size)
+        plaintext = self._crypt(index, ciphertext)
+        return plaintext
+
+    def write(self, plaintext):
+        index = self.file.tell()
+        ciphertext = self._crypt(index, plaintext)
+        self.file.write(ciphertext)
+
+    def truncate(self, newsize):
+        self.file.truncate(newsize)
+
+
 def make_dirs(dirname, mode=0777):
     """
     An idempotent version of os.makedirs().  If the dir already exists, do
}
[fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile.
david-sarah@jacaranda.org**20100717054647
 Ignore-this: 46d8fc10782fa8ec2b6c5b168c841943
] {
hunk ./src/allmydata/util/fileutil.py 147
         return offset
 
     def read(self, size=-1):
+        """A read must not follow a write, or vice-versa, without an intervening seek."""
         index = self.file.tell()
         ciphertext = self.file.read(size)
         plaintext = self._crypt(index, ciphertext)
hunk ./src/allmydata/util/fileutil.py 154
         return plaintext
 
     def write(self, plaintext):
+        """A read must not follow a write, or vice-versa, without an intervening seek.
+        If seeking and then writing causes a 'hole' in the file, the contents of the
+        hole are unspecified."""
         index = self.file.tell()
         ciphertext = self._crypt(index, plaintext)
         self.file.write(ciphertext)
hunk ./src/allmydata/util/fileutil.py 162
 
     def truncate(self, newsize):
+        """Truncate or extend the file to 'newsize'. If it is extended, the contents after the
+        old end-of-file are unspecified. The file position after this operation is unspecified."""
         self.file.truncate(newsize)
 
 
}

Context:

[cli.py: make command descriptions consistently end with a full stop.
david-sarah@jacaranda.org**20100714014538
 Ignore-this: 9ee7fa29ca2d1631db4049c2a389a97a
] 
[SFTP: address some of the comments in zooko's review (#1106).
david-sarah@jacaranda.org**20100712025537
 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e
] 
[docs/logging.txt: note that setting flogging vars might affect tests with race conditions.
david-sarah@jacaranda.org**20100712050721
 Ignore-this: fc1609d215fcd5561a57fd1226206f27
] 
[test_storage.py: potential fix for failures when logging is enabled.
david-sarah@jacaranda.org**19700713040546
 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7
] 
[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
0b1bde36c9d61d62ac06a2649406c59c725cc48a