Sun Jul 11 22:37:21 GMT Daylight Time 2010 david-sarah@jacaranda.org * Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083). Sat Jul 17 06:46:47 GMT Daylight Time 2010 david-sarah@jacaranda.org * fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile. New patches: [Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083). david-sarah@jacaranda.org**20100711213721 Ignore-this: e452e8ca66391aa2a1a49afe0114f317 ] { hunk ./src/allmydata/frontends/ftpd.py 2 -import tempfile from zope.interface import implements from twisted.application import service, strports from twisted.internet import defer hunk ./src/allmydata/frontends/ftpd.py 12 from allmydata.interfaces import IDirectoryNode, ExistingChildError, \ NoSuchChildError from allmydata.immutable.upload import FileHandle +from allmydata.util.fileutil import EncryptedTemporaryFile class ReadFile: implements(ftp.IReadFile) hunk ./src/allmydata/frontends/ftpd.py 30 raise NotImplementedError("Non-streaming producer not supported.") # we write the data to a temporary file, since Tahoe can't do # streaming upload yet. - self.f = tempfile.TemporaryFile() + self.f = EncryptedTemporaryFile() return None def unregisterProducer(self): hunk ./src/allmydata/frontends/sftpd.py 2 -import os, tempfile, heapq, binascii, traceback, array, stat, struct +import heapq, traceback, array, stat, struct from types import NoneType from stat import S_IFREG, S_IFDIR from time import time, strftime, localtime hunk ./src/allmydata/frontends/sftpd.py 35 from allmydata.mutable.common import NotWriteableError from allmydata.immutable.upload import FileHandle from allmydata.dirnode import update_metadata - -from pycryptopp.cipher.aes import AES +from allmydata.util.fileutil import EncryptedTemporaryFile noisy = True use_foolscap_logging = True hunk ./src/allmydata/frontends/sftpd.py 290 return None -class EncryptedTemporaryFile(PrefixingLogMixin): - # not implemented: next, readline, readlines, xreadlines, writelines - - def __init__(self): - PrefixingLogMixin.__init__(self, facility="tahoe.sftp") - self.file = tempfile.TemporaryFile() - self.key = os.urandom(16) # AES-128 - - def _crypt(self, offset, data): - # TODO: use random-access AES (pycryptopp ticket #18) - offset_big = offset // 16 - offset_small = offset % 16 - iv = binascii.unhexlify("%032x" % offset_big) - cipher = AES(self.key, iv=iv) - cipher.process("\x00"*offset_small) - return cipher.process(data) - - def close(self): - self.file.close() - - def flush(self): - self.file.flush() - - def seek(self, offset, whence=0): # 0 = SEEK_SET - if noisy: self.log(".seek(%r, %r)" % (offset, whence), level=NOISY) - self.file.seek(offset, whence) - - def tell(self): - offset = self.file.tell() - if noisy: self.log(".tell() = %r" % (offset,), level=NOISY) - return offset - - def read(self, size=-1): - if noisy: self.log(".read(%r)" % (size,), level=NOISY) - index = self.file.tell() - ciphertext = self.file.read(size) - plaintext = self._crypt(index, ciphertext) - return plaintext - - def write(self, plaintext): - if noisy: self.log(".write()" % (len(plaintext),), level=NOISY) - index = self.file.tell() - ciphertext = self._crypt(index, plaintext) - self.file.write(ciphertext) - - def truncate(self, newsize): - if noisy: self.log(".truncate(%r)" % (newsize,), level=NOISY) - self.file.truncate(newsize) - - class OverwriteableFileConsumer(PrefixingLogMixin): implements(IFinishableConsumer) """I act both as a consumer for the download of the original file contents, and as a hunk ./src/allmydata/util/fileutil.py 5 Futz with files like a pro. """ -import sys, exceptions, os, stat, tempfile, time +import sys, exceptions, os, stat, tempfile, time, binascii from twisted.python import log hunk ./src/allmydata/util/fileutil.py 9 +from pycryptopp.cipher.aes import AES + + def rename(src, dst, tries=4, basedelay=0.1): """ Here is a superkludge to workaround the fact that occasionally on Windows some other process (e.g. an anti-virus scanner, a local search hunk ./src/allmydata/util/fileutil.py 118 if self.cleanup and hasattr(self, 'name'): rm_dir(self.name) +class EncryptedTemporaryFile: + # not implemented: next, readline, readlines, xreadlines, writelines + + def __init__(self): + self.file = tempfile.TemporaryFile() + self.key = os.urandom(16) # AES-128 + + def _crypt(self, offset, data): + offset_big = offset // 16 + offset_small = offset % 16 + iv = binascii.unhexlify("%032x" % offset_big) + cipher = AES(self.key, iv=iv) + cipher.process("\x00"*offset_small) + return cipher.process(data) + + def close(self): + self.file.close() + + def flush(self): + self.file.flush() + + def seek(self, offset, whence=0): # 0 = SEEK_SET + self.file.seek(offset, whence) + + def tell(self): + offset = self.file.tell() + return offset + + def read(self, size=-1): + index = self.file.tell() + ciphertext = self.file.read(size) + plaintext = self._crypt(index, ciphertext) + return plaintext + + def write(self, plaintext): + index = self.file.tell() + ciphertext = self._crypt(index, plaintext) + self.file.write(ciphertext) + + def truncate(self, newsize): + self.file.truncate(newsize) + + def make_dirs(dirname, mode=0777): """ An idempotent version of os.makedirs(). If the dir already exists, do } [fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile. david-sarah@jacaranda.org**20100717054647 Ignore-this: 46d8fc10782fa8ec2b6c5b168c841943 ] { hunk ./src/allmydata/util/fileutil.py 147 return offset def read(self, size=-1): + """A read must not follow a write, or vice-versa, without an intervening seek.""" index = self.file.tell() ciphertext = self.file.read(size) plaintext = self._crypt(index, ciphertext) hunk ./src/allmydata/util/fileutil.py 154 return plaintext def write(self, plaintext): + """A read must not follow a write, or vice-versa, without an intervening seek. + If seeking and then writing causes a 'hole' in the file, the contents of the + hole are unspecified.""" index = self.file.tell() ciphertext = self._crypt(index, plaintext) self.file.write(ciphertext) hunk ./src/allmydata/util/fileutil.py 162 def truncate(self, newsize): + """Truncate or extend the file to 'newsize'. If it is extended, the contents after the + old end-of-file are unspecified. The file position after this operation is unspecified.""" self.file.truncate(newsize) } Context: [cli.py: make command descriptions consistently end with a full stop. david-sarah@jacaranda.org**20100714014538 Ignore-this: 9ee7fa29ca2d1631db4049c2a389a97a ] [SFTP: address some of the comments in zooko's review (#1106). david-sarah@jacaranda.org**20100712025537 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e ] [docs/logging.txt: note that setting flogging vars might affect tests with race conditions. david-sarah@jacaranda.org**20100712050721 Ignore-this: fc1609d215fcd5561a57fd1226206f27 ] [test_storage.py: potential fix for failures when logging is enabled. david-sarah@jacaranda.org**19700713040546 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7 ] [upcase_since_on_welcome terrellrussell@gmail.com**20100708193903] [server_version_on_welcome_page.dpatch.txt freestorm77@gmail.com**20100605191721 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a - The storage server version is 0 for all storage nodes in the Welcome Page ] [NEWS: add NEWS snippets about two recent patches zooko@zooko.com**20100708162058 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 ] [directory_html_top_banner.dpatch freestorm77@gmail.com**20100622205301 Ignore-this: 1d770d975e0c414c996564774f049bca The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct ] [tahoe_css_toolbar.dpatch freestorm77@gmail.com**20100622210046 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0 CSS modification to be correctly diplayed with Internet Explorer 8 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox. ] [runnin_test_tahoe_css.dpatch freestorm77@gmail.com**20100622214714 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c Runnin test for changes in tahoe.css file ] [runnin_test_directory_xhtml.dpatch freestorm77@gmail.com**20100622201403 Ignore-this: f8962463fce50b9466405cb59fe11d43 Runnin test for diretory.xhtml top banner ] [stringutils.py: tolerate sys.stdout having no 'encoding' attribute. david-sarah@jacaranda.org**20100626040817 Ignore-this: f42cad81cef645ee38ac1df4660cc850 ] [quickstart.html: python 2.5 -> 2.6 as recommended version david-sarah@jacaranda.org**20100705175858 Ignore-this: bc3a14645ea1d5435002966ae903199f ] [SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader). david-sarah@jacaranda.org**20100628231926 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f ] [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. david-sarah@jacaranda.org**20100625223929 Ignore-this: 99a5459cac51bd867cc11ad06927ff30 ] [setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk" zooko@zooko.com**20100619034928 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6 ] [trivial: tiny update to in-line comment zooko@zooko.com**20100614045715 Ignore-this: 10851b0ed2abfed542c97749e5d280bc (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.) ] [docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around zooko@zooko.com**20100619065318 Ignore-this: dc6db03f696e5b6d2848699e754d8053 ] [docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates" zooko@zooko.com**20100619065124 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c ] [TAG allmydata-tahoe-1.7.0 zooko@zooko.com**20100619052631 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1 ] Patch bundle hash: 0b1bde36c9d61d62ac06a2649406c59c725cc48a