Wed May 12 06:54:07 GMT Daylight Time 2010 david-sarah@jacaranda.org * New SFTP implementation: mutable files, read/write support, streaming download, Unicode filenames, and more Wed May 12 07:05:52 GMT Daylight Time 2010 david-sarah@jacaranda.org * Tests for new SFTP implementation Wed May 12 07:07:54 GMT Daylight Time 2010 david-sarah@jacaranda.org * Change shouldFail to avoid Unicode errors when converting Failure to str New patches: [New SFTP implementation: mutable files, read/write support, streaming download, Unicode filenames, and more david-sarah@jacaranda.org**20100512055407 Ignore-this: 906f51c48d974ba9cf360c27845c55eb ] { hunk ./src/allmydata/frontends/sftpd.py 2 -import tempfile +import os, tempfile, heapq, binascii, traceback, sys +from stat import S_IFREG, S_IFDIR + from zope.interface import implements from twisted.python import components from twisted.application import service, strports hunk ./src/allmydata/frontends/sftpd.py 8 -from twisted.internet import defer from twisted.conch.ssh import factory, keys, session hunk ./src/allmydata/frontends/sftpd.py 9 +from twisted.conch.ssh.filetransfer import FileTransferServer, SFTPError, \ + FX_NO_SUCH_FILE, FX_OP_UNSUPPORTED, FX_PERMISSION_DENIED, FX_EOF, \ + FX_BAD_MESSAGE, FX_FAILURE +from twisted.conch.ssh.filetransfer import FXF_READ, FXF_WRITE, FXF_APPEND, \ + FXF_CREAT, FXF_TRUNC, FXF_EXCL from twisted.conch.interfaces import ISFTPServer, ISFTPFile, IConchUser from twisted.conch.avatar import ConchUser from twisted.conch.openssh_compat import primes hunk ./src/allmydata/frontends/sftpd.py 17 -from twisted.conch import ls from twisted.cred import portal hunk ./src/allmydata/frontends/sftpd.py 19 -from allmydata.interfaces import IDirectoryNode, ExistingChildError, \ +from twisted.internet import defer +from twisted.python.failure import Failure +from twisted.internet.interfaces import IFinishableConsumer +from foolscap.api import eventually +from allmydata.util import deferredutil + +from allmydata.util.consumer import download_to_data +from allmydata.interfaces import IFileNode, IDirectoryNode, ExistingChildError, \ NoSuchChildError hunk ./src/allmydata/frontends/sftpd.py 28 +from allmydata.mutable.common import NotWriteableError from allmydata.immutable.upload import FileHandle hunk ./src/allmydata/frontends/sftpd.py 30 -from allmydata.util.consumer import download_to_data hunk ./src/allmydata/frontends/sftpd.py 31 -class ReadFile: +from pycryptopp.cipher.aes import AES + +# twisted.conch.ssh.filetransfer generates this warning, but not when it is imported, +# only on an error. +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning, + message="BaseException.message has been deprecated as of Python 2.6", + module=".*filetransfer", append=True) + +debug = False + +if debug: + def eventually_callback(d): + s = traceback.format_stack() + def _cb(res): + try: + print "CALLBACK %r %r" % (d, res) + d.callback(res) + except: # pragma: no cover + print "Failed to callback %r" % (d,) + print "with %r" % (res,) + print "Original stack:" + print '!' + '!'.join(s) + traceback.print_exc() + raise + return lambda res: eventually(_cb, res) + + def eventually_errback(d): + s = traceback.format_stack() + def _eb(err): + try: + print "ERRBACK", d, err + d.errback(err) + except: # pragma: no cover + print "Failed to errback %r" % (d,) + print "with %r" % (err,) + print "Original stack:" + print '!' + '!'.join(s) + traceback.print_exc() + raise + return lambda err: eventually(_eb, err) +else: + def eventually_callback(d): + return lambda res: eventually(d.callback, res) + + def eventually_errback(d): + return lambda err: eventually(d.errback, err) + + +def _raise_error(err): + if err is None: + return None + if debug: + print "TRACEBACK %r" % (err,) + #traceback.print_exc(err) + + # The message argument to SFTPError must not reveal information that + # might compromise anonymity. + + if err.check(SFTPError): + # original raiser of SFTPError has responsibility to ensure anonymity + raise err + if err.check(NoSuchChildError): + childname = err.value.args[0].encode('utf-8') + raise SFTPError(FX_NO_SUCH_FILE, childname) + if err.check(ExistingChildError): + msg = err.value.args[0].encode('utf-8') + # later versions of SFTP define FX_FILE_ALREADY_EXISTS, but version 3 doesn't + raise SFTPError(FX_PERMISSION_DENIED, msg) + if err.check(NotWriteableError): + msg = err.value.args[0].encode('utf-8') + raise SFTPError(FX_PERMISSION_DENIED, msg) + if err.check(NotImplementedError): + raise SFTPError(FX_OP_UNSUPPORTED, str(err.value)) + if err.check(EOFError): + raise SFTPError(FX_EOF, "end of file reached") + if err.check(defer.FirstError): + _raise_error(err.value.subFailure) + + # We assume that the type of error is not anonymity-sensitive. + raise SFTPError(FX_FAILURE, str(err.type)) + +def _repr_flags(flags): + return "|".join([f for f in + [(flags & FXF_READ) and "FXF_READ" or None, + (flags & FXF_WRITE) and "FXF_WRITE" or None, + (flags & FXF_APPEND) and "FXF_APPEND" or None, + (flags & FXF_CREAT) and "FXF_CREAT" or None, + (flags & FXF_TRUNC) and "FXF_TRUNC" or None, + (flags & FXF_EXCL) and "FXF_EXCL" or None, + ] + if f]) + +def _populate_attrs(childnode, metadata, writeable, size=None): + attrs = {} + + # see webapi.txt for what these times mean + if metadata: + if "linkmotime" in metadata.get("tahoe", {}): + attrs["mtime"] = int(metadata["tahoe"]["linkmotime"]) + elif "mtime" in metadata: + attrs["mtime"] = int(metadata["mtime"]) + + if "linkcrtime" in metadata.get("tahoe", {}): + attrs["createtime"] = int(metadata["tahoe"]["linkcrtime"]) + + if "ctime" in metadata: + attrs["ctime"] = int(metadata["ctime"]) + + # We would prefer to omit atime, but SFTP version 3 can only + # accept mtime if atime is also set. + attrs["atime"] = attrs["mtime"] + + # The permissions must have the extra bits (040000 or 0100000), + # otherwise the client will not call openDirectory. + + # Directories and unknown nodes have no size, and SFTP doesn't + # require us to make one up. + # childnode might be None, meaning that the file doesn't exist yet, + # but we're going to write it later. + + if childnode and childnode.is_unknown(): + perms = 0 + elif childnode and IDirectoryNode.providedBy(childnode): + perms = S_IFDIR | 0770 + else: + # For files, omit the size if we don't immediately know it. + if childnode and size is None: + size = childnode.get_size() + if size is not None: + assert isinstance(size, (int, long)), repr(size) + attrs["size"] = size + perms = S_IFREG | 0660 + + if not writeable: + perms &= S_IFDIR | S_IFREG | 0555 # clear 'w' bits + + attrs["permissions"] = perms + + # We could set the SSH_FILEXFER_ATTR_FLAGS here: + # ENCRYPTED would always be true ("The file is stored on disk + # using file-system level transparent encryption.") + # SYSTEM, HIDDEN, ARCHIVE and SYNC would always be false. + # READONLY and IMMUTABLE would be set according to + # childnode.is_readonly() and childnode.is_immutable() + # for known nodes. + # However, twisted.conch.ssh.filetransfer only implements + # SFTP version 3, which doesn't include these flags. + + return attrs + + +class EncryptedTemporaryFile: + # not implemented: next, readline, readlines, xreadlines, writelines + + def _crypt(self, offset, data): + # FIXME: use random-access AES (pycryptopp ticket #18) + offset_big = offset // 16 + offset_small = offset % 16 + iv = binascii.unhexlify("%032x" % offset_big) + cipher = AES(self.key, iv=iv) + cipher.process("\x00"*offset_small) + return cipher.process(data) + + def __init__(self): + self.file = tempfile.TemporaryFile() + self.key = os.urandom(16) # AES-128 + + def close(self): + self.file.close() + + def flush(self): + self.file.flush() + + def seek(self, offset, whence=os.SEEK_SET): + if debug: print ".seek(%r, %r)" % (offset, whence) + self.file.seek(offset, whence) + + def tell(self): + offset = self.file.tell() + if debug: print ".offset = %r" % (offset,) + return offset + + def read(self, size=-1): + if debug: print ".read(%r)" % (size,) + index = self.file.tell() + ciphertext = self.file.read(size) + plaintext = self._crypt(index, ciphertext) + return plaintext + + def write(self, plaintext): + if debug: print ".write(%r)" % (plaintext,) + index = self.file.tell() + ciphertext = self._crypt(index, plaintext) + self.file.write(ciphertext) + + def truncate(self, newsize): + if debug: print ".truncate(%r)" % (newsize,) + self.file.truncate(newsize) + + +class OverwriteableFileConsumer: + implements(IFinishableConsumer) + """I act both as a consumer for the download of the original file contents, and as a + wrapper for a temporary file that records the downloaded data and any overwrites. + I use a priority queue to keep track of which regions of the file have been overwritten + but not yet downloaded, so that the download does not clobber overwritten data. + I use another priority queue to record milestones at which to make callbacks + indicating that a given number of bytes have been downloaded. + + The temporary file reflects the contents of the file that I represent, except that: + - regions that have neither been downloaded nor overwritten, if present, + contain zeroes. + - the temporary file may be shorter than the represented file (it is never longer). + The latter's current size is stored in self.current_size. + + This abstraction is mostly independent of SFTP. Consider moving it, if it is found + useful for other frontends.""" + + def __init__(self, check_abort, download_size, tempfile_maker): + self.check_abort = check_abort + self.download_size = download_size + self.current_size = download_size + self.f = tempfile_maker() + self.downloaded = 0 + self.milestones = [] # empty heap of (offset, d) + self.overwrites = [] # empty heap of (start, end) + self.done = self.when_reached(download_size) # adds a milestone + self.producer = None + + def get_file(self): + return self.f + + def get_current_size(self): + return self.current_size + + def set_current_size(self, size): + if debug: print "set_current_size(%r), current_size = %r, downloaded = %r" % (size, self.current_size, self.downloaded) + if size < self.current_size or size < self.downloaded: + self.f.truncate(size) + self.current_size = size + if size < self.download_size: + self.download_size = size + if self.downloaded >= self.download_size: + self.finish() + + def registerProducer(self, p, streaming): + self.producer = p + if streaming: + # call resumeProducing once to start things off + p.resumeProducing() + else: + while not self.done: + p.resumeProducing() + + def write(self, data): + if debug: print "write(%r)" % (data,) + if self.check_abort(): + self.close() + return + + if self.downloaded >= self.download_size: + return + + next_downloaded = self.downloaded + len(data) + if next_downloaded > self.download_size: + data = data[:(self.download_size - self.downloaded)] + + while len(self.overwrites) > 0: + (start, end) = self.overwrites[0] + if start >= next_downloaded: + # This and all remaining overwrites are after the data we just downloaded. + break + if start > self.downloaded: + # The data we just downloaded has been partially overwritten. + # Write the prefix of it that precedes the overwritten region. + self.f.seek(self.downloaded) + self.f.write(data[:(start - self.downloaded)]) + + # This merges consecutive overwrites if possible, which allows us to detect the + # case where the download can be stopped early because the remaining region + # to download has already been fully overwritten. + heapq.heappop(self.overwrites) + while len(self.overwrites) > 0: + (start1, end1) = self.overwrites[0] + if start1 > end: + break + end = end1 + heapq.heappop(self.overwrites) + + if end >= next_downloaded: + # This overwrite extends past the downloaded data, so there is no + # more data to consider on this call. + heapq.heappush(self.overwrites, (next_downloaded, end)) + self._update_downloaded(next_downloaded) + return + elif end >= self.downloaded: + data = data[(end - self.downloaded):] + self._update_downloaded(end) + + self.f.seek(self.downloaded) + self.f.write(data) + self._update_downloaded(next_downloaded) + + def _update_downloaded(self, new_downloaded): + self.downloaded = new_downloaded + milestone = new_downloaded + if len(self.overwrites) > 0: + (start, end) = self.overwrites[0] + if start <= new_downloaded and end > milestone: + milestone = end + + while len(self.milestones) > 0: + (next, d) = self.milestones[0] + if next > milestone: + return + if debug: print "MILESTONE %r %r" % (next, d) + heapq.heappop(self.milestones) + eventually_callback(d)(None) + + if milestone >= self.download_size: + self.finish() + + def overwrite(self, offset, data): + if debug: print "overwrite(%r, %r)" % (offset, data) + if offset > self.download_size and offset > self.current_size: + # Normally writing at an offset beyond the current end-of-file + # would leave a hole that appears filled with zeroes. However, an + # EncryptedTemporaryFile doesn't behave like that (if there is a + # hole in the file on disk, the zeroes that are read back will be + # XORed with the keystream). So we must explicitly write zeroes in + # the gap between the current EOF and the offset. + + self.f.seek(self.current_size) + self.f.write("\x00" * (offset - self.current_size)) + else: + self.f.seek(offset) + self.f.write(data) + end = offset + len(data) + self.current_size = max(self.current_size, end) + if end > self.downloaded: + heapq.heappush(self.overwrites, (offset, end)) + + def read(self, offset, length): + """When the data has been read, callback the Deferred that we return with this data. + Otherwise errback the Deferred that we return. + The caller must perform no more overwrites until the Deferred has fired.""" + + if debug: print "read(%r, %r), current_size = %r" % (offset, length, self.current_size) + if offset >= self.current_size: + def _eof(): raise EOFError("read past end of file") + return defer.execute(_eof) + + if offset + length > self.current_size: + length = self.current_size - offset + + needed = min(offset + length, self.download_size) + d = self.when_reached(needed) + def _reached(ign): + # It is not necessarily the case that self.downloaded >= needed, because + # the file might have been truncated (thus truncating the download) and + # then extended. + + assert self.current_size >= offset + length, (self.current_size, offset, length) + if debug: print "!!! %r" % (self.f,) + self.f.seek(offset) + return self.f.read(length) + d.addCallback(_reached) + return d + + def when_reached(self, index): + if debug: print "when_reached(%r)" % (index,) + if index <= self.downloaded: # already reached + if debug: print "already reached %r" % (index,) + return defer.succeed(None) + d = defer.Deferred() + def _reached(ign): + if debug: print "reached %r" % (index,) + return ign + d.addCallback(_reached) + heapq.heappush(self.milestones, (index, d)) + return d + + def when_done(self): + return self.done + + def finish(self): + while len(self.milestones) > 0: + (next, d) = self.milestones[0] + if debug: print "MILESTONE FINISH %r %r" % (next, d) + heapq.heappop(self.milestones) + # The callback means that the milestone has been reached if + # it is ever going to be. Note that the file may have been + # truncated to before the milestone. + eventually_callback(d)(None) + + # FIXME: causes spurious failures + #self.unregisterProducer() + + def close(self): + self.finish() + self.f.close() + + def unregisterProducer(self): + if self.producer: + self.producer.stopProducing() + self.producer = None + + +SIZE_THRESHOLD = 1000 + +def _make_sftp_file(check_abort, flags, convergence, parent=None, childname=None, filenode=None, metadata=None): + if not (flags & (FXF_WRITE | FXF_CREAT)) and (flags & FXF_READ) and filenode and \ + not filenode.is_mutable() and filenode.get_size() <= SIZE_THRESHOLD: + return ShortReadOnlySFTPFile(filenode, metadata) + else: + return GeneralSFTPFile(check_abort, flags, convergence, + parent=parent, childname=childname, filenode=filenode, metadata=metadata) + + +class ShortReadOnlySFTPFile: implements(ISFTPFile) hunk ./src/allmydata/frontends/sftpd.py 453 - def __init__(self, node): - self.node = node + """I represent a file handle to a particular file on an SFTP connection. + I am used only for short immutable files opened in read-only mode. + The file contents are downloaded to memory when I am created.""" + + def __init__(self, filenode, metadata): + assert IFileNode.providedBy(filenode), filenode + self.filenode = filenode + self.metadata = metadata + self.async = download_to_data(filenode) + self.closed = False + def readChunk(self, offset, length): hunk ./src/allmydata/frontends/sftpd.py 465 - d = download_to_data(self.node, offset, length) - def _got(data): + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle") + return defer.execute(_closed) + + d = defer.Deferred() + def _read(data): + if debug: print "_read(%r) in readChunk(%r, %r)" % (data, offset, length) + + # "In response to this request, the server will read as many bytes as it + # can from the file (up to 'len'), and return them in a SSH_FXP_DATA + # message. If an error occurs or EOF is encountered before reading any + # data, the server will respond with SSH_FXP_STATUS. For normal disk + # files, it is guaranteed that this will read the specified number of + # bytes, or up to end of file." + # + # i.e. we respond with an EOF error iff offset is already at EOF. + + if offset >= len(data): + eventually_errback(d)(SFTPError(FX_EOF, "read at or past end of file")) + else: + eventually_callback(d)(data[offset:min(offset+length, len(data))]) return data hunk ./src/allmydata/frontends/sftpd.py 487 - d.addCallback(_got) + self.async.addCallbacks(_read, eventually_errback(d)) return d hunk ./src/allmydata/frontends/sftpd.py 489 + + def writeChunk(self, offset, data): + def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing") + return defer.execute(_denied) + def close(self): hunk ./src/allmydata/frontends/sftpd.py 495 - pass + self.closed = True + return defer.succeed(None) + def getAttrs(self): hunk ./src/allmydata/frontends/sftpd.py 499 - print "GETATTRS(file)" - raise NotImplementedError + if debug: print "GETATTRS(file)" + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle") + return defer.execute(_closed) + + return defer.succeed(_populate_attrs(self.filenode, self.metadata, False)) + def setAttrs(self, attrs): hunk ./src/allmydata/frontends/sftpd.py 507 - print "SETATTRS(file)", attrs - raise NotImplementedError + if debug: print "SETATTRS(file) %r" % (attrs,) + def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing") + return defer.execute(_denied) + hunk ./src/allmydata/frontends/sftpd.py 512 -class WriteFile: +class GeneralSFTPFile: implements(ISFTPFile) hunk ./src/allmydata/frontends/sftpd.py 514 + """I represent a file handle to a particular file on an SFTP connection. + I wrap an instance of OverwriteableFileConsumer, which is responsible for + storing the file contents. In order to allow write requests to be satisfied + immediately, there is effectively a FIFO queue between requests made to this + file handle, and requests to my OverwriteableFileConsumer. This queue is + implemented by the callback chain of self.async.""" hunk ./src/allmydata/frontends/sftpd.py 521 - def __init__(self, parent, childname, convergence): + def __init__(self, check_abort, flags, convergence, parent=None, childname=None, filenode=None, metadata=None): + self.check_abort = check_abort + self.flags = flags + self.convergence = convergence self.parent = parent self.childname = childname hunk ./src/allmydata/frontends/sftpd.py 527 - self.convergence = convergence - self.f = tempfile.TemporaryFile() + self.filenode = filenode + self.metadata = metadata + self.async = defer.succeed(None) + self.closed = False + + # self.consumer should only be relied on in callbacks for self.async, since it might + # not be set before then. + self.consumer = None + + if (flags & FXF_TRUNC) or not filenode: + # We're either truncating or creating the file, so we don't need the old contents. + assert flags & FXF_CREAT, flags + self.consumer = OverwriteableFileConsumer(self.check_abort, 0, + tempfile_maker=EncryptedTemporaryFile) + self.consumer.finish() + else: + assert IFileNode.providedBy(filenode), filenode + + # TODO: use download interface described in #993 when implemented. + if filenode.is_mutable(): + self.async.addCallback(lambda ign: filenode.download_best_version()) + def _downloaded(data): + self.consumer = OverwriteableFileConsumer(self.check_abort, len(data), + tempfile_maker=tempfile.TemporaryFile) + self.consumer.write(data) + self.consumer.finish() + return None + self.async.addCallback(_downloaded) + else: + download_size = filenode.get_size() + assert download_size is not None + self.consumer = OverwriteableFileConsumer(self.check_abort, download_size, + tempfile_maker=tempfile.TemporaryFile) + self.async.addCallback(lambda ign: filenode.read(self.consumer, 0, None)) + + + def readChunk(self, offset, length): + if not (self.flags & FXF_READ): + return defer.fail(SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for reading")) + + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle") + return defer.execute(_closed) + + d = defer.Deferred() + def _read(ign): + if debug: print "_read in readChunk(%r, %r)" % (offset, length) + d2 = self.consumer.read(offset, length) + d2.addErrback(_raise_error) + d2.addCallbacks(eventually_callback(d), eventually_errback(d)) + # It is correct to drop d2 here. + return None + self.async.addCallbacks(_read, eventually_errback(d)) + return d + def writeChunk(self, offset, data): hunk ./src/allmydata/frontends/sftpd.py 583 - self.f.seek(offset) - self.f.write(data) + if debug: print "writeChunk(%r, %r)" % (offset, data) + if not (self.flags & FXF_WRITE): + def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing") + return defer.execute(_denied) + + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot write to a closed file handle") + return defer.execute(_closed) + + # Note that we return without waiting for the write to occur. Reads and + # close wait for prior writes, and will fail if any prior operation failed. + # This is ok because SFTP makes no guarantee that the request completes + # before the write. In fact it explicitly allows write errors to be delayed + # until close: + # "One should note that on some server platforms even a close can fail. + # This can happen e.g. if the server operating system caches writes, + # and an error occurs while flushing cached writes during the close." + + def _write(ign): + # FXF_APPEND means that we should always write at the current end of file. + write_offset = offset + if self.flags & FXF_APPEND: + write_offset = self.consumer.get_current_size() + + self.consumer.overwrite(write_offset, data) + return None + self.async.addCallback(_write) + # don't addErrback to self.async, just allow subsequent async ops to fail. + return defer.succeed(None) def close(self): hunk ./src/allmydata/frontends/sftpd.py 614 - u = FileHandle(self.f, self.convergence) - d = self.parent.add_file(self.childname, u) + if self.closed: + return defer.succeed(None) + + # This means that close has been called, not that the close has succeeded. + self.closed = True + + if not (self.flags & (FXF_WRITE | FXF_CREAT)): + return defer.execute(self.consumer.close) + + def _close(ign): + d2 = self.consumer.when_done() + if self.filenode and self.filenode.is_mutable(): + d2.addCallback(lambda ign: self.consumer.get_current_size()) + d2.addCallback(lambda size: self.consumer.read(0, size)) + d2.addCallback(lambda new_contents: self.filenode.overwrite(new_contents)) + else: + def _add_file(ign): + u = FileHandle(self.consumer.get_file(), self.convergence) + return self.parent.add_file(self.childname, u) + d2.addCallback(_add_file) + + d2.addCallback(lambda ign: self.consumer.close()) + return d2 + self.async.addCallback(_close) + + d = defer.Deferred() + self.async.addCallbacks(eventually_callback(d), eventually_errback(d)) return d def getAttrs(self): hunk ./src/allmydata/frontends/sftpd.py 644 - print "GETATTRS(file)" - raise NotImplementedError + if debug: print "GETATTRS(file)" + + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle") + return defer.execute(_closed) + + # Optimization for read-only handles, when we already know the metadata. + if not(self.flags & (FXF_WRITE | FXF_CREAT)) and self.metadata and self.filenode and not self.filenode.is_mutable(): + return defer.succeed(_populate_attrs(self.filenode, self.metadata, False)) + + d = defer.Deferred() + def _get(ign): + # FIXME: pass correct value for writeable + # self.filenode might be None, but that's ok. + attrs = _populate_attrs(self.filenode, self.metadata, False, + size=self.consumer.get_current_size()) + eventually_callback(d)(attrs) + return None + self.async.addCallbacks(_get, eventually_errback(d)) + return d + def setAttrs(self, attrs): hunk ./src/allmydata/frontends/sftpd.py 666 - print "SETATTRS(file)", attrs - raise NotImplementedError + if debug: print "SETATTRS(file) %r" % (attrs,) + if not (self.flags & FXF_WRITE): + def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing") + return defer.execute(_denied) hunk ./src/allmydata/frontends/sftpd.py 671 + if self.closed: + def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot set attributes for a closed file handle") + return defer.execute(_closed) hunk ./src/allmydata/frontends/sftpd.py 675 -class NoParentError(Exception): - pass + if not "size" in attrs: + return defer.succeed(None) hunk ./src/allmydata/frontends/sftpd.py 678 -class PermissionError(Exception): - pass + size = attrs["size"] + if not isinstance(size, (int, long)) or size < 0: + def _bad(): raise SFTPError(FX_BAD_MESSAGE, "new size is not a valid nonnegative integer") + return defer.execute(_bad) hunk ./src/allmydata/frontends/sftpd.py 683 -from twisted.conch.ssh.filetransfer import FileTransferServer, SFTPError, \ - FX_NO_SUCH_FILE, FX_FILE_ALREADY_EXISTS, FX_OP_UNSUPPORTED, \ - FX_PERMISSION_DENIED -from twisted.conch.ssh.filetransfer import FXF_READ, FXF_WRITE, FXF_APPEND, FXF_CREAT, FXF_TRUNC, FXF_EXCL + d = defer.Deferred() + def _resize(ign): + self.consumer.set_current_size(size) + eventually_callback(d)(None) + return None + self.async.addCallbacks(_resize, eventually_errback(d)) + return d class SFTPUser(ConchUser): hunk ./src/allmydata/frontends/sftpd.py 692 - def __init__(self, client, rootnode, username, convergence): + def __init__(self, check_abort, client, rootnode, username, convergence): ConchUser.__init__(self) self.channelLookup["session"] = session.SSHSession self.subsystemLookup["sftp"] = FileTransferServer hunk ./src/allmydata/frontends/sftpd.py 697 + self.check_abort = check_abort self.client = client self.root = rootnode self.username = username hunk ./src/allmydata/frontends/sftpd.py 712 def close(self): pass -class FakeStat: - pass +import array +import stat + +from time import time, strftime, localtime + +def lsLine(name, attrs): + st_uid = "tahoe" + st_gid = "tahoe" + st_mtime = attrs.get("mtime", 0) + st_mode = attrs["permissions"] + # TODO: check that clients are okay with this being a "?". + # (They should be because the longname is intended for human + # consumption.) + st_size = attrs.get("size", "?") + # We don't know how many links there really are to this object. + st_nlink = 1 + + # From . + # We can't call the version in Twisted because we might have a version earlier than + # (released in Twisted 8.2). + + mode = st_mode + perms = array.array('c', '-'*10) + ft = stat.S_IFMT(mode) + if stat.S_ISDIR(ft): perms[0] = 'd' + elif stat.S_ISCHR(ft): perms[0] = 'c' + elif stat.S_ISBLK(ft): perms[0] = 'b' + elif stat.S_ISREG(ft): perms[0] = '-' + elif stat.S_ISFIFO(ft): perms[0] = 'f' + elif stat.S_ISLNK(ft): perms[0] = 'l' + elif stat.S_ISSOCK(ft): perms[0] = 's' + else: perms[0] = '?' + # user + if mode&stat.S_IRUSR:perms[1] = 'r' + if mode&stat.S_IWUSR:perms[2] = 'w' + if mode&stat.S_IXUSR:perms[3] = 'x' + # group + if mode&stat.S_IRGRP:perms[4] = 'r' + if mode&stat.S_IWGRP:perms[5] = 'w' + if mode&stat.S_IXGRP:perms[6] = 'x' + # other + if mode&stat.S_IROTH:perms[7] = 'r' + if mode&stat.S_IWOTH:perms[8] = 'w' + if mode&stat.S_IXOTH:perms[9] = 'x' + # suid/sgid never set + + l = perms.tostring() + l += str(st_nlink).rjust(5) + ' ' + un = str(st_uid) + l += un.ljust(9) + gr = str(st_gid) + l += gr.ljust(9) + sz = str(st_size) + l += sz.rjust(8) + l += ' ' + sixmo = 60 * 60 * 24 * 7 * 26 + if st_mtime + sixmo < time(): # last edited more than 6mo ago + l += strftime("%b %d %Y ", localtime(st_mtime)) + else: + l += strftime("%b %d %H:%M ", localtime(st_mtime)) + l += name + return l hunk ./src/allmydata/frontends/sftpd.py 775 -class BadRemoveRequest(Exception): - pass class SFTPHandler: implements(ISFTPServer) hunk ./src/allmydata/frontends/sftpd.py 779 def __init__(self, user): - print "Creating SFTPHandler from", user + if debug: print "Creating SFTPHandler from", user + self.check_abort = user.check_abort self.client = user.client self.root = user.root self.username = user.username hunk ./src/allmydata/frontends/sftpd.py 787 self.convergence = user.convergence def gotVersion(self, otherVersion, extData): + if debug: print "GOTVERSION %r %r" % (otherVersion, extData) return {} hunk ./src/allmydata/frontends/sftpd.py 790 - def openFile(self, filename, flags, attrs): - f = "|".join([f for f in - [(flags & FXF_READ) and "FXF_READ" or None, - (flags & FXF_WRITE) and "FXF_WRITE" or None, - (flags & FXF_APPEND) and "FXF_APPEND" or None, - (flags & FXF_CREAT) and "FXF_CREAT" or None, - (flags & FXF_TRUNC) and "FXF_TRUNC" or None, - (flags & FXF_EXCL) and "FXF_EXCL" or None, - ] - if f]) - print "OPENFILE", filename, flags, f, attrs + def openFile(self, pathstring, flags, attrs): + if debug: print "OPENFILE %r %r %r %r" % (pathstring, flags, _repr_flags(flags), attrs) # this is used for both reading and writing. hunk ./src/allmydata/frontends/sftpd.py 794 -# createPlease = False -# exclusive = False -# openFlags = 0 -# -# if flags & FXF_READ == FXF_READ and flags & FXF_WRITE == 0: -# openFlags = os.O_RDONLY -# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == 0: -# createPlease = True -# openFlags = os.O_WRONLY -# if flags & FXF_WRITE == FXF_WRITE and flags & FXF_READ == FXF_READ: -# createPlease = True -# openFlags = os.O_RDWR -# if flags & FXF_APPEND == FXF_APPEND: -# createPlease = True -# openFlags |= os.O_APPEND -# if flags & FXF_CREAT == FXF_CREAT: -# createPlease = True -# openFlags |= os.O_CREAT -# if flags & FXF_TRUNC == FXF_TRUNC: -# openFlags |= os.O_TRUNC -# if flags & FXF_EXCL == FXF_EXCL: -# exclusive = True + # First exclude invalid combinations of flags. # /usr/bin/sftp 'get' gives us FXF_READ, while 'put' on a new file hunk ./src/allmydata/frontends/sftpd.py 797 - # gives FXF_WRITE,FXF_CREAT,FXF_TRUNC . I'm guessing that 'put' on an + # gives FXF_WRITE | FXF_CREAT | FXF_TRUNC. I'm guessing that 'put' on an # existing file gives the same. hunk ./src/allmydata/frontends/sftpd.py 800 - path = self._convert_sftp_path(filename) + if not (flags & (FXF_READ | FXF_WRITE)): + raise SFTPError(FX_BAD_MESSAGE, + "invalid file open flags: at least one of FXF_READ and FXF_WRITE must be set") hunk ./src/allmydata/frontends/sftpd.py 804 - if flags & FXF_READ: - if flags & FXF_WRITE: - raise NotImplementedError - d = self._get_node_and_metadata_for_path(path) - d.addCallback(lambda (node,metadata): ReadFile(node)) - d.addErrback(self._convert_error) - return d + if not (flags & FXF_CREAT): + if flags & FXF_TRUNC: + raise SFTPError(FX_BAD_MESSAGE, + "invalid file open flags: FXF_TRUNC cannot be set without FXF_CREAT") + if flags & FXF_EXCL: + raise SFTPError(FX_BAD_MESSAGE, + "invalid file open flags: FXF_EXCL cannot be set without FXF_CREAT") + + path = self._path_from_string(pathstring) + if not path: + raise SFTPError(FX_NO_SUCH_FILE, "path cannot be empty") hunk ./src/allmydata/frontends/sftpd.py 816 - if flags & FXF_WRITE: - if not (flags & FXF_CREAT) or not (flags & FXF_TRUNC): - raise NotImplementedError + # The combination of flags is potentially valid. Now there are two major cases: + # + # 1. The path is specified as /uri/FILECAP, with no parent directory. + # If the FILECAP is mutable and writeable, then we can open it in write-only + # or read/write mode (non-exclusively), otherwise we can only open it in + # read-only mode. The open should succeed immediately as long as FILECAP is + # a valid known filecap that grants the required permission. + # + # 2. The path is specified relative to a parent. We find the parent dirnode and + # get the child's URI and metadata if it exists. There are four subcases: + # a. the child does not exist: FXF_CREAT must be set, and we must be able + # to write to the parent directory. + # b. the child exists but is not a valid known filecap: fail + # c. the child is mutable: if we are trying to open it write-only or + # read/write, then we must be able to write to the file. + # d. the child is immutable: if we are trying to open it write-only or + # read/write, then we must be able to write to the parent directory. + # + # To reduce latency, open succeeds as soon as these conditions are met, even + # though there might be a failure in downloading the existing file or uploading + # a new one. + # + # Note that the permission checks below are for more precise error reporting on + # the open call; later operations would fail even if we did not make these checks. + + stash = {'parent': None} + d = self._get_root(path) + def _got_root((root, path)): + if root.is_unknown(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open an unknown cap (or child of an unknown directory). " + "Upgrading the gateway to a later Tahoe-LAFS version may help") if not path: hunk ./src/allmydata/frontends/sftpd.py 849 - raise PermissionError("cannot STOR to root directory") - childname = path[-1] - d = self._get_root(path) - def _got_root((root, path)): - if not path: - raise PermissionError("cannot STOR to root directory") - return root.get_child_at_path(path[:-1]) - d.addCallback(_got_root) - def _got_parent(parent): - return WriteFile(parent, childname, self.convergence) - d.addCallback(_got_parent) - return d - raise NotImplementedError + # case 1 + if not IFileNode.providedBy(root): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open a directory cap") + if (flags & FXF_WRITE) and root.is_readonly(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot write to a non-writeable filecap without a parent directory") + if flags & FXF_EXCL: + raise SFTPError(FX_PERMISSION_DENIED, + "cannot create a file exclusively when it already exists") + + return _make_sftp_file(self.check_abort, flags, self.convergence, filenode=root) + else: + # case 2 + childname = path[-1] + if debug: print "case 2: childname = %r, path[:-1] = %r" % (childname, path[:-1]) + d2 = root.get_child_at_path(path[:-1]) + def _got_parent(parent): + if debug: print "_got_parent(%r)" % (parent,) + stash['parent'] = parent + + if flags & FXF_EXCL: + # FXF_EXCL means that the link to the file (not the file itself) must + # be created atomically wrt updates by this storage client. + # That is, we need to create the link before returning success to the + # SFTP open request (and not just on close, as would normally be the + # case). We make the link initially point to a zero-length LIT file, + # which is consistent with what might happen on a POSIX filesystem. + + if parent.is_readonly(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot create a file exclusively when the parent directory is read-only") + + # 'overwrite=False' ensures failure if the link already exists. + # FIXME: should use a single call to set_uri and return (child, metadata) (#1035) + d3 = parent.set_uri(childname, None, "URI:LIT:", overwrite=False) + def _seturi_done(child): + stash['child'] = child + return parent.get_metadata_for(childname) + d3.addCallback(_seturi_done) + d3.addCallback(lambda metadata: (stash['child'], metadata)) + return d3 + else: + if debug: print "get_child_and_metadata" + return parent.get_child_and_metadata(childname) + d2.addCallback(_got_parent) + + def _got_child( (filenode, metadata) ): + if debug: print "_got_child((%r, %r))" % (filenode, metadata) + parent = stash['parent'] + if filenode.is_unknown(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open an unknown cap. Upgrading the gateway " + "to a later Tahoe-LAFS version may help") + if not IFileNode.providedBy(filenode): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open a directory as if it were a file") + if (flags & FXF_WRITE) and filenode.is_mutable() and filenode.is_readonly(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open a read-only mutable file for writing") + if (flags & FXF_WRITE) and parent.is_readonly(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot open a file for writing when the parent directory is read-only") + + return _make_sftp_file(self.check_abort, flags, self.convergence, parent=parent, + childname=childname, filenode=filenode, metadata=metadata) + def _no_child(f): + if debug: print "_no_child(%r)" % (f,) + f.trap(NoSuchChildError) + parent = stash['parent'] + if parent is None: + return f + if not (flags & FXF_CREAT): + raise SFTPError(FX_NO_SUCH_FILE, + "the file does not exist, and was not opened with the creation (CREAT) flag") + if parent.is_readonly(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot create a file when the parent directory is read-only") + + return _make_sftp_file(self.check_abort, flags, self.convergence, parent=parent, + childname=childname) + d2.addCallbacks(_got_child, _no_child) + return d2 + d.addCallback(_got_root) + d.addErrback(_raise_error) + return d + + def removeFile(self, pathstring): + if debug: print "REMOVEFILE %r" % (pathstring,) + path = self._path_from_string(pathstring) + return self._remove_object(path, must_be_file=True) hunk ./src/allmydata/frontends/sftpd.py 941 - def removeFile(self, path): - print "REMOVEFILE", path - path = self._convert_sftp_path(path) - return self._remove_thing(path, must_be_file=True) + def renameFile(self, oldpathstring, newpathstring): + if debug: print "RENAMEFILE %r %r" % (oldpathstring, newpathstring) + fromPath = self._path_from_string(oldpathstring) + toPath = self._path_from_string(newpathstring) hunk ./src/allmydata/frontends/sftpd.py 946 - def renameFile(self, oldpath, newpath): - print "RENAMEFILE", oldpath, newpath - fromPath = self._convert_sftp_path(oldpath) - toPath = self._convert_sftp_path(newpath) # the target directory must already exist hunk ./src/allmydata/frontends/sftpd.py 947 - d = self._get_parent(fromPath) - def _got_from_parent( (fromparent, childname) ): - d = self._get_parent(toPath) - d.addCallback(lambda (toparent, tochildname): - fromparent.move_child_to(childname, - toparent, tochildname, - overwrite=False)) + d = deferredutil.gatherResults([self._get_parent(fromPath), + self._get_parent(toPath)]) + def _got( (fromPair, toPair) ): + (fromParent, fromChildname) = fromPair + (toParent, toChildname) = toPair + + # + # "It is an error if there already exists a file with the name specified + # by newpath." + # FIXME: use move_child_to_path to avoid possible data loss due to #943 + d = fromParent.move_child_to(fromChildname, toParent, toChildname, overwrite=False) + #d = parent.move_child_to_path(fromChildname, toRoot, toPath[:-1], + # toPath[-1], overwrite=False) return d hunk ./src/allmydata/frontends/sftpd.py 961 - d.addCallback(_got_from_parent) - d.addErrback(self._convert_error) + d.addCallback(_got) + d.addErrback(_raise_error) return d hunk ./src/allmydata/frontends/sftpd.py 965 - def makeDirectory(self, path, attrs): - print "MAKEDIRECTORY", path, attrs - # TODO: extract attrs["mtime"], use it to set the parent metadata. - # Maybe also copy attrs["ext_*"] . - path = self._convert_sftp_path(path) + def makeDirectory(self, pathstring, attrs): + if debug: print "MAKEDIRECTORY %r %r" % (pathstring, attrs) + path = self._path_from_string(pathstring) + metadata = self._attrs_to_metadata(attrs) d = self._get_root(path) d.addCallback(lambda (root,path): hunk ./src/allmydata/frontends/sftpd.py 971 - self._get_or_create_directories(root, path)) + self._get_or_create_directories(root, path, metadata)) + d.addErrback(_raise_error) return d hunk ./src/allmydata/frontends/sftpd.py 975 - def _get_or_create_directories(self, node, path): + def _get_or_create_directories(self, node, path, metadata): if not IDirectoryNode.providedBy(node): # unfortunately it is too late to provide the name of the hunk ./src/allmydata/frontends/sftpd.py 978 - # blocking directory in the error message. - raise ExistingChildError("cannot create directory because there " - "is a file in the way") # close enough + # blocking file in the error message. + raise SFTPError(FX_PERMISSION_DENIED, + "cannot create directory because there " + "is a file in the way") # close enough if not path: return defer.succeed(node) d = node.get(path[0]) hunk ./src/allmydata/frontends/sftpd.py 989 f.trap(NoSuchChildError) return node.create_subdirectory(path[0]) d.addErrback(_maybe_create) - d.addCallback(self._get_or_create_directories, path[1:]) + d.addCallback(self._get_or_create_directories, path[1:], metadata) + d.addErrback(_raise_error) return d hunk ./src/allmydata/frontends/sftpd.py 993 - def removeDirectory(self, path): - print "REMOVEDIRECTORY", path - path = self._convert_sftp_path(path) - return self._remove_thing(path, must_be_directory=True) + def removeDirectory(self, pathstring): + if debug: print "REMOVEDIRECTORY %r" % (pathstring,) + path = self._path_from_string(pathstring) + return self._remove_object(path, must_be_directory=True) hunk ./src/allmydata/frontends/sftpd.py 998 - def _remove_thing(self, path, must_be_directory=False, must_be_file=False): + def _remove_object(self, path, must_be_directory=False, must_be_file=False): d = defer.maybeDeferred(self._get_parent, path) hunk ./src/allmydata/frontends/sftpd.py 1000 - def _convert_error(f): - f.trap(NoParentError) - raise PermissionError("cannot delete root directory") - d.addErrback(_convert_error) def _got_parent( (parent, childname) ): hunk ./src/allmydata/frontends/sftpd.py 1001 - d = parent.get(childname) + d2 = parent.get(childname) def _got_child(child): hunk ./src/allmydata/frontends/sftpd.py 1003 - if must_be_directory and not IDirectoryNode.providedBy(child): - raise BadRemoveRequest("rmdir called on a file") + # Unknown children can be removed by either removeFile or removeDirectory. + if must_be_directory and IFileNode.providedBy(child): + raise SFTPError(FX_PERMISSION_DENIED, "rmdir called on a file") if must_be_file and IDirectoryNode.providedBy(child): hunk ./src/allmydata/frontends/sftpd.py 1007 - raise BadRemoveRequest("rmfile called on a directory") + raise SFTPError(FX_PERMISSION_DENIED, "rmfile called on a directory") return parent.delete(childname) hunk ./src/allmydata/frontends/sftpd.py 1009 - d.addCallback(_got_child) - d.addErrback(self._convert_error) - return d + d2.addCallback(_got_child) + return d2 d.addCallback(_got_parent) hunk ./src/allmydata/frontends/sftpd.py 1012 + d.addErrback(_raise_error) return d hunk ./src/allmydata/frontends/sftpd.py 1015 - - def openDirectory(self, path): - print "OPENDIRECTORY", path - path = self._convert_sftp_path(path) + def openDirectory(self, pathstring): + if debug: print "OPENDIRECTORY %r" % (pathstring,) + path = self._path_from_string(pathstring) + if debug: print " PATH %r" % (path,) d = self._get_node_and_metadata_for_path(path) hunk ./src/allmydata/frontends/sftpd.py 1020 - d.addCallback(lambda (dirnode,metadata): dirnode.list()) - def _render(children): - results = [] - for filename, (node, metadata) in children.iteritems(): - s = FakeStat() - if IDirectoryNode.providedBy(node): - s.st_mode = 040700 - s.st_size = 0 - else: - s.st_mode = 0100600 - s.st_size = node.get_size() - s.st_nlink = 1 - s.st_uid = 0 - s.st_gid = 0 - s.st_mtime = int(metadata.get("mtime", 0)) - longname = ls.lsLine(filename.encode("utf-8"), s) - attrs = self._populate_attrs(node, metadata) - results.append( (filename.encode("utf-8"), longname, attrs) ) - return StoppableList(results) - d.addCallback(_render) + def _list( (dirnode, metadata) ): + if dirnode.is_unknown(): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot list an unknown cap as a directory. Upgrading the gateway " + "to a later Tahoe-LAFS version may help") + if not IDirectoryNode.providedBy(dirnode): + raise SFTPError(FX_PERMISSION_DENIED, + "cannot list a file as if it were a directory") + d2 = dirnode.list() + def _render(children): + parent_writeable = not dirnode.is_readonly() + results = [] + for filename, (node, metadata) in children.iteritems(): + # The file size may be cached or absent. + writeable = parent_writeable and (node.is_unknown() or + not (node.is_mutable() and node.is_readonly())) + attrs = _populate_attrs(node, metadata, writeable) + filename_utf8 = filename.encode('utf-8') + longname = lsLine(filename_utf8, attrs) + results.append( (filename_utf8, longname, attrs) ) + return StoppableList(results) + d2.addCallback(_render) + return d2 + d.addCallback(_list) + d.addErrback(_raise_error) return d hunk ./src/allmydata/frontends/sftpd.py 1047 - def getAttrs(self, path, followLinks): - print "GETATTRS", path, followLinks - # from ftp.stat - d = self._get_node_and_metadata_for_path(self._convert_sftp_path(path)) - def _render((node,metadata)): - return self._populate_attrs(node, metadata) + def getAttrs(self, pathstring, followLinks): + if debug: print "GETATTRS %r %r" % (pathstring, followLinks) + d = self._get_node_and_metadata_for_path(self._path_from_string(pathstring)) + def _render( (node, metadata) ): + # When asked about a specific file, report its current size. + # TODO: the modification time for a mutable file should be + # reported as the update time of the best version. But that + # information isn't currently stored in mutable shares, I think. + d2 = node.get_current_size() + def _got_size(size): + # FIXME: pass correct value for writeable + attrs = _populate_attrs(node, metadata, False, size=size) + return attrs + d2.addCallback(_got_size) + return d2 d.addCallback(_render) hunk ./src/allmydata/frontends/sftpd.py 1063 - d.addErrback(self._convert_error) + d.addErrback(_raise_error) def _done(res): hunk ./src/allmydata/frontends/sftpd.py 1065 - print " DONE", res + if debug: print " DONE %r" % (res,) return res d.addBoth(_done) return d hunk ./src/allmydata/frontends/sftpd.py 1070 - def _convert_sftp_path(self, pathstring): - assert pathstring[0] == "/" + def setAttrs(self, pathstring, attrs): + if debug: print "SETATTRS %r %r" % (pathstring, attrs) + if "size" in attrs: + # this would require us to download and re-upload the truncated/extended + # file contents + raise SFTPError(FX_OP_UNSUPPORTED, "setAttrs wth size attribute") + return None + + def readLink(self, pathstring): + if debug: print "READLINK %r" % (pathstring,) + raise SFTPError(FX_OP_UNSUPPORTED, "readLink") + + def makeLink(self, linkPathstring, targetPathstring): + if debug: print "MAKELINK %r %r" % (linkPathstring, targetPathstring) + raise SFTPError(FX_OP_UNSUPPORTED, "makeLink") + + def extendedRequest(self, extendedName, extendedData): + if debug: print "EXTENDEDREQUEST %r %r" % (extendedName, extendedData) + # Client 'df' command requires 'statvfs@openssh.com' extension + # (but there's little point to implementing that since we would only + # have faked values to report). + raise SFTPError(FX_OP_UNSUPPORTED, "extendedRequest %r" % extendedName) + + def realPath(self, pathstring): + if debug: print "REALPATH %r" % (pathstring,) + return "/" + "/".join(self._path_from_string(pathstring)) + + def _path_from_string(self, pathstring): + if debug: print "CONVERT %r" % (pathstring,) + + # The home directory is the root directory. pathstring = pathstring.strip("/") hunk ./src/allmydata/frontends/sftpd.py 1102 - if pathstring == "": - path = [] + if pathstring == "" or pathstring == ".": + path_utf8 = [] else: hunk ./src/allmydata/frontends/sftpd.py 1105 - path = pathstring.split("/") - print "CONVERT", pathstring, path - path = [unicode(p) for p in path] + path_utf8 = pathstring.split("/") + + # + # "Servers SHOULD interpret a path name component ".." as referring to + # the parent directory, and "." as referring to the current directory." + path = [] + for p_utf8 in path_utf8: + if p_utf8 == "..": + # ignore excess .. components at the root + if len(path) > 0: + path = path[:-1] + elif p_utf8 != ".": + try: + p = p_utf8.decode('utf-8', 'strict') + except UnicodeError: + raise SFTPError(FX_NO_SUCH_FILE, "path could not be decoded as UTF-8") + path.append(p) + + if debug: print " PATH %r" % (path,) return path def _get_node_and_metadata_for_path(self, path): hunk ./src/allmydata/frontends/sftpd.py 1128 d = self._get_root(path) - def _got_root((root,path)): - print "ROOT", root - print "PATH", path + def _got_root( (root, path) ): + if debug: print " ROOT %r" % (root,) + if debug: print " PATH %r" % (path,) if path: return root.get_child_and_metadata_at_path(path) else: hunk ./src/allmydata/frontends/sftpd.py 1140 def _get_root(self, path): # return (root, remaining_path) - path = [unicode(p) for p in path] - if path and path[0] == "uri": + if path and path[0] == u"uri": d = defer.maybeDeferred(self.client.create_node_from_uri, str(path[1])) d.addCallback(lambda root: (root, path[2:])) hunk ./src/allmydata/frontends/sftpd.py 1148 d = defer.succeed((self.root,path)) return d - def _populate_attrs(self, childnode, metadata): - attrs = {} - attrs["uid"] = 1000 - attrs["gid"] = 1000 - attrs["atime"] = 0 - attrs["mtime"] = int(metadata.get("mtime", 0)) - isdir = bool(IDirectoryNode.providedBy(childnode)) - if isdir: - attrs["size"] = 1 - # the permissions must have the extra bits (040000 or 0100000), - # otherwise the client will not call openDirectory - attrs["permissions"] = 040700 # S_IFDIR - else: - attrs["size"] = childnode.get_size() - attrs["permissions"] = 0100600 # S_IFREG - return attrs - - def _convert_error(self, f): - if f.check(NoSuchChildError): - childname = f.value.args[0].encode("utf-8") - raise SFTPError(FX_NO_SUCH_FILE, childname) - if f.check(ExistingChildError): - msg = f.value.args[0].encode("utf-8") - raise SFTPError(FX_FILE_ALREADY_EXISTS, msg) - if f.check(PermissionError): - raise SFTPError(FX_PERMISSION_DENIED, str(f.value)) - if f.check(NotImplementedError): - raise SFTPError(FX_OP_UNSUPPORTED, str(f.value)) - return f - - - def setAttrs(self, path, attrs): - print "SETATTRS", path, attrs - # ignored - return None - - def readLink(self, path): - print "READLINK", path - raise NotImplementedError - - def makeLink(self, linkPath, targetPath): - print "MAKELINK", linkPath, targetPath - raise NotImplementedError - - def extendedRequest(self, extendedName, extendedData): - print "EXTENDEDREQUEST", extendedName, extendedData - # client 'df' command requires 'statvfs@openssh.com' extension - raise NotImplementedError - def realPath(self, path): - print "REALPATH", path - if path == ".": - return "/" - return path - - def _get_parent(self, path): # fire with (parentnode, childname) hunk ./src/allmydata/frontends/sftpd.py 1150 - path = [unicode(p) for p in path] if not path: hunk ./src/allmydata/frontends/sftpd.py 1151 - raise NoParentError + def _nosuch(): raise SFTPError(FX_NO_SUCH_FILE, "path does not exist") + return defer.execute(_nosuch) + childname = path[-1] hunk ./src/allmydata/frontends/sftpd.py 1155 + assert isinstance(childname, unicode), repr(childname) d = self._get_root(path) hunk ./src/allmydata/frontends/sftpd.py 1157 - def _got_root((root, path)): + def _got_root( (root, path) ): if not path: hunk ./src/allmydata/frontends/sftpd.py 1159 - raise NoParentError + raise SFTPError(FX_NO_SUCH_FILE, "path does not exist") return root.get_child_at_path(path[:-1]) d.addCallback(_got_root) def _got_parent(parent): hunk ./src/allmydata/frontends/sftpd.py 1167 d.addCallback(_got_parent) return d + def _attrs_to_metadata(self, attrs): + metadata = {} + + for key in attrs: + if key == "mtime" or key == "ctime" or key == "createtime": + metadata[key] = long(attrs[key]) + elif key.startswith("ext_"): + metadata[key] = str(attrs[key]) + + return metadata + # if you have an SFTPUser, and you want something that provides ISFTPServer, # then you get SFTPHandler(user) hunk ./src/allmydata/frontends/sftpd.py 1183 components.registerAdapter(SFTPHandler, SFTPUser, ISFTPServer) -from auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme +from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme class Dispatcher: implements(portal.IRealm) hunk ./src/allmydata/frontends/sftpd.py 1194 assert interface == IConchUser rootnode = self.client.create_node_from_uri(avatarID.rootcap) convergence = self.client.convergence - s = SFTPUser(self.client, rootnode, avatarID.username, convergence) - def logout(): pass + logged_out = {'flag': False} + def check_abort(): + return logged_out['flag'] + def logout(): + logged_out['flag'] = True + s = SFTPUser(check_abort, self.client, rootnode, avatarID.username, convergence) return (interface, s, logout) class SFTPServer(service.MultiService): } [Tests for new SFTP implementation david-sarah@jacaranda.org**20100512060552 Ignore-this: 20308d4a59b3ebc868aad55ae0a7a981 ] { addfile ./src/allmydata/test/test_sftp.py hunk ./src/allmydata/test/test_sftp.py 1 + +import re +from stat import S_IFREG, S_IFDIR + +from twisted.trial import unittest +from twisted.internet import defer +from twisted.python.failure import Failure + +sftp = None +sftpd = None +have_pycrypto = False +try: + from Crypto import Util + Util # hush pyflakes + have_pycrypto = True +except ImportError: + pass + +if have_pycrypto: + from twisted.conch.ssh import filetransfer as sftp + from allmydata.frontends import sftpd + +# FIXME remove this +#from twisted.internet.base import DelayedCall +#DelayedCall.debug = True + +import sys, traceback + +""" +def trace_exceptions(frame, event, arg): + if event != 'exception': + return + co = frame.f_code + func_name = co.co_name + line_no = frame.f_lineno + filename = co.co_filename + exc_type, exc_value, exc_traceback = arg + print 'Tracing exception: %r %r on line %r of %r in %r' % \ + (exc_type.__name__, exc_value, line_no, func_name, filename) + +def trace_calls(frame, event, arg): + if event != 'call': + return + return trace_exceptions + +sys.settrace(trace_calls) +""" + +timeout = 30 + +from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError +from allmydata.mutable.common import NotWriteableError + +from allmydata.util.consumer import download_to_data +from allmydata.immutable import upload +from allmydata.test.no_network import GridTestMixin +from allmydata.test.common import ShouldFailMixin + +class Handler(GridTestMixin, ShouldFailMixin, unittest.TestCase): + """This is a no-network unit test of the SFTPHandler class.""" + + if not have_pycrypto: + skip = "SFTP support requires pycrypto, which is not installed" + + def shouldFailWithSFTPError(self, expected_code, which, callable, *args, **kwargs): + assert isinstance(expected_code, int), repr(expected_code) + assert isinstance(which, str), repr(which) + s = traceback.format_stack() + d = defer.maybeDeferred(callable, *args, **kwargs) + def _done(res): + if isinstance(res, Failure): + res.trap(sftp.SFTPError) + self.failUnlessEqual(res.value.code, expected_code, + "%s was supposed to raise SFTPError(%d), not SFTPError(%d): %s" % + (which, expected_code, res.value.code, res)) + else: + print '@' + '@'.join(s) + self.fail("%s was supposed to raise SFTPError(%d), not get '%s'" % + (which, expected_code, res)) + d.addBoth(_done) + return d + + def _set_up(self, basedir, num_clients=1, num_servers=10): + self.basedir = "sftp/" + basedir + self.set_up_grid(num_clients=num_clients, num_servers=num_servers) + + def check_abort(): + pass + self.client = self.g.clients[0] + self.username = "alice" + self.convergence = "convergence" + + d = self.client.create_dirnode() + def _created_root(node): + self.root = node + self.root_uri = node.get_uri() + self.user = sftpd.SFTPUser(check_abort, self.client, self.root, self.username, self.convergence) + self.handler = sftpd.SFTPHandler(self.user) + d.addCallback(_created_root) + return d + + def _set_up_tree(self): + d = self.client.create_mutable_file("mutable file contents") + d.addCallback(lambda node: self.root.set_node(u"mutable", node)) + def _created_mutable(n): + self.mutable = n + self.mutable_uri = n.get_uri() + d.addCallback(_created_mutable) + + d.addCallback(lambda ign: + self.root._create_and_validate_node(None, self.mutable.get_readonly_uri(), name=u"readonly")) + d.addCallback(lambda node: self.root.set_node(u"readonly", node)) + def _created_readonly(n): + self.readonly = n + self.readonly_uri = n.get_uri() + d.addCallback(_created_readonly) + + gross = upload.Data("0123456789" * 101, None) + d.addCallback(lambda ign: self.root.add_file(u"gro\u00DF", gross)) + def _created_gross(n): + self.gross = n + self.gross_uri = n.get_uri() + d.addCallback(_created_gross) + + small = upload.Data("0123456789", None) + d.addCallback(lambda ign: self.root.add_file(u"small", small)) + def _created_small(n): + self.small = n + self.small_uri = n.get_uri() + d.addCallback(_created_small) + + small2 = upload.Data("Small enough for a LIT too", None) + d.addCallback(lambda ign: self.root.add_file(u"small2", small2)) + def _created_small2(n): + self.small2 = n + self.small2_uri = n.get_uri() + d.addCallback(_created_small2) + + empty_litdir_uri = "URI:DIR2-LIT:" + + # contains one child which is itself also LIT: + tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" + + unknown_uri = "x-tahoe-crazy://I_am_from_the_future." + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"empty_lit_dir")) + def _created_empty_lit_dir(n): + self.empty_lit_dir = n + self.empty_lit_dir_uri = n.get_uri() + self.root.set_node(u"empty_lit_dir", n) + d.addCallback(_created_empty_lit_dir) + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"tiny_lit_dir")) + def _created_tiny_lit_dir(n): + self.tiny_lit_dir = n + self.tiny_lit_dir_uri = n.get_uri() + self.root.set_node(u"tiny_lit_dir", n) + d.addCallback(_created_tiny_lit_dir) + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, unknown_uri, name=u"unknown")) + def _created_unknown(n): + self.unknown = n + self.unknown_uri = n.get_uri() + self.root.set_node(u"unknown", n) + d.addCallback(_created_unknown) + + d.addCallback(lambda ign: self.root.set_node(u"loop", self.root)) + return d + + def test_basic(self): + d = self._set_up("basic") + def _check(ign): + # Test operations that have no side-effects, and don't need the tree. + + version = self.handler.gotVersion(3, {}) + self.failUnless(isinstance(version, dict)) + + self.failUnlessEqual(self.handler._path_from_string(""), []) + self.failUnlessEqual(self.handler._path_from_string("/"), []) + self.failUnlessEqual(self.handler._path_from_string("."), []) + self.failUnlessEqual(self.handler._path_from_string("//"), []) + self.failUnlessEqual(self.handler._path_from_string("/."), []) + self.failUnlessEqual(self.handler._path_from_string("/./"), []) + self.failUnlessEqual(self.handler._path_from_string("foo"), [u"foo"]) + self.failUnlessEqual(self.handler._path_from_string("/foo"), [u"foo"]) + self.failUnlessEqual(self.handler._path_from_string("foo/"), [u"foo"]) + self.failUnlessEqual(self.handler._path_from_string("/foo/"), [u"foo"]) + self.failUnlessEqual(self.handler._path_from_string("foo/bar"), [u"foo", u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("/foo/bar"), [u"foo", u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("foo/bar//"), [u"foo", u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("/foo/bar//"), [u"foo", u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("foo/../bar"), [u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("/foo/../bar"), [u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("../bar"), [u"bar"]) + self.failUnlessEqual(self.handler._path_from_string("/../bar"), [u"bar"]) + + self.failUnlessEqual(self.handler.realPath(""), "/") + self.failUnlessEqual(self.handler.realPath("/"), "/") + self.failUnlessEqual(self.handler.realPath("."), "/") + self.failUnlessEqual(self.handler.realPath("//"), "/") + self.failUnlessEqual(self.handler.realPath("/."), "/") + self.failUnlessEqual(self.handler.realPath("/./"), "/") + self.failUnlessEqual(self.handler.realPath("foo"), "/foo") + self.failUnlessEqual(self.handler.realPath("/foo"), "/foo") + self.failUnlessEqual(self.handler.realPath("foo/"), "/foo") + self.failUnlessEqual(self.handler.realPath("/foo/"), "/foo") + self.failUnlessEqual(self.handler.realPath("foo/bar"), "/foo/bar") + self.failUnlessEqual(self.handler.realPath("/foo/bar"), "/foo/bar") + self.failUnlessEqual(self.handler.realPath("foo/bar//"), "/foo/bar") + self.failUnlessEqual(self.handler.realPath("/foo/bar//"), "/foo/bar") + self.failUnlessEqual(self.handler.realPath("foo/../bar"), "/bar") + self.failUnlessEqual(self.handler.realPath("/foo/../bar"), "/bar") + self.failUnlessEqual(self.handler.realPath("../bar"), "/bar") + self.failUnlessEqual(self.handler.realPath("/../bar"), "/bar") + d.addCallback(_check) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_path_from_string invalid UTF-8", + self.handler._path_from_string, "\xFF")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "realPath invalid UTF-8", + self.handler.realPath, "\xFF")) + + return d + + def test_raise_error(self): + self.failUnlessEqual(sftpd._raise_error(None), None) + + d = defer.succeed(None) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_raise_error SFTPError", + sftpd._raise_error, Failure(sftp.SFTPError(sftp.FX_FAILURE, "foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "_raise_error NoSuchChildError", + sftpd._raise_error, Failure(NoSuchChildError("foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "_raise_error ExistingChildError", + sftpd._raise_error, Failure(ExistingChildError("foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "_raise_error NotWriteableError", + sftpd._raise_error, Failure(NotWriteableError("foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "_raise_error NotImplementedError", + sftpd._raise_error, Failure(NotImplementedError("foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "_raise_error EOFError", + sftpd._raise_error, Failure(EOFError("foo")))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "_raise_error defer.FirstError", + sftpd._raise_error, Failure(defer.FirstError( + Failure(sftp.SFTPError(sftp.FX_EOF, "foo")), 0)))) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_FAILURE, "_raise_error AssertionError", + sftpd._raise_error, Failure(AssertionError("foo")))) + + return d + + def test_not_implemented(self): + d = self._set_up("not_implemented") + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "readLink link", + self.handler.readLink, "link")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "makeLink link file", + self.handler.makeLink, "link", "file")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "extendedRequest foo bar", + self.handler.extendedRequest, "foo", "bar")) + + return d + + def _compareDirLists(self, actual, expected): + actual_list = sorted(actual) + expected_list = sorted(expected) + self.failUnlessEqual(len(actual_list), len(expected_list), + "%r is wrong length, expecting %r" % (actual_list, expected_list)) + for (a, b) in zip(actual_list, expected_list): + (name, text, attrs) = a + (expected_name, expected_text_re, expected_attrs) = b + self.failUnlessEqual(name, expected_name) + self.failUnless(re.match(expected_text_re, text), "%r does not match %r" % (text, expected_text_re)) + # it is ok for there to be extra actual attributes + # TODO: check times + for e in expected_attrs: + self.failUnlessEqual(attrs[e], expected_attrs[e]) + + def test_openDirectory_and_attrs(self): + d = self._set_up("openDirectory") + d.addCallback(lambda ign: self._set_up_tree()) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory small", + self.handler.openDirectory, "small")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openDirectory unknown", + self.handler.openDirectory, "unknown")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir", + self.handler.openDirectory, "nodir")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openDirectory nodir/nodir", + self.handler.openDirectory, "nodir/nodir")) + + gross = u"gro\u00DF".encode("utf-8") + expected_root = [ + ('empty_lit_dir', r'drwxrwx--- .* \? .* empty_lit_dir$', {'permissions': S_IFDIR | 0770}), + (gross, r'-rw-rw---- .* 1010 .* '+gross+'$', {'permissions': S_IFREG | 0660, 'size': 1010}), + ('loop', r'drwxrwx--- .* \? .* loop$', {'permissions': S_IFDIR | 0770}), + ('mutable', r'-rw-rw---- .* \? .* mutable$', {'permissions': S_IFREG | 0660}), + ('readonly', r'-r--r----- .* \? .* readonly$', {'permissions': S_IFREG | 0440}), + ('small', r'-rw-rw---- .* 10 .* small$', {'permissions': S_IFREG | 0660, 'size': 10}), + ('small2', r'-rw-rw---- .* 26 .* small2$', {'permissions': S_IFREG | 0660, 'size': 26}), + ('tiny_lit_dir', r'drwxrwx--- .* \? .* tiny_lit_dir$', {'permissions': S_IFDIR | 0770}), + ('unknown', r'\?--------- .* \? .* unknown$', {'permissions': 0}), + ] + + d.addCallback(lambda ign: self.handler.openDirectory("")) + d.addCallback(lambda res: self._compareDirLists(res, expected_root)) + + d.addCallback(lambda ign: self.handler.openDirectory("loop")) + d.addCallback(lambda res: self._compareDirLists(res, expected_root)) + + d.addCallback(lambda ign: self.handler.openDirectory("loop/loop")) + d.addCallback(lambda res: self._compareDirLists(res, expected_root)) + + d.addCallback(lambda ign: self.handler.openDirectory("empty_lit_dir")) + d.addCallback(lambda res: self._compareDirLists(res, [])) + + expected_tiny_lit = [ + ('short', r'-r--r----- .* 8 Jan 01 1970 short$', {'permissions': S_IFREG | 0440, 'size': 8}), + ] + + d.addCallback(lambda ign: self.handler.openDirectory("tiny_lit_dir")) + d.addCallback(lambda res: self._compareDirLists(res, expected_tiny_lit)) + + d.addCallback(lambda ign: self.handler.getAttrs("small", True)) + def _check_attrs(attrs): + self.failUnlessEqual(attrs['permissions'], S_IFREG | 0440) #FIXME + self.failUnlessEqual(attrs['size'], 10) + d.addCallback(_check_attrs) + + d.addCallback(lambda ign: + self.failUnlessEqual(self.handler.setAttrs("small", {}), None)) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_OP_UNSUPPORTED, "setAttrs size", + self.handler.setAttrs, "small", {'size': 0})) + + return d + + def test_openFile_read(self): + d = self._set_up("openFile") + d.addCallback(lambda ign: self._set_up_tree()) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small 0", + self.handler.openFile, "small", 0, {})) + + # attempting to open a non-existent file should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nofile READ", + self.handler.openFile, "nofile", sftp.FXF_READ, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile nodir/file READ", + self.handler.openFile, "nodir/file", sftp.FXF_READ, {})) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown READ denied", + self.handler.openFile, "unknown", sftp.FXF_READ, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir READ denied", + self.handler.openFile, "tiny_lit_dir", sftp.FXF_READ, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown uri READ denied", + self.handler.openFile, "uri/"+self.unknown_uri, sftp.FXF_READ, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir uri READ denied", + self.handler.openFile, "uri/"+self.tiny_lit_dir_uri, sftp.FXF_READ, {})) + + # reading an existing file should succeed + d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {})) + def _read_small(rf): + d2 = rf.readChunk(0, 10) + d2.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + + d2.addCallback(lambda ign: rf.readChunk(2, 6)) + d2.addCallback(lambda data: self.failUnlessEqual(data, "234567")) + + d2.addCallback(lambda ign: rf.readChunk(8, 4)) # read that starts before EOF is OK + d2.addCallback(lambda data: self.failUnlessEqual(data, "89")) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)", + rf.readChunk, 10, 0)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF", + rf.readChunk, 10, 1)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF", + rf.readChunk, 11, 1)) + + d2.addCallback(lambda ign: rf.getAttrs()) + def _check_attrs(attrs): + self.failUnlessEqual(attrs['permissions'], S_IFREG | 0440) #FIXME + self.failUnlessEqual(attrs['size'], 10) + d2.addCallback(_check_attrs) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied", + rf.writeChunk, 0, "a")) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied", + rf.setAttrs, {})) + + d2.addCallback(lambda ign: rf.close()) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file", + rf.readChunk, 0, 1)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file", + rf.getAttrs)) + + d2.addCallback(lambda ign: rf.close()) # should be no-op + return d2 + d.addCallback(_read_small) + + # repeat for a large file + gross = u"gro\u00DF".encode("utf-8") + d.addCallback(lambda ign: self.handler.openFile(gross, sftp.FXF_READ, {})) + def _read_gross(rf): + d2 = rf.readChunk(0, 10) + d2.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + + d2.addCallback(lambda ign: rf.readChunk(2, 6)) + d2.addCallback(lambda data: self.failUnlessEqual(data, "234567")) + + d2.addCallback(lambda ign: rf.readChunk(1008, 4)) # read that starts before EOF is OK + d2.addCallback(lambda data: self.failUnlessEqual(data, "89")) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF (0-byte)", + rf.readChunk, 1010, 0)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting at EOF", + rf.readChunk, 1010, 1)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_EOF, "readChunk starting after EOF", + rf.readChunk, 1011, 1)) + + d2.addCallback(lambda ign: rf.getAttrs()) + def _check_attrs(attrs): + self.failUnlessEqual(attrs['permissions'], S_IFREG | 0440) #FIXME + self.failUnlessEqual(attrs['size'], 1010) + d2.addCallback(_check_attrs) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "writeChunk on read-only handle denied", + rf.writeChunk, 0, "a")) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "setAttrs on read-only handle denied", + rf.setAttrs, {})) + + d2.addCallback(lambda ign: rf.close()) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "readChunk on closed file", + rf.readChunk, 0, 1)) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "getAttrs on closed file", + rf.getAttrs)) + + d2.addCallback(lambda ign: rf.close()) # should be no-op + return d2 + d.addCallback(_read_gross) + + # reading an existing small file via uri/ should succeed + d.addCallback(lambda ign: self.handler.openFile("uri/"+self.small_uri, sftp.FXF_READ, {})) + def _read_small_uri(rf): + d2 = rf.readChunk(0, 10) + d2.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + d2.addCallback(lambda ign: rf.close()) + return d2 + d.addCallback(_read_small_uri) + + # repeat for a large file + d.addCallback(lambda ign: self.handler.openFile("uri/"+self.gross_uri, sftp.FXF_READ, {})) + def _read_gross_uri(rf): + d2 = rf.readChunk(0, 10) + d2.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + d2.addCallback(lambda ign: rf.close()) + return d2 + d.addCallback(_read_gross_uri) + + # repeat for a mutable file + d.addCallback(lambda ign: self.handler.openFile("uri/"+self.mutable_uri, sftp.FXF_READ, {})) + def _read_mutable_uri(rf): + d2 = rf.readChunk(0, 100) + d2.addCallback(lambda data: self.failUnlessEqual(data, "mutable file contents")) + d2.addCallback(lambda ign: rf.close()) + return d2 + d.addCallback(_read_mutable_uri) + + return d + + def test_openFile_write(self): + d = self._set_up("openFile") + d.addCallback(lambda ign: self._set_up_tree()) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "openFile '' WRITE|CREAT|TRUNC", + self.handler.openFile, "", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile newfile WRITE|TRUNC", + self.handler.openFile, "newfile", sftp.FXF_WRITE | sftp.FXF_TRUNC, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "openFile small WRITE|EXCL", + self.handler.openFile, "small", sftp.FXF_WRITE | sftp.FXF_EXCL, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir WRITE", + self.handler.openFile, "tiny_lit_dir", sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile unknown WRITE", + self.handler.openFile, "unknown", sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/newfile WRITE|CREAT|TRUNC", + self.handler.openFile, "tiny_lit_dir/newfile", + sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE", + self.handler.openFile, "tiny_lit_dir/short", sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile tiny_lit_dir/short WRITE|CREAT|EXCL", + self.handler.openFile, "tiny_lit_dir/short", + sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly WRITE", + self.handler.openFile, "readonly", sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small WRITE|CREAT|EXCL", + self.handler.openFile, "small", + sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile readonly uri WRITE", + self.handler.openFile, "uri/"+self.readonly_uri, sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE", + self.handler.openFile, "uri/"+self.small_uri, sftp.FXF_WRITE, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile small uri WRITE|CREAT|TRUNC", + self.handler.openFile, "uri/"+self.small_uri, + sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {})) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "openFile mutable uri WRITE|CREAT|EXCL", + self.handler.openFile, "uri/"+self.mutable_uri, + sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_EXCL, {})) + + d.addCallback(lambda ign: + self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT | sftp.FXF_TRUNC, {})) + def _write(wf): + d2 = wf.writeChunk(0, "0123456789") + d2.addCallback(lambda res: self.failUnlessEqual(res, None)) + + d2.addCallback(lambda ign: wf.writeChunk(8, "0123")) + d2.addCallback(lambda ign: wf.writeChunk(13, "abc")) + + d2.addCallback(lambda ign: wf.getAttrs()) + def _check_attrs(attrs): + self.failUnlessEqual(attrs['permissions'], S_IFREG | 0440) #FIXME + self.failUnlessEqual(attrs['size'], 16) + d2.addCallback(_check_attrs) + + d2.addCallback(lambda ign: wf.setAttrs({})) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs with negative size", + wf.setAttrs, {'size': -1})) + + d2.addCallback(lambda ign: wf.setAttrs({'size': 14})) + d2.addCallback(lambda ign: wf.getAttrs()) + d2.addCallback(lambda attrs: self.failUnlessEqual(attrs['size'], 14)) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "readChunk on write-only handle denied", + wf.readChunk, 0, 1)) + + d2.addCallback(lambda ign: wf.close()) + + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "writeChunk on closed file", + wf.writeChunk, 0, "a")) + d2.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_BAD_MESSAGE, "setAttrs on closed file", + wf.setAttrs, {'size': 0})) + + d2.addCallback(lambda ign: wf.close()) # should be no-op + return d2 + d.addCallback(_write) + d.addCallback(lambda ign: self.root.get(u"newfile")) + d.addCallback(lambda node: download_to_data(node)) + d.addCallback(lambda data: self.failUnlessEqual(data, "012345670123\x00a")) + + # test APPEND flag, and also replacing an existing file ("newfile") + d.addCallback(lambda ign: + self.handler.openFile("newfile", sftp.FXF_WRITE | sftp.FXF_CREAT | + sftp.FXF_TRUNC | sftp.FXF_APPEND, {})) + def _write_append(wf): + d2 = wf.writeChunk(0, "0123456789") + d2.addCallback(lambda ign: wf.writeChunk(8, "0123")) + d2.addCallback(lambda ign: wf.close()) + return d2 + d.addCallback(_write_append) + d.addCallback(lambda ign: self.root.get(u"newfile")) + d.addCallback(lambda node: download_to_data(node)) + d.addCallback(lambda data: self.failUnlessEqual(data, "01234567890123")) + + # test EXCL flag + d.addCallback(lambda ign: + self.handler.openFile("excl", sftp.FXF_WRITE | sftp.FXF_CREAT | + sftp.FXF_TRUNC | sftp.FXF_EXCL, {})) + def _write_excl(wf): + d2 = self.root.get(u"excl") + d2.addCallback(lambda node: download_to_data(node)) + d2.addCallback(lambda data: self.failUnlessEqual(data, "")) + + d2.addCallback(lambda ign: wf.writeChunk(0, "0123456789")) + d2.addCallback(lambda ign: wf.close()) + return d2 + d.addCallback(_write_excl) + d.addCallback(lambda ign: self.root.get(u"excl")) + d.addCallback(lambda node: download_to_data(node)) + d.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + + # test WRITE | CREAT without TRUNC + d.addCallback(lambda ign: + self.handler.openFile("newfile2", sftp.FXF_WRITE | sftp.FXF_CREAT, {})) + def _write_notrunc(wf): + d2 = wf.writeChunk(0, "0123456789") + d2.addCallback(lambda ign: wf.close()) + return d2 + d.addCallback(_write_notrunc) + d.addCallback(lambda ign: self.root.get(u"newfile2")) + d.addCallback(lambda node: download_to_data(node)) + d.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + + # test writing to a mutable file + d.addCallback(lambda ign: + self.handler.openFile("mutable", sftp.FXF_WRITE, {})) + def _write_mutable(wf): + d2 = wf.writeChunk(8, "new!") + d2.addCallback(lambda ign: wf.close()) + return d2 + d.addCallback(_write_mutable) + d.addCallback(lambda ign: self.root.get(u"mutable")) + def _check_same_file(node): + self.failUnless(node.is_mutable()) + self.failUnlessEqual(node.get_uri(), self.mutable_uri) + return node.download_best_version() + d.addCallback(_check_same_file) + d.addCallback(lambda data: self.failUnlessEqual(data, "mutable new! contents")) + + """ + # test READ | WRITE without CREAT or TRUNC + d.addCallback(lambda ign: + self.handler.openFile("small", sftp.FXF_READ | sftp.FXF_WRITE, {})) + def _read_write(rwf): + d2 = rwf.writeChunk(8, "0123") + d2.addCallback(lambda ign: rwf.readChunk(0, 100)) + d2.addCallback(lambda data: self.failUnlessEqual(data, "012345670123")) + d2.addCallback(lambda ign: rwf.close()) + return d2 + d.addCallback(_read_write) + d.addCallback(lambda ign: self.root.get(u"small")) + d.addCallback(lambda node: download_to_data(node)) + d.addCallback(lambda data: self.failUnlessEqual(data, "012345670123")) + """ + return d + + def test_removeFile(self): + d = self._set_up("removeFile") + d.addCallback(lambda ign: self._set_up_tree()) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile", + self.handler.removeFile, "nofile")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nofile", + self.handler.removeFile, "nofile")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeFile nodir/file", + self.handler.removeFile, "nodir/file")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removefile ''", + self.handler.removeFile, "")) + + # removing a directory should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeFile tiny_lit_dir", + self.handler.removeFile, "tiny_lit_dir")) + + # removing a file should succeed + d.addCallback(lambda ign: self.root.get(u"gro\u00DF")) + d.addCallback(lambda ign: self.handler.removeFile(u"gro\u00DF".encode('utf-8'))) + d.addCallback(lambda ign: + self.shouldFail(NoSuchChildError, "removeFile gross", "gro\\xdf", + self.root.get, u"gro\u00DF")) + + # removing an unknown should succeed + d.addCallback(lambda ign: self.root.get(u"unknown")) + d.addCallback(lambda ign: self.handler.removeFile("unknown")) + d.addCallback(lambda ign: + self.shouldFail(NoSuchChildError, "removeFile unknown", "unknown", + self.root.get, u"unknown")) + + # removing a link to an open file should not prevent it from being read + d.addCallback(lambda ign: self.handler.openFile("small", sftp.FXF_READ, {})) + def _remove_and_read_small(rf): + d2= self.handler.removeFile("small") + d2.addCallback(lambda ign: + self.shouldFail(NoSuchChildError, "removeFile small", "small", + self.root.get, u"small")) + d2.addCallback(lambda ign: rf.readChunk(0, 10)) + d2.addCallback(lambda data: self.failUnlessEqual(data, "0123456789")) + d2.addCallback(lambda ign: rf.close()) + return d2 + d.addCallback(_remove_and_read_small) + + return d + + def test_removeDirectory(self): + d = self._set_up("removeDirectory") + d.addCallback(lambda ign: self._set_up_tree()) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir", + self.handler.removeDirectory, "nodir")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory nodir/nodir", + self.handler.removeDirectory, "nodir/nodir")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "removeDirectory ''", + self.handler.removeDirectory, "")) + + # removing a file should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "removeDirectory gross", + self.handler.removeDirectory, u"gro\u00DF".encode('utf-8'))) + + # removing a directory should succeed + d.addCallback(lambda ign: self.root.get(u"tiny_lit_dir")) + d.addCallback(lambda ign: self.handler.removeDirectory("tiny_lit_dir")) + d.addCallback(lambda ign: + self.shouldFail(NoSuchChildError, "removeDirectory tiny_lit_dir", "tiny_lit_dir", + self.root.get, u"tiny_lit_dir")) + + # removing an unknown should succeed + d.addCallback(lambda ign: self.root.get(u"unknown")) + d.addCallback(lambda ign: self.handler.removeDirectory("unknown")) + d.addCallback(lambda err: + self.shouldFail(NoSuchChildError, "removeDirectory unknown", "unknown", + self.root.get, u"unknown")) + + return d + + def test_renameFile(self): + d = self._set_up("renameFile") + d.addCallback(lambda ign: self._set_up_tree()) + + # renaming a non-existent file should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile nofile newfile", + self.handler.renameFile, "nofile", "newfile")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile '' newfile", + self.handler.renameFile, "", "newfile")) + + # renaming a file to a non-existent path should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small nodir/small", + self.handler.renameFile, "small", "nodir/small")) + + # renaming a file to an invalid UTF-8 name should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small invalid", + self.handler.renameFile, "small", "\xFF")) + + # renaming a file to or from an URI should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small from uri", + self.handler.renameFile, "uri/"+self.small_uri, "new")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "renameFile small to uri", + self.handler.renameFile, "small", "uri/fake_uri")) + + # renaming a file onto an existing file, directory or unknown should fail + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small small2", + self.handler.renameFile, "small", "small2")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small tiny_lit_dir", + self.handler.renameFile, "small", "tiny_lit_dir")) + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "renameFile small unknown", + self.handler.renameFile, "small", "unknown")) + + # renaming a file to a correct path should succeed + d.addCallback(lambda ign: self.handler.renameFile("small", "new_small")) + d.addCallback(lambda ign: self.root.get(u"new_small")) + d.addCallback(lambda node: self.failUnlessEqual(node.get_uri(), self.small_uri)) + + # renaming a file into a subdirectory should succeed (also tests Unicode names) + d.addCallback(lambda ign: self.handler.renameFile(u"gro\u00DF".encode('utf-8'), + u"loop/neue_gro\u00DF".encode('utf-8'))) + d.addCallback(lambda ign: self.root.get(u"neue_gro\u00DF")) + d.addCallback(lambda node: self.failUnlessEqual(node.get_uri(), self.gross_uri)) + + # renaming a directory to a correct path should succeed + d.addCallback(lambda ign: self.handler.renameFile("tiny_lit_dir", "new_tiny_lit_dir")) + d.addCallback(lambda ign: self.root.get(u"new_tiny_lit_dir")) + d.addCallback(lambda node: self.failUnlessEqual(node.get_uri(), self.tiny_lit_dir_uri)) + + # renaming an unknown to a correct path should succeed + d.addCallback(lambda ign: self.handler.renameFile("unknown", "new_unknown")) + d.addCallback(lambda ign: self.root.get(u"new_unknown")) + d.addCallback(lambda node: self.failUnlessEqual(node.get_uri(), self.unknown_uri)) + + return d + + def test_makeDirectory(self): + d = self._set_up("makeDirectory") + d.addCallback(lambda ign: self._set_up_tree()) + + # making a directory at a correct path should succeed + d.addCallback(lambda ign: self.handler.makeDirectory("newdir", {'ext_foo': 'bar', 'ctime': 42})) + + d.addCallback(lambda ign: self.root.get_child_and_metadata(u"newdir")) + def _got( (child, metadata) ): + self.failUnless(IDirectoryNode.providedBy(child)) + self.failUnless(child.is_mutable()) + # FIXME + #self.failUnless('ctime' in metadata, metadata) + #self.failUnlessEqual(metadata['ctime'], 42) + #self.failUnless('ext_foo' in metadata, metadata) + #self.failUnlessEqual(metadata['ext_foo'], 'bar') + # TODO: child should be empty + d.addCallback(_got) + + # making intermediate directories should also succeed + d.addCallback(lambda ign: self.handler.makeDirectory("newparent/newchild", {})) + + d.addCallback(lambda ign: self.root.get(u"newparent")) + def _got_newparent(newparent): + self.failUnless(IDirectoryNode.providedBy(newparent)) + self.failUnless(newparent.is_mutable()) + return newparent.get(u"newchild") + d.addCallback(_got_newparent) + + def _got_newchild(newchild): + self.failUnless(IDirectoryNode.providedBy(newchild)) + self.failUnless(newchild.is_mutable()) + d.addCallback(_got_newchild) + + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_NO_SUCH_FILE, "makeDirectory invalid UTF-8", + self.handler.makeDirectory, "\xFF", {})) + + # should fail because there is an existing file "small" + d.addCallback(lambda ign: + self.shouldFailWithSFTPError(sftp.FX_PERMISSION_DENIED, "makeDirectory small", + self.handler.makeDirectory, "small", {})) + return d } [Change shouldFail to avoid Unicode errors when converting Failure to str david-sarah@jacaranda.org**20100512060754 Ignore-this: 86ed419d332d9c33090aae2cde1dc5df ] { hunk ./src/allmydata/test/common.py 1090 error message, if any, because Deferred chains frequently make it difficult to tell which assertion was tripped. - The substring= argument, if not None, must appear inside the - stringified Failure, or the test will fail. + The substring= argument, if not None, must appear in the 'repr' + of the message wrapped by this Failure, or the test will fail. """ assert substring is None or isinstance(substring, str) hunk ./src/allmydata/test/common.py 1100 if isinstance(res, failure.Failure): res.trap(expected_failure) if substring: - self.failUnless(substring in str(res), + message = repr(res.value.args[0]) + self.failUnless(substring in message, "substring '%s' not in '%s'" hunk ./src/allmydata/test/common.py 1103 - % (substring, str(res))) + % (substring, message)) else: self.fail("%s was supposed to raise %s, not get '%s'" % (which, expected_failure, res)) } Context: [Clarify quickstart instructions for installing pywin32 david-sarah@jacaranda.org**20100511180300 Ignore-this: d4668359673600d2acbc7cd8dd44b93c ] [web: add a simple test that you can load directory.xhtml zooko@zooko.com**20100510063729 Ignore-this: e49b25fa3c67b3c7a56c8b1ae01bb463 ] [setup: fix typos in misc/show-tool-versions.py zooko@zooko.com**20100510063615 Ignore-this: 2181b1303a0e288e7a9ebd4c4855628 ] [setup: show code-coverage tool versions in show-tools-versions.py zooko@zooko.com**20100510062955 Ignore-this: 4b4c68eb3780b762c8dbbd22b39df7cf ] [docs: update README, mv it to README.txt, update setup.py zooko@zooko.com**20100504094340 Ignore-this: 40e28ca36c299ea1fd12d3b91e5b421c ] [tests: pass z to tar so that BSD tar will know to ungzip zooko@zooko.com**20100504090628 Ignore-this: 1339e493f255e8fc0b01b70478f23a09 ] [setup: update comments and URLs in setup.cfg zooko@zooko.com**20100504061653 Ignore-this: f97692807c74bcab56d33100c899f829 ] [setup: reorder and extend the show-tool-versions script, the better to glean information about our new buildslaves zooko@zooko.com**20100504045643 Ignore-this: 836084b56b8d4ee8f1de1f4efb706d36 ] [Dependency on Windmill test framework is not needed yet. david-sarah@jacaranda.org**20100504161043 Ignore-this: be088712bec650d4ef24766c0026ebc8 ] [CLI: Support for https url in option --node-url Francois Deppierraz **20100430185609 Ignore-this: 1717176b4d27c877e6bc67a944d9bf34 This patch modifies the regular expression used for verifying of '--node-url' parameter. Support for accessing a Tahoe gateway over HTTPS was already present, thanks to Python's urllib. ] [backupdb.did_create_directory: use REPLACE INTO, not INSERT INTO + ignore error Brian Warner **20100428050803 Ignore-this: 1fca7b8f364a21ae413be8767161e32f This handles the case where we upload a new tahoe directory for a previously-processed local directory, possibly creating a new dircap (if the metadata had changed). Now we replace the old dirhash->dircap record. The previous behavior left the old record in place (with the old dircap and timestamps), so we'd never stop creating new directories and never converge on a null backup. ] ["tahoe webopen": add --info flag, to get ?t=info Brian Warner **20100424233003 Ignore-this: 126b0bb6db340fabacb623d295eb45fa Also fix some trailing whitespace. ] [docs: install.html http-equiv refresh to quickstart.html zooko@zooko.com**20100421165708 Ignore-this: 52b4b619f9dde5886ae2cd7f1f3b734b ] [docs: install.html -> quickstart.html zooko@zooko.com**20100421155757 Ignore-this: 6084e203909306bed93efb09d0e6181d It is not called "installing" because that implies that it is going to change the configuration of your operating system. It is not called "building" because that implies that you need developer tools like a compiler. Also I added a stern warning against looking at the "InstallDetails" wiki page, which I have renamed to "AdvancedInstall". ] [Fix another typo in tahoe_storagespace munin plugin david-sarah@jacaranda.org**20100416220935 Ignore-this: ad1f7aa66b554174f91dfb2b7a3ea5f3 ] [licensing: phrase the OpenSSL-exemption in the vocabulary of copyright instead of computer technology, and replicate the exemption from the GPL to the TGPPL zooko@zooko.com**20100414232521 Ignore-this: a5494b2f582a295544c6cad3f245e91 ] [munin-tahoe_storagespace freestorm77@gmail.com**20100221203626 Ignore-this: 14d6d6a587afe1f8883152bf2e46b4aa Plugin configuration rename ] [Add dependency on windmill >= 1.3 david-sarah@jacaranda.org**20100416190404 Ignore-this: 4437a7a464e92d6c9012926b18676211 ] [setup: add licensing declaration for setuptools (noticed by the FSF compliance folks) zooko@zooko.com**20100309184415 Ignore-this: 2dfa7d812d65fec7c72ddbf0de609ccb ] [setup: fix error in licensing declaration from Shawn Willden, as noted by the FSF compliance division zooko@zooko.com**20100309163736 Ignore-this: c0623d27e469799d86cabf67921a13f8 ] [CREDITS to Jacob Appelbaum zooko@zooko.com**20100304015616 Ignore-this: 70db493abbc23968fcc8db93f386ea54 ] [desert-island-build-with-proper-versions jacob@appelbaum.net**20100304013858] [docs: a few small edits to try to guide newcomers through the docs zooko@zooko.com**20100303231902 Ignore-this: a6aab44f5bf5ad97ea73e6976bc4042d These edits were suggested by my watching over Jake Appelbaum's shoulder as he completely ignored/skipped/missed install.html and also as he decided that debian.txt wouldn't help him with basic installation. Then I threw in a few docs edits that have been sitting around in my sandbox asking to be committed for months. ] [TAG allmydata-tahoe-1.6.1 david-sarah@jacaranda.org**20100228062314 Ignore-this: eb5f03ada8ea953ee7780e7fe068539 ] Patch bundle hash: af606a89f9b9e87a70e34177bdb8b84c8d25e4f4