Ticket #1108: abspath-and-unicode-basedirs.dpatch

Thu Jul 22 00:15:07 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * util.fileutil, test.test_util: add abspath_expanduser_unicode function, to work around <http://bugs.python.org/issue3426>. util.encodingutil: add a convenience function argv_to_abspath.

Thu Jul 22 01:14:18 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Replace uses of os.path.abspath with abspath_expanduser_unicode where necessary. This makes basedir paths consistently represented as Unicode.

New patches:

[util.fileutil, test.test_util: add abspath_expanduser_unicode function, to work around <http://bugs.python.org/issue3426>. util.encodingutil: add a convenience function argv_to_abspath.
david-sarah@jacaranda.org**20100721231507
 Ignore-this: eee6904d1f65a733ff35190879844d08
] {
hunk ./src/allmydata/test/test_util.py 4
 
 def foo(): pass # keep the line number constant
 
-import os, time
+import os, time, sys
 from StringIO import StringIO
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
hunk ./src/allmydata/test/test_util.py 473
         used = fileutil.du(basedir)
         self.failUnlessEqual(10+11+12+13, used)
 
+    def test_abspath_expanduser_unicode(self):
+        self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, "bytestring")
+
+        saved_cwd = os.path.normpath(os.getcwdu())
+        abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
+        self.failUnless(isinstance(saved_cwd, unicode), saved_cwd)
+        self.failUnless(isinstance(abspath_cwd, unicode), abspath_cwd)
+        self.failUnlessEqual(abspath_cwd, saved_cwd)
+
+        # adapted from <http://svn.python.org/view/python/branches/release26-maint/Lib/test/test_posixpath.py?view=markup&pathrev=78279#test_abspath>
+
+        self.failUnlessIn(u"foo", fileutil.abspath_expanduser_unicode(u"foo"))
+        self.failIfIn(u"~", fileutil.abspath_expanduser_unicode(u"~"))
+
+        cwds = ['cwd']
+        try:
+            cwds.append(u'\xe7w\xf0'.encode(sys.getfilesystemencoding()
+                                            or 'ascii'))
+        except UnicodeEncodeError:
+            pass # the cwd can't be encoded -- test with ascii cwd only
+
+        for cwd in cwds:
+            try:
+                os.mkdir(cwd)
+                os.chdir(cwd)
+                for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\', u'~'):
+                    uabspath = fileutil.abspath_expanduser_unicode(upath)
+                    self.failUnless(isinstance(uabspath, unicode), uabspath)
+            finally:
+                os.chdir(saved_cwd)
+
 class PollMixinTests(unittest.TestCase):
     def setUp(self):
         self.pm = pollmixin.PollMixin()
hunk ./src/allmydata/util/encodingutil.py 13
 from twisted.python import usage
 import locale
 from allmydata.util import log
+from allmydata.util.fileutil import abspath_expanduser_unicode
 
 
 def _canonical_encoding(encoding):
hunk ./src/allmydata/util/encodingutil.py 95
         raise usage.UsageError("Argument %s cannot be decoded as %s." %
                                (quote_output(s), argv_encoding))
 
+def argv_to_abspath(s):
+    """
+    Convenience function to decode an argv element to an absolute path, with ~ expanded.
+    If this fails, raise a UsageError.
+    """
+    return abspath_expanduser_unicode(argv_to_unicode(s))
+
 def unicode_to_url(s):
     """
     Encode an unicode object used in an URL.
hunk ./src/allmydata/util/fileutil.py 276
     finally:
         outf.close()
 
+
+# Work around <http://bugs.python.org/issue3426>. This code is adapted from
+# <http://svn.python.org/view/python/trunk/Lib/ntpath.py?revision=78247&view=markup>
+# with some simplifications.
+
+_getfullpathname = None
+try:
+    from nt import _getfullpathname
+except ImportError:
+    pass
+
+def abspath_expanduser_unicode(path):
+    """Return the absolute version of a path."""
+    assert isinstance(path, unicode), path
+
+    path = os.path.expanduser(path)
+
+    if _getfullpathname:
+        # On Windows, os.path.isabs will return True for paths without a drive letter,
+        # e.g. "\\". See <http://bugs.python.org/issue1669539>.
+        try:
+            path = _getfullpathname(path or u".")
+        except WindowsError:
+            pass
+
+    if not os.path.isabs(path):
+        path = os.path.join(os.getcwdu(), path)
+
+    # We won't hit <http://bugs.python.org/issue5827> because
+    # there is always at least one Unicode path component.
+    return os.path.normpath(path)
+
}
[Replace uses of os.path.abspath with abspath_expanduser_unicode where necessary. This makes basedir paths consistently represented as Unicode.
david-sarah@jacaranda.org**20100722001418
 Ignore-this: 9f8cb706540e695550e0dbe303c01f52
] {
hunk ./src/allmydata/node.py 14
 from allmydata.util import log
 from allmydata.util import fileutil, iputil, observer
 from allmydata.util.assertutil import precondition, _assert
+from allmydata.util.fileutil import abspath_expanduser_unicode
+from allmydata.util.encodingutil import get_filesystem_encoding
 
 # Add our application versions to the data that Foolscap's LogPublisher
 # reports.
hunk ./src/allmydata/node.py 54
     PORTNUMFILE = None
     CERTFILE = "node.pem"
 
-    def __init__(self, basedir="."):
+    def __init__(self, basedir=u"."):
         service.MultiService.__init__(self)
hunk ./src/allmydata/node.py 56
-        self.basedir = os.path.abspath(basedir)
+        self.basedir = abspath_expanduser_unicode(unicode(basedir))
         self._portnumfile = os.path.join(self.basedir, self.PORTNUMFILE)
         self._tub_ready_observerlist = observer.OneShotObserverList()
         fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700)
hunk ./src/allmydata/node.py 79
         iputil.increase_rlimits()
 
     def init_tempdir(self):
-        local_tempdir = "tmp" # default is NODEDIR/tmp/
-        tempdir = self.get_config("node", "tempdir", local_tempdir)
+        local_tempdir_utf8 = "tmp" # default is NODEDIR/tmp/
+        tempdir = self.get_config("node", "tempdir", local_tempdir_utf8).decode('utf-8')
         tempdir = os.path.join(self.basedir, tempdir)
         if not os.path.exists(tempdir):
             fileutil.make_dirs(tempdir)
hunk ./src/allmydata/node.py 84
-        tempfile.tempdir = os.path.abspath(tempdir)
+        tempfile.tempdir = abspath_expanduser_unicode(tempdir)
         # this should cause twisted.web.http (which uses
         # tempfile.TemporaryFile) to put large request bodies in the given
         # directory. Without this, the default temp dir is usually /tmp/,
hunk ./src/allmydata/node.py 172
     def setup_ssh(self):
         ssh_port = self.get_config("node", "ssh.port", "")
         if ssh_port:
-            ssh_keyfile = self.get_config("node", "ssh.authorized_keys_file")
+            ssh_keyfile = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8')
             from allmydata import manhole
hunk ./src/allmydata/node.py 174
-            m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile)
+            m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile.encode(get_filesystem_encoding()))
             m.setServiceParent(self)
             self.log("AuthorizedKeysManhole listening on %s" % ssh_port)
 
hunk ./src/allmydata/scripts/backupdb.py 12
 from allmydata.util.netstring import netstring
 from allmydata.util.hashutil import backupdb_dirhash
 from allmydata.util import base32
+from allmydata.util.fileutil import abspath_expanduser_unicode
+from allmydata.util.encodingutil import to_str
 
 DAY = 24*60*60
 MONTH = 30*DAY
hunk ./src/allmydata/scripts/backupdb.py 208
         current working directory. The database stores absolute pathnames.
         """
 
-        path = os.path.abspath(path)
+        path = abspath_expanduser_unicode(path)
         s = os.stat(path)
         size = s[stat.ST_SIZE]
         ctime = s[stat.ST_CTIME]
hunk ./src/allmydata/scripts/backupdb.py 251
         probability = min(max(probability, 0.0), 1.0)
         should_check = bool(random.random() < probability)
 
-        return FileResult(self, str(filecap), should_check,
+        return FileResult(self, to_str(filecap), should_check,
                           path, mtime, ctime, size)
 
     def get_or_allocate_fileid_for_cap(self, filecap):
hunk ./src/allmydata/scripts/backupdb.py 354
         probability = min(max(probability, 0.0), 1.0)
         should_check = bool(random.random() < probability)
 
-        return DirectoryResult(self, dirhash_s, str(dircap), should_check)
+        return DirectoryResult(self, dirhash_s, to_str(dircap), should_check)
 
     def did_create_directory(self, dircap, dirhash):
         now = time.time()
hunk ./src/allmydata/scripts/create_node.py 87
 
 
 def create_node(basedir, config, out=sys.stdout, err=sys.stderr):
+    # This should always be called with an absolute Unicode basedir.
+    precondition(isinstance(basedir, unicode), basedir)
+
     if os.path.exists(basedir):
hunk ./src/allmydata/scripts/create_node.py 91
-        if os.listdir(basedir):
-            print >>err, "The base directory \"%s\", which is \"%s\" is not empty." % (basedir, os.path.abspath(basedir))
+        if listdir_unicode(basedir):
+            print >>err, "The base directory %s is not empty." % quote_output(basedir)
             print >>err, "To avoid clobbering anything, I am going to quit now."
             print >>err, "Please use a different directory, or empty this one."
             return -1
hunk ./src/allmydata/scripts/create_node.py 135
 
     from allmydata.util import fileutil
     fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
-    print >>out, "Node created in %s" % basedir
+    print >>out, "Node created in %s" % quote_output(basedir)
     if not config.get("introducer", ""):
         print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
         print >>out, " The node cannot connect to a grid without it."
hunk ./src/allmydata/scripts/create_node.py 149
 
 
 def create_introducer(basedir, config, out=sys.stdout, err=sys.stderr):
+    # This should always be called with an absolute Unicode basedir.
+    precondition(isinstance(basedir, unicode), basedir)
+
     if os.path.exists(basedir):
hunk ./src/allmydata/scripts/create_node.py 153
-        if os.listdir(basedir):
-            print >>err, "The base directory \"%s\", which is \"%s\" is not empty." % (basedir, os.path.abspath(basedir))
+        if listdir_unicode(basedir):
+            print >>err, "The base directory %s is not empty." % quote_output(basedir)
             print >>err, "To avoid clobbering anything, I am going to quit now."
             print >>err, "Please use a different directory, or empty this one."
             return -1
hunk ./src/allmydata/scripts/create_node.py 169
     write_node_config(c, config)
     c.close()
 
-    print >>out, "Introducer created in %s" % basedir
+    print >>out, "Introducer created in %s" % quote_output(basedir)
 
 
 subCommands = [
hunk ./src/allmydata/scripts/debug.py 8
 from twisted.python import usage, failure
 from twisted.internet import defer
 
+
 class DumpOptions(usage.Options):
     def getSynopsis(self):
         return "Usage: tahoe debug dump-share SHARE_FILENAME"
hunk ./src/allmydata/scripts/debug.py 32
         return t
 
     def parseArgs(self, filename):
-        self['filename'] = filename
+        from allmydata.util.encodingutil import argv_to_abspath
+        self['filename'] = argv_to_abspath(filename)
 
 def dump_share(options):
     from allmydata.storage.mutable import MutableShareFile
hunk ./src/allmydata/scripts/debug.py 37
+    from allmydata.util.encodingutil import quote_output
 
     out = options.stdout
 
hunk ./src/allmydata/scripts/debug.py 42
     # check the version, to see if we have a mutable or immutable share
-    print >>out, "share filename: %s" % options['filename']
+    print >>out, "share filename: %s" % quote_output(options['filename'])
 
     f = open(options['filename'], "rb")
     prefix = f.read(32)
hunk ./src/allmydata/scripts/debug.py 67
     from allmydata import uri
     from allmydata.util import base32
     from allmydata.immutable.layout import ReadBucketProxy
+    from allmydata.util.encodingutil import quote_output, to_str
+
     # use a ReadBucketProxy to parse the bucket and find the uri extension
     bp = ReadBucketProxy(None, '', '')
     offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
hunk ./src/allmydata/scripts/debug.py 112
     # the storage index isn't stored in the share itself, so we depend upon
     # knowing the parent directory name to get it
     pieces = options['filename'].split(os.sep)
-    if len(pieces) >= 2 and base32.could_be_base32_encoded(pieces[-2]):
-        storage_index = base32.a2b(pieces[-2])
-        uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
-        u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
-                                   unpacked["needed_shares"],
-                                   unpacked["total_shares"], unpacked["size"])
-        verify_cap = u.to_string()
-        print >>out, "%20s: %s" % ("verify-cap", verify_cap)
+    if len(pieces) >= 2:
+        piece = to_str(pieces[-2])
+        if base32.could_be_base32_encoded(piece):
+            storage_index = base32.a2b(piece)
+            uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
+            u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
+                                      unpacked["needed_shares"],
+                                      unpacked["total_shares"], unpacked["size"])
+            verify_cap = u.to_string()
+            print >>out, "%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False))
 
     sizes = {}
     sizes['data'] = (offsets['plaintext_hash_tree'] -
hunk ./src/allmydata/scripts/debug.py 220
     from allmydata.mutable.common import NeedMoreDataError
     from allmydata.util import base32, hashutil
     from allmydata.uri import SSKVerifierURI
+    from allmydata.util.encodingutil import quote_output, to_str
 
     offset = m.DATA_OFFSET
 
hunk ./src/allmydata/scripts/debug.py 267
     # the storage index isn't stored in the share itself, so we depend upon
     # knowing the parent directory name to get it
     pieces = options['filename'].split(os.sep)
-    if len(pieces) >= 2 and base32.could_be_base32_encoded(pieces[-2]):
-        storage_index = base32.a2b(pieces[-2])
-        fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
-        u = SSKVerifierURI(storage_index, fingerprint)
-        verify_cap = u.to_string()
-        print >>out, "  verify-cap:", verify_cap
+    if len(pieces) >= 2:
+        piece = to_str(pieces[-2])
+        if base32.could_be_base32_encoded(piece):
+            storage_index = base32.a2b(piece)
+            fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
+            u = SSKVerifierURI(storage_index, fingerprint)
+            verify_cap = u.to_string()
+            print >>out, "  verify-cap:", quote_output(verify_cap, quotemarks=False)
 
     if options['offsets']:
         # NOTE: this offset-calculation code is fragile, and needs to be
hunk ./src/allmydata/scripts/debug.py 393
     from allmydata import uri
     from allmydata.storage.server import si_b2a
     from allmydata.util import base32, hashutil
+    from allmydata.util.encodingutil import quote_output
 
     if isinstance(u, uri.CHKFileURI):
         if show_header:
hunk ./src/allmydata/scripts/debug.py 415
     elif isinstance(u, uri.LiteralFileURI):
         if show_header:
             print >>out, "Literal File URI:"
-        print >>out, " data:", u.data
+        print >>out, " data:", quote_output(u.data)
 
     elif isinstance(u, uri.WriteableSSKFileURI):
         if show_header:
hunk ./src/allmydata/scripts/debug.py 461
 class FindSharesOptions(usage.Options):
     def getSynopsis(self):
         return "Usage: tahoe debug find-shares STORAGE_INDEX NODEDIRS.."
+
     def parseArgs(self, storage_index_s, *nodedirs):
hunk ./src/allmydata/scripts/debug.py 463
+        from allmydata.util.encodingutil import argv_to_abspath
         self.si_s = storage_index_s
hunk ./src/allmydata/scripts/debug.py 465
-        self.nodedirs = nodedirs
+        self.nodedirs = map(argv_to_abspath, nodedirs)
+
     def getUsage(self, width=None):
         t = usage.Options.getUsage(self, width)
         t += """
hunk ./src/allmydata/scripts/debug.py 495
     /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
     """
     from allmydata.storage.server import si_a2b, storage_index_to_dir
+    from allmydata.util.encodingutil import listdir_unicode
 
     out = options.stdout
     sharedir = storage_index_to_dir(si_a2b(options.si_s))
hunk ./src/allmydata/scripts/debug.py 500
     for d in options.nodedirs:
-        d = os.path.join(os.path.expanduser(d), "storage/shares", sharedir)
+        d = os.path.join(d, "storage/shares", sharedir)
         if os.path.exists(d):
hunk ./src/allmydata/scripts/debug.py 502
-            for shnum in os.listdir(d):
+            for shnum in listdir_unicode(d):
                 print >>out, os.path.join(d, shnum)
 
     return 0
hunk ./src/allmydata/scripts/debug.py 513
 
     """
     def parseArgs(self, *nodedirs):
-        self.nodedirs = nodedirs
+        from allmydata.util.encodingutil import argv_to_abspath
+        self.nodedirs = map(argv_to_abspath, nodedirs)
         if not nodedirs:
             raise usage.UsageError("must specify at least one node directory")
 
hunk ./src/allmydata/scripts/debug.py 562
     from allmydata.mutable.common import NeedMoreDataError
     from allmydata.immutable.layout import ReadBucketProxy
     from allmydata.util import base32
+    from allmydata.util.encodingutil import quote_output
     import struct
 
     f = open(abs_sharefile, "rb")
hunk ./src/allmydata/scripts/debug.py 602
             print >>out, "SDMF %s %d/%d %d #%d:%s %d %s" % \
                   (si_s, k, N, datalen,
                    seqnum, base32.b2a(root_hash),
-                   expiration, abs_sharefile)
+                   expiration, quote_output(abs_sharefile))
         else:
hunk ./src/allmydata/scripts/debug.py 604
-            print >>out, "UNKNOWN mutable %s" % (abs_sharefile,)
+            print >>out, "UNKNOWN mutable %s" % quote_output(abs_sharefile)
 
     elif struct.unpack(">L", prefix[:4]) == (1,):
         # immutable
hunk ./src/allmydata/scripts/debug.py 636
 
         print >>out, "CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
                                                    ueb_hash, expiration,
-                                                   abs_sharefile)
+                                                   quote_output(abs_sharefile))
 
     else:
hunk ./src/allmydata/scripts/debug.py 639
-        print >>out, "UNKNOWN really-unknown %s" % (abs_sharefile,)
+        print >>out, "UNKNOWN really-unknown %s" % quote_output(abs_sharefile)
 
     f.close()
 
hunk ./src/allmydata/scripts/debug.py 644
 def catalog_shares(options):
+    from allmydata.util.encodingutil import listdir_unicode, quote_output
+
     out = options.stdout
     err = options.stderr
     now = time.time()
hunk ./src/allmydata/scripts/debug.py 650
     for d in options.nodedirs:
-        d = os.path.join(os.path.expanduser(d), "storage/shares")
+        d = os.path.join(d, "storage/shares")
         try:
hunk ./src/allmydata/scripts/debug.py 652
-            abbrevs = os.listdir(d)
+            abbrevs = listdir_unicode(d)
         except EnvironmentError:
             # ignore nodes that have storage turned off altogether
             pass
hunk ./src/allmydata/scripts/debug.py 662
                     continue
                 abbrevdir = os.path.join(d, abbrevdir)
                 # this tool may get run against bad disks, so we can't assume
-                # that os.listdir will always succeed. Try to catalog as much
+                # that listdir_unicode will always succeed. Try to catalog as much
                 # as possible.
                 try:
hunk ./src/allmydata/scripts/debug.py 665
-                    sharedirs = os.listdir(abbrevdir)
+                    sharedirs = listdir_unicode(abbrevdir)
                     for si_s in sharedirs:
                         si_dir = os.path.join(abbrevdir, si_s)
                         catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
hunk ./src/allmydata/scripts/debug.py 670
                 except:
-                    print >>err, "Error processing %s" % abbrevdir
+                    print >>err, "Error processing %s" % quote_output(abbrevdir)
                     failure.Failure().printTraceback(err)
 
     return 0
hunk ./src/allmydata/scripts/debug.py 676
 
 def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
+    from allmydata.util.encodingutil import listdir_unicode, quote_output
+
     try:
hunk ./src/allmydata/scripts/debug.py 679
-        for shnum_s in os.listdir(si_dir):
+        for shnum_s in listdir_unicode(si_dir):
             abs_sharefile = os.path.join(si_dir, shnum_s)
hunk ./src/allmydata/scripts/debug.py 681
-            abs_sharefile = os.path.abspath(abs_sharefile)
             assert os.path.isfile(abs_sharefile)
             try:
                 describe_share(abs_sharefile, si_s, shnum_s, now,
hunk ./src/allmydata/scripts/debug.py 686
                                out)
             except:
-                print >>err, "Error processing %s" % abs_sharefile
+                print >>err, "Error processing %s" % quote_output(abs_sharefile)
                 failure.Failure().printTraceback(err)
     except:
hunk ./src/allmydata/scripts/debug.py 689
-        print >>err, "Error processing %s" % si_dir
+        print >>err, "Error processing %s" % quote_output(si_dir)
         failure.Failure().printTraceback(err)
 
 class CorruptShareOptions(usage.Options):
hunk ./src/allmydata/scripts/keygen.py 32
         print >>err, "a basedir was not provided, please use --basedir or -C"
         return -1
     if os.path.exists(basedir):
-        if os.listdir(basedir):
-            print >>err, "The base directory \"%s\", which is \"%s\" is not empty." % (basedir, os.path.abspath(basedir))
+        if listdir_unicode(basedir):
+            print >>err, "The base directory %s is not empty." % quote_output(basedir)
             print >>err, "To avoid clobbering anything, I am going to quit now."
             print >>err, "Please use a different directory, or empty this one."
             return -1
hunk ./src/allmydata/scripts/tahoe_backup.py 14
 from allmydata.scripts import backupdb
 from allmydata.util.encodingutil import listdir_unicode, quote_output, to_str, FilenameEncodingError
 from allmydata.util.assertutil import precondition
+from allmydata.util.fileutil import abspath_expanduser_unicode
 
 
 def get_local_metadata(path):
hunk ./src/allmydata/scripts/tahoe_backup.py 88
         self.backupdb = None
         bdbfile = os.path.join(options["node-directory"],
                                "private", "backupdb.sqlite")
-        bdbfile = os.path.abspath(bdbfile)
+        bdbfile = abspath_expanduser_unicode(bdbfile)
         self.backupdb = backupdb.get_backupdb(bdbfile, stderr)
         if not self.backupdb:
             print >>stderr, "ERROR: Unable to load backup db."
hunk ./src/allmydata/scripts/tahoe_cp.py 12
 from allmydata.scripts.common_http import do_http, HTTPError
 from allmydata import uri
 from allmydata.util import fileutil
+from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, to_str
 from allmydata.util.assertutil import precondition
 
hunk ./src/allmydata/scripts/tahoe_cp.py 515
         rootcap, path = get_alias(self.aliases, destination_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = os.path.abspath(os.path.expanduser(path.decode('utf-8')))
+            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
             if not os.path.exists(pathname):
                 t = LocalMissingTarget(pathname)
             elif os.path.isdir(pathname):
hunk ./src/allmydata/scripts/tahoe_cp.py 555
         rootcap, path = get_alias(self.aliases, source_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = os.path.abspath(os.path.expanduser(path.decode('utf-8')))
+            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
             name = os.path.basename(pathname)
             if not os.path.exists(pathname):
                 raise MissingSourceError(source_spec)
hunk ./src/allmydata/test/check_grid.py 68
         ]
 
     def parseArgs(self, nodedir, tahoe):
-        self.nodedir = nodedir
-        self.tahoe = os.path.abspath(tahoe)
+        # Note: does not support Unicode arguments.
+        self.nodedir = os.path.expanduser(nodedir)
+        self.tahoe = os.path.abspath(os.path.expanduser(tahoe))
 
 class CommandFailed(Exception):
     pass
hunk ./src/allmydata/test/check_memory.py 12
 from allmydata.immutable import upload
 from allmydata.scripts import create_node
 from allmydata.util import fileutil, pollmixin
+from allmydata.util.fileutil import abspath_expanduser_unicode
+from allmydata.util.encodingutil import get_filesystem_encoding
 from foolscap.api import Tub, fireEventually, flushEventualQueue
 from twisted.python import log
 
hunk ./src/allmydata/test/check_memory.py 68
     numnodes = 7
 
     def __init__(self, basedir, mode):
-        self.basedir = basedir = os.path.abspath(basedir)
-        if not basedir.startswith(os.path.abspath(".")):
+        self.basedir = basedir = abspath_expanduser_unicode(unicode(basedir))
+        if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
             raise AssertionError("safety issue: basedir must be a subdir")
         self.testdir = testdir = os.path.join(basedir, "test")
         if os.path.exists(testdir):
hunk ./src/allmydata/test/check_memory.py 231
     def start_client(self):
         # this returns a Deferred that fires with the client's control.furl
         log.msg("MAKING CLIENT")
-        clientdir = self.clientdir = os.path.join(self.testdir, "client")
+        # self.testdir is an absolute Unicode path
+        clientdir = self.clientdir = os.path.join(self.testdir, u"client")
+        clientdir_str = clientdir.encode(get_filesystem_encoding())
         quiet = StringIO()
         create_node.create_node(clientdir, {}, out=quiet)
         log.msg("DONE MAKING CLIENT")
hunk ./src/allmydata/test/check_memory.py 272
         logfile = os.path.join(self.basedir, "client.log")
         cmd = ["twistd", "-n", "-y", "tahoe-client.tac", "-l", logfile]
         env = os.environ.copy()
-        self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir)
+        self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str)
         log.msg("CLIENT STARTED")
 
         # now we wait for the client to get started. we're looking for the
hunk ./src/allmydata/test/test_cli.py 36
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import listdir_unicode, unicode_platform, \
     quote_output, get_output_encoding, get_argv_encoding, get_filesystem_encoding, \
-    unicode_to_output, to_str
+    unicode_to_output, to_str, to_argv
+from allmydata.util.fileutil import abspath_expanduser_unicode
 
 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
 
hunk ./src/allmydata/test/test_cli.py 185
         u = uri.LiteralFileURI("this is some data")
         output = self._dump_cap(u.to_string())
         self.failUnless("Literal File URI:" in output, output)
-        self.failUnless("data: this is some data" in output, output)
+        self.failUnless("data: 'this is some data'" in output, output)
 
     def test_dump_cap_ssk(self):
         writekey = "\x01" * 16
hunk ./src/allmydata/test/test_cli.py 778
         self.set_up_grid()
 
         rel_fn = os.path.join(self.basedir, "DATAFILE")
-        abs_fn = os.path.abspath(rel_fn)
+        abs_fn = to_argv(abspath_expanduser_unicode(unicode(rel_fn)))
         # we make the file small enough to fit in a LIT file, for speed
         fileutil.write(rel_fn, "short file")
         d = self.do_cli("put", rel_fn)
hunk ./src/allmydata/test/test_encodingutil.py 262
             raise unittest.SkipTest("%r\nIt is possible that the filesystem on which this test is being run "
                                     "does not support Unicode, even though the platform does." % (e,))
 
-        fn = lumiere_nfc + '/' + lumiere_nfc + '.txt'
+        fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt'
         open(fn, 'wb').close()
         self.failUnless(os.path.exists(fn))
hunk ./src/allmydata/test/test_encodingutil.py 265
-        self.failUnless(os.path.exists(os.path.abspath(fn)))
+        self.failUnless(os.path.exists(os.path.join(os.getcwdu(), fn)))
         filenames = listdir_unicode(lumiere_nfc)
 
         # We only require that the listing includes a filename that is canonically equivalent
hunk ./src/allmydata/test/test_runner.py 196
         rc, out, err = self.run_tahoe(argv)
         self.failIfEqual(rc, 0, str((out, err, rc)))
         self.failUnlessEqual(out, "")
-        self.failUnless("is not empty." in err)
+        self.failUnlessIn("is not empty.", err)
 
         # make sure it rejects too many arguments
         argv = ["create-key-generator", "basedir", "extraarg"]
hunk ./src/allmydata/test/test_system.py 15
 from allmydata.immutable.filenode import ImmutableFileNode, LiteralFileNode
 from allmydata.util import idlib, mathutil
 from allmydata.util import log, base32
+from allmydata.util.encodingutil import quote_output
+from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.consumer import MemoryConsumer, download_to_data
 from allmydata.scripts import runner
 from allmydata.interfaces import IDirectoryNode, IFileNode, \
hunk ./src/allmydata/test/test_system.py 1290
                 if magic == '\x00\x00\x00\x01':
                     break
         else:
-            self.fail("unable to find any uri_extension files in %s"
+            self.fail("unable to find any uri_extension files in %r"
                       % self.basedir)
hunk ./src/allmydata/test/test_system.py 1292
-        log.msg("test_system.SystemTest._test_runner using %s" % filename)
+        log.msg("test_system.SystemTest._test_runner using %r" % filename)
 
         out,err = StringIO(), StringIO()
         rc = runner.runner(["debug", "dump-share", "--offsets",
hunk ./src/allmydata/test/test_system.py 1303
 
         # we only upload a single file, so we can assert some things about
         # its size and shares.
-        self.failUnless(("share filename: %s" % filename) in output)
-        self.failUnless("size: %d\n" % len(self.data) in output)
-        self.failUnless("num_segments: 1\n" in output)
+        self.failUnlessIn("share filename: %s" % quote_output(abspath_expanduser_unicode(filename)), output)
+        self.failUnlessIn("size: %d\n" % len(self.data), output)
+        self.failUnlessIn("num_segments: 1\n", output)
         # segment_size is always a multiple of needed_shares
hunk ./src/allmydata/test/test_system.py 1307
-        self.failUnless("segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3) in output)
-        self.failUnless("total_shares: 10\n" in output)
+        self.failUnlessIn("segment_size: %d\n" % mathutil.next_multiple(len(self.data), 3), output)
+        self.failUnlessIn("total_shares: 10\n", output)
         # keys which are supposed to be present
         for key in ("size", "num_segments", "segment_size",
                     "needed_shares", "total_shares",
hunk ./src/allmydata/test/test_system.py 1316
                     #"plaintext_hash", "plaintext_root_hash",
                     "crypttext_hash", "crypttext_root_hash",
                     "share_root_hash", "UEB_hash"):
-            self.failUnless("%s: " % key in output, key)
-        self.failUnless("  verify-cap: URI:CHK-Verifier:" in output)
+            self.failUnlessIn("%s: " % key, output)
+        self.failUnlessIn("  verify-cap: URI:CHK-Verifier:", output)
 
         # now use its storage index to find the other shares using the
         # 'find-shares' tool
}

Context:

[__init__.py: silence DeprecationWarning about BaseException.message globally. fixes #1129
david-sarah@jacaranda.org**20100720011939
 Ignore-this: 38808986ba79cb2786b010504a22f89
] 
[test_runner: test that 'tahoe --version' outputs no noise (e.g. DeprecationWarnings).
david-sarah@jacaranda.org**20100720011345
 Ignore-this: dd358b7b2e5d57282cbe133e8069702e
] 
[TAG allmydata-tahoe-1.7.1
zooko@zooko.com**20100719131352
 Ignore-this: 6942056548433dc653a746703819ad8c
] 
Patch bundle hash:
7701b52497cb185827602a19acd985d87395e356