Ticket #1072: rename-stringutils-drop-listdir_unicode-and-open_unicode.dpatch

Sun Jul 11 20:55:25 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.

Mon Jul 12 01:30:15 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil.

New patches:

[Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.
david-sarah@jacaranda.org**20100711195525
 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93
] {
hunk ./src/allmydata/web/common.py 14
      EmptyPathnameComponentError, MustBeDeepImmutableError, \
      MustBeReadonlyError, MustNotBeUnknownRWError
 from allmydata.mutable.common import UnrecoverableFileError
-from allmydata.util import abbreviate # TODO: consolidate
+from allmydata.util import abbreviate
+from allmydata.util.stringutils import to_str
 
 class IOpHandleTable(Interface):
     pass
hunk ./src/allmydata/web/common.py 68
     children = {}
     if children_json:
         data = simplejson.loads(children_json)
-        for (name, (ctype, propdict)) in data.iteritems():
-            name = unicode(name)
-            writecap = propdict.get("rw_uri")
-            if writecap is not None:
-                writecap = str(writecap)
-            readcap = propdict.get("ro_uri")
-            if readcap is not None:
-                readcap = str(readcap)
+        for (namex, (ctype, propdict)) in data.iteritems():
+            namex = unicode(namex)
+            writecap = to_str(propdict.get("rw_uri"))
+            readcap = to_str(propdict.get("ro_uri"))
             metadata = propdict.get("metadata", {})
hunk ./src/allmydata/web/common.py 73
-            childnode = nodemaker.create_from_cap(writecap, readcap)
-            children[name] = (childnode, metadata)
+            # name= argument is just for error reporting
+            childnode = nodemaker.create_from_cap(writecap, readcap, name=namex)
+            children[namex] = (childnode, metadata)
     return children
 
 def abbreviate_time(data):
}
[Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil.
david-sarah@jacaranda.org**20100712003015
 Ignore-this: 103b809d180df17a7283077c3104c7be
] {
move ./src/allmydata/test/test_stringutils.py ./src/allmydata/test/test_encodingutil.py
move ./src/allmydata/util/stringutils.py ./src/allmydata/util/encodingutil.py
hunk ./src/allmydata/dirnode.py 19
      DeepCheckAndRepairResults
 from allmydata.monitor import Monitor
 from allmydata.util import hashutil, mathutil, base32, log
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 from allmydata.util.assertutil import precondition
 from allmydata.util.netstring import netstring, split_netstring
 from allmydata.util.consumer import download_to_data
hunk ./src/allmydata/scripts/cli.py 4
 import os.path, re, sys, fnmatch
 from twisted.python import usage
 from allmydata.scripts.common import BaseOptions, get_aliases
-from allmydata.util.stringutils import argv_to_unicode
+from allmydata.util.encodingutil import argv_to_unicode
 
 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
 
hunk ./src/allmydata/scripts/common.py 5
 import os, sys, urllib
 import codecs
 from twisted.python import usage
-from allmydata.util.stringutils import unicode_to_url, quote_output
+from allmydata.util.encodingutil import unicode_to_url, quote_output
 from allmydata.util.assertutil import precondition
 
 class BaseOptions:
hunk ./src/allmydata/scripts/common_http.py 6
 import urlparse, httplib
 import allmydata # for __full_version__
 
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 from allmydata.scripts.common import TahoeError
 
 
hunk ./src/allmydata/scripts/slow_operation.py 7
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
 from allmydata.util import base32
-from allmydata.util.stringutils import quote_output, is_printable_ascii
+from allmydata.util.encodingutil import quote_output, is_printable_ascii
 import urllib
 import simplejson
 
hunk ./src/allmydata/scripts/tahoe_add_alias.py 8
 from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_aliases
 from allmydata.util.fileutil import move_into_place
-from allmydata.util.stringutils import unicode_to_output, quote_output
+from allmydata.util.encodingutil import unicode_to_output, quote_output
 
 
 def add_line_to_aliasfile(aliasfile, alias, cap):
hunk ./src/allmydata/scripts/tahoe_backup.py 12
 from allmydata.scripts.common_http import do_http, HTTPError, format_http_error
 from allmydata.util import time_format
 from allmydata.scripts import backupdb
-from allmydata.util.stringutils import listdir_unicode, open_unicode, quote_output, to_str
+from allmydata.util.encodingutil import quote_output, to_str
+from allmydata.util.fileutil import open_expanduser
 from allmydata.util.assertutil import precondition
 
 
hunk ./src/allmydata/scripts/tahoe_backup.py 170
         compare_contents = {} # childname -> rocap
 
         try:
-            children = listdir_unicode(localpath)
+            children = os.listdir(localpath)
         except EnvironmentError:
             self.directories_skipped += 1
             self.warn("WARNING: permission denied on directory %s" % quote_output(localpath))
hunk ./src/allmydata/scripts/tahoe_backup.py 175
             children = []
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            self.directories_skipped += 1
+            self.warn("WARNING: could not list directory %s due to an encoding error" % quote_output(localpath))
+            children = []
 
         for child in self.options.filter_listdir(children):
             assert isinstance(child, unicode), child
hunk ./src/allmydata/scripts/tahoe_backup.py 300
 
         if must_upload:
             self.verboseprint("uploading %s.." % quote_output(childpath))
-            infileobj = open_unicode(childpath, "rb")
+            infileobj = open_expanduser(childpath, "rb")
             url = self.options['node-url'] + "uri"
             resp = do_http("PUT", url, infileobj)
             if resp.status not in (200, 201):
hunk ./src/allmydata/scripts/tahoe_check.py 8
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import quote_output, quote_path
+from allmydata.util.encodingutil import quote_output, quote_path
 
 class Checker:
     pass
hunk ./src/allmydata/scripts/tahoe_cp.py 11
                                      DefaultAliasMarker, TahoeError
 from allmydata.scripts.common_http import do_http, HTTPError
 from allmydata import uri
-from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode, \
-    abspath_expanduser_unicode, quote_output, to_str
+from allmydata.util.encodingutil import unicode_to_url, quote_output, to_str
+from allmydata.util import fileutil
+from allmydata.util.fileutil import open_expanduser, abspath_expanduser
 from allmydata.util.assertutil import precondition
 
 
hunk ./src/allmydata/scripts/tahoe_cp.py 17
-def _put_local_file(pathname, inf):
-    # TODO: create temporary file and move into place?
-    # TODO: move this to fileutil.
-    outf = open_unicode(pathname, "wb")
-    try:
-        while True:
-            data = inf.read(32768)
-            if not data:
-                break
-            outf.write(data)
-    finally:
-        outf.close()
-
-
 class MissingSourceError(TahoeError):
     def __init__(self, name):
         TahoeError.__init__(self, "No such file or directory %s" % quote_output(name))
hunk ./src/allmydata/scripts/tahoe_cp.py 71
         return True
 
     def open(self, caps_only):
-        return open_unicode(self.pathname, "rb")
+        return open_expanduser(self.pathname, "rb")
 
 
 class LocalFileTarget:
hunk ./src/allmydata/scripts/tahoe_cp.py 80
         self.pathname = pathname
 
     def put_file(self, inf):
-        _put_local_file(self.pathname, inf)
+        fileutil.put_file(self.pathname, inf)
 
 
 class LocalMissingTarget:
hunk ./src/allmydata/scripts/tahoe_cp.py 89
         self.pathname = pathname
 
     def put_file(self, inf):
-        _put_local_file(self.pathname, inf)
+        fileutil.put_file(self.pathname, inf)
 
 
 class LocalDirectorySource:
hunk ./src/allmydata/scripts/tahoe_cp.py 104
         if self.children is not None:
             return
         self.children = {}
-        children = listdir_unicode(self.pathname)
+        children = os.listdir(self.pathname)
         for i,n in enumerate(children):
             self.progressfunc("examining %d of %d" % (i, len(children)))
             pn = os.path.join(self.pathname, n)
hunk ./src/allmydata/scripts/tahoe_cp.py 132
         if self.children is not None:
             return
         self.children = {}
-        children = listdir_unicode(self.pathname)
+        children = os.listdir(self.pathname)
         for i,n in enumerate(children):
             self.progressfunc("examining %d of %d" % (i, len(children)))
             n = unicode(n)
hunk ./src/allmydata/scripts/tahoe_cp.py 158
     def put_file(self, name, inf):
         precondition(isinstance(name, unicode), name)
         pathname = os.path.join(self.pathname, name)
-        _put_local_file(pathname, inf)
+        fileutil.put_file(pathname, inf)
 
     def set_children(self):
         pass
hunk ./src/allmydata/scripts/tahoe_cp.py 515
         rootcap, path = get_alias(self.aliases, destination_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
+            pathname = abspath_expanduser(path.decode('utf-8'))
             if not os.path.exists(pathname):
                 t = LocalMissingTarget(pathname)
             elif os.path.isdir(pathname):
hunk ./src/allmydata/scripts/tahoe_cp.py 555
         rootcap, path = get_alias(self.aliases, source_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
+            pathname = abspath_expanduser(path.decode('utf-8'))
             name = os.path.basename(pathname)
             if not os.path.exists(pathname):
                 raise MissingSourceError(source_spec)
hunk ./src/allmydata/scripts/tahoe_get.py 6
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import open_unicode
+from allmydata.util.fileutil import open_expanduser
 
 def get(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_get.py 30
     resp = do_http("GET", url)
     if resp.status in (200, 201,):
         if to_file:
-            outf = open_unicode(to_file, "wb")
+            outf = open_expanduser(to_file, "wb")
         else:
             outf = stdout
         while True:
hunk ./src/allmydata/scripts/tahoe_ls.py 7
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import unicode_to_output, quote_output, is_printable_ascii, to_str
+from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_str
 
 def list(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_manifest.py 9
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import quote_output, quote_path
+from allmydata.util.encodingutil import quote_output, quote_path
 
 class FakeTransport:
     disconnecting = False
hunk ./src/allmydata/scripts/tahoe_mkdir.py 5
 import urllib
 from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 
 def mkdir(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_mv.py 8
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import to_str
+from allmydata.util.encodingutil import to_str
 
 # this script is used for both 'mv' and 'ln'
 
hunk ./src/allmydata/scripts/tahoe_put.py 7
 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
-from allmydata.util.stringutils import quote_output, open_unicode
+from allmydata.util.encodingutil import quote_output
+from allmydata.util.fileutil import open_expanduser
 
 def put(options):
     """
hunk ./src/allmydata/scripts/tahoe_put.py 68
     if mutable:
         url += "?mutable=true"
     if from_file:
-        infileobj = open_unicode(from_file, "rb")
+        infileobj = open_expanduser(from_file, "rb")
     else:
         # do_http() can't use stdin directly: for one thing, we need a
         # Content-Length field. So we currently must copy it.
hunk ./src/allmydata/test/test_backupdb.py 7
 from twisted.trial import unittest
 
 from allmydata.util import fileutil
-from allmydata.util.stringutils import listdir_unicode, get_filesystem_encoding, unicode_platform
+from allmydata.util.encodingutil import get_filesystem_encoding, unicode_platform
 from allmydata.util.assertutil import precondition
 from allmydata.scripts import backupdb
 
hunk ./src/allmydata/test/test_backupdb.py 252
         self.failUnless(bdb)
 
         self.writeto(u"f\u00f6\u00f6.txt", "foo.txt")
-        files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")]
+        files = [fn for fn in os.listdir(unicode(basedir)) if fn.endswith(".txt")]
         self.failUnlessEqual(len(files), 1)
         foo_fn = os.path.join(basedir, files[0])
         #print foo_fn, type(foo_fn)
hunk ./src/allmydata/test/test_cli.py 34
 from twisted.python import usage
 
 from allmydata.util.assertutil import precondition
-from allmydata.util.stringutils import listdir_unicode, open_unicode, unicode_platform, \
-    quote_output, get_output_encoding, get_argv_encoding, get_filesystem_encoding, \
+from allmydata.util.encodingutil import unicode_platform, quote_output, \
+    get_output_encoding, get_argv_encoding, get_filesystem_encoding, \
     unicode_to_output, to_str
 
 timeout = 480 # deep_check takes 360s on Zandr's linksys box, others take > 240s
hunk ./src/allmydata/test/test_cli.py 442
         fileutil.make_dirs(basedir)
 
         for name in filenames:
-            open_unicode(os.path.join(unicode(basedir), name), "wb").close()
+            open(os.path.join(unicode(basedir), name), "wb").close()
 
hunk ./src/allmydata/test/test_cli.py 444
-        for file in listdir_unicode(unicode(basedir)):
+        for file in os.listdir(unicode(basedir)):
             self.failUnlessIn(normalize(file), filenames)
 
 
hunk ./src/allmydata/test/test_cli.py 977
         rel_fn = os.path.join(unicode(self.basedir), u"à trier.txt")
         # we make the file small enough to fit in a LIT file, for speed
         DATA = "short file"
-        f = open_unicode(rel_fn, "wb")
-        try:
-            f.write(DATA)
-        finally:
-            f.close()
+        fileutil.write(rel_fn, DATA)
 
         d = self.do_cli("create-alias", "tahoe")
 
hunk ./src/allmydata/test/test_cli.py 1348
         self.set_up_grid()
 
         DATA1 = "unicode file content"
-        f = open_unicode(fn1, "wb")
-        try:
-            f.write(DATA1)
-        finally:
-            f.close()
+        fileutil.write(fn1, DATA1)
 
         fn2 = os.path.join(self.basedir, "Metallica")
         DATA2 = "non-unicode file content"
hunk ./src/allmydata/test/test_encodingutil.py 16
 # systems.
 
 if __name__ == "__main__":
-    import sys, os
-    import tempfile
-    import shutil
+    import sys
     import platform
 
     if len(sys.argv) != 2:
hunk ./src/allmydata/test/test_encodingutil.py 32
     print "    filesystem_encoding = '%s'" % sys.getfilesystemencoding()
     print "    output_encoding = '%s'" % sys.stdout.encoding
     print "    argv_encoding = '%s'" % (sys.platform == "win32" and 'ascii' or sys.stdout.encoding)
-
-    try:
-        tmpdir = tempfile.mkdtemp()
-        for fname in TEST_FILENAMES:
-            open(os.path.join(tmpdir, fname), 'w').close() 
-
-        # Use Unicode API under Windows or MacOS X
-        if sys.platform in ('win32', 'darwin'):
-            dirlist = os.listdir(unicode(tmpdir))
-        else:
-            dirlist = os.listdir(tmpdir)
-
-        print "    dirlist = %s" % repr(dirlist)
-    except:
-        print "    # Oops, I cannot write filenames containing non-ascii characters"
     print
 
hunk ./src/allmydata/test/test_encodingutil.py 34
-    shutil.rmtree(tmpdir)
     sys.exit(0)
 
 from twisted.trial import unittest
hunk ./src/allmydata/test/test_encodingutil.py 41
 import sys
 
 from allmydata.test.common_util import ReallyEqualMixin
-from allmydata.util.stringutils import argv_to_unicode, unicode_to_url, \
-    unicode_to_output, unicode_platform, listdir_unicode, open_unicode, \
-    FilenameEncodingError, get_output_encoding, _reload
-from allmydata.dirnode import normalize
+from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
+    unicode_to_output, unicode_platform, get_output_encoding, _reload
 
 from twisted.python import usage
 
hunk ./src/allmydata/test/test_encodingutil.py 85
         _reload()
         self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc)
 
-    @patch('os.listdir')
-    def test_no_unicode_normalization(self, mock):
-        # Pretend to run on a Unicode platform.
-        # We normalized to NFC in 1.7beta, but we now don't.
-        orig_platform = sys.platform
-        try:
-            sys.platform = 'darwin'
-            mock.return_value = [Artonwall_nfd]
-            _reload()
-            self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
-        finally:
-            sys.platform = orig_platform
-
-# The following tests applies only to platforms which don't store filenames as
-# Unicode entities on the filesystem.
-class StringUtilsNonUnicodePlatform(unittest.TestCase):
-    def setUp(self):
-        # Mock sys.platform because unicode_platform() uses it
-        self.original_platform = sys.platform
-        sys.platform = 'linux'
-
-    def tearDown(self):
-        sys.platform = self.original_platform
-        _reload()
-
-    @patch('sys.getfilesystemencoding')
-    @patch('os.listdir')
-    def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding):
-        # What happens if latin1-encoded filenames are encountered on an UTF-8
-        # filesystem?
-        mock_listdir.return_value = [
-            lumiere_nfc.encode('utf-8'),
-            lumiere_nfc.encode('latin1')]
-
-        mock_getfilesystemencoding.return_value = 'utf-8'
-        _reload()
-        self.failUnlessRaises(FilenameEncodingError,
-                              listdir_unicode,
-                              u'/dummy')
-        
-        # We're trying to list a directory whose name cannot be represented in
-        # the filesystem encoding.  This should fail.
-        mock_getfilesystemencoding.return_value = 'ascii'
-        _reload()
-        self.failUnlessRaises(FilenameEncodingError,
-                              listdir_unicode,
-                              u'/' + lumiere_nfc)
-
-    @patch('sys.getfilesystemencoding')
-    def test_open_unicode(self, mock):
-        mock.return_value = 'ascii'
-        _reload()
-        self.failUnlessRaises(FilenameEncodingError,
-                              open_unicode,
-                              lumiere_nfc, 'rb')
 
 class StringUtils(ReallyEqualMixin):
     def setUp(self):
hunk ./src/allmydata/test/test_encodingutil.py 130
         _reload()
         self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform])
  
-    @patch('sys.getfilesystemencoding')
-    @patch('os.listdir')
-    def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding):
-        if 'dirlist' not in dir(self):
-            return
-
-        try:
-            u"test".encode(self.filesystem_encoding)
-        except (LookupError, AttributeError):
-            raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding "
-                                    "that we are testing for the benefit of a different platform."
-                                    % (self.filesystem_encoding,))
-
-        mock_listdir.return_value = self.dirlist
-        mock_getfilesystemencoding.return_value = self.filesystem_encoding
-       
-        _reload()
-        filenames = listdir_unicode(u'/dummy')
-
-        self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
-                             set(TEST_FILENAMES))
-
-    @patch('sys.getfilesystemencoding')
-    @patch('__builtin__.open')
-    def test_open_unicode(self, mock_open, mock_getfilesystemencoding):
-        mock_getfilesystemencoding.return_value = self.filesystem_encoding
-        fn = u'/dummy_directory/" + lumiere_nfc + ".txt'
-
-        try:
-            u"test".encode(self.filesystem_encoding)
-        except (LookupError, AttributeError):
-            raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding "
-                                    "that we are testing for the benefit of a different platform."
-                                    % (self.filesystem_encoding,))
-
-        _reload()
-        try:
-            open_unicode(fn, 'rb')
-        except FilenameEncodingError:
-            return
-
-        # Pass Unicode string to open() on Unicode platforms
-        if unicode_platform():
-            mock_open.assert_called_with(fn, 'rb')
-
-        # Pass correctly encoded bytestrings to open() on non-Unicode platforms
-        else:
-            fn_bytestring = fn.encode(self.filesystem_encoding)
-            mock_open.assert_called_with(fn_bytestring, 'rb')
-
 
 class UbuntuKarmicUTF8(StringUtils, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
hunk ./src/allmydata/test/test_encodingutil.py 139
     filesystem_encoding = 'UTF-8'
     output_encoding = 'UTF-8'
     argv_encoding = 'UTF-8'
-    dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt']
 
 class UbuntuKarmicLatin1(StringUtils, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
hunk ./src/allmydata/test/test_encodingutil.py 148
     filesystem_encoding = 'ISO-8859-1'
     output_encoding = 'ISO-8859-1'
     argv_encoding = 'ISO-8859-1'
-    dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3']
 
 class WindowsXP(StringUtils, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
hunk ./src/allmydata/test/test_encodingutil.py 156
     filesystem_encoding = 'mbcs'
     output_encoding = 'cp850'
     argv_encoding = 'ascii'
-    dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
 class WindowsXP_UTF8(StringUtils, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
hunk ./src/allmydata/test/test_encodingutil.py 164
     filesystem_encoding = 'mbcs'
     output_encoding = 'cp65001'
     argv_encoding = 'ascii'
-    dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
 class WindowsVista(StringUtils, unittest.TestCase):
     uname = 'Windows Vista 6.0.6000 x86 x86 Family 6 Model 15 Stepping 11, GenuineIntel'
hunk ./src/allmydata/test/test_encodingutil.py 172
     filesystem_encoding = 'mbcs'
     output_encoding = 'cp850'
     argv_encoding = 'ascii'
-    dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
 class MacOSXLeopard(StringUtils, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
hunk ./src/allmydata/test/test_encodingutil.py 181
     filesystem_encoding = 'utf-8'
     output_encoding = 'UTF-8'
     argv_encoding = 'UTF-8'
-    dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
 
 class MacOSXLeopard7bit(StringUtils, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
hunk ./src/allmydata/test/test_encodingutil.py 188
     filesystem_encoding = 'utf-8'
     output_encoding = 'US-ASCII'
     argv_encoding = 'US-ASCII'
-    dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
 
 class OpenBSD(StringUtils, unittest.TestCase):
     uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)'
hunk ./src/allmydata/util/encodingutil.py 7
 """
 
 import sys
-import os
 import re
 from allmydata.util.assertutil import precondition
 from twisted.python import usage
hunk ./src/allmydata/util/encodingutil.py 176
     Does the current platform handle Unicode filenames natively?
     """
     return is_unicode_platform
-
-class FilenameEncodingError(Exception):
-    """
-    Filename cannot be encoded using the current encoding of your filesystem
-    (%s). Please configure your locale correctly or rename this file.
-    """
-    pass
-
-def listdir_unicode_fallback(path):
-    """
-    This function emulates a fallback Unicode API similar to one available
-    under Windows or MacOS X.
-
-    If badly encoded filenames are encountered, an exception is raised.
-    """
-    precondition(isinstance(path, unicode), path)
-
-    try:
-        byte_path = path.encode(filesystem_encoding)
-    except (UnicodeEncodeError, UnicodeDecodeError):
-        raise FilenameEncodingError(path)
-
-    try:
-        return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)]
-    except UnicodeDecodeError:
-        raise FilenameEncodingError(fn)
-
-def listdir_unicode(path):
-    """
-    Wrapper around listdir() which provides safe access to the convenient
-    Unicode API even under platforms that don't provide one natively.
-    """
-    precondition(isinstance(path, unicode), path)
-
-    # On Windows and MacOS X, the Unicode API is used
-    # On other platforms (ie. Unix systems), the byte-level API is used
-
-    if is_unicode_platform:
-        return os.listdir(path)
-    else:
-        return listdir_unicode_fallback(path)
-
-def open_unicode(path, mode):
-    """
-    Wrapper around open() which provides safe access to the convenient Unicode
-    API even under Unix.
-    """
-    precondition(isinstance(path, unicode), path)
-
-    if is_unicode_platform:
-        return open(os.path.expanduser(path), mode)
-    else:
-        try:
-            return open(os.path.expanduser(path.encode(filesystem_encoding)), mode)
-        except UnicodeEncodeError:
-            raise FilenameEncodingError(path)
-
-def abspath_expanduser_unicode(path):
-    precondition(isinstance(path, unicode), path)
-
-    if is_unicode_platform:
-        return os.path.abspath(os.path.expanduser(path))
-    else:
-        try:
-            pathstr = path.encode(filesystem_encoding)
-            return os.path.abspath(os.path.expanduser(pathstr)).decode(filesystem_encoding)
-        except (UnicodeEncodeError, UnicodeDecodeError):
-            raise FilenameEncodingError(path)
hunk ./src/allmydata/util/fileutil.py 212
     finally:
         rf.close()
 
+def put_file(pathname, inf):
+    # TODO: create temporary file and move into place?
+    outf = open_expanduser(pathname, "wb")
+    try:
+        while True:
+            data = inf.read(32768)
+            if not data:
+                break
+            outf.write(data)
+    finally:
+        outf.close()
+
+def open_expanduser(path, mode):
+    assert isinstance(path, unicode), path
+    return open(os.path.expanduser(path), mode)
+
+def abspath_expanduser(path):
+    assert isinstance(path, unicode), path
+    return os.path.abspath(os.path.expanduser(path))
+
hunk ./src/allmydata/web/common.py 15
      MustBeReadonlyError, MustNotBeUnknownRWError
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.util import abbreviate
-from allmydata.util.stringutils import to_str
+from allmydata.util.encodingutil import to_str
 
 class IOpHandleTable(Interface):
     pass
}

Context:

[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
5dd0b81c02ed648d4eb4ab565a237c53ccbf4ff3