Ticket #1072: rename-stringutils-drop-open_unicode.dpatch

Sun Jul 11 20:55:25 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.

Tue Jul 13 05:27:15 GMT Daylight Time 2010  david-sarah@jacaranda.org
  * Rename stringutils to encodingutil, and drop open_unicode (since the Python 'open' function works fine with Unicode paths).

New patches:

[Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.
david-sarah@jacaranda.org**20100711195525
 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93
] {
hunk ./src/allmydata/web/common.py 14
      EmptyPathnameComponentError, MustBeDeepImmutableError, \
      MustBeReadonlyError, MustNotBeUnknownRWError
 from allmydata.mutable.common import UnrecoverableFileError
-from allmydata.util import abbreviate # TODO: consolidate
+from allmydata.util import abbreviate
+from allmydata.util.stringutils import to_str
 
 class IOpHandleTable(Interface):
     pass
hunk ./src/allmydata/web/common.py 68
     children = {}
     if children_json:
         data = simplejson.loads(children_json)
-        for (name, (ctype, propdict)) in data.iteritems():
-            name = unicode(name)
-            writecap = propdict.get("rw_uri")
-            if writecap is not None:
-                writecap = str(writecap)
-            readcap = propdict.get("ro_uri")
-            if readcap is not None:
-                readcap = str(readcap)
+        for (namex, (ctype, propdict)) in data.iteritems():
+            namex = unicode(namex)
+            writecap = to_str(propdict.get("rw_uri"))
+            readcap = to_str(propdict.get("ro_uri"))
             metadata = propdict.get("metadata", {})
hunk ./src/allmydata/web/common.py 73
-            childnode = nodemaker.create_from_cap(writecap, readcap)
-            children[name] = (childnode, metadata)
+            # name= argument is just for error reporting
+            childnode = nodemaker.create_from_cap(writecap, readcap, name=namex)
+            children[namex] = (childnode, metadata)
     return children
 
 def abbreviate_time(data):
}
[Rename stringutils to encodingutil, and drop open_unicode (since the Python 'open' function works fine with Unicode paths).
david-sarah@jacaranda.org**20100713042715
 Ignore-this: fa2bb6b5d48ce9ba7ea2b1afc9f3b7b4
] {
move ./src/allmydata/test/test_stringutils.py ./src/allmydata/test/test_encodingutil.py
move ./src/allmydata/util/stringutils.py ./src/allmydata/util/encodingutil.py
hunk ./src/allmydata/dirnode.py 19
      DeepCheckAndRepairResults
 from allmydata.monitor import Monitor
 from allmydata.util import hashutil, mathutil, base32, log
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 from allmydata.util.assertutil import precondition
 from allmydata.util.netstring import netstring, split_netstring
 from allmydata.util.consumer import download_to_data
hunk ./src/allmydata/scripts/cli.py 4
 import os.path, re, sys, fnmatch
 from twisted.python import usage
 from allmydata.scripts.common import BaseOptions, get_aliases
-from allmydata.util.stringutils import argv_to_unicode
+from allmydata.util.encodingutil import argv_to_unicode
 
 NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
 
hunk ./src/allmydata/scripts/common.py 5
 import os, sys, urllib
 import codecs
 from twisted.python import usage
-from allmydata.util.stringutils import unicode_to_url, quote_output
+from allmydata.util.encodingutil import unicode_to_url, quote_output
 from allmydata.util.assertutil import precondition
 
 class BaseOptions:
hunk ./src/allmydata/scripts/common_http.py 6
 import urlparse, httplib
 import allmydata # for __full_version__
 
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 from allmydata.scripts.common import TahoeError
 
 
hunk ./src/allmydata/scripts/slow_operation.py 7
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
 from allmydata.util import base32
-from allmydata.util.stringutils import quote_output, is_printable_ascii
+from allmydata.util.encodingutil import quote_output, is_printable_ascii
 import urllib
 import simplejson
 
hunk ./src/allmydata/scripts/tahoe_add_alias.py 8
 from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_aliases
 from allmydata.util.fileutil import move_into_place
-from allmydata.util.stringutils import unicode_to_output, quote_output
+from allmydata.util.encodingutil import unicode_to_output, quote_output
 
 
 def add_line_to_aliasfile(aliasfile, alias, cap):
hunk ./src/allmydata/scripts/tahoe_backup.py 12
 from allmydata.scripts.common_http import do_http, HTTPError, format_http_error
 from allmydata.util import time_format
 from allmydata.scripts import backupdb
-from allmydata.util.stringutils import listdir_unicode, open_unicode, quote_output, to_str
+from allmydata.util.encodingutil import listdir_unicode, quote_output, to_str, FilenameEncodingError
 from allmydata.util.assertutil import precondition
 
 
hunk ./src/allmydata/scripts/tahoe_backup.py 174
             self.directories_skipped += 1
             self.warn("WARNING: permission denied on directory %s" % quote_output(localpath))
             children = []
+        except FilenameEncodingError:
+            self.directories_skipped += 1
+            self.warn("WARNING: could not list directory %s due to a filename encoding error" % quote_output(localpath))
+            children = []
 
         for child in self.options.filter_listdir(children):
             assert isinstance(child, unicode), child
hunk ./src/allmydata/scripts/tahoe_backup.py 299
 
         if must_upload:
             self.verboseprint("uploading %s.." % quote_output(childpath))
-            infileobj = open_unicode(childpath, "rb")
+            infileobj = open(childpath, "rb")
             url = self.options['node-url'] + "uri"
             resp = do_http("PUT", url, infileobj)
             if resp.status not in (200, 201):
hunk ./src/allmydata/scripts/tahoe_check.py 8
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import quote_output, quote_path
+from allmydata.util.encodingutil import quote_output, quote_path
 
 class Checker:
     pass
hunk ./src/allmydata/scripts/tahoe_cp.py 11
                                      DefaultAliasMarker, TahoeError
 from allmydata.scripts.common_http import do_http, HTTPError
 from allmydata import uri
-from allmydata.util.stringutils import unicode_to_url, listdir_unicode, open_unicode, \
-    abspath_expanduser_unicode, quote_output, to_str
+from allmydata.util import fileutil
+from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, to_str
 from allmydata.util.assertutil import precondition
 
 
hunk ./src/allmydata/scripts/tahoe_cp.py 16
-def _put_local_file(pathname, inf):
-    # TODO: create temporary file and move into place?
-    # TODO: move this to fileutil.
-    outf = open_unicode(pathname, "wb")
-    try:
-        while True:
-            data = inf.read(32768)
-            if not data:
-                break
-            outf.write(data)
-    finally:
-        outf.close()
-
-
 class MissingSourceError(TahoeError):
     def __init__(self, name):
         TahoeError.__init__(self, "No such file or directory %s" % quote_output(name))
hunk ./src/allmydata/scripts/tahoe_cp.py 70
         return True
 
     def open(self, caps_only):
-        return open_unicode(self.pathname, "rb")
+        return open(os.path.expanduser(self.pathname), "rb")
 
 
 class LocalFileTarget:
hunk ./src/allmydata/scripts/tahoe_cp.py 79
         self.pathname = pathname
 
     def put_file(self, inf):
-        _put_local_file(self.pathname, inf)
+        fileutil.put_file(self.pathname, inf)
 
 
 class LocalMissingTarget:
hunk ./src/allmydata/scripts/tahoe_cp.py 88
         self.pathname = pathname
 
     def put_file(self, inf):
-        _put_local_file(self.pathname, inf)
+        fileutil.put_file(self.pathname, inf)
 
 
 class LocalDirectorySource:
hunk ./src/allmydata/scripts/tahoe_cp.py 157
     def put_file(self, name, inf):
         precondition(isinstance(name, unicode), name)
         pathname = os.path.join(self.pathname, name)
-        _put_local_file(pathname, inf)
+        fileutil.put_file(pathname, inf)
 
     def set_children(self):
         pass
hunk ./src/allmydata/scripts/tahoe_cp.py 514
         rootcap, path = get_alias(self.aliases, destination_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
+            pathname = os.path.abspath(os.path.expanduser(path.decode('utf-8')))
             if not os.path.exists(pathname):
                 t = LocalMissingTarget(pathname)
             elif os.path.isdir(pathname):
hunk ./src/allmydata/scripts/tahoe_cp.py 554
         rootcap, path = get_alias(self.aliases, source_spec, None)
         if rootcap == DefaultAliasMarker:
             # no alias, so this is a local file
-            pathname = abspath_expanduser_unicode(path.decode('utf-8'))
+            pathname = os.path.abspath(os.path.expanduser(path.decode('utf-8')))
             name = os.path.basename(pathname)
             if not os.path.exists(pathname):
                 raise MissingSourceError(source_spec)
hunk ./src/allmydata/scripts/tahoe_get.py 2
 
-import urllib
+import os, urllib
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
hunk ./src/allmydata/scripts/tahoe_get.py 6
-from allmydata.util.stringutils import open_unicode
 
 def get(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_get.py 29
     resp = do_http("GET", url)
     if resp.status in (200, 201,):
         if to_file:
-            outf = open_unicode(to_file, "wb")
+            outf = open(os.path.expanduser(to_file), "wb")
         else:
             outf = stdout
         while True:
hunk ./src/allmydata/scripts/tahoe_ls.py 7
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import unicode_to_output, quote_output, is_printable_ascii, to_str
+from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_str
 
 def list(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_manifest.py 9
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import quote_output, quote_path
+from allmydata.util.encodingutil import quote_output, quote_path
 
 class FakeTransport:
     disconnecting = False
hunk ./src/allmydata/scripts/tahoe_mkdir.py 5
 import urllib
 from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError
-from allmydata.util.stringutils import quote_output
+from allmydata.util.encodingutil import quote_output
 
 def mkdir(options):
     nodeurl = options['node-url']
hunk ./src/allmydata/scripts/tahoe_mv.py 8
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
-from allmydata.util.stringutils import to_str
+from allmydata.util.encodingutil import to_str
 
 # this script is used for both 'mv' and 'ln'
 
hunk ./src/allmydata/scripts/tahoe_put.py 2
 
+import os
 from cStringIO import StringIO
 import urllib
 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
hunk ./src/allmydata/scripts/tahoe_put.py 8
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
-from allmydata.util.stringutils import quote_output, open_unicode
+from allmydata.util.encodingutil import quote_output
 
 def put(options):
     """
hunk ./src/allmydata/scripts/tahoe_put.py 68
     if mutable:
         url += "?mutable=true"
     if from_file:
-        infileobj = open_unicode(from_file, "rb")
+        infileobj = open(os.path.expanduser(from_file), "rb")
     else:
         # do_http() can't use stdin directly: for one thing, we need a
         # Content-Length field. So we currently must copy it.
hunk ./src/allmydata/test/test_backupdb.py 7
 from twisted.trial import unittest
 
 from allmydata.util import fileutil
-from allmydata.util.stringutils import listdir_unicode, get_filesystem_encoding, unicode_platform
+from allmydata.util.encodingutil import listdir_unicode, get_filesystem_encoding, unicode_platform
 from allmydata.util.assertutil import precondition
 from allmydata.scripts import backupdb
 
hunk ./src/allmydata/test/test_cli.py 34
 from twisted.python import usage
 
 from allmydata.util.assertutil import precondition
-from allmydata.util.stringutils import listdir_unicode, open_unicode, unicode_platform, \
+from allmydata.util.encodingutil import listdir_unicode, unicode_platform, \
     quote_output, get_output_encoding, get_argv_encoding, get_filesystem_encoding, \
     unicode_to_output, to_str
 
hunk ./src/allmydata/test/test_cli.py 442
         fileutil.make_dirs(basedir)
 
         for name in filenames:
-            open_unicode(os.path.join(unicode(basedir), name), "wb").close()
+            open(os.path.join(unicode(basedir), name), "wb").close()
 
         for file in listdir_unicode(unicode(basedir)):
             self.failUnlessIn(normalize(file), filenames)
hunk ./src/allmydata/test/test_cli.py 977
         rel_fn = os.path.join(unicode(self.basedir), u"à trier.txt")
         # we make the file small enough to fit in a LIT file, for speed
         DATA = "short file"
-        f = open_unicode(rel_fn, "wb")
-        try:
-            f.write(DATA)
-        finally:
-            f.close()
+        fileutil.write(rel_fn, DATA)
 
         d = self.do_cli("create-alias", "tahoe")
 
hunk ./src/allmydata/test/test_cli.py 1348
         self.set_up_grid()
 
         DATA1 = "unicode file content"
-        f = open_unicode(fn1, "wb")
-        try:
-            f.write(DATA1)
-        finally:
-            f.close()
+        fileutil.write(fn1, DATA1)
 
         fn2 = os.path.join(self.basedir, "Metallica")
         DATA2 = "non-unicode file content"
hunk ./src/allmydata/test/test_encodingutil.py 26
         sys.exit(1)
     
     print
-    print "class MyWeirdOS(StringUtils, unittest.TestCase):"
+    print "class MyWeirdOS(EncodingUtil, unittest.TestCase):"
     print "    uname = '%s'" % ' '.join(platform.uname())
     if sys.platform != "win32":
         print "    argv = %s" % repr(sys.argv[1])
hunk ./src/allmydata/test/test_encodingutil.py 56
 
 from twisted.trial import unittest
 from mock import patch
-import sys
+import os, sys, locale
 
 from allmydata.test.common_util import ReallyEqualMixin
hunk ./src/allmydata/test/test_encodingutil.py 59
-from allmydata.util.stringutils import argv_to_unicode, unicode_to_url, \
-    unicode_to_output, unicode_platform, listdir_unicode, open_unicode, \
-    FilenameEncodingError, get_output_encoding, _reload
+from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
+    unicode_to_output, unicode_platform, listdir_unicode, FilenameEncodingError, \
+    get_output_encoding, get_filesystem_encoding, _reload
 from allmydata.dirnode import normalize
 
 from twisted.python import usage
hunk ./src/allmydata/test/test_encodingutil.py 66
 
-class StringUtilsErrors(ReallyEqualMixin, unittest.TestCase):
+class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
     def tearDown(self):
         _reload()
 
hunk ./src/allmydata/test/test_encodingutil.py 87
         mock_stdout.encoding = 'nonexistent_encoding'
         self.failUnlessRaises(AssertionError, _reload)
 
-        # TODO: mock_stdout.encoding = None
+    @patch('locale.getpreferredencoding')
+    def test_get_output_encoding_not_from_stdout(self, mock_locale_getpreferredencoding):
+        locale  # hush pyflakes
+        mock_locale_getpreferredencoding.return_value = 'koi8-r'
+
+        class DummyStdout:
+            pass
+        old_stdout = sys.stdout
+        sys.stdout = DummyStdout()
+        try:
+            _reload()
+            self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r')
+
+            sys.stdout.encoding = None
+            _reload()
+            self.failUnlessReallyEqual(get_output_encoding(), 'koi8-r')
+
+            mock_locale_getpreferredencoding.return_value = None
+            _reload()
+            self.failUnlessReallyEqual(get_output_encoding(), 'utf-8')
+        finally:
+            sys.stdout = old_stdout
 
     @patch('sys.stdout')
     def test_argv_to_unicode(self, mock):
hunk ./src/allmydata/test/test_encodingutil.py 139
         finally:
             sys.platform = orig_platform
 
-# The following tests applies only to platforms which don't store filenames as
+# The following tests apply only to platforms that don't store filenames as
 # Unicode entities on the filesystem.
hunk ./src/allmydata/test/test_encodingutil.py 141
-class StringUtilsNonUnicodePlatform(unittest.TestCase):
+class EncodingUtilNonUnicodePlatform(unittest.TestCase):
     def setUp(self):
         # Mock sys.platform because unicode_platform() uses it
         self.original_platform = sys.platform
hunk ./src/allmydata/test/test_encodingutil.py 174
                               listdir_unicode,
                               u'/' + lumiere_nfc)
 
-    @patch('sys.getfilesystemencoding')
-    def test_open_unicode(self, mock):
-        mock.return_value = 'ascii'
-        _reload()
-        self.failUnlessRaises(FilenameEncodingError,
-                              open_unicode,
-                              lumiere_nfc, 'rb')
-
-class StringUtils(ReallyEqualMixin):
+class EncodingUtil(ReallyEqualMixin):
     def setUp(self):
         # Mock sys.platform because unicode_platform() uses it
         self.original_platform = sys.platform
hunk ./src/allmydata/test/test_encodingutil.py 233
 
         mock_listdir.return_value = self.dirlist
         mock_getfilesystemencoding.return_value = self.filesystem_encoding
-       
+
         _reload()
         filenames = listdir_unicode(u'/dummy')
 
hunk ./src/allmydata/test/test_encodingutil.py 240
         self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
                              set(TEST_FILENAMES))
 
-    @patch('sys.getfilesystemencoding')
-    @patch('__builtin__.open')
-    def test_open_unicode(self, mock_open, mock_getfilesystemencoding):
-        mock_getfilesystemencoding.return_value = self.filesystem_encoding
-        fn = u'/dummy_directory/" + lumiere_nfc + ".txt'
 
hunk ./src/allmydata/test/test_encodingutil.py 241
-        try:
-            u"test".encode(self.filesystem_encoding)
-        except (LookupError, AttributeError):
-            raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding "
-                                    "that we are testing for the benefit of a different platform."
-                                    % (self.filesystem_encoding,))
+class StdlibUnicode(unittest.TestCase):
+    """This mainly tests that some of the stdlib functions support Unicode paths, but also that
+    listdir_unicode works for valid filenames."""
+
+    def skip_if_cannot_represent_filename(self, u):
+        enc = get_filesystem_encoding()
+        if not unicode_platform():
+            try:
+                u.encode(enc)
+            except UnicodeEncodeError:
+                raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
+
+    def test_mkdir_open_exists_abspath_listdir_expanduser(self):
+        self.skip_if_cannot_represent_filename(lumiere_nfc)
 
hunk ./src/allmydata/test/test_encodingutil.py 256
-        _reload()
         try:
hunk ./src/allmydata/test/test_encodingutil.py 257
-            open_unicode(fn, 'rb')
-        except FilenameEncodingError:
-            return
+            os.mkdir(lumiere_nfc)
+        except EnvironmentError, e:
+            raise unittest.SkipTest("%r\nIt is possible that the filesystem on which this test is being run "
+                                    "does not support Unicode, even though the platform does." % (e,))
+
+        fn = lumiere_nfc + '/' + lumiere_nfc + '.txt'
+        open(fn, 'wb').close()
+        self.failUnless(os.path.exists(fn))
+        self.failUnless(os.path.exists(os.path.abspath(fn)))
+        filenames = listdir_unicode(lumiere_nfc)
+
+        # We only require that the listing includes a filename that is canonically equivalent
+        # to lumiere_nfc (on Mac OS X, it will be the NFD equivalent).
+        self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames]))
 
hunk ./src/allmydata/test/test_encodingutil.py 272
-        # Pass Unicode string to open() on Unicode platforms
+        expanded = os.path.expanduser("~/" + lumiere_nfc)
+        self.failIfIn("~", expanded)
+        self.failUnless(expanded.endswith(lumiere_nfc), expanded)
+
+    @patch('sys.getfilesystemencoding')
+    def test_open_unrepresentable(self, mock):
         if unicode_platform():
hunk ./src/allmydata/test/test_encodingutil.py 279
-            mock_open.assert_called_with(fn, 'rb')
+            raise unittest.SkipTest("This test is not applicable to platforms that represent filenames as Unicode.")
 
hunk ./src/allmydata/test/test_encodingutil.py 281
-        # Pass correctly encoded bytestrings to open() on non-Unicode platforms
-        else:
-            fn_bytestring = fn.encode(self.filesystem_encoding)
-            mock_open.assert_called_with(fn_bytestring, 'rb')
+        mock.return_value = 'ascii'
+        self.failUnlessRaises(UnicodeEncodeError, open, lumiere_nfc, 'rb')
 
 
hunk ./src/allmydata/test/test_encodingutil.py 285
-class UbuntuKarmicUTF8(StringUtils, unittest.TestCase):
+class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
     output = 'lumi\xc3\xa8re'
     argv = 'lumi\xc3\xa8re'
hunk ./src/allmydata/test/test_encodingutil.py 295
     argv_encoding = 'UTF-8'
     dirlist = ['test_file', '\xc3\x84rtonwall.mp3', 'Blah blah.txt']
 
-class UbuntuKarmicLatin1(StringUtils, unittest.TestCase):
+class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
     uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
     output = 'lumi\xe8re'
     argv = 'lumi\xe8re'
hunk ./src/allmydata/test/test_encodingutil.py 305
     argv_encoding = 'ISO-8859-1'
     dirlist = ['test_file', 'Blah blah.txt', '\xc4rtonwall.mp3']
 
-class WindowsXP(StringUtils, unittest.TestCase):
+class WindowsXP(EncodingUtil, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
     output = 'lumi\x8are'
     platform = 'win32'
hunk ./src/allmydata/test/test_encodingutil.py 314
     argv_encoding = 'ascii'
     dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
-class WindowsXP_UTF8(StringUtils, unittest.TestCase):
+class WindowsXP_UTF8(EncodingUtil, unittest.TestCase):
     uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
     output = 'lumi\xc3\xa8re'
     platform = 'win32'
hunk ./src/allmydata/test/test_encodingutil.py 323
     argv_encoding = 'ascii'
     dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
-class WindowsVista(StringUtils, unittest.TestCase):
+class WindowsVista(EncodingUtil, unittest.TestCase):
     uname = 'Windows Vista 6.0.6000 x86 x86 Family 6 Model 15 Stepping 11, GenuineIntel'
     output = 'lumi\x8are'
     platform = 'win32'
hunk ./src/allmydata/test/test_encodingutil.py 332
     argv_encoding = 'ascii'
     dirlist = [u'Blah blah.txt', u'test_file', u'\xc4rtonwall.mp3']
 
-class MacOSXLeopard(StringUtils, unittest.TestCase):
+class MacOSXLeopard(EncodingUtil, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
     output = 'lumi\xc3\xa8re'
     argv = 'lumi\xc3\xa8re'
hunk ./src/allmydata/test/test_encodingutil.py 342
     argv_encoding = 'UTF-8'
     dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
 
-class MacOSXLeopard7bit(StringUtils, unittest.TestCase):
+class MacOSXLeopard7bit(EncodingUtil, unittest.TestCase):
     uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
     platform = 'darwin'
     filesystem_encoding = 'utf-8'
hunk ./src/allmydata/test/test_encodingutil.py 350
     argv_encoding = 'US-ASCII'
     dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
 
-class OpenBSD(StringUtils, unittest.TestCase):
+class OpenBSD(EncodingUtil, unittest.TestCase):
     uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)'
     platform = 'openbsd4'
     filesystem_encoding = '646'
hunk ./src/allmydata/util/encodingutil.py 218
         return os.listdir(path)
     else:
         return listdir_unicode_fallback(path)
-
-def open_unicode(path, mode):
-    """
-    Wrapper around open() which provides safe access to the convenient Unicode
-    API even under Unix.
-    """
-    precondition(isinstance(path, unicode), path)
-
-    if is_unicode_platform:
-        return open(os.path.expanduser(path), mode)
-    else:
-        try:
-            return open(os.path.expanduser(path.encode(filesystem_encoding)), mode)
-        except UnicodeEncodeError:
-            raise FilenameEncodingError(path)
-
-def abspath_expanduser_unicode(path):
-    precondition(isinstance(path, unicode), path)
-
-    if is_unicode_platform:
-        return os.path.abspath(os.path.expanduser(path))
-    else:
-        try:
-            pathstr = path.encode(filesystem_encoding)
-            return os.path.abspath(os.path.expanduser(pathstr)).decode(filesystem_encoding)
-        except (UnicodeEncodeError, UnicodeDecodeError):
-            raise FilenameEncodingError(path)
hunk ./src/allmydata/util/fileutil.py 212
     finally:
         rf.close()
 
+def put_file(pathname, inf):
+    # TODO: create temporary file and move into place?
+    outf = open(os.path.expanduser(pathname), "wb")
+    try:
+        while True:
+            data = inf.read(32768)
+            if not data:
+                break
+            outf.write(data)
+    finally:
+        outf.close()
+
hunk ./src/allmydata/web/common.py 15
      MustBeReadonlyError, MustNotBeUnknownRWError
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.util import abbreviate
-from allmydata.util.stringutils import to_str
+from allmydata.util.encodingutil import to_str
 
 class IOpHandleTable(Interface):
     pass
}

Context:

[SFTP: address some of the comments in zooko's review (#1106).
david-sarah@jacaranda.org**20100712025537
 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e
] 
[docs/logging.txt: note that setting flogging vars might affect tests with race conditions.
david-sarah@jacaranda.org**20100712050721
 Ignore-this: fc1609d215fcd5561a57fd1226206f27
] 
[test_storage.py: potential fix for failures when logging is enabled.
david-sarah@jacaranda.org**19700713040546
 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7
] 
[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
29fe9a0c6399795c94c07f81cb2771f0425bd895