Ticket #534: unicode-v3-minus-the-stdout-and-aliases-and-argv-parts.patch.txt

File unicode-v3-minus-the-stdout-and-aliases-and-argv-parts.patch.txt, 23.0 KB (added by zooko, at 2009-04-09T04:09:10Z)
Line 
1diff -rN -u old-unicode/docs/frontends/CLI.txt new-unicode/docs/frontends/CLI.txt
2--- old-unicode/docs/frontends/CLI.txt  2009-04-08 22:04:16.000000000 -0600
3+++ new-unicode/docs/frontends/CLI.txt  2009-04-08 22:04:24.000000000 -0600
4@@ -91,9 +91,21 @@
5 These commands also use a table of "aliases" to figure out which directory
6 they ought to use a starting point. This is explained in more detail below.
7 
8-In Tahoe v1.3.0, passing non-ascii characters to the cli is not guaranteed to
9-work, although it might work on your platform, especially if your platform
10-uses utf-8 encoding.
11+As of Tahoe v1.3.1, filenames containing non-ascii characters are
12+supported on the commande line if your terminal is correctly configured
13+for UTF-8 support. This is usually the case on moderns GNU/Linux
14+distributions.
15+
16+If your terminal doesn't support UTF-8, you will still be able to list
17+directories but non-ascii characters will be replaced by a question mark
18+(?) on display.
19+
20+Reading from and writing to files whose name contain non-ascii
21+characters is also supported when your system correctly understand them.
22+Under Unix, this is usually handled by locale settings. If Tahoe cannot
23+correctly decode a filename, it will raise an error. In such case,
24+you'll need to correct the name of your file, possibly with help from
25+tools such as convmv.
26 
27 === Starting Directories ===
28 
29diff -rN -u old-unicode/src/allmydata/scripts/common.py new-unicode/src/allmydata/scripts/common.py
30--- old-unicode/src/allmydata/scripts/common.py 2009-04-08 22:04:19.000000000 -0600
31+++ new-unicode/src/allmydata/scripts/common.py 2009-04-08 22:04:25.000000000 -0600
32@@ -1,7 +1,8 @@
33 
34 import os, sys, urllib
35+import codecs
36 from twisted.python import usage
37-
38+from allmydata.util.stringutils import unicode_to_url
39 
40 class BaseOptions:
41     # unit tests can override these to point at StringIO instances
42@@ -107,7 +108,7 @@
43                 continue
44             name, cap = line.split(":", 1)
45             # normalize it: remove http: prefix, urldecode
46-            cap = cap.strip()
47+            cap = cap.strip().encode('ascii')
48             aliases[name] = uri.from_string_dirnode(cap).to_string()
49     except EnvironmentError:
50         pass
51@@ -163,4 +164,4 @@
52 
53 def escape_path(path):
54     segments = path.split("/")
55-    return "/".join([urllib.quote(s) for s in segments])
56+    return "/".join([urllib.quote(unicode_to_url(s)) for s in segments])
57diff -rN -u old-unicode/src/allmydata/scripts/tahoe_backup.py new-unicode/src/allmydata/scripts/tahoe_backup.py
58--- old-unicode/src/allmydata/scripts/tahoe_backup.py   2009-04-08 22:04:19.000000000 -0600
59+++ new-unicode/src/allmydata/scripts/tahoe_backup.py   2009-04-08 22:04:25.000000000 -0600
60@@ -4,11 +4,15 @@
61 import urllib
62 import simplejson
63 import datetime
64+import sys
65 from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS
66 from allmydata.scripts.common_http import do_http
67 from allmydata import uri
68 from allmydata.util import time_format
69 from allmydata.scripts import backupdb
70+from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs
71+from allmydata.util.assertutil import precondition
72+from twisted.python import usage
73 
74 class HTTPError(Exception):
75     pass
76@@ -248,6 +252,7 @@
77             print >>self.options.stdout, msg
78 
79     def process(self, localpath, olddircap):
80+        precondition(isinstance(localpath, unicode), localpath)
81         # returns newdircap
82 
83         self.verboseprint("processing %s, olddircap %s" % (localpath, olddircap))
84@@ -256,7 +261,8 @@
85             olddircontents = self.readdir(olddircap)
86 
87         newdircontents = {} # childname -> (type, rocap, metadata)
88-        for child in self.options.filter_listdir(os.listdir(localpath)):
89+        for child in self.options.filter_listdir(os.listdir(unicode_to_fs(localpath))):
90+            child = fs_to_unicode(child)
91             childpath = os.path.join(localpath, child)
92             if os.path.isdir(childpath):
93                 metadata = get_local_metadata(childpath)
94@@ -342,6 +348,8 @@
95         return contents
96 
97     def upload(self, childpath):
98+        precondition(isinstance(childpath, unicode), childpath)
99+
100         #self.verboseprint("uploading %s.." % childpath)
101         metadata = get_local_metadata(childpath)
102 
103@@ -350,7 +358,7 @@
104 
105         if must_upload:
106             self.verboseprint("uploading %s.." % childpath)
107-            infileobj = open(os.path.expanduser(childpath), "rb")
108+            infileobj = open(unicode_to_fs(os.path.expanduser(childpath)), "rb")
109             url = self.options['node-url'] + "uri"
110             resp = do_http("PUT", url, infileobj)
111             if resp.status not in (200, 201):
112diff -rN -u old-unicode/src/allmydata/scripts/tahoe_cp.py new-unicode/src/allmydata/scripts/tahoe_cp.py
113--- old-unicode/src/allmydata/scripts/tahoe_cp.py       2009-04-08 22:04:19.000000000 -0600
114+++ new-unicode/src/allmydata/scripts/tahoe_cp.py       2009-04-08 22:04:25.000000000 -0600
115@@ -4,9 +4,13 @@
116 import simplejson
117 from cStringIO import StringIO
118 from twisted.python.failure import Failure
119+import sys
120 from allmydata.scripts.common import get_alias, escape_path, DefaultAliasMarker
121 from allmydata.scripts.common_http import do_http
122 from allmydata import uri
123+from twisted.python import usage
124+from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs, unicode_to_url
125+from allmydata.util.assertutil import precondition
126 
127 def ascii_or_none(s):
128     if s is None:
129@@ -69,6 +73,7 @@
130 
131 class LocalFileSource:
132     def __init__(self, pathname):
133+        precondition(isinstance(pathname, unicode), pathname)
134         self.pathname = pathname
135 
136     def need_to_copy_bytes(self):
137@@ -79,6 +84,7 @@
138 
139 class LocalFileTarget:
140     def __init__(self, pathname):
141+        precondition(isinstance(pathname, unicode), pathname)
142         self.pathname = pathname
143     def put_file(self, inf):
144         outf = open(self.pathname, "wb")
145@@ -91,6 +97,7 @@
146 
147 class LocalMissingTarget:
148     def __init__(self, pathname):
149+        precondition(isinstance(pathname, unicode), pathname)
150         self.pathname = pathname
151 
152     def put_file(self, inf):
153@@ -104,6 +111,8 @@
154 
155 class LocalDirectorySource:
156     def __init__(self, progressfunc, pathname):
157+        precondition(isinstance(pathname, unicode), pathname)
158+
159         self.progressfunc = progressfunc
160         self.pathname = pathname
161         self.children = None
162@@ -112,8 +121,9 @@
163         if self.children is not None:
164             return
165         self.children = {}
166-        children = os.listdir(self.pathname)
167+        children = os.listdir(unicode_to_fs(self.pathname))
168         for i,n in enumerate(children):
169+            n = fs_to_unicode(n)
170             self.progressfunc("examining %d of %d" % (i, len(children)))
171             pn = os.path.join(self.pathname, n)
172             if os.path.isdir(pn):
173@@ -129,6 +139,8 @@
174 
175 class LocalDirectoryTarget:
176     def __init__(self, progressfunc, pathname):
177+        precondition(isinstance(pathname, unicode), pathname)
178+
179         self.progressfunc = progressfunc
180         self.pathname = pathname
181         self.children = None
182@@ -137,8 +149,9 @@
183         if self.children is not None:
184             return
185         self.children = {}
186-        children = os.listdir(self.pathname)
187+        children = os.listdir(unicode_to_fs(self.pathname))
188         for i,n in enumerate(children):
189+            n = fs_to_unicode(n)
190             self.progressfunc("examining %d of %d" % (i, len(children)))
191             pn = os.path.join(self.pathname, n)
192             if os.path.isdir(pn):
193@@ -160,8 +173,9 @@
194         return LocalDirectoryTarget(self.progressfunc, pathname)
195 
196     def put_file(self, name, inf):
197+        precondition(isinstance(name, unicode), name)
198         pathname = os.path.join(self.pathname, name)
199-        outf = open(pathname, "wb")
200+        outf = open(unicode_to_fs(pathname), "wb")
201         while True:
202             data = inf.read(32768)
203             if not data:
204@@ -350,7 +364,7 @@
205                 if self.writecap:
206                     url = self.nodeurl + "/".join(["uri",
207                                                    urllib.quote(self.writecap),
208-                                                   urllib.quote(name.encode('utf-8'))])
209+                                                   urllib.quote(unicode_to_url(name))])
210                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
211                                                       writecap, readcap, url)
212             else:
213diff -rN -u old-unicode/src/allmydata/scripts/tahoe_ls.py new-unicode/src/allmydata/scripts/tahoe_ls.py
214--- old-unicode/src/allmydata/scripts/tahoe_ls.py       2009-04-08 22:04:19.000000000 -0600
215+++ new-unicode/src/allmydata/scripts/tahoe_ls.py       2009-04-08 22:04:25.000000000 -0600
216@@ -82,17 +82,17 @@
217         if childtype == "dirnode":
218             t0 = "d"
219             size = "-"
220-            classify = "/"
221+            classify = u"/"
222         elif childtype == "filenode":
223             t0 = "-"
224             size = str(child[1]['size'])
225-            classify = ""
226+            classify = u""
227             if rw_uri:
228-                classify = "*"
229+                classify = u"*"
230         else:
231             t0 = "?"
232             size = "?"
233-            classify = "?"
234+            classify = u"?"
235         t1 = "-"
236         if ro_uri:
237             t1 = "r"
238@@ -111,7 +111,7 @@
239             line.append(size)
240             line.append(ctime_s)
241         if not options["classify"]:
242-            classify = ""
243+            classify = u""
244         line.append(name + classify)
245         if options["uri"]:
246             line.append(uri)
247@@ -135,13 +135,13 @@
248         left_justifys[0] = True
249     fmt_pieces = []
250     for i in range(len(max_widths)):
251-        piece = "%"
252+        piece = u"%"
253         if left_justifys[i]:
254-            piece += "-"
255+            piece += u"-"
256         piece += str(max_widths[i])
257-        piece += "s"
258+        piece += u"s"
259         fmt_pieces.append(piece)
260-    fmt = " ".join(fmt_pieces)
261+    fmt = u" ".join(fmt_pieces)
262     for row in rows:
263         print >>stdout, (fmt % tuple(row)).rstrip()
264 
265diff -rN -u old-unicode/src/allmydata/scripts/tahoe_manifest.py new-unicode/src/allmydata/scripts/tahoe_manifest.py
266--- old-unicode/src/allmydata/scripts/tahoe_manifest.py 2009-04-08 22:04:19.000000000 -0600
267+++ new-unicode/src/allmydata/scripts/tahoe_manifest.py 2009-04-08 22:04:25.000000000 -0600
268@@ -78,10 +78,15 @@
269                     print >>stdout, vc
270             else:
271                 try:
272-                    print >>stdout, d["cap"], "/".join(d["path"])
273+                    print >>stdout, d["cap"], u"/".join(d["path"])
274                 except UnicodeEncodeError:
275-                    print >>stdout, d["cap"], "/".join([p.encode("utf-8")
276-                                                        for p in d["path"]])
277+                    # Perhaps python and/or the local system is misconfigured
278+                    # and actually it should have used utf-8.  See ticket #534
279+                    # about the questionable practice of second-guessing
280+                    # python+system-config like this.  (And how 'utf-16le'
281+                    # might be a better second-guess on Windows.)
282+                    print >>stdout, d["cap"].encode('utf-8'),
283+                        "/".join([p.encode('utf-8') for p in d["path"]])
284 
285 def manifest(options):
286     return ManifestStreamer().run(options)
287diff -rN -u old-unicode/src/allmydata/scripts/tahoe_mkdir.py new-unicode/src/allmydata/scripts/tahoe_mkdir.py
288--- old-unicode/src/allmydata/scripts/tahoe_mkdir.py    2009-04-08 22:04:19.000000000 -0600
289+++ new-unicode/src/allmydata/scripts/tahoe_mkdir.py    2009-04-08 22:04:25.000000000 -0600
290@@ -2,6 +2,7 @@
291 import urllib
292 from allmydata.scripts.common_http import do_http, check_http_error
293 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS
294+from allmydata.util.stringutils import unicode_to_url
295 
296 def mkdir(options):
297     nodeurl = options['node-url']
298@@ -31,7 +32,7 @@
299         path = path[:-1]
300     # path (in argv) must be "/".join([s.encode("utf-8") for s in segments])
301     url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap),
302-                                           urllib.quote(path))
303+                                           urllib.quote(unicode_to_url(path)))
304     resp = do_http("POST", url)
305     check_http_error(resp, stderr)
306     new_uri = resp.read().strip()
307diff -rN -u old-unicode/src/allmydata/test/test_cli.py new-unicode/src/allmydata/test/test_cli.py
308--- old-unicode/src/allmydata/test/test_cli.py  2009-04-08 22:04:20.000000000 -0600
309+++ new-unicode/src/allmydata/test/test_cli.py  2009-04-08 22:04:25.000000000 -0600
310@@ -1,5 +1,6 @@
311 # coding=utf-8
312 
313+import sys
314 import os.path
315 from twisted.trial import unittest
316 from cStringIO import StringIO
317@@ -518,6 +519,41 @@
318             self._test_webopen(["two:"], self.two_url)
319         d.addCallback(_test_urls)
320 
321+        d.addCallback(lambda res: self.do_cli("create-alias", "études"))
322+        def _check_create_unicode((rc,stdout,stderr)):
323+            self.failUnlessEqual(rc, 0)
324+            self.failIf(stderr)
325+
326+            # If stdout only supports ascii, accentuated characters are
327+            # being replaced by '?'
328+            if sys.stdout.encoding == "ANSI_X3.4-1968":
329+                self.failUnless("Alias '?tudes' created" in stdout)
330+            else:
331+                self.failUnless("Alias 'études' created" in stdout)
332+
333+            aliases = get_aliases(self.get_clientdir())
334+            self.failUnless(aliases[u"études"].startswith("URI:DIR2:"))
335+        d.addCallback(_check_create_unicode)
336+
337+        d.addCallback(lambda res: self.do_cli("ls", "études:"))
338+        def _check_ls1((rc, stdout, stderr)):
339+            self.failUnlessEqual(rc, 0)
340+            self.failIf(stderr)
341+
342+            self.failUnlessEqual(stdout, "")
343+        d.addCallback(_check_ls1)
344+
345+        d.addCallback(lambda res: self.do_cli("put", "-", "études:uploaded.txt",
346+          stdin="Blah blah blah"))
347+
348+        d.addCallback(lambda res: self.do_cli("ls", "études:"))
349+        def _check_ls2((rc, stdout, stderr)):
350+            self.failUnlessEqual(rc, 0)
351+            self.failIf(stderr)
352+
353+            self.failUnlessEqual(stdout, "uploaded.txt\n")
354+        d.addCallback(_check_ls2)
355+
356         return d
357 
358 class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
359@@ -739,6 +775,37 @@
360         d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2))
361         return d
362 
363+    def test_immutable_from_file_unicode(self):
364+        # tahoe put file.txt "à trier.txt"
365+        self.basedir = os.path.dirname(self.mktemp())
366+        self.set_up_grid()
367+
368+        rel_fn = os.path.join(self.basedir, "DATAFILE")
369+        abs_fn = os.path.abspath(rel_fn)
370+        # we make the file small enough to fit in a LIT file, for speed
371+        DATA = "short file"
372+        f = open(rel_fn, "w")
373+        f.write(DATA)
374+        f.close()
375+
376+        d = self.do_cli("create-alias", "tahoe")
377+
378+        d.addCallback(lambda res:
379+                      self.do_cli("put", rel_fn, "à trier.txt"))
380+        def _uploaded((rc,stdout,stderr)):
381+            readcap = stdout.strip()
382+            self.failUnless(readcap.startswith("URI:LIT:"))
383+            self.failUnless("201 Created" in stderr, stderr)
384+            self.readcap = readcap
385+        d.addCallback(_uploaded)
386+
387+        d.addCallback(lambda res:
388+                      self.do_cli("get", "tahoe:à trier.txt"))
389+        d.addCallback(lambda (rc,stdout,stderr):
390+                      self.failUnlessEqual(stdout, DATA))
391+
392+        return d
393+
394 class List(GridTestMixin, CLITestMixin, unittest.TestCase):
395     def test_list(self):
396         self.basedir = "cli/List/list"
397@@ -795,30 +862,37 @@
398     def test_unicode_filename(self):
399         self.basedir = "cli/Cp/unicode_filename"
400         self.set_up_grid()
401+        d = self.do_cli("create-alias", "tahoe")
402+
403+        # Use unicode strings when calling os functions
404+        if sys.getfilesystemencoding() == "ANSI_X3.4-1968":
405+            fn1 = os.path.join(self.basedir, u"Artonwall")
406+        else:
407+            fn1 = os.path.join(self.basedir, u"Ärtonwall")
408 
409-        fn1 = os.path.join(self.basedir, "Ärtonwall")
410         DATA1 = "unicode file content"
411         open(fn1, "wb").write(DATA1)
412+        d.addCallback(lambda res: self.do_cli("cp", fn1.encode('utf-8'), "tahoe:Ärtonwall"))
413+
414+        d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall"))
415+        d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1))
416 
417-        fn2 = os.path.join(self.basedir, "Metallica")
418+
419+        fn2 = os.path.join(self.basedir, u"Metallica")
420         DATA2 = "non-unicode file content"
421         open(fn2, "wb").write(DATA2)
422 
423         # Bug #534
424         # Assure that uploading a file whose name contains unicode character doesn't
425         # prevent further uploads in the same directory
426-        d = self.do_cli("create-alias", "tahoe")
427-        d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:"))
428-        d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
429-
430-        d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall"))
431-        d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1))
432+        d.addCallback(lambda res: self.do_cli("cp", fn2.encode('utf-8'), "tahoe:"))
433 
434         d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
435         d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2))
436 
437+        d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
438+
439         return d
440-    test_unicode_filename.todo = "This behavior is not yet supported, although it does happen to work (for reasons that are ill-understood) on many platforms.  See issue ticket #534."
441 
442     def test_dangling_symlink_vs_recursion(self):
443         if not hasattr(os, 'symlink'):
444@@ -837,6 +911,17 @@
445                                               dn, "tahoe:"))
446         return d
447 
448+class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
449+    def test_unicode_mkdir(self):
450+        self.basedir = os.path.dirname(self.mktemp())
451+        self.set_up_grid()
452+
453+        d = self.do_cli("create-alias", "tahoe")
454+        d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:Motörhead"))
455+
456+        return d
457+
458+
459 class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
460 
461     def writeto(self, path, data):
462@@ -871,6 +956,11 @@
463         self.writeto("parent/subdir/bar.txt", "bar\n" * 1000)
464         self.writeto("parent/blah.txt", "blah")
465 
466+        if sys.getfilesystemencoding() == "ANSI_X3.4-1968":
467+            self.writeto(u"parent/artonwall.txt", "Marmelade Jacuzzi")
468+        else:
469+            self.writeto(u"parent/ärtonwall.txt", "Marmelade Jacuzzi")
470+
471         def do_backup(use_backupdb=True, verbose=False):
472             cmd = ["backup"]
473             if not have_bdb or not use_backupdb:
474@@ -895,8 +985,8 @@
475             self.failUnlessEqual(err, "")
476             self.failUnlessEqual(rc, 0)
477             fu, fr, dc, dr = self.count_output(out)
478-            # foo.txt, bar.txt, blah.txt
479-            self.failUnlessEqual(fu, 3)
480+            # foo.txt, bar.txt, blah.txt, ärtonwall.txt
481+            self.failUnlessEqual(fu, 4)
482             self.failUnlessEqual(fr, 0)
483             # empty, home, home/parent, home/parent/subdir
484             self.failUnlessEqual(dc, 4)
485@@ -945,9 +1035,9 @@
486             self.failUnlessEqual(rc, 0)
487             if have_bdb:
488                 fu, fr, dc, dr = self.count_output(out)
489-                # foo.txt, bar.txt, blah.txt
490+                # foo.txt, bar.txt, blah.txt, ärtonwall.txt
491                 self.failUnlessEqual(fu, 0)
492-                self.failUnlessEqual(fr, 3)
493+                self.failUnlessEqual(fr, 4)
494                 # empty, home, home/parent, home/parent/subdir
495                 self.failUnlessEqual(dc, 0)
496                 self.failUnlessEqual(dr, 4)
497@@ -975,9 +1065,9 @@
498                 self.failUnlessEqual(rc, 0)
499                 fu, fr, dc, dr = self.count_output(out)
500                 fchecked, dchecked, dread = self.count_output2(out)
501-                self.failUnlessEqual(fchecked, 3)
502+                self.failUnlessEqual(fchecked, 4)
503                 self.failUnlessEqual(fu, 0)
504-                self.failUnlessEqual(fr, 3)
505+                self.failUnlessEqual(fr, 4)
506                 # TODO: backupdb doesn't do dirs yet; when it does, this will
507                 # change to dchecked=4, and maybe dread=0
508                 self.failUnlessEqual(dchecked, 0)
509@@ -1023,8 +1113,8 @@
510                 fu, fr, dc, dr = self.count_output(out)
511                 # new foo.txt, surprise file, subfile, empty
512                 self.failUnlessEqual(fu, 4)
513-                # old bar.txt
514-                self.failUnlessEqual(fr, 1)
515+                # old bar.txt, ärtonwall.txt
516+                self.failUnlessEqual(fr, 2)
517                 # home, parent, subdir, blah.txt, surprisedir
518                 self.failUnlessEqual(dc, 5)
519                 self.failUnlessEqual(dr, 0)
520@@ -1063,7 +1153,7 @@
521             self.failUnlessEqual(err, "")
522             self.failUnlessEqual(rc, 0)
523             fu, fr, dc, dr = self.count_output(out)
524-            self.failUnlessEqual(fu, 5)
525+            self.failUnlessEqual(fu, 6)
526             self.failUnlessEqual(fr, 0)
527             self.failUnlessEqual(dc, 0)
528             self.failUnlessEqual(dr, 5)
529diff -rN -u old-unicode/src/allmydata/util/stringutils.py new-unicode/src/allmydata/util/stringutils.py
530--- old-unicode/src/allmydata/util/stringutils.py       1969-12-31 17:00:00.000000000 -0700
531+++ new-unicode/src/allmydata/util/stringutils.py       2009-04-08 22:04:25.000000000 -0600
532@@ -0,0 +1,48 @@
533+"""
534+Functions used to convert inputs from whatever encoding used in the system to
535+unicode and back.
536+
537+TODO:
538+  * Accept two cli arguments --argv-encoding and --filesystem-encoding
539+"""
540+
541+import sys
542+from allmydata.util.assertutil import precondition
543+from twisted.python import usage
544+
545+def fs_to_unicode(s):
546+    """
547+    Decode a filename (or a directory name) to unicode using the same encoding
548+    as the filesystem.
549+    """
550+    # Filename encoding detection is a little bit better thanks to
551+    # getfilesystemencoding() in the sys module. However, filenames can be
552+    # encoded using another encoding than the one used on the filesystem.
553+
554+    precondition(isinstance(s, str), s)
555+    encoding = sys.getfilesystemencoding()
556+    try:
557+        return unicode(s, encoding)
558+    except UnicodeDecodeError:
559+        raise usage.UsageError("Filename '%s' cannot be decoded using the current encoding of your filesystem (%s). Please rename this file." % (s, encoding))
560+
561+def unicode_to_fs(s):
562+    """
563+    Encode an unicode object used in file or directoy name.
564+    """
565+
566+    precondition(isinstance(s, unicode), s)
567+    encoding = sys.getfilesystemencoding()
568+    try:
569+        return s.encode(encoding)
570+    except UnicodeEncodeError:
571+        raise usage.UsageError("Filename '%s' cannot be encoded using the current encoding of your filesystem (%s). Please configure your locale correctly or rename this file." % (s, encoding))
572+
573+def unicode_to_url(s):
574+    """
575+    Encode an unicode object used in an URL.
576+    """
577+    # According to RFC 2718, non-ascii characters in url's must be UTF-8 encoded.
578+
579+    precondition(isinstance(s, unicode), s)
580+    return s.encode('utf-8')
581