1 | diff -rN -u old-unicode/docs/frontends/CLI.txt new-unicode/docs/frontends/CLI.txt |
---|
2 | --- old-unicode/docs/frontends/CLI.txt 2009-04-08 22:04:16.000000000 -0600 |
---|
3 | +++ new-unicode/docs/frontends/CLI.txt 2009-04-08 22:04:24.000000000 -0600 |
---|
4 | @@ -91,9 +91,21 @@ |
---|
5 | These commands also use a table of "aliases" to figure out which directory |
---|
6 | they ought to use a starting point. This is explained in more detail below. |
---|
7 | |
---|
8 | -In Tahoe v1.3.0, passing non-ascii characters to the cli is not guaranteed to |
---|
9 | -work, although it might work on your platform, especially if your platform |
---|
10 | -uses utf-8 encoding. |
---|
11 | +As of Tahoe v1.3.1, filenames containing non-ascii characters are |
---|
12 | +supported on the commande line if your terminal is correctly configured |
---|
13 | +for UTF-8 support. This is usually the case on moderns GNU/Linux |
---|
14 | +distributions. |
---|
15 | + |
---|
16 | +If your terminal doesn't support UTF-8, you will still be able to list |
---|
17 | +directories but non-ascii characters will be replaced by a question mark |
---|
18 | +(?) on display. |
---|
19 | + |
---|
20 | +Reading from and writing to files whose name contain non-ascii |
---|
21 | +characters is also supported when your system correctly understand them. |
---|
22 | +Under Unix, this is usually handled by locale settings. If Tahoe cannot |
---|
23 | +correctly decode a filename, it will raise an error. In such case, |
---|
24 | +you'll need to correct the name of your file, possibly with help from |
---|
25 | +tools such as convmv. |
---|
26 | |
---|
27 | === Starting Directories === |
---|
28 | |
---|
29 | diff -rN -u old-unicode/src/allmydata/scripts/common.py new-unicode/src/allmydata/scripts/common.py |
---|
30 | --- old-unicode/src/allmydata/scripts/common.py 2009-04-08 22:04:19.000000000 -0600 |
---|
31 | +++ new-unicode/src/allmydata/scripts/common.py 2009-04-08 22:04:25.000000000 -0600 |
---|
32 | @@ -1,7 +1,8 @@ |
---|
33 | |
---|
34 | import os, sys, urllib |
---|
35 | +import codecs |
---|
36 | from twisted.python import usage |
---|
37 | - |
---|
38 | +from allmydata.util.stringutils import unicode_to_url |
---|
39 | |
---|
40 | class BaseOptions: |
---|
41 | # unit tests can override these to point at StringIO instances |
---|
42 | @@ -107,7 +108,7 @@ |
---|
43 | continue |
---|
44 | name, cap = line.split(":", 1) |
---|
45 | # normalize it: remove http: prefix, urldecode |
---|
46 | - cap = cap.strip() |
---|
47 | + cap = cap.strip().encode('ascii') |
---|
48 | aliases[name] = uri.from_string_dirnode(cap).to_string() |
---|
49 | except EnvironmentError: |
---|
50 | pass |
---|
51 | @@ -163,4 +164,4 @@ |
---|
52 | |
---|
53 | def escape_path(path): |
---|
54 | segments = path.split("/") |
---|
55 | - return "/".join([urllib.quote(s) for s in segments]) |
---|
56 | + return "/".join([urllib.quote(unicode_to_url(s)) for s in segments]) |
---|
57 | diff -rN -u old-unicode/src/allmydata/scripts/tahoe_backup.py new-unicode/src/allmydata/scripts/tahoe_backup.py |
---|
58 | --- old-unicode/src/allmydata/scripts/tahoe_backup.py 2009-04-08 22:04:19.000000000 -0600 |
---|
59 | +++ new-unicode/src/allmydata/scripts/tahoe_backup.py 2009-04-08 22:04:25.000000000 -0600 |
---|
60 | @@ -4,11 +4,15 @@ |
---|
61 | import urllib |
---|
62 | import simplejson |
---|
63 | import datetime |
---|
64 | +import sys |
---|
65 | from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS |
---|
66 | from allmydata.scripts.common_http import do_http |
---|
67 | from allmydata import uri |
---|
68 | from allmydata.util import time_format |
---|
69 | from allmydata.scripts import backupdb |
---|
70 | +from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs |
---|
71 | +from allmydata.util.assertutil import precondition |
---|
72 | +from twisted.python import usage |
---|
73 | |
---|
74 | class HTTPError(Exception): |
---|
75 | pass |
---|
76 | @@ -248,6 +252,7 @@ |
---|
77 | print >>self.options.stdout, msg |
---|
78 | |
---|
79 | def process(self, localpath, olddircap): |
---|
80 | + precondition(isinstance(localpath, unicode), localpath) |
---|
81 | # returns newdircap |
---|
82 | |
---|
83 | self.verboseprint("processing %s, olddircap %s" % (localpath, olddircap)) |
---|
84 | @@ -256,7 +261,8 @@ |
---|
85 | olddircontents = self.readdir(olddircap) |
---|
86 | |
---|
87 | newdircontents = {} # childname -> (type, rocap, metadata) |
---|
88 | - for child in self.options.filter_listdir(os.listdir(localpath)): |
---|
89 | + for child in self.options.filter_listdir(os.listdir(unicode_to_fs(localpath))): |
---|
90 | + child = fs_to_unicode(child) |
---|
91 | childpath = os.path.join(localpath, child) |
---|
92 | if os.path.isdir(childpath): |
---|
93 | metadata = get_local_metadata(childpath) |
---|
94 | @@ -342,6 +348,8 @@ |
---|
95 | return contents |
---|
96 | |
---|
97 | def upload(self, childpath): |
---|
98 | + precondition(isinstance(childpath, unicode), childpath) |
---|
99 | + |
---|
100 | #self.verboseprint("uploading %s.." % childpath) |
---|
101 | metadata = get_local_metadata(childpath) |
---|
102 | |
---|
103 | @@ -350,7 +358,7 @@ |
---|
104 | |
---|
105 | if must_upload: |
---|
106 | self.verboseprint("uploading %s.." % childpath) |
---|
107 | - infileobj = open(os.path.expanduser(childpath), "rb") |
---|
108 | + infileobj = open(unicode_to_fs(os.path.expanduser(childpath)), "rb") |
---|
109 | url = self.options['node-url'] + "uri" |
---|
110 | resp = do_http("PUT", url, infileobj) |
---|
111 | if resp.status not in (200, 201): |
---|
112 | diff -rN -u old-unicode/src/allmydata/scripts/tahoe_cp.py new-unicode/src/allmydata/scripts/tahoe_cp.py |
---|
113 | --- old-unicode/src/allmydata/scripts/tahoe_cp.py 2009-04-08 22:04:19.000000000 -0600 |
---|
114 | +++ new-unicode/src/allmydata/scripts/tahoe_cp.py 2009-04-08 22:04:25.000000000 -0600 |
---|
115 | @@ -4,9 +4,13 @@ |
---|
116 | import simplejson |
---|
117 | from cStringIO import StringIO |
---|
118 | from twisted.python.failure import Failure |
---|
119 | +import sys |
---|
120 | from allmydata.scripts.common import get_alias, escape_path, DefaultAliasMarker |
---|
121 | from allmydata.scripts.common_http import do_http |
---|
122 | from allmydata import uri |
---|
123 | +from twisted.python import usage |
---|
124 | +from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs, unicode_to_url |
---|
125 | +from allmydata.util.assertutil import precondition |
---|
126 | |
---|
127 | def ascii_or_none(s): |
---|
128 | if s is None: |
---|
129 | @@ -69,6 +73,7 @@ |
---|
130 | |
---|
131 | class LocalFileSource: |
---|
132 | def __init__(self, pathname): |
---|
133 | + precondition(isinstance(pathname, unicode), pathname) |
---|
134 | self.pathname = pathname |
---|
135 | |
---|
136 | def need_to_copy_bytes(self): |
---|
137 | @@ -79,6 +84,7 @@ |
---|
138 | |
---|
139 | class LocalFileTarget: |
---|
140 | def __init__(self, pathname): |
---|
141 | + precondition(isinstance(pathname, unicode), pathname) |
---|
142 | self.pathname = pathname |
---|
143 | def put_file(self, inf): |
---|
144 | outf = open(self.pathname, "wb") |
---|
145 | @@ -91,6 +97,7 @@ |
---|
146 | |
---|
147 | class LocalMissingTarget: |
---|
148 | def __init__(self, pathname): |
---|
149 | + precondition(isinstance(pathname, unicode), pathname) |
---|
150 | self.pathname = pathname |
---|
151 | |
---|
152 | def put_file(self, inf): |
---|
153 | @@ -104,6 +111,8 @@ |
---|
154 | |
---|
155 | class LocalDirectorySource: |
---|
156 | def __init__(self, progressfunc, pathname): |
---|
157 | + precondition(isinstance(pathname, unicode), pathname) |
---|
158 | + |
---|
159 | self.progressfunc = progressfunc |
---|
160 | self.pathname = pathname |
---|
161 | self.children = None |
---|
162 | @@ -112,8 +121,9 @@ |
---|
163 | if self.children is not None: |
---|
164 | return |
---|
165 | self.children = {} |
---|
166 | - children = os.listdir(self.pathname) |
---|
167 | + children = os.listdir(unicode_to_fs(self.pathname)) |
---|
168 | for i,n in enumerate(children): |
---|
169 | + n = fs_to_unicode(n) |
---|
170 | self.progressfunc("examining %d of %d" % (i, len(children))) |
---|
171 | pn = os.path.join(self.pathname, n) |
---|
172 | if os.path.isdir(pn): |
---|
173 | @@ -129,6 +139,8 @@ |
---|
174 | |
---|
175 | class LocalDirectoryTarget: |
---|
176 | def __init__(self, progressfunc, pathname): |
---|
177 | + precondition(isinstance(pathname, unicode), pathname) |
---|
178 | + |
---|
179 | self.progressfunc = progressfunc |
---|
180 | self.pathname = pathname |
---|
181 | self.children = None |
---|
182 | @@ -137,8 +149,9 @@ |
---|
183 | if self.children is not None: |
---|
184 | return |
---|
185 | self.children = {} |
---|
186 | - children = os.listdir(self.pathname) |
---|
187 | + children = os.listdir(unicode_to_fs(self.pathname)) |
---|
188 | for i,n in enumerate(children): |
---|
189 | + n = fs_to_unicode(n) |
---|
190 | self.progressfunc("examining %d of %d" % (i, len(children))) |
---|
191 | pn = os.path.join(self.pathname, n) |
---|
192 | if os.path.isdir(pn): |
---|
193 | @@ -160,8 +173,9 @@ |
---|
194 | return LocalDirectoryTarget(self.progressfunc, pathname) |
---|
195 | |
---|
196 | def put_file(self, name, inf): |
---|
197 | + precondition(isinstance(name, unicode), name) |
---|
198 | pathname = os.path.join(self.pathname, name) |
---|
199 | - outf = open(pathname, "wb") |
---|
200 | + outf = open(unicode_to_fs(pathname), "wb") |
---|
201 | while True: |
---|
202 | data = inf.read(32768) |
---|
203 | if not data: |
---|
204 | @@ -350,7 +364,7 @@ |
---|
205 | if self.writecap: |
---|
206 | url = self.nodeurl + "/".join(["uri", |
---|
207 | urllib.quote(self.writecap), |
---|
208 | - urllib.quote(name.encode('utf-8'))]) |
---|
209 | + urllib.quote(unicode_to_url(name))]) |
---|
210 | self.children[name] = TahoeFileTarget(self.nodeurl, mutable, |
---|
211 | writecap, readcap, url) |
---|
212 | else: |
---|
213 | diff -rN -u old-unicode/src/allmydata/scripts/tahoe_ls.py new-unicode/src/allmydata/scripts/tahoe_ls.py |
---|
214 | --- old-unicode/src/allmydata/scripts/tahoe_ls.py 2009-04-08 22:04:19.000000000 -0600 |
---|
215 | +++ new-unicode/src/allmydata/scripts/tahoe_ls.py 2009-04-08 22:04:25.000000000 -0600 |
---|
216 | @@ -82,17 +82,17 @@ |
---|
217 | if childtype == "dirnode": |
---|
218 | t0 = "d" |
---|
219 | size = "-" |
---|
220 | - classify = "/" |
---|
221 | + classify = u"/" |
---|
222 | elif childtype == "filenode": |
---|
223 | t0 = "-" |
---|
224 | size = str(child[1]['size']) |
---|
225 | - classify = "" |
---|
226 | + classify = u"" |
---|
227 | if rw_uri: |
---|
228 | - classify = "*" |
---|
229 | + classify = u"*" |
---|
230 | else: |
---|
231 | t0 = "?" |
---|
232 | size = "?" |
---|
233 | - classify = "?" |
---|
234 | + classify = u"?" |
---|
235 | t1 = "-" |
---|
236 | if ro_uri: |
---|
237 | t1 = "r" |
---|
238 | @@ -111,7 +111,7 @@ |
---|
239 | line.append(size) |
---|
240 | line.append(ctime_s) |
---|
241 | if not options["classify"]: |
---|
242 | - classify = "" |
---|
243 | + classify = u"" |
---|
244 | line.append(name + classify) |
---|
245 | if options["uri"]: |
---|
246 | line.append(uri) |
---|
247 | @@ -135,13 +135,13 @@ |
---|
248 | left_justifys[0] = True |
---|
249 | fmt_pieces = [] |
---|
250 | for i in range(len(max_widths)): |
---|
251 | - piece = "%" |
---|
252 | + piece = u"%" |
---|
253 | if left_justifys[i]: |
---|
254 | - piece += "-" |
---|
255 | + piece += u"-" |
---|
256 | piece += str(max_widths[i]) |
---|
257 | - piece += "s" |
---|
258 | + piece += u"s" |
---|
259 | fmt_pieces.append(piece) |
---|
260 | - fmt = " ".join(fmt_pieces) |
---|
261 | + fmt = u" ".join(fmt_pieces) |
---|
262 | for row in rows: |
---|
263 | print >>stdout, (fmt % tuple(row)).rstrip() |
---|
264 | |
---|
265 | diff -rN -u old-unicode/src/allmydata/scripts/tahoe_manifest.py new-unicode/src/allmydata/scripts/tahoe_manifest.py |
---|
266 | --- old-unicode/src/allmydata/scripts/tahoe_manifest.py 2009-04-08 22:04:19.000000000 -0600 |
---|
267 | +++ new-unicode/src/allmydata/scripts/tahoe_manifest.py 2009-04-08 22:04:25.000000000 -0600 |
---|
268 | @@ -78,10 +78,15 @@ |
---|
269 | print >>stdout, vc |
---|
270 | else: |
---|
271 | try: |
---|
272 | - print >>stdout, d["cap"], "/".join(d["path"]) |
---|
273 | + print >>stdout, d["cap"], u"/".join(d["path"]) |
---|
274 | except UnicodeEncodeError: |
---|
275 | - print >>stdout, d["cap"], "/".join([p.encode("utf-8") |
---|
276 | - for p in d["path"]]) |
---|
277 | + # Perhaps python and/or the local system is misconfigured |
---|
278 | + # and actually it should have used utf-8. See ticket #534 |
---|
279 | + # about the questionable practice of second-guessing |
---|
280 | + # python+system-config like this. (And how 'utf-16le' |
---|
281 | + # might be a better second-guess on Windows.) |
---|
282 | + print >>stdout, d["cap"].encode('utf-8'), |
---|
283 | + "/".join([p.encode('utf-8') for p in d["path"]]) |
---|
284 | |
---|
285 | def manifest(options): |
---|
286 | return ManifestStreamer().run(options) |
---|
287 | diff -rN -u old-unicode/src/allmydata/scripts/tahoe_mkdir.py new-unicode/src/allmydata/scripts/tahoe_mkdir.py |
---|
288 | --- old-unicode/src/allmydata/scripts/tahoe_mkdir.py 2009-04-08 22:04:19.000000000 -0600 |
---|
289 | +++ new-unicode/src/allmydata/scripts/tahoe_mkdir.py 2009-04-08 22:04:25.000000000 -0600 |
---|
290 | @@ -2,6 +2,7 @@ |
---|
291 | import urllib |
---|
292 | from allmydata.scripts.common_http import do_http, check_http_error |
---|
293 | from allmydata.scripts.common import get_alias, DEFAULT_ALIAS |
---|
294 | +from allmydata.util.stringutils import unicode_to_url |
---|
295 | |
---|
296 | def mkdir(options): |
---|
297 | nodeurl = options['node-url'] |
---|
298 | @@ -31,7 +32,7 @@ |
---|
299 | path = path[:-1] |
---|
300 | # path (in argv) must be "/".join([s.encode("utf-8") for s in segments]) |
---|
301 | url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap), |
---|
302 | - urllib.quote(path)) |
---|
303 | + urllib.quote(unicode_to_url(path))) |
---|
304 | resp = do_http("POST", url) |
---|
305 | check_http_error(resp, stderr) |
---|
306 | new_uri = resp.read().strip() |
---|
307 | diff -rN -u old-unicode/src/allmydata/test/test_cli.py new-unicode/src/allmydata/test/test_cli.py |
---|
308 | --- old-unicode/src/allmydata/test/test_cli.py 2009-04-08 22:04:20.000000000 -0600 |
---|
309 | +++ new-unicode/src/allmydata/test/test_cli.py 2009-04-08 22:04:25.000000000 -0600 |
---|
310 | @@ -1,5 +1,6 @@ |
---|
311 | # coding=utf-8 |
---|
312 | |
---|
313 | +import sys |
---|
314 | import os.path |
---|
315 | from twisted.trial import unittest |
---|
316 | from cStringIO import StringIO |
---|
317 | @@ -518,6 +519,41 @@ |
---|
318 | self._test_webopen(["two:"], self.two_url) |
---|
319 | d.addCallback(_test_urls) |
---|
320 | |
---|
321 | + d.addCallback(lambda res: self.do_cli("create-alias", "études")) |
---|
322 | + def _check_create_unicode((rc,stdout,stderr)): |
---|
323 | + self.failUnlessEqual(rc, 0) |
---|
324 | + self.failIf(stderr) |
---|
325 | + |
---|
326 | + # If stdout only supports ascii, accentuated characters are |
---|
327 | + # being replaced by '?' |
---|
328 | + if sys.stdout.encoding == "ANSI_X3.4-1968": |
---|
329 | + self.failUnless("Alias '?tudes' created" in stdout) |
---|
330 | + else: |
---|
331 | + self.failUnless("Alias 'études' created" in stdout) |
---|
332 | + |
---|
333 | + aliases = get_aliases(self.get_clientdir()) |
---|
334 | + self.failUnless(aliases[u"études"].startswith("URI:DIR2:")) |
---|
335 | + d.addCallback(_check_create_unicode) |
---|
336 | + |
---|
337 | + d.addCallback(lambda res: self.do_cli("ls", "études:")) |
---|
338 | + def _check_ls1((rc, stdout, stderr)): |
---|
339 | + self.failUnlessEqual(rc, 0) |
---|
340 | + self.failIf(stderr) |
---|
341 | + |
---|
342 | + self.failUnlessEqual(stdout, "") |
---|
343 | + d.addCallback(_check_ls1) |
---|
344 | + |
---|
345 | + d.addCallback(lambda res: self.do_cli("put", "-", "études:uploaded.txt", |
---|
346 | + stdin="Blah blah blah")) |
---|
347 | + |
---|
348 | + d.addCallback(lambda res: self.do_cli("ls", "études:")) |
---|
349 | + def _check_ls2((rc, stdout, stderr)): |
---|
350 | + self.failUnlessEqual(rc, 0) |
---|
351 | + self.failIf(stderr) |
---|
352 | + |
---|
353 | + self.failUnlessEqual(stdout, "uploaded.txt\n") |
---|
354 | + d.addCallback(_check_ls2) |
---|
355 | + |
---|
356 | return d |
---|
357 | |
---|
358 | class Put(GridTestMixin, CLITestMixin, unittest.TestCase): |
---|
359 | @@ -739,6 +775,37 @@ |
---|
360 | d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2)) |
---|
361 | return d |
---|
362 | |
---|
363 | + def test_immutable_from_file_unicode(self): |
---|
364 | + # tahoe put file.txt "à trier.txt" |
---|
365 | + self.basedir = os.path.dirname(self.mktemp()) |
---|
366 | + self.set_up_grid() |
---|
367 | + |
---|
368 | + rel_fn = os.path.join(self.basedir, "DATAFILE") |
---|
369 | + abs_fn = os.path.abspath(rel_fn) |
---|
370 | + # we make the file small enough to fit in a LIT file, for speed |
---|
371 | + DATA = "short file" |
---|
372 | + f = open(rel_fn, "w") |
---|
373 | + f.write(DATA) |
---|
374 | + f.close() |
---|
375 | + |
---|
376 | + d = self.do_cli("create-alias", "tahoe") |
---|
377 | + |
---|
378 | + d.addCallback(lambda res: |
---|
379 | + self.do_cli("put", rel_fn, "à trier.txt")) |
---|
380 | + def _uploaded((rc,stdout,stderr)): |
---|
381 | + readcap = stdout.strip() |
---|
382 | + self.failUnless(readcap.startswith("URI:LIT:")) |
---|
383 | + self.failUnless("201 Created" in stderr, stderr) |
---|
384 | + self.readcap = readcap |
---|
385 | + d.addCallback(_uploaded) |
---|
386 | + |
---|
387 | + d.addCallback(lambda res: |
---|
388 | + self.do_cli("get", "tahoe:à trier.txt")) |
---|
389 | + d.addCallback(lambda (rc,stdout,stderr): |
---|
390 | + self.failUnlessEqual(stdout, DATA)) |
---|
391 | + |
---|
392 | + return d |
---|
393 | + |
---|
394 | class List(GridTestMixin, CLITestMixin, unittest.TestCase): |
---|
395 | def test_list(self): |
---|
396 | self.basedir = "cli/List/list" |
---|
397 | @@ -795,30 +862,37 @@ |
---|
398 | def test_unicode_filename(self): |
---|
399 | self.basedir = "cli/Cp/unicode_filename" |
---|
400 | self.set_up_grid() |
---|
401 | + d = self.do_cli("create-alias", "tahoe") |
---|
402 | + |
---|
403 | + # Use unicode strings when calling os functions |
---|
404 | + if sys.getfilesystemencoding() == "ANSI_X3.4-1968": |
---|
405 | + fn1 = os.path.join(self.basedir, u"Artonwall") |
---|
406 | + else: |
---|
407 | + fn1 = os.path.join(self.basedir, u"Ärtonwall") |
---|
408 | |
---|
409 | - fn1 = os.path.join(self.basedir, "Ärtonwall") |
---|
410 | DATA1 = "unicode file content" |
---|
411 | open(fn1, "wb").write(DATA1) |
---|
412 | + d.addCallback(lambda res: self.do_cli("cp", fn1.encode('utf-8'), "tahoe:Ärtonwall")) |
---|
413 | + |
---|
414 | + d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall")) |
---|
415 | + d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1)) |
---|
416 | |
---|
417 | - fn2 = os.path.join(self.basedir, "Metallica") |
---|
418 | + |
---|
419 | + fn2 = os.path.join(self.basedir, u"Metallica") |
---|
420 | DATA2 = "non-unicode file content" |
---|
421 | open(fn2, "wb").write(DATA2) |
---|
422 | |
---|
423 | # Bug #534 |
---|
424 | # Assure that uploading a file whose name contains unicode character doesn't |
---|
425 | # prevent further uploads in the same directory |
---|
426 | - d = self.do_cli("create-alias", "tahoe") |
---|
427 | - d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:")) |
---|
428 | - d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:")) |
---|
429 | - |
---|
430 | - d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall")) |
---|
431 | - d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1)) |
---|
432 | + d.addCallback(lambda res: self.do_cli("cp", fn2.encode('utf-8'), "tahoe:")) |
---|
433 | |
---|
434 | d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica")) |
---|
435 | d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA2)) |
---|
436 | |
---|
437 | + d.addCallback(lambda res: self.do_cli("ls", "tahoe:")) |
---|
438 | + |
---|
439 | return d |
---|
440 | - test_unicode_filename.todo = "This behavior is not yet supported, although it does happen to work (for reasons that are ill-understood) on many platforms. See issue ticket #534." |
---|
441 | |
---|
442 | def test_dangling_symlink_vs_recursion(self): |
---|
443 | if not hasattr(os, 'symlink'): |
---|
444 | @@ -837,6 +911,17 @@ |
---|
445 | dn, "tahoe:")) |
---|
446 | return d |
---|
447 | |
---|
448 | +class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): |
---|
449 | + def test_unicode_mkdir(self): |
---|
450 | + self.basedir = os.path.dirname(self.mktemp()) |
---|
451 | + self.set_up_grid() |
---|
452 | + |
---|
453 | + d = self.do_cli("create-alias", "tahoe") |
---|
454 | + d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:Motörhead")) |
---|
455 | + |
---|
456 | + return d |
---|
457 | + |
---|
458 | + |
---|
459 | class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): |
---|
460 | |
---|
461 | def writeto(self, path, data): |
---|
462 | @@ -871,6 +956,11 @@ |
---|
463 | self.writeto("parent/subdir/bar.txt", "bar\n" * 1000) |
---|
464 | self.writeto("parent/blah.txt", "blah") |
---|
465 | |
---|
466 | + if sys.getfilesystemencoding() == "ANSI_X3.4-1968": |
---|
467 | + self.writeto(u"parent/artonwall.txt", "Marmelade Jacuzzi") |
---|
468 | + else: |
---|
469 | + self.writeto(u"parent/ärtonwall.txt", "Marmelade Jacuzzi") |
---|
470 | + |
---|
471 | def do_backup(use_backupdb=True, verbose=False): |
---|
472 | cmd = ["backup"] |
---|
473 | if not have_bdb or not use_backupdb: |
---|
474 | @@ -895,8 +985,8 @@ |
---|
475 | self.failUnlessEqual(err, "") |
---|
476 | self.failUnlessEqual(rc, 0) |
---|
477 | fu, fr, dc, dr = self.count_output(out) |
---|
478 | - # foo.txt, bar.txt, blah.txt |
---|
479 | - self.failUnlessEqual(fu, 3) |
---|
480 | + # foo.txt, bar.txt, blah.txt, ärtonwall.txt |
---|
481 | + self.failUnlessEqual(fu, 4) |
---|
482 | self.failUnlessEqual(fr, 0) |
---|
483 | # empty, home, home/parent, home/parent/subdir |
---|
484 | self.failUnlessEqual(dc, 4) |
---|
485 | @@ -945,9 +1035,9 @@ |
---|
486 | self.failUnlessEqual(rc, 0) |
---|
487 | if have_bdb: |
---|
488 | fu, fr, dc, dr = self.count_output(out) |
---|
489 | - # foo.txt, bar.txt, blah.txt |
---|
490 | + # foo.txt, bar.txt, blah.txt, ärtonwall.txt |
---|
491 | self.failUnlessEqual(fu, 0) |
---|
492 | - self.failUnlessEqual(fr, 3) |
---|
493 | + self.failUnlessEqual(fr, 4) |
---|
494 | # empty, home, home/parent, home/parent/subdir |
---|
495 | self.failUnlessEqual(dc, 0) |
---|
496 | self.failUnlessEqual(dr, 4) |
---|
497 | @@ -975,9 +1065,9 @@ |
---|
498 | self.failUnlessEqual(rc, 0) |
---|
499 | fu, fr, dc, dr = self.count_output(out) |
---|
500 | fchecked, dchecked, dread = self.count_output2(out) |
---|
501 | - self.failUnlessEqual(fchecked, 3) |
---|
502 | + self.failUnlessEqual(fchecked, 4) |
---|
503 | self.failUnlessEqual(fu, 0) |
---|
504 | - self.failUnlessEqual(fr, 3) |
---|
505 | + self.failUnlessEqual(fr, 4) |
---|
506 | # TODO: backupdb doesn't do dirs yet; when it does, this will |
---|
507 | # change to dchecked=4, and maybe dread=0 |
---|
508 | self.failUnlessEqual(dchecked, 0) |
---|
509 | @@ -1023,8 +1113,8 @@ |
---|
510 | fu, fr, dc, dr = self.count_output(out) |
---|
511 | # new foo.txt, surprise file, subfile, empty |
---|
512 | self.failUnlessEqual(fu, 4) |
---|
513 | - # old bar.txt |
---|
514 | - self.failUnlessEqual(fr, 1) |
---|
515 | + # old bar.txt, ärtonwall.txt |
---|
516 | + self.failUnlessEqual(fr, 2) |
---|
517 | # home, parent, subdir, blah.txt, surprisedir |
---|
518 | self.failUnlessEqual(dc, 5) |
---|
519 | self.failUnlessEqual(dr, 0) |
---|
520 | @@ -1063,7 +1153,7 @@ |
---|
521 | self.failUnlessEqual(err, "") |
---|
522 | self.failUnlessEqual(rc, 0) |
---|
523 | fu, fr, dc, dr = self.count_output(out) |
---|
524 | - self.failUnlessEqual(fu, 5) |
---|
525 | + self.failUnlessEqual(fu, 6) |
---|
526 | self.failUnlessEqual(fr, 0) |
---|
527 | self.failUnlessEqual(dc, 0) |
---|
528 | self.failUnlessEqual(dr, 5) |
---|
529 | diff -rN -u old-unicode/src/allmydata/util/stringutils.py new-unicode/src/allmydata/util/stringutils.py |
---|
530 | --- old-unicode/src/allmydata/util/stringutils.py 1969-12-31 17:00:00.000000000 -0700 |
---|
531 | +++ new-unicode/src/allmydata/util/stringutils.py 2009-04-08 22:04:25.000000000 -0600 |
---|
532 | @@ -0,0 +1,48 @@ |
---|
533 | +""" |
---|
534 | +Functions used to convert inputs from whatever encoding used in the system to |
---|
535 | +unicode and back. |
---|
536 | + |
---|
537 | +TODO: |
---|
538 | + * Accept two cli arguments --argv-encoding and --filesystem-encoding |
---|
539 | +""" |
---|
540 | + |
---|
541 | +import sys |
---|
542 | +from allmydata.util.assertutil import precondition |
---|
543 | +from twisted.python import usage |
---|
544 | + |
---|
545 | +def fs_to_unicode(s): |
---|
546 | + """ |
---|
547 | + Decode a filename (or a directory name) to unicode using the same encoding |
---|
548 | + as the filesystem. |
---|
549 | + """ |
---|
550 | + # Filename encoding detection is a little bit better thanks to |
---|
551 | + # getfilesystemencoding() in the sys module. However, filenames can be |
---|
552 | + # encoded using another encoding than the one used on the filesystem. |
---|
553 | + |
---|
554 | + precondition(isinstance(s, str), s) |
---|
555 | + encoding = sys.getfilesystemencoding() |
---|
556 | + try: |
---|
557 | + return unicode(s, encoding) |
---|
558 | + except UnicodeDecodeError: |
---|
559 | + raise usage.UsageError("Filename '%s' cannot be decoded using the current encoding of your filesystem (%s). Please rename this file." % (s, encoding)) |
---|
560 | + |
---|
561 | +def unicode_to_fs(s): |
---|
562 | + """ |
---|
563 | + Encode an unicode object used in file or directoy name. |
---|
564 | + """ |
---|
565 | + |
---|
566 | + precondition(isinstance(s, unicode), s) |
---|
567 | + encoding = sys.getfilesystemencoding() |
---|
568 | + try: |
---|
569 | + return s.encode(encoding) |
---|
570 | + except UnicodeEncodeError: |
---|
571 | + raise usage.UsageError("Filename '%s' cannot be encoded using the current encoding of your filesystem (%s). Please configure your locale correctly or rename this file." % (s, encoding)) |
---|
572 | + |
---|
573 | +def unicode_to_url(s): |
---|
574 | + """ |
---|
575 | + Encode an unicode object used in an URL. |
---|
576 | + """ |
---|
577 | + # According to RFC 2718, non-ascii characters in url's must be UTF-8 encoded. |
---|
578 | + |
---|
579 | + precondition(isinstance(s, unicode), s) |
---|
580 | + return s.encode('utf-8') |
---|
581 | |
---|