Ticket #534: unicode-v2.dpatch

File unicode-v2.dpatch, 39.2 KB (added by francois, at 2009-02-25T09:31:55Z)
Line 
1
2New patches:
3
4[CLI.txt: Document behaviour in presence of unicode filename
5francois@ctrlaltdel.ch**20090224000657] hunk ./docs/frontends/CLI.txt 94
6-In Tahoe v1.3.0, passing non-ascii characters to the cli is not guaranteed to
7-work, although it might work on your platform, especially if your platform
8-uses utf-8 encoding.
9+Filenames containing non-ascii characters are supported on the commande
10+line if your terminal is correctly configured for UTF-8 support. This is
11+usually the case on moderns GNU/Linux distributions.
12+
13+If your terminal doesn't support UTF-8, you will still be able to list
14+directories but non-ascii characters will be replaced by a question mark
15+(?) on display.
16+
17+Reading from and writing to files whose name contain accentuated
18+characters is also supported when your system correctly understand them.
19+Under Unix, this is usually handled by locale settings. If Tahoe cannot
20+correctly decode a filename, it will raise an error. In such case,
21+you'll need to correct the name of your file, perhaps with help from
22+tools such as convmv.
23
24[cli.py: Convert filenames and filepaths argument into unicode objects
25francois@ctrlaltdel.ch**20090224001251] {
26hunk ./src/allmydata/scripts/cli.py 4
27+from allmydata.util.stringutils import argv_to_unicode
28hunk ./src/allmydata/scripts/cli.py 53
29-        self.where = where
30+        self.where = argv_to_unicode(where)
31hunk ./src/allmydata/scripts/cli.py 58
32-        self.alias = alias
33+        self.alias = argv_to_unicode(alias)
34hunk ./src/allmydata/scripts/cli.py 68
35-        self.alias = alias
36+        self.alias = argv_to_unicode(alias)
37hunk ./src/allmydata/scripts/cli.py 87
38-        self.where = where
39+        self.where = argv_to_unicode(where)
40hunk ./src/allmydata/scripts/cli.py 98
41-        self.from_file = arg1
42-        self.to_file = arg2
43+        self.from_file = argv_to_unicode(arg1)
44+
45+        if arg2:
46+            self.to_file = argv_to_unicode(arg2)
47+        else:
48+            self.to_file = None
49+
50hunk ./src/allmydata/scripts/cli.py 140
51-            self.from_file = arg1
52-            self.to_file = arg2
53+            self.from_file = argv_to_unicode(arg1)
54+            self.to_file =  argv_to_unicode(arg2)
55hunk ./src/allmydata/scripts/cli.py 143
56-            self.from_file = arg1 # might be "-"
57+            self.from_file = argv_to_unicode(arg1) # might be "-"
58hunk ./src/allmydata/scripts/cli.py 148
59-        if self.from_file == "-":
60+        if self.from_file == u"-":
61hunk ./src/allmydata/scripts/cli.py 182
62-        self.sources = args[:-1]
63-        self.destination = args[-1]
64+        self.sources = map(argv_to_unicode, args[:-1])
65+        self.destination = argv_to_unicode(args[-1])
66hunk ./src/allmydata/scripts/cli.py 187
67-        self.where = where
68+        self.where = argv_to_unicode(where)
69hunk ./src/allmydata/scripts/cli.py 194
70-        self.from_file = frompath
71-        self.to_file = topath
72+        self.from_file = argv_to_unicode(frompath)
73+        self.to_file = argv_to_unicode(topath)
74hunk ./src/allmydata/scripts/cli.py 202
75-        self.from_file = frompath
76-        self.to_file = topath
77+        self.from_file = argv_to_unicode(frompath)
78+        self.to_file = argv_to_unicode(topath)
79hunk ./src/allmydata/scripts/cli.py 227
80-        self.from_dir = localdir
81-        self.to_dir = topath
82+        self.from_dir = argv_to_unicode(localdir)
83+        self.to_dir = argv_to_unicode(topath)
84hunk ./src/allmydata/scripts/cli.py 276
85-        self.where = where
86+        self.where = argv_to_unicode(where)
87hunk ./src/allmydata/scripts/cli.py 291
88-        self.where = where
89+        self.where = argv_to_unicode(where)
90hunk ./src/allmydata/scripts/cli.py 303
91-        self.where = where
92+        self.where = argv_to_unicode(where)
93hunk ./src/allmydata/scripts/cli.py 318
94-        self.where = where
95+        self.where = argv_to_unicode(where)
96hunk ./src/allmydata/scripts/cli.py 334
97-        self.where = where
98+        self.where = argv_to_unicode(where)
99hunk ./src/allmydata/scripts/common.py 3
100+import codecs
101}
102
103[common.py: support unicode alias names
104francois@ctrlaltdel.ch**20090224001554] {
105hunk ./src/allmydata/scripts/common.py 5
106-
107+from allmydata.util.stringutils import unicode_to_url
108hunk ./src/allmydata/scripts/common.py 104
109-        f = open(aliasfile, "r")
110+        f = codecs.open(aliasfile, "r", "utf-8")
111hunk ./src/allmydata/scripts/common.py 111
112-            cap = cap.strip()
113+            cap = cap.strip().encode('ascii')
114hunk ./src/allmydata/scripts/common.py 167
115-    return "/".join([urllib.quote(s) for s in segments])
116+    return "/".join([urllib.quote(unicode_to_url(s)) for s in segments])
117}
118
119[tahoe_add_alias.py: support unicode alias names
120francois@ctrlaltdel.ch**20090224001659] {
121hunk ./src/allmydata/scripts/tahoe_add_alias.py 3
122+import codecs
123+import sys
124hunk ./src/allmydata/scripts/tahoe_add_alias.py 8
125+from allmydata.util.stringutils import unicode_to_stdout
126hunk ./src/allmydata/scripts/tahoe_add_alias.py 58
127-    f = open(aliasfile, "a")
128+    f = codecs.open(aliasfile, "a", "utf-8")
129hunk ./src/allmydata/scripts/tahoe_add_alias.py 61
130-    print >>stdout, "Alias '%s' created" % (alias,)
131+    print >>stdout, "Alias '%s' created" % (unicode_to_stdout(alias),)
132}
133
134[tahoe_ls.py: convert filenames using unicode_to_stdout()
135francois@ctrlaltdel.ch**20090224002118] {
136hunk ./src/allmydata/scripts/tahoe_ls.py 6
137+from allmydata.util.stringutils import unicode_to_stdout
138hunk ./src/allmydata/scripts/tahoe_ls.py 116
139-        line.append(name + classify)
140+        line.append(unicode_to_stdout(name) + classify)
141}
142
143[tahoe_manifest.py: use unicode_to_stdout() instead of .encode("utf-8") to avoid an exception if terminal doesn't support utf-8
144francois@ctrlaltdel.ch**20090224002248] hunk ./src/allmydata/scripts/tahoe_manifest.py 83
145-                    print >>stdout, d["cap"], "/".join([p.encode("utf-8")
146+                    print >>stdout, d["cap"], "/".join([unicode_to_stdout(p)
147
148[tahoe_mkdir.py: path is now unicode and needs conversion before ending up in an url
149francois@ctrlaltdel.ch**20090224002345] {
150hunk ./src/allmydata/scripts/tahoe_mkdir.py 5
151+from allmydata.util.stringutils import unicode_to_url
152hunk ./src/allmydata/scripts/tahoe_mkdir.py 35
153-                                           urllib.quote(path))
154+                                           urllib.quote(unicode_to_url(path)))
155}
156
157[test_cli.py: add a bunch of unicode tests
158francois@ctrlaltdel.ch**20090224002513] {
159hunk ./src/allmydata/test/test_cli.py 3
160+import sys
161hunk ./src/allmydata/test/test_cli.py 522
162+        d.addCallback(lambda res: self.do_cli("create-alias", "études"))
163+        def _check_create_unicode((rc,stdout,stderr)):
164+            self.failUnlessEqual(rc, 0)
165+            self.failIf(stderr)
166+
167+            # If stdout only supports ascii, accentuated characters are
168+            # being replaced by '?'
169+            if sys.stdout.encoding == "ANSI_X3.4-1968":
170+                self.failUnless("Alias '?tudes' created" in stdout)
171+            else:
172+                self.failUnless("Alias 'études' created" in stdout)
173+
174+            aliases = get_aliases(self.get_clientdir())
175+            self.failUnless(aliases[u"études"].startswith("URI:DIR2:"))
176+        d.addCallback(_check_create_unicode)
177+
178+        d.addCallback(lambda res: self.do_cli("ls", "études:"))
179+        def _check_ls1((rc, stdout, stderr)):
180+            self.failUnlessEqual(rc, 0)
181+            self.failIf(stderr)
182+
183+            self.failUnlessEqual(stdout, "")
184+        d.addCallback(_check_ls1)
185+
186+        d.addCallback(lambda res: self.do_cli("put", "-", "études:uploaded.txt",
187+          stdin="Blah blah blah"))
188+
189+        d.addCallback(lambda res: self.do_cli("ls", "études:"))
190+        def _check_ls2((rc, stdout, stderr)):
191+            self.failUnlessEqual(rc, 0)
192+            self.failIf(stderr)
193+
194+            self.failUnlessEqual(stdout, "uploaded.txt\n")
195+        d.addCallback(_check_ls2)
196+
197hunk ./src/allmydata/test/test_cli.py 781
198+    def test_immutable_from_file_unicode(self):
199+        # tahoe put file.txt "à trier.txt"
200+        self.basedir = os.path.dirname(self.mktemp())
201+        self.set_up_grid()
202+
203+        rel_fn = os.path.join(self.basedir, "DATAFILE")
204+        abs_fn = os.path.abspath(rel_fn)
205+        # we make the file small enough to fit in a LIT file, for speed
206+        DATA = "short file"
207+        f = open(rel_fn, "w")
208+        f.write(DATA)
209+        f.close()
210+
211+        d = self.do_cli("create-alias", "tahoe")
212+
213+        d.addCallback(lambda res:
214+                      self.do_cli("put", rel_fn, "à trier.txt"))
215+        def _uploaded((rc,stdout,stderr)):
216+            readcap = stdout.strip()
217+            self.failUnless(readcap.startswith("URI:LIT:"))
218+            self.failUnless("201 Created" in stderr, stderr)
219+            self.readcap = readcap
220+        d.addCallback(_uploaded)
221+
222+        d.addCallback(lambda res:
223+                      self.do_cli("get", "tahoe:à trier.txt"))
224+        d.addCallback(lambda (rc,stdout,stderr):
225+                      self.failUnlessEqual(stdout, DATA))
226+
227+        return d
228+
229hunk ./src/allmydata/test/test_cli.py 822
230+        d = self.do_cli("create-alias", "tahoe")
231+
232+        # Use unicode strings when calling os functions
233+        if sys.getfilesystemencoding() == "ANSI_X3.4-1968":
234+            fn1 = os.path.join(self.basedir, u"Artonwall")
235+        else:
236+            fn1 = os.path.join(self.basedir, u"Ärtonwall")
237hunk ./src/allmydata/test/test_cli.py 830
238-        fn1 = os.path.join(self.basedir, "Ärtonwall")
239hunk ./src/allmydata/test/test_cli.py 832
240+        d.addCallback(lambda res: self.do_cli("cp", fn1.encode('utf-8'), "tahoe:Ärtonwall"))
241+
242+        d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall"))
243+        d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1))
244hunk ./src/allmydata/test/test_cli.py 837
245-        fn2 = os.path.join(self.basedir, "Metallica")
246+
247+        fn2 = os.path.join(self.basedir, u"Metallica")
248hunk ./src/allmydata/test/test_cli.py 845
249-        d = self.do_cli("create-alias", "tahoe")
250-        d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:"))
251-        d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
252-
253-        d.addCallback(lambda res: self.do_cli("get", "tahoe:Ärtonwall"))
254-        d.addCallback(lambda (rc,out,err): self.failUnlessEqual(out, DATA1))
255+        d.addCallback(lambda res: self.do_cli("cp", fn2.encode('utf-8'), "tahoe:"))
256hunk ./src/allmydata/test/test_cli.py 850
257+        d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
258+
259hunk ./src/allmydata/test/test_cli.py 853
260-    test_unicode_filename.todo = "This behavior is not yet supported, although it does happen to work (for reasons that are ill-understood) on many platforms.  See issue ticket #534."
261hunk ./src/allmydata/test/test_cli.py 871
262+class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
263+    def test_unicode_mkdir(self):
264+        self.basedir = os.path.dirname(self.mktemp())
265+        self.set_up_grid()
266+
267+        d = self.do_cli("create-alias", "tahoe")
268+        d.addCallback(lambda res: self.do_cli("mkdir", "tahoe:Motörhead"))
269+
270+        return d
271+
272+
273hunk ./src/allmydata/test/test_cli.py 916
274+        if sys.getfilesystemencoding() == "ANSI_X3.4-1968":
275+            self.writeto(u"parent/artonwall.txt", "Marmelade Jacuzzi")
276+        else:
277+            self.writeto(u"parent/ärtonwall.txt", "Marmelade Jacuzzi")
278+
279hunk ./src/allmydata/test/test_cli.py 945
280-            # foo.txt, bar.txt, blah.txt
281-            self.failUnlessEqual(fu, 3)
282+            # foo.txt, bar.txt, blah.txt, ärtonwall.txt
283+            self.failUnlessEqual(fu, 4)
284hunk ./src/allmydata/test/test_cli.py 995
285-                # foo.txt, bar.txt, blah.txt
286+                # foo.txt, bar.txt, blah.txt, ärtonwall.txt
287hunk ./src/allmydata/test/test_cli.py 997
288-                self.failUnlessEqual(fr, 3)
289+                self.failUnlessEqual(fr, 4)
290hunk ./src/allmydata/test/test_cli.py 1025
291-                self.failUnlessEqual(fchecked, 3)
292+                self.failUnlessEqual(fchecked, 4)
293hunk ./src/allmydata/test/test_cli.py 1027
294-                self.failUnlessEqual(fr, 3)
295+                self.failUnlessEqual(fr, 4)
296hunk ./src/allmydata/test/test_cli.py 1073
297-                # old bar.txt
298-                self.failUnlessEqual(fr, 1)
299+                # old bar.txt, ärtonwall.txt
300+                self.failUnlessEqual(fr, 2)
301hunk ./src/allmydata/test/test_cli.py 1113
302-            self.failUnlessEqual(fu, 5)
303+            self.failUnlessEqual(fu, 6)
304}
305
306[stringutils.py: common functions taking care of encoding conversions
307francois@ctrlaltdel.ch**20090224010021] {
308addfile ./src/allmydata/util/stringutils.py
309hunk ./src/allmydata/util/stringutils.py 1
310+"""
311+Functions used to convert inputs from whatever encoding used in the system to
312+unicode and back.
313+
314+TODO:
315+  * Accept two cli arguments --argv-encoding and --filesystem-encoding
316+"""
317+
318+import sys
319+from allmydata.util.assertutil import precondition
320+from twisted.python import usage
321+
322+def argv_to_unicode(s):
323+    """
324+    Decode given argv element to unicode.
325+    """
326+    # sys.argv encoding detection in Python is not trivial so utf-8 is
327+    # currently used by default and an informative error message is given if
328+    # the argument cannot be correctly decoded.
329+
330+    precondition(isinstance(s, str), s)
331+    try:
332+        return unicode(s, 'utf-8')
333+    except UnicodeEncodeError:
334+        raise usageError("Argument '%s' cannot be decoded as UTF-8." % s)
335+
336+def fs_to_unicode(s):
337+    """
338+    Decode a filename (or a directory name) to unicode using the same encoding
339+    as the filesystem.
340+    """
341+    # Filename encoding detection is a little bit better thanks to
342+    # getfilesystemencoding() in the sys module. However, filenames can be
343+    # encoded using another encoding than the one used on the filesystem.
344+   
345+    precondition(isinstance(s, str), s)
346+    encoding = sys.getfilesystemencoding()
347+    try:
348+        return unicode(s, encoding)
349+    except UnicodeDecodeError:
350+        raise usage.UsageError("Filename '%s' cannot be decoded using the current encoding of your filesystem (%s). Please rename this file." % (s, encoding))
351+
352+def unicode_to_fs(s):
353+    """
354+    Encode an unicode object used in file or directoy name.
355+    """
356+
357+    precondition(isinstance(s, unicode), s)
358+    encoding = sys.getfilesystemencoding()
359+    try:
360+        return s.encode(encoding)
361+    except UnicodeEncodeError:
362+        raise usage.UsageError("Filename '%s' cannot be encoded using the current encoding of your filesystem (%s). Please configure your locale correctly or rename this file." % (s, encoding))
363+
364+def unicode_to_url(s):
365+    """
366+    Encode an unicode object used in an URL.
367+    """
368+    # According to RFC 2718, non-ascii characters in url's must be UTF-8 encoded.
369+
370+    precondition(isinstance(s, unicode), s)
371+    return s.encode('utf-8')
372+
373+def unicode_to_stdout(s):
374+    """
375+    Encode an unicode object for representation on stdout.
376+    """
377+
378+    precondition(isinstance(s, unicode), s)
379+    return s.encode(sys.stdout.encoding, 'replace')
380}
381
382[tahoe_cp.py: handle unicode filename if the system locale supports it, print an error message otherwise
383francois@ctrlaltdel.ch**20090224010247] {
384hunk ./src/allmydata/scripts/tahoe_cp.py 5
385+import sys
386hunk ./src/allmydata/scripts/tahoe_cp.py 9
387+from twisted.python import usage
388+from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs, unicode_to_url
389+from allmydata.util.assertutil import precondition
390hunk ./src/allmydata/scripts/tahoe_cp.py 72
391+        precondition(isinstance(pathname, unicode), pathname)
392hunk ./src/allmydata/scripts/tahoe_cp.py 83
393+        precondition(isinstance(pathname, unicode), pathname)
394hunk ./src/allmydata/scripts/tahoe_cp.py 96
395+        precondition(isinstance(pathname, unicode), pathname)
396hunk ./src/allmydata/scripts/tahoe_cp.py 110
397+        precondition(isinstance(pathname, unicode), pathname)
398+
399hunk ./src/allmydata/scripts/tahoe_cp.py 120
400-        children = os.listdir(self.pathname)
401+        children = os.listdir(unicode_to_fs(self.pathname))
402hunk ./src/allmydata/scripts/tahoe_cp.py 122
403+            n = fs_to_unicode(n)
404hunk ./src/allmydata/scripts/tahoe_cp.py 138
405+        precondition(isinstance(pathname, unicode), pathname)
406+
407hunk ./src/allmydata/scripts/tahoe_cp.py 148
408-        children = os.listdir(self.pathname)
409+        children = os.listdir(unicode_to_fs(self.pathname))
410hunk ./src/allmydata/scripts/tahoe_cp.py 150
411+            n = fs_to_unicode(n)
412hunk ./src/allmydata/scripts/tahoe_cp.py 172
413+        precondition(isinstance(name, unicode), name)
414hunk ./src/allmydata/scripts/tahoe_cp.py 174
415-        outf = open(pathname, "wb")
416+        outf = open(unicode_to_fs(pathname), "wb")
417hunk ./src/allmydata/scripts/tahoe_cp.py 359
418-                                                   urllib.quote(name.encode('utf-8'))])
419+                                                   urllib.quote(unicode_to_url(name))])
420}
421
422[tahoe_backup.py: handle unicode filenames if locale settings supports it, print an error message otherwise
423francois@ctrlaltdel.ch**20090224010308] {
424merger 0.0 (
425hunk ./src/allmydata/scripts/tahoe_backup.py 6
426+import datetime
427hunk ./src/allmydata/scripts/tahoe_backup.py 6
428+import sys
429)
430hunk ./src/allmydata/scripts/tahoe_backup.py 11
431+from allmydata.util.stringutils import fs_to_unicode, unicode_to_fs, unicode_to_stdout
432+from allmydata.util.assertutil import precondition
433+from twisted.python import usage
434hunk ./src/allmydata/scripts/tahoe_backup.py 239
435-            print >>self.options.stdout, msg
436+            print >>self.options.stdout, unicode_to_stdout(msg)
437hunk ./src/allmydata/scripts/tahoe_backup.py 242
438+        precondition(isinstance(localpath, unicode), localpath)
439merger 0.0 (
440hunk ./src/allmydata/scripts/tahoe_backup.py 251
441-        for child in os.listdir(localpath):
442+        for child in self.options.filter_listdir(os.listdir(localpath)):
443hunk ./src/allmydata/scripts/tahoe_backup.py 251
444-        for child in os.listdir(localpath):
445-            childpath = os.path.join(localpath, child)
446+        for child in os.listdir(unicode_to_fs(localpath)):
447+            child = fs_to_unicode(child)
448+            childpath = os.path.join(localpath, child)
449)
450hunk ./src/allmydata/scripts/tahoe_backup.py 337
451+        precondition(isinstance(childpath, unicode), childpath)
452+
453hunk ./src/allmydata/scripts/tahoe_backup.py 347
454-            infileobj = open(os.path.expanduser(childpath), "rb")
455+            infileobj = open(unicode_to_fs(os.path.expanduser(childpath)), "rb")
456}
457
458[tahoe_backup.py: fix conflict
459francois@ctrlaltdel.ch**20090224114549] hunk ./src/allmydata/scripts/tahoe_backup.py 251
460-        for child in os.listdir(localpath):
461+        for child in self.options.filter_listdir(os.listdir(unicode_to_fs(localpath))):
462+            child = fs_to_unicode(child)
463
464[tahoe_backup.py: fix conflict
465francois.deppierraz@camptocamp.com**20090225092653] {
466hunk ./src/allmydata/scripts/tahoe_backup.py 6
467+import datetime
468+import sys
469}
470
471Context:
472
473[web: fix the ERROR: line to work the same in python2.4 and 2.5
474warner@lothar.com**20090225074621] 
475[test_cli/test_web: fix spurious test failure on solaris (maybe python2.4?) due to variations in the way that exceptions are stringified
476warner@allmydata.com**20090225060128
477 Ignore-this: 829fb7d67fd199babdde1443b64a5381
478] 
479[CLI: modify 'tahoe manifest' and 'tahoe deep-check' to report ERROR: properly. For #590.
480warner@allmydata.com**20090225054415
481 Ignore-this: 99162f894fdd24112a869e14848c3dea
482] 
483[webapi: modify streaming deep-manifest/deep-checker to emit an ERROR: line if they encounter an unrecoverable+untraversable directory. For #590.
484warner@allmydata.com**20090225051335
485 Ignore-this: e6bc49368fb0e7ada7cff477fd63ffe6
486] 
487[scripts/common: fix alias handling on windows again, emit slightly nicer error message in response to an unknown alias
488warner@allmydata.com**20090225042136
489 Ignore-this: 76df800d131aed6701b5c7408105b134
490] 
491[#165: make 'tahoe restart --force' the default behavior: warn but do not stop if restart is used on something that wasn't a running node, and always try to start it afterwards. This is particularly important for #315 (restart -m), because otherwise a single not-already-running node will prevent all nodes from being restarted, resulting in longer downtime than necessary
492warner@allmydata.com**20090225034213
493 Ignore-this: 26d8b0eea0279ca793faf23c5e948d90
494] 
495[test_deepcheck: switch deep-check tests to use no-network too. This cuts the runtime down by about 50%
496warner@allmydata.com**20090225030457
497 Ignore-this: b3a98ed18c5752c9016c047e95d42b
498] 
499[test_deepcheck: convert MutableChecker to no-network GridTest
500warner@allmydata.com**20090225020010
501 Ignore-this: eccba7fda129330b642886271a61a573
502] 
503[tests/no_network: move GET into the GridTestMixin class
504warner@allmydata.com**20090225003300
505 Ignore-this: 7779ad38c2d687ae328ba3cb6164a7a4
506] 
507[common_web.py: oops, add .fields to the other FakeRequest
508warner@allmydata.com**20090225000459
509 Ignore-this: 7144269b5e083553ee2c3e7afea00604
510] 
511[test_cli: exercise the recent tolerate-'c:\dir\file.txt' fix in scripts/common, recorded in a separate match to make it easier to merge the fix to prod
512warner@allmydata.com**20090224235620
513 Ignore-this: 2cae196dd4ccb578b2abae085376e0d7
514] 
515[scripts/common: on windows, tolerate paths like 'c:\dir\file.txt', by treating single-letter aliases on windows/cygwin as non-aliases
516warner@allmydata.com**20090224235522
517 Ignore-this: 96d37644b7f81ac768ff4a1d1915eb46
518] 
519[test/common_web.py: add a .fields attribute to our FakeRequest, since we support versions of Nevow that are old enough to not do it themselves
520warner@allmydata.com**20090224232050
521 Ignore-this: 239aaed746128e54e886762782541a1f
522] 
523[Two small fixes on documentation for cli backup command.
524Alberto Berti <alberto@metapensiero.it>**20090224223634
525 Ignore-this: 5634a6dadad6e4e43a112de7fe5c74c
526] 
527[test_web: add (disabled) test to see what happens when deep-check encounters an unrecoverable directory. We still need code changes to improve this behavior.
528warner@lothar.com**20090224214017
529 Ignore-this: e839f1b0ec40f53fedcd809c2a30d5f9
530] 
531[Add elapsed timestamp to cli backup command final summary.
532Alberto Berti <alberto@metapensiero.it>**20090224171425
533 Ignore-this: 9a042d11f95ee9f6858a5096d513c0bc
534] 
535[Added documentation for '--exclude' and friends cli backup command.
536Alberto Berti <alberto@metapensiero.it>**20090224153049
537 Ignore-this: bbc791fa56e38535bb82cc3077ffde90
538] 
539[test_repairer: change to use faster no_network.GridTestMixin, split Verifier tests into separate cases, refactor judgement funcs into shared methods
540warner@lothar.com**20090224041506
541 Ignore-this: 584ce72d6276da5edc00562793d4ee53
542] 
543[immutable/checker.py: trap ShareVersionIncompatible too. Also, use f.check
544warner@lothar.com**20090224041405
545 Ignore-this: b667e8d3192116293babcacdeed42898
546 instead of examining the value returned by f.trap, because the latter appears
547 to squash exception types down into their base classes (i.e. since
548 ShareVersionIncompatible is a subclass of LayoutInvalid,
549 f.trap(Failure(ShareVersionIncompatible)) == LayoutInvalid).
550 
551 All this resulted in 'incompatible' shares being misclassified as 'corrupt'.
552] 
553[immutable/layout.py: wrap to 80 cols, no functional changes
554warner@lothar.com**20090224005837
555 Ignore-this: 40019480180ec34141506a28d7711608
556] 
557[test_repairer: change Repairer to use much-faster no_network.GridTestMixin. As a side-effect, fix what I think was a bug: some of the assert-minimal-effort-expended checks were mixing write counts and allocate counts
558warner@lothar.com**20090223234227
559 Ignore-this: d58bd0a909f9939775730cda4a858cae
560] 
561[test/no_network.py: add a basic stats provider
562warner@lothar.com**20090223233937
563 Ignore-this: c9f3cc4eed99cfc36f68938ceff4162c
564] 
565[tests: stop using setUpClass/tearDownClass, since they've been deprecated in Twisted-8.2.0
566warner@lothar.com**20090223204312
567 Ignore-this: 24c6592141cf64103530c024f93a5b88
568] 
569[test_checker: improve test coverage for checker results
570warner@lothar.com**20090223201943
571 Ignore-this: 83e173602f0f4c811a7a9893d85385df
572] 
573[Fixed tests again so they will pass on windows.
574Alberto Berti <alberto@metapensiero.it>**20090223003502
575 Ignore-this: 80d5074e7153642a2fa2a77958bfb50d
576] 
577[misc/*: remove RuntimeError too
578warner@lothar.com**20090222233401
579 Ignore-this: b76f8a184f75bb28eb9d8002f957936a
580] 
581[scripts: stop using RuntimeError, for #639
582warner@lothar.com**20090222233106
583 Ignore-this: 686a424442670fffbd4d1816c284a601
584] 
585[mutable/publish: stop using RuntimeError, for #639
586warner@lothar.com**20090222233056
587 Ignore-this: 2a80a661c7850d97357caddad48c6e9d
588] 
589[remove more RuntimeError from unit tests, for #639
590warner@lothar.com**20090222232855
591 Ignore-this: 1a1c3e1457f3f29ba7101fe406ee5f43
592] 
593[stop using RuntimeError in unit tests, for #639
594warner@lothar.com**20090222232722
595 Ignore-this: 475ce0c0dcd7a1f5ed83ef460312efea
596] 
597[ftpd/sftpd: stop using RuntimeError, for #639
598warner@lothar.com**20090222232426
599 Ignore-this: 97001362c4ba9e94b2e254e229b79987
600] 
601[Added tests for the cse when listdir is an iterator
602Alberto Berti <alberto@metapensiero.it>**20090222224356
603 Ignore-this: 218fb2aba02c28b4b1e5324bdb5adeaa
604] 
605[Fixed tests so that they pass also on buildbots.
606Alberto Berti <alberto@metapensiero.it>**20090222224311
607 Ignore-this: fcb91cd6acf028382411d23d380a4576
608] 
609[Use failUnlessEqual instead of failUnless(a == b)
610Alberto Berti <alberto@metapensiero.it>**20090222224214
611 Ignore-this: 8f9144632e3ac9acb4726fb48a083bf4
612] 
613[Better implementation of filtering algorithm.
614Alberto Berti <alberto@metapensiero.it>**20090222224049
615 Ignore-this: 67a8bd2f99bcc87ca2443bef13370a87
616] 
617[Removed '.hgrags' from vcs excludes
618Alberto Berti <alberto@metapensiero.it>**20090222223946
619 Ignore-this: 3e94c22fc9d85f380ee11fb8bdb4d1e9
620] 
621[docs: CREDITS to Alberto Berti
622zooko@zooko.com**20090222193314
623 Ignore-this: 74d370ada3234cce9e58aec15d739f71
624] 
625[Added tests for the --exclude* options of backup command.
626Alberto Berti <alberto@metapensiero.it>**20090222165106
627 Ignore-this: f1b931cf2e7929ce47b737c022bca707
628] 
629[Added --exclude, --exclude-from and --exclude-vcs options to backup command.
630Alberto Berti <alberto@metapensiero.it>**20090222170829
631 Ignore-this: 4912890229cd54a2f61f14f06bc4afcc
632 
633 It is still impossible to specify absolute exclusion path, only
634 relative. I must check with tar or rsync how they allow them to be
635 specified.
636] 
637[Raise a more explanatory exception for errors encountered during backup processing.
638Alberto Berti <alberto@metapensiero.it>**20090222170252
639 Ignore-this: f6b8ffe2a903ba07a2c1c59130dac1e4
640] 
641[Added tests for the fixed alias related command's synopsis
642Alberto Berti <alberto@metapensiero.it>**20090222163732
643 Ignore-this: 4432b4e88e990ba53a5b3fe0f12db2ac
644] 
645[Add missing synopsis and descriptions for alias commands.
646Alberto Berti <alberto@metapensiero.it>**20090221003106
647 Ignore-this: 8aedd03d36d92d912102c7f29e4ca697
648] 
649[docs: move many specification-like documents into specifications/
650warner@lothar.com**20090222054054
651 Ignore-this: a4110cc478198c0611205aba1ccf54f4
652] 
653[test_web.py: increase test coverage of web.status.plural()
654warner@lothar.com**20090222000116
655 Ignore-this: 3138c9d5d2410d8e1121e9b2ed694169
656] 
657[crawler: fix performance problems: only save state once per timeslice (not after every bucket), don't start the crawler until 5 minutes after node startup
658warner@lothar.com**20090221205649
659 Ignore-this: e6551569982bd31d19779ff15c2d6f58
660] 
661[test_system: oops, don't assume that all files in storage/ are in a deep storage/shares/prefix/si/shnum path, since now the crawler pickle has a short path
662warner@lothar.com**20090221061710
663 Ignore-this: fde76d0e5cae853014d1bb18b5f17dae
664] 
665[crawler: tolerate low-resolution system clocks (i.e. windows)
666warner@lothar.com**20090221061533
667 Ignore-this: 57286a3abcaf44f6d1a78c3c1ad547a5
668] 
669[BucketCountingCrawler: store just the count, not cycle+count, since it's too easy to make usage mistakes otherwise
670warner@lothar.com**20090221035831
671 Ignore-this: 573b6f651af74380cdd64059fbbdda4b
672] 
673[test_storage: startService the server, as is now the standard practice
674warner@lothar.com**20090221035755
675 Ignore-this: 3999889bd628fe4039bbcf1b29160453
676] 
677[crawler: load state from the pickle in init, rather than waiting until startService, so get_state() can be called early
678warner@lothar.com**20090221035720
679 Ignore-this: ecd128a5f4364c0daf4b72d791340b66
680] 
681[BucketCountingCrawler: rename status and state keys to use 'bucket' instead of 'share', because the former is more accurate
682warner@lothar.com**20090221034606
683 Ignore-this: cf819f63fac9506c878d6c9715ce35b7
684] 
685[storage: also report space-free-for-root and space-free-for-nonroot, since that helps users understand the space-left-for-tahoe number better
686warner@lothar.com**20090221032856
687 Ignore-this: 9fdf0475f758acd98b73026677170b45
688] 
689[storage: add bucket-counting share crawler, add its output (number of files+directories maintained by a storage server) and status to the webapi /storage page
690warner@lothar.com**20090221030408
691 Ignore-this: 28761c5e076648026bc5f518506db65c
692] 
693[storage: move si_b2a/si_a2b/storage_index_to_dir out of server.py and into common.py
694warner@lothar.com**20090221030309
695 Ignore-this: 645056428ab797f0b542831c82bf192a
696] 
697[crawler: add get_progress, clean up get_state
698warner@lothar.com**20090221002743
699 Ignore-this: 9bea69f154c75b31a53425a8ea67789b
700] 
701[web/storage: make sure we can handle platforms without os.statvfs too
702warner@lothar.com**20090220220353
703 Ignore-this: 79d4cb8482a8543b9759dc949c86c587
704] 
705[crawler: provide for one-shot crawlers, which stop after their first full cycle, for share-upgraders and database-populaters
706warner@lothar.com**20090220211911
707 Ignore-this: fcdf72c5ffcafa374d376388be6fa5c5
708] 
709[web: add Storage status page, improve tests
710warner@lothar.com**20090220202926
711 Ignore-this: e34d5270dcf0237fe72f573f717c7a4
712] 
713[storage: include reserved_space in stats
714warner@lothar.com**20090220202920
715 Ignore-this: b5b480fe0abad0148ecad0c1fb47ecae
716] 
717[web/check_results: sort share identifiers in the sharemap display
718warner@lothar.com**20090220182922
719 Ignore-this: 5c7bfcee3e15c7082c3653eb8a460960
720] 
721[webapi: pass client through constructor arguments, remove IClient, should make it easier to test web renderers in isolation
722warner@lothar.com**20090220181554
723 Ignore-this: e7848cd1bee8faf2ce7aaf040b9bf8e3
724] 
725[test/no_network: do startService on the storage servers, make it easier to customize the storage servers
726warner@lothar.com**20090220022254
727 Ignore-this: e62f328721c007e4c5ee023a6efdf66d
728] 
729[crawler: modify API to support upcoming bucket-counting crawler
730warner@lothar.com**20090220013142
731 Ignore-this: 808f8382837b13082f8b245db2ebee06
732] 
733[test_backupdb: make the not-a-database file larger, since the older sqlite-2.3.2 on OS-X is easily fooled
734warner@lothar.com**20090220000409
735 Ignore-this: 694d2ca5053bb96e91670765d0cedf2e
736] 
737[web/reliability: add parameter descriptions, adapted from a patch from Terrell Russell.
738warner@lothar.com**20090219222918
739 Ignore-this: 835f5ab01e1aff31b2ff9febb9a51f3
740] 
741[test_crawler: hush pyflakes
742warner@lothar.com**20090219202340
743 Ignore-this: 765d22c9c9682cc86c5205dc130500af
744] 
745[test_crawler: disable the percentage-of-cpu-used test, since it is too unreliable on our slow buildslaves. But leave the code in place for developers to run by hand.
746warner@lothar.com**20090219201654
747 Ignore-this: ff7cf5cfa79c6f2ef0cf959495dd989a
748] 
749[reliability.py: fix the numpy conversion, it was completely broken. Thanks to Terrell Russell for the help.
750warner@lothar.com**20090219195515
751 Ignore-this: f2b1eb65855111b338e1487feee1bbcf
752] 
753[reliability: switch to NumPy, since Numeric is deprecated
754warner@lothar.com**20090219074435
755 Ignore-this: f588a68e9bcd3b0bc3653570882b6fd5
756] 
757[setup.py: fix pyflakes complaints
758warner@lothar.com**20090219073643
759 Ignore-this: a314e5456b0a796bc9f70232a119ec68
760] 
761[move show-tool-versions out of setup.py and into a separate script in misc/ , since setuptools is trying to build and install a bunch of stuff first
762warner@lothar.com**20090219073558
763 Ignore-this: 9e56bc43026379212e6b6671ed6a1fd4
764] 
765[test_crawler: don't require >=1 cycle on cygwin
766warner@lothar.com**20090219065818
767 Ignore-this: b8d2d40f26aeb30a7622479840a04635
768] 
769[setup.py: add show_tool_versions command, for the benefit of a new buildbot step
770warner@lothar.com**20090219062436
771 Ignore-this: 21d761c76a033e481831584bedc60c86
772] 
773[setup.py: wrap to 80 cols, no functional changes
774warner@lothar.com**20090219055751
775 Ignore-this: d29e57c6ee555f2ee435667b7e13e60b
776] 
777[crawler: use fileutil.move_info_place in preference to our own version
778warner@lothar.com**20090219051342
779 Ignore-this: ee4e46f3de965610503ba36b28184db9
780] 
781[fileutil: add move_into_place(), to perform the standard unix trick of atomically replacing a file, with a fallback for windows
782warner@lothar.com**20090219051310
783 Ignore-this: c1d35e8ca88fcb223ea194513611c511
784] 
785[crawler: fix problems on windows and our slow cygwin slave
786warner@lothar.com**20090219042431
787 Ignore-this: 8019cb0da79ba00c536183a6f57b4cab
788] 
789[#633: first version of a rate-limited interruptable share-crawler
790warner@lothar.com**20090219034633
791 Ignore-this: 5d2d30c743e3b096a8e775d5a9b33601
792] 
793[change StorageServer to take nodeid in the constructor, instead of assigning it later, since it's cleaner and because the original problem (Tubs not being ready until later) went away
794warner@lothar.com**20090218222301
795 Ignore-this: 740d582f20c93bebf60e21d9a446d3d2
796] 
797[test_system: split off checker tests to test_deepcheck.py, this file is too big
798warner@lothar.com**20090218214234
799 Ignore-this: 82bf8db81dfbc98224bbf694054a8761
800] 
801[break storage.py into smaller pieces in storage/*.py . No behavioral changes.
802warner@lothar.com**20090218204655
803 Ignore-this: 312d408d1cacc5a764d791b53ebf8f91
804] 
805[immutable/layout: minor change to repr name
806warner@lothar.com**20090218204648
807 Ignore-this: c8781ef15b7dea63b39236a1899b86ce
808] 
809[docs: add lease-tradeoffs diagram
810warner@lothar.com**20090218204137
811 Ignore-this: c22a589ad465dac846da834c30dc4083
812] 
813[interfaces.py: allow add/renew/cancel-lease to return Any, so that 1.3.1 clients (the first to use these calls) can tolerate future storage servers which might return something other than None
814warner@lothar.com**20090218192903
815 Ignore-this: dcbb704a05416ecc66d90fb486c3d75b
816] 
817[docs/debian.txt: minor edit
818warner@lothar.com**20090218032212
819 Ignore-this: 64ff1fb163ffca4bcfd920254f1cf866
820] 
821[add --add-lease to 'tahoe check', 'tahoe deep-check', and webapi.
822warner@lothar.com**20090218013243
823 Ignore-this: 176b2006cef5041adcb592ee83e084dd
824] 
825[change RIStorageServer.remote_add_lease to exit silently in case of no-such-bucket, instead of raising IndexError, because that makes the upcoming --add-lease feature faster and less noisy
826warner@lothar.com**20090218013053
827 Ignore-this: 6fdfcea2c832178f1ce72ab0ff510f3a
828] 
829[CLI #590: convert 'tahoe deep-check' to streaming form, improve display, add tests
830warner@lothar.com**20090217231511
831 Ignore-this: 6d88eb94b1c877eacc8c5ca7d0aac776
832] 
833[interfaces.py: document behavior of add_lease/renew_lease/cancel_lease, before I change it
834warner@lothar.com**20090217194809
835 Ignore-this: 703c6712926b8edb19d55d790b65a400
836] 
837[test_backupdb: improve error messages if the test fails
838warner@lothar.com**20090217170838
839 Ignore-this: ef657e87c66e4304d3e0aca9831b84c
840] 
841[webapi #590: add streaming deep-check. Still need a CLI tool to use it.
842warner@lothar.com**20090217053553
843 Ignore-this: a0edd3d2a531c48a64d8397f7e4b208c
844] 
845[test_web.Grid: change the CHECK() function to make it easier to test t= values with hyphens in them
846warner@lothar.com**20090217050034
847 Ignore-this: 410c08735347c2057df52f6716520228
848] 
849[test_web: improve checker-results coverage with a no-network -based test, enhance no-network harness to assist, fix some bugs in web/check_results.py that were exposed
850warner@lothar.com**20090217041242
851 Ignore-this: fe54bb66a9ae073c002a7af51cd1e18
852] 
853[web: fix handling of reliability page when Numeric is not available
854warner@lothar.com**20090217015658
855 Ignore-this: 9d329182f1b2e5f812e5e7eb5f4cf2ed
856] 
857[test/no_network: update comments with setup timing: no_network takes 50ms, SystemTestMixin takes 2s (on my laptop)
858warner@lothar.com**20090217000643
859 Ignore-this: cc778fa3219775b25057bfc9491f8f34
860] 
861[test_upload: rewrite in terms of no-network GridTestMixin, improve no_network.py as necessary
862warner@lothar.com**20090216234457
863 Ignore-this: 80a341d5aa3036d24de98e267499d70d
864] 
865[test_download: rewrite in terms of no-network GridTestMixin, improve no_network.py as necessary
866warner@lothar.com**20090216233658
867 Ignore-this: ec2febafd2403830519120fb3f3ca04e
868] 
869[test_dirnode.py: convert Deleter to new no-network gridtest
870warner@lothar.com**20090216232348
871 Ignore-this: 8041739442ec4db726675e48f9775ae9
872] 
873[test_cli.py: modify to use the new 'no-network' gridtest instead of SystemTestMixin, which speeds it up from 73s to 43s on my system
874warner@lothar.com**20090216232005
875 Ignore-this: ec6d010c9182aa72049d1fb894cf890e
876] 
877[tests: fix no_network framework to work with upload/download and checker
878warner@lothar.com**20090216231947
879 Ignore-this: 74b4dbd66b8384ae7c7544969fe4f744
880] 
881[client.py: improve docstring
882warner@lothar.com**20090216231532
883 Ignore-this: bbaa9e3f63fdb0048e3125c4681b2d1f
884] 
885[test_cli: add test coverage for help strings
886warner@lothar.com**20090216210833
887 Ignore-this: d2020849107f687448e159a19d0e5dab
888] 
889[test/no_network: new test harness, like system-test but doesn't use the network so it's faster
890warner@lothar.com**20090216205844
891 Ignore-this: 31678f7bdef30b0216fd657fc6145534
892] 
893[interfaces.py: minor docstring edit
894warner@lothar.com**20090216205816
895 Ignore-this: cec3855070197f7920b370f95e8b07bd
896] 
897[setup: if you sdist_dsc (to produce the input files for dpkg-buildpackage) then run darcsver first
898zooko@zooko.com**20090216201558
899 Ignore-this: b85be51b3d4a9a19a3366e690f1063e2
900] 
901[doc: a few edits to docs made after the 1.3.0 release
902zooko@zooko.com**20090216201539
903 Ignore-this: dbff3b929d88134d862f1dffd1ef068a
904] 
905[test_cli: improve test coverage slightly
906warner@lothar.com**20090216030451
907 Ignore-this: e01ccc6a6fb44aaa4fb14fe8669e2065
908] 
909[test_util: get almost full test coverage of dictutil, starting with the original pyutil tests as a base. The remaining three uncovered lines involve funny cases of ValueOrderedDict that I can't figure out how to get at
910warner@lothar.com**20090216023210
911 Ignore-this: dc1f0c6d8c003c0ade38bc8f8516b04d
912] 
913[provisioning/reliability: add tests, hush pyflakes, remove dead code, fix web links
914warner@lothar.com**20090215222451
915 Ignore-this: 7854df3e0130d9388f06efd4c797262f
916] 
917[util/statistics: add tests, fix mean_repair_cost
918warner@lothar.com**20090215222326
919 Ignore-this: c576eabc74c23b170702018fc3c122d9
920] 
921[test_repairer: hush pyflakes
922warner@lothar.com**20090215222310
923 Ignore-this: 875eb52e86077cda77efd02da77f8cfa
924] 
925[lossmodel.lyx: move draft paper into docs/proposed/, since it's unfinished
926warner@lothar.com**20090215221905
927 Ignore-this: 7f7ee204e47fd66932759c94deefe68
928] 
929[build a 'reliability' web page, with a simulation of file decay and repair over time
930warner@lothar.com**20090213234234
931 Ignore-this: 9e9623eaac7b0637bbd0071f082bd345
932] 
933[More lossmodel work, on repair.
934Shawn Willden <shawn-tahoe@willden.org>**20090116025648] 
935[Loss model work (temp1)
936Shawn Willden <shawn@willden.org>**20090115030058] 
937[Statistics module
938Shawn Willden <shawn-tahoe@willden.org>**20090114021235
939 
940 Added a statistics module for calculating various facets of
941 share survival statistics.
942] 
943[docs: relnotes-short.txt
944zooko@zooko.com**20090215163510
945 Ignore-this: 683649bb13499bbe0e5cea2e1716ff59
946 linkedin.com imposed a strict limit on the number of characters I could post.  This forced me to prune and prune and edit and edit until relnotes.txt was a quarter of its former size.  Here's the short version.
947] 
948[TAG allmydata-tahoe-1.3.0
949zooko@zooko.com**20090214000556
950 Ignore-this: aa6c9a31a14a58ad2298cb7b08d3ea70
951] 
952Patch bundle hash:
953764aa2a99d1bd688d00dfb850176910a0f2b0124