Ticket #1118: happy_share_distribution.dpatch.txt

Sun Jul 18 16:15:37 MDT 2010  zooko@zooko.com
  * immutable: test for #1118

Sun Jul 18 22:50:47 MDT 2010  zooko@zooko.com
  * immutable: extend the tests to check that the shares that got uploaded really do make a sufficiently Happy distribution
  This patch also renames some instances of "find_shares()" to "find_all_shares()" and other instances to "find_uri_shares()" as appropriate -- the conflation between those names confused me at first when writing these tests.

New patches:

[immutable: test for #1118
zooko@zooko.com**20100718221537
 Ignore-this: 8882aabe2aaec6a0148c87e735d817ad
] {
hunk ./src/allmydata/immutable/upload.py 919
             for shnum in peer.buckets:
                 self._peer_trackers[shnum] = peer
                 servermap.setdefault(shnum, set()).add(peer.peerid)
-        assert len(buckets) == sum([len(peer.buckets) for peer in used_peers])
+        assert len(buckets) == sum([len(peer.buckets) for peer in used_peers]), "%s (%s) != %s (%s)" % (len(buckets), buckets, sum([len(peer.buckets) for peer in used_peers]), [(p.buckets, p.peerid) for p in used_peers])
         encoder.set_shareholders(buckets, servermap)
 
     def _encrypted_done(self, verifycap):
hunk ./src/allmydata/test/test_upload.py 1789
         return d
     test_problem_layout_comment_187.todo = "this isn't fixed yet"
 
+    def test_problem_layout_ticket_1118(self):
+        # #1118 includes a report from a user who hit an assertion in
+        # the upload code with this layout.
+        self.basedir = self.mktemp()
+        d = self._setup_and_upload(k=2, n=4)
+
+        # server 0: no shares
+        # server 1: shares 0, 3
+        # server 3: share 1
+        # server 2: share 2
+        # The order that they get queries is 0, 1, 3, 2
+        def _setup(ign):
+            self._add_server(server_number=0)
+            self._add_server_with_share(server_number=1, share_number=0)
+            self._add_server_with_share(server_number=2, share_number=2)
+            self._add_server_with_share(server_number=3, share_number=1)
+            # Copy shares
+            self._copy_share_to_server(3, 1)
+            storedir = self.get_serverdir(0)
+            # remove the storedir, wiping out any existing shares
+            shutil.rmtree(storedir)
+            # create an empty storedir to replace the one we just removed
+            os.mkdir(storedir)
+            client = self.g.clients[0]
+            client.DEFAULT_ENCODING_PARAMETERS['happy'] = 4
+            return client
+
+        d.addCallback(_setup)
+        d.addCallback(lambda client:
+                          client.upload(upload.Data("data" * 10000, convergence="")))
+        return d
 
     def test_upload_succeeds_with_some_homeless_shares(self):
         # If the upload is forced to stop trying to place shares before
}
[immutable: extend the tests to check that the shares that got uploaded really do make a sufficiently Happy distribution
zooko@zooko.com**20100719045047
 Ignore-this: 89c33a7b795e23018667351045a8d5d0
 This patch also renames some instances of "find_shares()" to "find_all_shares()" and other instances to "find_uri_shares()" as appropriate -- the conflation between those names confused me at first when writing these tests.
] {
hunk ./src/allmydata/test/common.py 959
         d.addCallback(_stash_it)
         return d
 
-    def find_shares(self, unused=None):
+    def find_all_shares(self, unused=None):
         """Locate shares on disk. Returns a dict that maps
         (clientnum,sharenum) to a string that contains the share container
         (copied directly from the disk, containing leases etc). You can
hunk ./src/allmydata/test/common.py 984
 
     def replace_shares(self, newshares, storage_index):
         """Replace shares on disk. Takes a dictionary in the same form
-        as find_shares() returns."""
+        as find_all_shares() returns."""
 
         for i, c in enumerate(self.clients):
             sharedir = c.getServiceNamed("storage").sharedir
hunk ./src/allmydata/test/common.py 1009
     def _delete_a_share(self, unused=None, sharenum=None):
         """ Delete one share. """
 
-        shares = self.find_shares()
+        shares = self.find_all_shares()
         ks = shares.keys()
         if sharenum is not None:
             k = [ key for key in shares.keys() if key[1] == sharenum ][0]
hunk ./src/allmydata/test/common.py 1021
         return unused
 
     def _corrupt_a_share(self, unused, corruptor_func, sharenum):
-        shares = self.find_shares()
+        shares = self.find_all_shares()
         ks = [ key for key in shares.keys() if key[1] == sharenum ]
         assert ks, (shares.keys(), sharenum)
         k = ks[0]
hunk ./src/allmydata/test/common.py 1031
 
     def _corrupt_all_shares(self, unused, corruptor_func):
         """ All shares on disk will be corrupted by corruptor_func. """
-        shares = self.find_shares()
+        shares = self.find_all_shares()
         for k in shares.keys():
             self._corrupt_a_share(unused, corruptor_func, k[1])
         return corruptor_func
hunk ./src/allmydata/test/common.py 1038
 
     def _corrupt_a_random_share(self, unused, corruptor_func):
         """ Exactly one share on disk will be corrupted by corruptor_func. """
-        shares = self.find_shares()
+        shares = self.find_all_shares()
         ks = shares.keys()
         k = random.choice(ks)
         self._corrupt_a_share(unused, corruptor_func, k[1])
hunk ./src/allmydata/test/no_network.py 305
             ss = self.g.servers_by_number[i]
             yield (i, ss, ss.storedir)
 
-    def find_shares(self, uri):
+    def find_uri_shares(self, uri):
         si = tahoe_uri.from_string(uri).get_storage_index()
         prefixdir = storage_index_to_dir(si)
         shares = []
hunk ./src/allmydata/test/no_network.py 326
         os.unlink(sharefile)
 
     def delete_shares_numbered(self, uri, shnums):
-        for (i_shnum, i_serverid, i_sharefile) in self.find_shares(uri):
+        for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri):
             if i_shnum in shnums:
                 os.unlink(i_sharefile)
 
hunk ./src/allmydata/test/no_network.py 336
         open(sharefile, "wb").write(corruptdata)
 
     def corrupt_shares_numbered(self, uri, shnums, corruptor, debug=False):
-        for (i_shnum, i_serverid, i_sharefile) in self.find_shares(uri):
+        for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri):
             if i_shnum in shnums:
                 sharedata = open(i_sharefile, "rb").read()
                 corruptdata = corruptor(sharedata, debug=debug)
hunk ./src/allmydata/test/test_cli.py 1998
 
         def _clobber_shares(ignored):
             # delete one, corrupt a second
-            shares = self.find_shares(self.uri)
+            shares = self.find_uri_shares(self.uri)
             self.failUnlessReallyEqual(len(shares), 10)
             os.unlink(shares[0][2])
             cso = debug.CorruptShareOptions()
hunk ./src/allmydata/test/test_cli.py 2123
         d.addCallback(_check_stats)
 
         def _clobber_shares(ignored):
-            shares = self.find_shares(self.uris[u"gööd"])
+            shares = self.find_uri_shares(self.uris[u"gööd"])
             self.failUnlessReallyEqual(len(shares), 10)
             os.unlink(shares[0][2])
 
hunk ./src/allmydata/test/test_cli.py 2127
-            shares = self.find_shares(self.uris["mutable"])
+            shares = self.find_uri_shares(self.uris["mutable"])
             cso = debug.CorruptShareOptions()
             cso.stdout = StringIO()
             cso.parseOptions([shares[1][2]])
hunk ./src/allmydata/test/test_deepcheck.py 994
         self.delete_shares_numbered(node.get_uri(), [0,1])
 
     def _corrupt_some_shares(self, node):
-        for (shnum, serverid, sharefile) in self.find_shares(node.get_uri()):
+        for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
             if shnum in (0,1):
                 self._run_cli(["debug", "corrupt-share", sharefile])
 
hunk ./src/allmydata/test/test_hung_server.py 67
             os.makedirs(si_dir)
         new_sharefile = os.path.join(si_dir, str(sharenum))
         shutil.copy(sharefile, new_sharefile)
-        self.shares = self.find_shares(self.uri)
+        self.shares = self.find_uri_shares(self.uri)
         # Make sure that the storage server has the share.
         self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile)
                         in self.shares)
hunk ./src/allmydata/test/test_hung_server.py 98
             d = nm.create_mutable_file(mutable_plaintext)
             def _uploaded_mutable(node):
                 self.uri = node.get_uri()
-                self.shares = self.find_shares(self.uri)
+                self.shares = self.find_uri_shares(self.uri)
             d.addCallback(_uploaded_mutable)
         else:
             data = upload.Data(immutable_plaintext, convergence="")
hunk ./src/allmydata/test/test_hung_server.py 105
             d = self.c0.upload(data)
             def _uploaded_immutable(upload_res):
                 self.uri = upload_res.uri
-                self.shares = self.find_shares(self.uri)
+                self.shares = self.find_uri_shares(self.uri)
             d.addCallback(_uploaded_immutable)
         return d
 
hunk ./src/allmydata/test/test_immutable.py 14
         # replace_shares, and asserting that the new set of shares equals the
         # old is more to test this test code than to test the Tahoe code...
         d = defer.succeed(None)
-        d.addCallback(self.find_shares)
+        d.addCallback(self.find_all_shares)
         stash = [None]
         def _stash_it(res):
             stash[0] = res
hunk ./src/allmydata/test/test_repairer.py 90
         d.addCallback(_check)
 
         def _remove_all(ignored):
-            for sh in self.find_shares(self.uri):
+            for sh in self.find_uri_shares(self.uri):
                 self.delete_share(sh)
         d.addCallback(_remove_all)
 
hunk ./src/allmydata/test/test_repairer.py 325
         def _grab_sh0(res):
             self.sh0_file = [sharefile
                              for (shnum, serverid, sharefile)
-                             in self.find_shares(self.uri)
+                             in self.find_uri_shares(self.uri)
                              if shnum == 0][0]
             self.sh0_orig = open(self.sh0_file, "rb").read()
         d.addCallback(_grab_sh0)
hunk ./src/allmydata/test/test_repairer.py 470
         self.set_up_grid(num_clients=2)
         d = self.upload_and_stash()
 
-        d.addCallback(lambda ignored: self.find_shares(self.uri))
+        d.addCallback(lambda ignored: self.find_uri_shares(self.uri))
         def _stash_shares(oldshares):
             self.oldshares = oldshares
         d.addCallback(_stash_shares)
hunk ./src/allmydata/test/test_repairer.py 474
-        d.addCallback(lambda ignored: self.find_shares(self.uri))
+        d.addCallback(lambda ignored: self.find_uri_shares(self.uri))
         def _compare(newshares):
             self.failUnlessEqual(newshares, self.oldshares)
         d.addCallback(_compare)
hunk ./src/allmydata/test/test_repairer.py 485
             for sh in self.oldshares[1:8]:
                 self.delete_share(sh)
         d.addCallback(_delete_8)
-        d.addCallback(lambda ignored: self.find_shares(self.uri))
+        d.addCallback(lambda ignored: self.find_uri_shares(self.uri))
         d.addCallback(lambda shares: self.failUnlessEqual(len(shares), 2))
 
         d.addCallback(lambda ignored:
hunk ./src/allmydata/test/test_repairer.py 502
         # test share corruption
         def _test_corrupt(ignored):
             olddata = {}
-            shares = self.find_shares(self.uri)
+            shares = self.find_uri_shares(self.uri)
             for (shnum, serverid, sharefile) in shares:
                 olddata[ (shnum, serverid) ] = open(sharefile, "rb").read()
             for sh in shares:
hunk ./src/allmydata/test/test_repairer.py 513
         d.addCallback(_test_corrupt)
 
         def _remove_all(ignored):
-            for sh in self.find_shares(self.uri):
+            for sh in self.find_uri_shares(self.uri):
                 self.delete_share(sh)
         d.addCallback(_remove_all)
hunk ./src/allmydata/test/test_repairer.py 516
-        d.addCallback(lambda ignored: self.find_shares(self.uri))
+        d.addCallback(lambda ignored: self.find_uri_shares(self.uri))
         d.addCallback(lambda shares: self.failUnlessEqual(shares, []))
 
         return d
hunk ./src/allmydata/test/test_repairer.py 547
 
             # Now we inspect the filesystem to make sure that it has 10
             # shares.
-            shares = self.find_shares(self.uri)
+            shares = self.find_uri_shares(self.uri)
             self.failIf(len(shares) < 10)
         d.addCallback(_check_results)
 
hunk ./src/allmydata/test/test_repairer.py 592
             self.failUnless(post.is_healthy(), post.data)
 
             # Make sure we really have 10 shares.
-            shares = self.find_shares(self.uri)
+            shares = self.find_uri_shares(self.uri)
             self.failIf(len(shares) < 10)
         d.addCallback(_check_results)
 
hunk ./src/allmydata/test/test_repairer.py 653
     def OFF_test_repair_from_corruption_of_1(self):
         d = defer.succeed(None)
 
-        d.addCallback(self.find_shares)
+        d.addCallback(self.find_all_shares)
         stash = [None]
         def _stash_it(res):
             stash[0] = res
hunk ./src/allmydata/test/test_repairer.py 688
 
                 # Now we inspect the filesystem to make sure that it has 10
                 # shares.
-                shares = self.find_shares()
+                shares = self.find_all_shares()
                 self.failIf(len(shares) < 10)
 
                 # Now we assert that the verifier reports the file as healthy.
hunk ./src/allmydata/test/test_system.py 386
 
         return d
 
-    def _find_shares(self, basedir):
+    def _find_all_shares(self, basedir):
         shares = []
         for (dirpath, dirnames, filenames) in os.walk(basedir):
             if "storage" not in dirpath:
hunk ./src/allmydata/test/test_system.py 478
         def _test_debug(res):
             # find a share. It is important to run this while there is only
             # one slot in the grid.
-            shares = self._find_shares(self.basedir)
+            shares = self._find_all_shares(self.basedir)
             (client_num, storage_index, filename, shnum) = shares[0]
             log.msg("test_system.SystemTest.test_mutable._test_debug using %s"
                     % filename)
hunk ./src/allmydata/test/test_system.py 581
         def _corrupt_shares(res):
             # run around and flip bits in all but k of the shares, to test
             # the hash checks
-            shares = self._find_shares(self.basedir)
+            shares = self._find_all_shares(self.basedir)
             ## sort by share number
             #shares.sort( lambda a,b: cmp(a[3], b[3]) )
             where = dict([ (shnum, filename)
hunk ./src/allmydata/test/test_upload.py 1
+# -*- coding: utf-8 -*-
+
 import os, shutil
 from cStringIO import StringIO
 from twisted.trial import unittest
hunk ./src/allmydata/test/test_upload.py 689
         d.addCallback(_done)
         return d
 
+# copied from python docs because itertools.combinations was added in
+# python 2.6 and we support >= 2.4.
+def combinations(iterable, r):
+    # combinations('ABCD', 2) --> AB AC AD BC BD CD
+    # combinations(range(4), 3) --> 012 013 023 123
+    pool = tuple(iterable)
+    n = len(pool)
+    if r > n:
+        return
+    indices = range(r)
+    yield tuple(pool[i] for i in indices)
+    while True:
+        for i in reversed(range(r)):
+            if indices[i] != i + n - r:
+                break
+        else:
+            return
+        indices[i] += 1
+        for j in range(i+1, r):
+            indices[j] = indices[j-1] + 1
+        yield tuple(pool[i] for i in indices)
+
+def is_happy_enough(servertoshnums, h, k):
+    """ I calculate whether servertoshnums achieves happiness level h. I do this with a naïve "brute force search" approach. (See src/allmydata/util/happinessutil.py for a better algorithm.) """
+    if len(servertoshnums) < h:
+        return False
+    # print "servertoshnums: ", servertoshnums, h, k
+    for happysetcombo in combinations(servertoshnums.iterkeys(), h):
+        # print "happysetcombo: ", happysetcombo
+        for subsetcombo in combinations(happysetcombo, k):
+            shnums = reduce(set.union, [ servertoshnums[s] for s in subsetcombo ])
+            # print "subsetcombo: ", subsetcombo, ", shnums: ", shnums
+            if len(shnums) < k:
+                # print "NOT HAAPP{Y", shnums, k
+                return False
+    # print "HAAPP{Y"
+    return True
+
 class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
     ShouldFailMixin):
hunk ./src/allmydata/test/test_upload.py 729
+    def find_all_shares(self, unused=None):
+        """Locate shares on disk. Returns a dict that maps
+        server to set of sharenums.
+        """
+        assert self.g, "I tried to find a grid at self.g, but failed"
+        servertoshnums = {} # k: server, v: set(shnum)
+
+        for i, c in self.g.servers_by_number.iteritems():
+            for (dirp, dirns, fns) in os.walk(c.sharedir):
+                for fn in fns:
+                    try:
+                        sharenum = int(fn)
+                    except TypeError:
+                        # Whoops, I guess that's not a share file then.
+                        pass
+                    else:
+                        servertoshnums.setdefault(i, set()).add(sharenum)
+
+        return servertoshnums
+
     def _do_upload_with_broken_servers(self, servers_to_break):
         """
         I act like a normal upload, but before I send the results of
hunk ./src/allmydata/test/test_upload.py 792
         d.addCallback(_have_shareholders)
         return d
 
+    def _has_happy_share_distribution(self):
+        servertoshnums = self.find_all_shares()
+        k = self.g.clients[0].DEFAULT_ENCODING_PARAMETERS['k']
+        h = self.g.clients[0].DEFAULT_ENCODING_PARAMETERS['happy']
+        return is_happy_enough(servertoshnums, h, k)
 
     def _add_server(self, server_number, readonly=False):
         assert self.g, "I tried to find a grid at self.g, but failed"
hunk ./src/allmydata/test/test_upload.py 828
                                           str(share_number))
         if old_share_location != new_share_location:
             shutil.copy(old_share_location, new_share_location)
-        shares = self.find_shares(self.uri)
+        shares = self.find_uri_shares(self.uri)
         # Make sure that the storage server has the share.
         self.failUnless((share_number, ss.my_nodeid, new_share_location)
                         in shares)
hunk ./src/allmydata/test/test_upload.py 858
             self.uri = ur.uri
         d.addCallback(_store_uri)
         d.addCallback(lambda ign:
-            self.find_shares(self.uri))
+            self.find_uri_shares(self.uri))
         def _store_shares(shares):
             self.shares = shares
         d.addCallback(_store_shares)
hunk ./src/allmydata/test/test_upload.py 944
         d.addCallback(lambda ign: self._add_server(4, False))
         # and this time the upload ought to succeed
         d.addCallback(lambda ign: c.upload(DATA))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_upload.py 1082
         d.addCallback(_reset_encoding_parameters)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
 
 
         # This scenario is basically comment:53, but changed so that the
hunk ./src/allmydata/test/test_upload.py 1122
         d.addCallback(_reset_encoding_parameters)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
 
 
         # Try the same thing, but with empty servers after the first one
hunk ./src/allmydata/test/test_upload.py 1155
         # servers of happiness were pushed.
         d.addCallback(lambda results:
             self.failUnlessEqual(results.pushed_shares, 3))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
     def test_problem_layout_ticket1124(self):
hunk ./src/allmydata/test/test_upload.py 1182
         d.addCallback(_setup)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
     test_problem_layout_ticket1124.todo = "Fix this after 1.7.1 release."
 
hunk ./src/allmydata/test/test_upload.py 1221
         d.addCallback(_reset_encoding_parameters)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_upload.py 1260
         d.addCallback(_reset_encoding_parameters)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_upload.py 1571
         d.addCallback(_prepare_client)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_upload.py 1867
         d.addCallback(_setup)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
     test_problem_layout_comment_187.todo = "this isn't fixed yet"
 
hunk ./src/allmydata/test/test_upload.py 1902
         d.addCallback(_setup)
         d.addCallback(lambda client:
                           client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
     def test_upload_succeeds_with_some_homeless_shares(self):
hunk ./src/allmydata/test/test_upload.py 1940
         d.addCallback(_server_setup)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_upload.py 1969
         d.addCallback(_server_setup)
         d.addCallback(lambda client:
             client.upload(upload.Data("data" * 10000, convergence="")))
+        d.addCallback(lambda ign:
+            self.failUnless(self._has_happy_share_distribution()))
         return d
 
 
hunk ./src/allmydata/test/test_web.py 3198
         d.addCallback(_compute_fileurls)
 
         def _clobber_shares(ignored):
-            good_shares = self.find_shares(self.uris["good"])
+            good_shares = self.find_uri_shares(self.uris["good"])
             self.failUnlessReallyEqual(len(good_shares), 10)
hunk ./src/allmydata/test/test_web.py 3200
-            sick_shares = self.find_shares(self.uris["sick"])
+            sick_shares = self.find_uri_shares(self.uris["sick"])
             os.unlink(sick_shares[0][2])
hunk ./src/allmydata/test/test_web.py 3202
-            dead_shares = self.find_shares(self.uris["dead"])
+            dead_shares = self.find_uri_shares(self.uris["dead"])
             for i in range(1, 10):
                 os.unlink(dead_shares[i][2])
hunk ./src/allmydata/test/test_web.py 3205
-            c_shares = self.find_shares(self.uris["corrupt"])
+            c_shares = self.find_uri_shares(self.uris["corrupt"])
             cso = CorruptShareOptions()
             cso.stdout = StringIO()
             cso.parseOptions([c_shares[0][2]])
hunk ./src/allmydata/test/test_web.py 3339
         d.addCallback(_compute_fileurls)
 
         def _clobber_shares(ignored):
-            good_shares = self.find_shares(self.uris["good"])
+            good_shares = self.find_uri_shares(self.uris["good"])
             self.failUnlessReallyEqual(len(good_shares), 10)
hunk ./src/allmydata/test/test_web.py 3341
-            sick_shares = self.find_shares(self.uris["sick"])
+            sick_shares = self.find_uri_shares(self.uris["sick"])
             os.unlink(sick_shares[0][2])
hunk ./src/allmydata/test/test_web.py 3343
-            dead_shares = self.find_shares(self.uris["dead"])
+            dead_shares = self.find_uri_shares(self.uris["dead"])
             for i in range(1, 10):
                 os.unlink(dead_shares[i][2])
hunk ./src/allmydata/test/test_web.py 3346
-            c_shares = self.find_shares(self.uris["corrupt"])
+            c_shares = self.find_uri_shares(self.uris["corrupt"])
             cso = CorruptShareOptions()
             cso.stdout = StringIO()
             cso.parseOptions([c_shares[0][2]])
hunk ./src/allmydata/test/test_web.py 3407
         d.addCallback(_compute_fileurls)
 
         def _clobber_shares(ignored):
-            sick_shares = self.find_shares(self.uris["sick"])
+            sick_shares = self.find_uri_shares(self.uris["sick"])
             os.unlink(sick_shares[0][2])
         d.addCallback(_clobber_shares)
 
hunk ./src/allmydata/test/test_web.py 3897
         #d.addCallback(_stash_uri, "corrupt")
 
         def _clobber_shares(ignored):
-            good_shares = self.find_shares(self.uris["good"])
+            good_shares = self.find_uri_shares(self.uris["good"])
             self.failUnlessReallyEqual(len(good_shares), 10)
hunk ./src/allmydata/test/test_web.py 3899
-            sick_shares = self.find_shares(self.uris["sick"])
+            sick_shares = self.find_uri_shares(self.uris["sick"])
             os.unlink(sick_shares[0][2])
hunk ./src/allmydata/test/test_web.py 3901
-            #dead_shares = self.find_shares(self.uris["dead"])
+            #dead_shares = self.find_uri_shares(self.uris["dead"])
             #for i in range(1, 10):
             #    os.unlink(dead_shares[i][2])
 
hunk ./src/allmydata/test/test_web.py 3905
-            #c_shares = self.find_shares(self.uris["corrupt"])
+            #c_shares = self.find_uri_shares(self.uris["corrupt"])
             #cso = CorruptShareOptions()
             #cso.stdout = StringIO()
             #cso.parseOptions([c_shares[0][2]])
hunk ./src/allmydata/test/test_web.py 3961
 
     def _count_leases(self, ignored, which):
         u = self.uris[which]
-        shares = self.find_shares(u)
+        shares = self.find_uri_shares(u)
         lease_counts = []
         for shnum, serverid, fn in shares:
             sf = get_share_file(fn)
}

Context:

[docs/logging.txt: document that _trial_temp/test.log does not receive messages below level=OPERATIONAL, due to <http://foolscap.lothar.com/trac/ticket/154>.
david-sarah@jacaranda.org**20100718230420
 Ignore-this: aef40f2e74ddeabee5e122e8d80893a1
] 
[immutable: test for #1124
zooko@zooko.com**20100718222907
 Ignore-this: 1766e3cbab92ea2a9e246f40eb6e770b
] 
[trivial: fix unused import (sorry about that, pyflakes)
zooko@zooko.com**20100718215133
 Ignore-this: c2414e443405072b51d552295f2c0e8c
] 
[tests, NEWS, CREDITS re: #1117
zooko@zooko.com**20100718203225
 Ignore-this: 1f08be2c692fb72cc0dd023259f11354
 Give Brian and Kevan promotions, move release date in NEWS to the 18th, commit Brian's test for #1117.
 fixes #1117
] 
[test/test_upload.py: test to see that aborted buckets are ignored by the storage server
Kevan Carstensen <kevan@isnotajoke.com>**20100716001046
 Ignore-this: cc075c24b1c86d737f3199af894cc780
] 
[test/test_storage.py: test for the new remote_abort semantics.
Kevan Carstensen <kevan@isnotajoke.com>**20100715232148
 Ignore-this: d3d6491f17bf670e770ca4b385007515
] 
[storage/immutable.py: make remote_abort btell the storage server about aborted buckets.
Kevan Carstensen <kevan@isnotajoke.com>**20100715232105
 Ignore-this: 16ab0090676355abdd5600ed44ff19c9
] 
[test/test_upload.py: changes to test plumbing for #1117 tests
Kevan Carstensen <kevan@isnotajoke.com>**20100715231820
 Ignore-this: 78a6d359d7bf8529d283e2815bf1e2de
 
     - Add a callRemoteOnly method to FakeBucketWriter.
     - Change the abort method in FakeBucketWriter to not return a
       RuntimeError.
] 
[immutable/upload.py: abort buckets if peer selection fails
Kevan Carstensen <kevan@isnotajoke.com>**20100715231714
 Ignore-this: 2a0b643a22284df292d8ed9d91b1fd37
] 
[test_encodingutil: correct an error in the previous patch to StdlibUnicode.test_open_representable.
david-sarah@jacaranda.org**20100718151420
 Ignore-this: af050955f623fbc0e4d78e15a0a8a144
] 
[NEWS: Forward-compatibility improvements for non-ASCII caps (#1051).
david-sarah@jacaranda.org**20100718143622
 Ignore-this: 1edfebc4bd38a3b5c35e75c99588153f
] 
[test_dirnode and test_web: don't use failUnlessReallyEqual in cases where the return type from simplejson.loads can vary between unicode and str. Use to_str when comparing URIs parsed from JSON.
david-sarah@jacaranda.org**20100718142915
 Ignore-this: c4e78ef4b1478dd400da71cf077ffa4a
] 
[test_encodingutil: StdlibUnicode.test_open_representable no longer uses a mock.
david-sarah@jacaranda.org**20100718125412
 Ignore-this: 4bf373a5e2dfe4209e5e364124af29a3
] 
[docs: add comment clarifying #1051
zooko@zooko.com**20100718053250
 Ignore-this: 6cfc0930434cbdbbc262dabb58f1505d
] 
[docs: update NEWS
zooko@zooko.com**20100718053225
 Ignore-this: 63d5c782ef84812e6d010f0590866831
] 
[Add tests of caps from the future that have non-ASCII characters in them (encoded as UTF-8). The changes to test_uri.py, test_client.py, and test_dirnode.py add tests of non-ASCII future caps in addition to the current tests. The changes to test_web.py just replace the tests of all-ASCII future caps with tests of non-ASCII future caps. We also change uses of failUnlessEqual to failUnlessReallyEqual, in order to catch cases where the type of a string is not as expected.
david-sarah@jacaranda.org**20100711200252
 Ignore-this: c2f193352369d32e06865f8f3e951894
] 
[Debian documentation update
jacob@appelbaum.net**20100305003004] 
[debian-docs-patch-final
jacob@appelbaum.net**20100304085955] 
[M-x whitespace-cleanup
zooko@zooko.com**20100718032739
 Ignore-this: babfd4af6ad2fc885c957fd5c8b10c3f
] 
[docs: tidy up NEWS a little
zooko@zooko.com**20100718032434
 Ignore-this: 54f2820fd1a37c8967609f6bfc4e5e18
] 
[benchmarking: update bench_dirnode.py to reflect the new directory interfaces
zooko@zooko.com**20100718031710
 Ignore-this: 368ba523dd3de80d9da29cd58afbe827
] 
[test_encodingutil: fix test_open_representable, which is only valid when run on a platform for which we know an unrepresentable filename.
david-sarah@jacaranda.org**20100718030333
 Ignore-this: c114d92c17714a5d4ae005c15267d60c
] 
[iputil.py: Add support for FreeBSD 7,8 and 9
francois@ctrlaltdel.ch**20100718022832
 Ignore-this: 1829b4cf4b91107f4cf87841e6167e99
 committed by: zooko@zooko.com
 date: 2010-07-17
 and I also patched: NEWS and CREDITS
] 
[NEWS: add snippet about #1083
zooko@zooko.com**20100718020653
 Ignore-this: d353a9d93cbc5a5e6ba4671f78d1e22b
] 
[fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile.
david-sarah@jacaranda.org**20100717054647
 Ignore-this: 46d8fc10782fa8ec2b6c5b168c841943
] 
[Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083).
david-sarah@jacaranda.org**20100711213721
 Ignore-this: e452e8ca66391aa2a1a49afe0114f317
] 
[NEWS: reorder NEWS snippets to be in descending order of interestingness
zooko@zooko.com**20100718015929
 Ignore-this: 146c42e88a9555a868a04a69dd0e5326
] 
[Correct stringutils->encodingutil patch to be the newer version, rather than the old version that was committed in error.
david-sarah@jacaranda.org**20100718013435
 Ignore-this: c8940c4e1aa2e9acc80cd4fe54753cd8
] 
[test_cli.py: fix error that crept in when rebasing the patch for #1072.
david-sarah@jacaranda.org**20100718000123
 Ignore-this: 3e8f6cc3a27b747c708221dd581934f4
] 
[stringutils: add test for when sys.stdout has no encoding attribute (fixes #1099).
david-sarah@jacaranda.org**20100717045816
 Ignore-this: f28dce6940e909f12f354086d17db54f
] 
[CLI: add 'tahoe unlink' as an alias to 'tahoe rm', for forward-compatibility.
david-sarah@jacaranda.org**20100717220411
 Ignore-this: 3ecdde7f2d0498514cef32e118e0b855
] 
[minor code clean-up in dirnode.py
zooko@zooko.com**20100714060255
 Ignore-this: bb0ab2783203e605024b3e2f798256a1
 Impose micro-POLA by passing only the writekey instead of the whole node object to {{{_encrypt_rw_uri()}}}. Remove DummyImmutableFileNode in nodemaker.py, which is obviated by this. Add micro-optimization by precomputing the netstring of the empty string and branching on whether the writekey is present or not outside of {{{_encrypt_rw_uri()}}}. Add doc about writekey to docstring.
 fixes #967
] 
[Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil.
david-sarah@jacaranda.org**20100712003015
 Ignore-this: 103b809d180df17a7283077c3104c7be
] 
[Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting.
david-sarah@jacaranda.org**20100711195525
 Ignore-this: deac32d8b91ba26ede18905d3f7d2b93
] 
[docs: CREDITS and NEWS
zooko@zooko.com**20100714060150
 Ignore-this: dc83e612f77d69e50ee975f07f6b16fe
] 
[CREDITS: more creds for Kevan, plus utf-8 BOM
zooko@zooko.com**20100619045503
 Ignore-this: 72d02bdd7a0f324f1cee8cd399c7c6de
] 
[cli.py: make command descriptions consistently end with a full stop.
david-sarah@jacaranda.org**20100714014538
 Ignore-this: 9ee7fa29ca2d1631db4049c2a389a97a
] 
[SFTP: address some of the comments in zooko's review (#1106).
david-sarah@jacaranda.org**20100712025537
 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e
] 
[docs/logging.txt: note that setting flogging vars might affect tests with race conditions.
david-sarah@jacaranda.org**20100712050721
 Ignore-this: fc1609d215fcd5561a57fd1226206f27
] 
[test_storage.py: potential fix for failures when logging is enabled.
david-sarah@jacaranda.org**19700713040546
 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7
] 
[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
7895cf593c9c13c42fabb11ffce5c26217e8d6b9