1 | Sun Jul 18 16:15:37 MDT 2010 zooko@zooko.com |
---|
2 | * immutable: test for #1118 |
---|
3 | |
---|
4 | Sun Jul 18 22:50:47 MDT 2010 zooko@zooko.com |
---|
5 | * immutable: extend the tests to check that the shares that got uploaded really do make a sufficiently Happy distribution |
---|
6 | This patch also renames some instances of "find_shares()" to "find_all_shares()" and other instances to "find_uri_shares()" as appropriate -- the conflation between those names confused me at first when writing these tests. |
---|
7 | |
---|
8 | Sun Jul 18 22:46:55 MDT 2010 david-sarah@jacaranda.org |
---|
9 | * upload.py: fix #1118 by aborting newly-homeless buckets when reassignment runs. This makes a previously failing assert correct. This version refactors 'abort' into two methods, rather than using a default argument. |
---|
10 | |
---|
11 | Mon Jul 19 00:56:29 MDT 2010 zooko@zooko.com |
---|
12 | * immutable: use PrefixingLogMixin to organize logging in Tahoe2PeerSelector and add more detailed messages about peer |
---|
13 | |
---|
14 | New patches: |
---|
15 | |
---|
16 | [immutable: test for #1118 |
---|
17 | zooko@zooko.com**20100718221537 |
---|
18 | Ignore-this: 8882aabe2aaec6a0148c87e735d817ad |
---|
19 | ] { |
---|
20 | hunk ./src/allmydata/immutable/upload.py 919 |
---|
21 | for shnum in peer.buckets: |
---|
22 | self._peer_trackers[shnum] = peer |
---|
23 | servermap.setdefault(shnum, set()).add(peer.peerid) |
---|
24 | - assert len(buckets) == sum([len(peer.buckets) for peer in used_peers]) |
---|
25 | + assert len(buckets) == sum([len(peer.buckets) for peer in used_peers]), "%s (%s) != %s (%s)" % (len(buckets), buckets, sum([len(peer.buckets) for peer in used_peers]), [(p.buckets, p.peerid) for p in used_peers]) |
---|
26 | encoder.set_shareholders(buckets, servermap) |
---|
27 | |
---|
28 | def _encrypted_done(self, verifycap): |
---|
29 | hunk ./src/allmydata/test/test_upload.py 1789 |
---|
30 | return d |
---|
31 | test_problem_layout_comment_187.todo = "this isn't fixed yet" |
---|
32 | |
---|
33 | + def test_problem_layout_ticket_1118(self): |
---|
34 | + # #1118 includes a report from a user who hit an assertion in |
---|
35 | + # the upload code with this layout. |
---|
36 | + self.basedir = self.mktemp() |
---|
37 | + d = self._setup_and_upload(k=2, n=4) |
---|
38 | + |
---|
39 | + # server 0: no shares |
---|
40 | + # server 1: shares 0, 3 |
---|
41 | + # server 3: share 1 |
---|
42 | + # server 2: share 2 |
---|
43 | + # The order that they get queries is 0, 1, 3, 2 |
---|
44 | + def _setup(ign): |
---|
45 | + self._add_server(server_number=0) |
---|
46 | + self._add_server_with_share(server_number=1, share_number=0) |
---|
47 | + self._add_server_with_share(server_number=2, share_number=2) |
---|
48 | + self._add_server_with_share(server_number=3, share_number=1) |
---|
49 | + # Copy shares |
---|
50 | + self._copy_share_to_server(3, 1) |
---|
51 | + storedir = self.get_serverdir(0) |
---|
52 | + # remove the storedir, wiping out any existing shares |
---|
53 | + shutil.rmtree(storedir) |
---|
54 | + # create an empty storedir to replace the one we just removed |
---|
55 | + os.mkdir(storedir) |
---|
56 | + client = self.g.clients[0] |
---|
57 | + client.DEFAULT_ENCODING_PARAMETERS['happy'] = 4 |
---|
58 | + return client |
---|
59 | + |
---|
60 | + d.addCallback(_setup) |
---|
61 | + d.addCallback(lambda client: |
---|
62 | + client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
63 | + return d |
---|
64 | |
---|
65 | def test_upload_succeeds_with_some_homeless_shares(self): |
---|
66 | # If the upload is forced to stop trying to place shares before |
---|
67 | } |
---|
68 | [immutable: extend the tests to check that the shares that got uploaded really do make a sufficiently Happy distribution |
---|
69 | zooko@zooko.com**20100719045047 |
---|
70 | Ignore-this: 89c33a7b795e23018667351045a8d5d0 |
---|
71 | This patch also renames some instances of "find_shares()" to "find_all_shares()" and other instances to "find_uri_shares()" as appropriate -- the conflation between those names confused me at first when writing these tests. |
---|
72 | ] { |
---|
73 | hunk ./src/allmydata/test/common.py 959 |
---|
74 | d.addCallback(_stash_it) |
---|
75 | return d |
---|
76 | |
---|
77 | - def find_shares(self, unused=None): |
---|
78 | + def find_all_shares(self, unused=None): |
---|
79 | """Locate shares on disk. Returns a dict that maps |
---|
80 | (clientnum,sharenum) to a string that contains the share container |
---|
81 | (copied directly from the disk, containing leases etc). You can |
---|
82 | hunk ./src/allmydata/test/common.py 984 |
---|
83 | |
---|
84 | def replace_shares(self, newshares, storage_index): |
---|
85 | """Replace shares on disk. Takes a dictionary in the same form |
---|
86 | - as find_shares() returns.""" |
---|
87 | + as find_all_shares() returns.""" |
---|
88 | |
---|
89 | for i, c in enumerate(self.clients): |
---|
90 | sharedir = c.getServiceNamed("storage").sharedir |
---|
91 | hunk ./src/allmydata/test/common.py 1009 |
---|
92 | def _delete_a_share(self, unused=None, sharenum=None): |
---|
93 | """ Delete one share. """ |
---|
94 | |
---|
95 | - shares = self.find_shares() |
---|
96 | + shares = self.find_all_shares() |
---|
97 | ks = shares.keys() |
---|
98 | if sharenum is not None: |
---|
99 | k = [ key for key in shares.keys() if key[1] == sharenum ][0] |
---|
100 | hunk ./src/allmydata/test/common.py 1021 |
---|
101 | return unused |
---|
102 | |
---|
103 | def _corrupt_a_share(self, unused, corruptor_func, sharenum): |
---|
104 | - shares = self.find_shares() |
---|
105 | + shares = self.find_all_shares() |
---|
106 | ks = [ key for key in shares.keys() if key[1] == sharenum ] |
---|
107 | assert ks, (shares.keys(), sharenum) |
---|
108 | k = ks[0] |
---|
109 | hunk ./src/allmydata/test/common.py 1031 |
---|
110 | |
---|
111 | def _corrupt_all_shares(self, unused, corruptor_func): |
---|
112 | """ All shares on disk will be corrupted by corruptor_func. """ |
---|
113 | - shares = self.find_shares() |
---|
114 | + shares = self.find_all_shares() |
---|
115 | for k in shares.keys(): |
---|
116 | self._corrupt_a_share(unused, corruptor_func, k[1]) |
---|
117 | return corruptor_func |
---|
118 | hunk ./src/allmydata/test/common.py 1038 |
---|
119 | |
---|
120 | def _corrupt_a_random_share(self, unused, corruptor_func): |
---|
121 | """ Exactly one share on disk will be corrupted by corruptor_func. """ |
---|
122 | - shares = self.find_shares() |
---|
123 | + shares = self.find_all_shares() |
---|
124 | ks = shares.keys() |
---|
125 | k = random.choice(ks) |
---|
126 | self._corrupt_a_share(unused, corruptor_func, k[1]) |
---|
127 | hunk ./src/allmydata/test/no_network.py 305 |
---|
128 | ss = self.g.servers_by_number[i] |
---|
129 | yield (i, ss, ss.storedir) |
---|
130 | |
---|
131 | - def find_shares(self, uri): |
---|
132 | + def find_uri_shares(self, uri): |
---|
133 | si = tahoe_uri.from_string(uri).get_storage_index() |
---|
134 | prefixdir = storage_index_to_dir(si) |
---|
135 | shares = [] |
---|
136 | hunk ./src/allmydata/test/no_network.py 326 |
---|
137 | os.unlink(sharefile) |
---|
138 | |
---|
139 | def delete_shares_numbered(self, uri, shnums): |
---|
140 | - for (i_shnum, i_serverid, i_sharefile) in self.find_shares(uri): |
---|
141 | + for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): |
---|
142 | if i_shnum in shnums: |
---|
143 | os.unlink(i_sharefile) |
---|
144 | |
---|
145 | hunk ./src/allmydata/test/no_network.py 336 |
---|
146 | open(sharefile, "wb").write(corruptdata) |
---|
147 | |
---|
148 | def corrupt_shares_numbered(self, uri, shnums, corruptor, debug=False): |
---|
149 | - for (i_shnum, i_serverid, i_sharefile) in self.find_shares(uri): |
---|
150 | + for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): |
---|
151 | if i_shnum in shnums: |
---|
152 | sharedata = open(i_sharefile, "rb").read() |
---|
153 | corruptdata = corruptor(sharedata, debug=debug) |
---|
154 | hunk ./src/allmydata/test/test_cli.py 1998 |
---|
155 | |
---|
156 | def _clobber_shares(ignored): |
---|
157 | # delete one, corrupt a second |
---|
158 | - shares = self.find_shares(self.uri) |
---|
159 | + shares = self.find_uri_shares(self.uri) |
---|
160 | self.failUnlessReallyEqual(len(shares), 10) |
---|
161 | os.unlink(shares[0][2]) |
---|
162 | cso = debug.CorruptShareOptions() |
---|
163 | hunk ./src/allmydata/test/test_cli.py 2123 |
---|
164 | d.addCallback(_check_stats) |
---|
165 | |
---|
166 | def _clobber_shares(ignored): |
---|
167 | - shares = self.find_shares(self.uris[u"gööd"]) |
---|
168 | + shares = self.find_uri_shares(self.uris[u"gööd"]) |
---|
169 | self.failUnlessReallyEqual(len(shares), 10) |
---|
170 | os.unlink(shares[0][2]) |
---|
171 | |
---|
172 | hunk ./src/allmydata/test/test_cli.py 2127 |
---|
173 | - shares = self.find_shares(self.uris["mutable"]) |
---|
174 | + shares = self.find_uri_shares(self.uris["mutable"]) |
---|
175 | cso = debug.CorruptShareOptions() |
---|
176 | cso.stdout = StringIO() |
---|
177 | cso.parseOptions([shares[1][2]]) |
---|
178 | hunk ./src/allmydata/test/test_deepcheck.py 994 |
---|
179 | self.delete_shares_numbered(node.get_uri(), [0,1]) |
---|
180 | |
---|
181 | def _corrupt_some_shares(self, node): |
---|
182 | - for (shnum, serverid, sharefile) in self.find_shares(node.get_uri()): |
---|
183 | + for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()): |
---|
184 | if shnum in (0,1): |
---|
185 | self._run_cli(["debug", "corrupt-share", sharefile]) |
---|
186 | |
---|
187 | hunk ./src/allmydata/test/test_hung_server.py 67 |
---|
188 | os.makedirs(si_dir) |
---|
189 | new_sharefile = os.path.join(si_dir, str(sharenum)) |
---|
190 | shutil.copy(sharefile, new_sharefile) |
---|
191 | - self.shares = self.find_shares(self.uri) |
---|
192 | + self.shares = self.find_uri_shares(self.uri) |
---|
193 | # Make sure that the storage server has the share. |
---|
194 | self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile) |
---|
195 | in self.shares) |
---|
196 | hunk ./src/allmydata/test/test_hung_server.py 98 |
---|
197 | d = nm.create_mutable_file(mutable_plaintext) |
---|
198 | def _uploaded_mutable(node): |
---|
199 | self.uri = node.get_uri() |
---|
200 | - self.shares = self.find_shares(self.uri) |
---|
201 | + self.shares = self.find_uri_shares(self.uri) |
---|
202 | d.addCallback(_uploaded_mutable) |
---|
203 | else: |
---|
204 | data = upload.Data(immutable_plaintext, convergence="") |
---|
205 | hunk ./src/allmydata/test/test_hung_server.py 105 |
---|
206 | d = self.c0.upload(data) |
---|
207 | def _uploaded_immutable(upload_res): |
---|
208 | self.uri = upload_res.uri |
---|
209 | - self.shares = self.find_shares(self.uri) |
---|
210 | + self.shares = self.find_uri_shares(self.uri) |
---|
211 | d.addCallback(_uploaded_immutable) |
---|
212 | return d |
---|
213 | |
---|
214 | hunk ./src/allmydata/test/test_immutable.py 14 |
---|
215 | # replace_shares, and asserting that the new set of shares equals the |
---|
216 | # old is more to test this test code than to test the Tahoe code... |
---|
217 | d = defer.succeed(None) |
---|
218 | - d.addCallback(self.find_shares) |
---|
219 | + d.addCallback(self.find_all_shares) |
---|
220 | stash = [None] |
---|
221 | def _stash_it(res): |
---|
222 | stash[0] = res |
---|
223 | hunk ./src/allmydata/test/test_repairer.py 90 |
---|
224 | d.addCallback(_check) |
---|
225 | |
---|
226 | def _remove_all(ignored): |
---|
227 | - for sh in self.find_shares(self.uri): |
---|
228 | + for sh in self.find_uri_shares(self.uri): |
---|
229 | self.delete_share(sh) |
---|
230 | d.addCallback(_remove_all) |
---|
231 | |
---|
232 | hunk ./src/allmydata/test/test_repairer.py 325 |
---|
233 | def _grab_sh0(res): |
---|
234 | self.sh0_file = [sharefile |
---|
235 | for (shnum, serverid, sharefile) |
---|
236 | - in self.find_shares(self.uri) |
---|
237 | + in self.find_uri_shares(self.uri) |
---|
238 | if shnum == 0][0] |
---|
239 | self.sh0_orig = open(self.sh0_file, "rb").read() |
---|
240 | d.addCallback(_grab_sh0) |
---|
241 | hunk ./src/allmydata/test/test_repairer.py 470 |
---|
242 | self.set_up_grid(num_clients=2) |
---|
243 | d = self.upload_and_stash() |
---|
244 | |
---|
245 | - d.addCallback(lambda ignored: self.find_shares(self.uri)) |
---|
246 | + d.addCallback(lambda ignored: self.find_uri_shares(self.uri)) |
---|
247 | def _stash_shares(oldshares): |
---|
248 | self.oldshares = oldshares |
---|
249 | d.addCallback(_stash_shares) |
---|
250 | hunk ./src/allmydata/test/test_repairer.py 474 |
---|
251 | - d.addCallback(lambda ignored: self.find_shares(self.uri)) |
---|
252 | + d.addCallback(lambda ignored: self.find_uri_shares(self.uri)) |
---|
253 | def _compare(newshares): |
---|
254 | self.failUnlessEqual(newshares, self.oldshares) |
---|
255 | d.addCallback(_compare) |
---|
256 | hunk ./src/allmydata/test/test_repairer.py 485 |
---|
257 | for sh in self.oldshares[1:8]: |
---|
258 | self.delete_share(sh) |
---|
259 | d.addCallback(_delete_8) |
---|
260 | - d.addCallback(lambda ignored: self.find_shares(self.uri)) |
---|
261 | + d.addCallback(lambda ignored: self.find_uri_shares(self.uri)) |
---|
262 | d.addCallback(lambda shares: self.failUnlessEqual(len(shares), 2)) |
---|
263 | |
---|
264 | d.addCallback(lambda ignored: |
---|
265 | hunk ./src/allmydata/test/test_repairer.py 502 |
---|
266 | # test share corruption |
---|
267 | def _test_corrupt(ignored): |
---|
268 | olddata = {} |
---|
269 | - shares = self.find_shares(self.uri) |
---|
270 | + shares = self.find_uri_shares(self.uri) |
---|
271 | for (shnum, serverid, sharefile) in shares: |
---|
272 | olddata[ (shnum, serverid) ] = open(sharefile, "rb").read() |
---|
273 | for sh in shares: |
---|
274 | hunk ./src/allmydata/test/test_repairer.py 513 |
---|
275 | d.addCallback(_test_corrupt) |
---|
276 | |
---|
277 | def _remove_all(ignored): |
---|
278 | - for sh in self.find_shares(self.uri): |
---|
279 | + for sh in self.find_uri_shares(self.uri): |
---|
280 | self.delete_share(sh) |
---|
281 | d.addCallback(_remove_all) |
---|
282 | hunk ./src/allmydata/test/test_repairer.py 516 |
---|
283 | - d.addCallback(lambda ignored: self.find_shares(self.uri)) |
---|
284 | + d.addCallback(lambda ignored: self.find_uri_shares(self.uri)) |
---|
285 | d.addCallback(lambda shares: self.failUnlessEqual(shares, [])) |
---|
286 | |
---|
287 | return d |
---|
288 | hunk ./src/allmydata/test/test_repairer.py 547 |
---|
289 | |
---|
290 | # Now we inspect the filesystem to make sure that it has 10 |
---|
291 | # shares. |
---|
292 | - shares = self.find_shares(self.uri) |
---|
293 | + shares = self.find_uri_shares(self.uri) |
---|
294 | self.failIf(len(shares) < 10) |
---|
295 | d.addCallback(_check_results) |
---|
296 | |
---|
297 | hunk ./src/allmydata/test/test_repairer.py 592 |
---|
298 | self.failUnless(post.is_healthy(), post.data) |
---|
299 | |
---|
300 | # Make sure we really have 10 shares. |
---|
301 | - shares = self.find_shares(self.uri) |
---|
302 | + shares = self.find_uri_shares(self.uri) |
---|
303 | self.failIf(len(shares) < 10) |
---|
304 | d.addCallback(_check_results) |
---|
305 | |
---|
306 | hunk ./src/allmydata/test/test_repairer.py 653 |
---|
307 | def OFF_test_repair_from_corruption_of_1(self): |
---|
308 | d = defer.succeed(None) |
---|
309 | |
---|
310 | - d.addCallback(self.find_shares) |
---|
311 | + d.addCallback(self.find_all_shares) |
---|
312 | stash = [None] |
---|
313 | def _stash_it(res): |
---|
314 | stash[0] = res |
---|
315 | hunk ./src/allmydata/test/test_repairer.py 688 |
---|
316 | |
---|
317 | # Now we inspect the filesystem to make sure that it has 10 |
---|
318 | # shares. |
---|
319 | - shares = self.find_shares() |
---|
320 | + shares = self.find_all_shares() |
---|
321 | self.failIf(len(shares) < 10) |
---|
322 | |
---|
323 | # Now we assert that the verifier reports the file as healthy. |
---|
324 | hunk ./src/allmydata/test/test_system.py 386 |
---|
325 | |
---|
326 | return d |
---|
327 | |
---|
328 | - def _find_shares(self, basedir): |
---|
329 | + def _find_all_shares(self, basedir): |
---|
330 | shares = [] |
---|
331 | for (dirpath, dirnames, filenames) in os.walk(basedir): |
---|
332 | if "storage" not in dirpath: |
---|
333 | hunk ./src/allmydata/test/test_system.py 478 |
---|
334 | def _test_debug(res): |
---|
335 | # find a share. It is important to run this while there is only |
---|
336 | # one slot in the grid. |
---|
337 | - shares = self._find_shares(self.basedir) |
---|
338 | + shares = self._find_all_shares(self.basedir) |
---|
339 | (client_num, storage_index, filename, shnum) = shares[0] |
---|
340 | log.msg("test_system.SystemTest.test_mutable._test_debug using %s" |
---|
341 | % filename) |
---|
342 | hunk ./src/allmydata/test/test_system.py 581 |
---|
343 | def _corrupt_shares(res): |
---|
344 | # run around and flip bits in all but k of the shares, to test |
---|
345 | # the hash checks |
---|
346 | - shares = self._find_shares(self.basedir) |
---|
347 | + shares = self._find_all_shares(self.basedir) |
---|
348 | ## sort by share number |
---|
349 | #shares.sort( lambda a,b: cmp(a[3], b[3]) ) |
---|
350 | where = dict([ (shnum, filename) |
---|
351 | hunk ./src/allmydata/test/test_upload.py 1 |
---|
352 | +# -*- coding: utf-8 -*- |
---|
353 | + |
---|
354 | import os, shutil |
---|
355 | from cStringIO import StringIO |
---|
356 | from twisted.trial import unittest |
---|
357 | hunk ./src/allmydata/test/test_upload.py 689 |
---|
358 | d.addCallback(_done) |
---|
359 | return d |
---|
360 | |
---|
361 | +# copied from python docs because itertools.combinations was added in |
---|
362 | +# python 2.6 and we support >= 2.4. |
---|
363 | +def combinations(iterable, r): |
---|
364 | + # combinations('ABCD', 2) --> AB AC AD BC BD CD |
---|
365 | + # combinations(range(4), 3) --> 012 013 023 123 |
---|
366 | + pool = tuple(iterable) |
---|
367 | + n = len(pool) |
---|
368 | + if r > n: |
---|
369 | + return |
---|
370 | + indices = range(r) |
---|
371 | + yield tuple(pool[i] for i in indices) |
---|
372 | + while True: |
---|
373 | + for i in reversed(range(r)): |
---|
374 | + if indices[i] != i + n - r: |
---|
375 | + break |
---|
376 | + else: |
---|
377 | + return |
---|
378 | + indices[i] += 1 |
---|
379 | + for j in range(i+1, r): |
---|
380 | + indices[j] = indices[j-1] + 1 |
---|
381 | + yield tuple(pool[i] for i in indices) |
---|
382 | + |
---|
383 | +def is_happy_enough(servertoshnums, h, k): |
---|
384 | + """ I calculate whether servertoshnums achieves happiness level h. I do this with a naïve "brute force search" approach. (See src/allmydata/util/happinessutil.py for a better algorithm.) """ |
---|
385 | + if len(servertoshnums) < h: |
---|
386 | + return False |
---|
387 | + # print "servertoshnums: ", servertoshnums, h, k |
---|
388 | + for happysetcombo in combinations(servertoshnums.iterkeys(), h): |
---|
389 | + # print "happysetcombo: ", happysetcombo |
---|
390 | + for subsetcombo in combinations(happysetcombo, k): |
---|
391 | + shnums = reduce(set.union, [ servertoshnums[s] for s in subsetcombo ]) |
---|
392 | + # print "subsetcombo: ", subsetcombo, ", shnums: ", shnums |
---|
393 | + if len(shnums) < k: |
---|
394 | + # print "NOT HAAPP{Y", shnums, k |
---|
395 | + return False |
---|
396 | + # print "HAAPP{Y" |
---|
397 | + return True |
---|
398 | + |
---|
399 | class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, |
---|
400 | ShouldFailMixin): |
---|
401 | hunk ./src/allmydata/test/test_upload.py 729 |
---|
402 | + def find_all_shares(self, unused=None): |
---|
403 | + """Locate shares on disk. Returns a dict that maps |
---|
404 | + server to set of sharenums. |
---|
405 | + """ |
---|
406 | + assert self.g, "I tried to find a grid at self.g, but failed" |
---|
407 | + servertoshnums = {} # k: server, v: set(shnum) |
---|
408 | + |
---|
409 | + for i, c in self.g.servers_by_number.iteritems(): |
---|
410 | + for (dirp, dirns, fns) in os.walk(c.sharedir): |
---|
411 | + for fn in fns: |
---|
412 | + try: |
---|
413 | + sharenum = int(fn) |
---|
414 | + except TypeError: |
---|
415 | + # Whoops, I guess that's not a share file then. |
---|
416 | + pass |
---|
417 | + else: |
---|
418 | + servertoshnums.setdefault(i, set()).add(sharenum) |
---|
419 | + |
---|
420 | + return servertoshnums |
---|
421 | + |
---|
422 | def _do_upload_with_broken_servers(self, servers_to_break): |
---|
423 | """ |
---|
424 | I act like a normal upload, but before I send the results of |
---|
425 | hunk ./src/allmydata/test/test_upload.py 792 |
---|
426 | d.addCallback(_have_shareholders) |
---|
427 | return d |
---|
428 | |
---|
429 | + def _has_happy_share_distribution(self): |
---|
430 | + servertoshnums = self.find_all_shares() |
---|
431 | + k = self.g.clients[0].DEFAULT_ENCODING_PARAMETERS['k'] |
---|
432 | + h = self.g.clients[0].DEFAULT_ENCODING_PARAMETERS['happy'] |
---|
433 | + return is_happy_enough(servertoshnums, h, k) |
---|
434 | |
---|
435 | def _add_server(self, server_number, readonly=False): |
---|
436 | assert self.g, "I tried to find a grid at self.g, but failed" |
---|
437 | hunk ./src/allmydata/test/test_upload.py 828 |
---|
438 | str(share_number)) |
---|
439 | if old_share_location != new_share_location: |
---|
440 | shutil.copy(old_share_location, new_share_location) |
---|
441 | - shares = self.find_shares(self.uri) |
---|
442 | + shares = self.find_uri_shares(self.uri) |
---|
443 | # Make sure that the storage server has the share. |
---|
444 | self.failUnless((share_number, ss.my_nodeid, new_share_location) |
---|
445 | in shares) |
---|
446 | hunk ./src/allmydata/test/test_upload.py 858 |
---|
447 | self.uri = ur.uri |
---|
448 | d.addCallback(_store_uri) |
---|
449 | d.addCallback(lambda ign: |
---|
450 | - self.find_shares(self.uri)) |
---|
451 | + self.find_uri_shares(self.uri)) |
---|
452 | def _store_shares(shares): |
---|
453 | self.shares = shares |
---|
454 | d.addCallback(_store_shares) |
---|
455 | hunk ./src/allmydata/test/test_upload.py 944 |
---|
456 | d.addCallback(lambda ign: self._add_server(4, False)) |
---|
457 | # and this time the upload ought to succeed |
---|
458 | d.addCallback(lambda ign: c.upload(DATA)) |
---|
459 | + d.addCallback(lambda ign: |
---|
460 | + self.failUnless(self._has_happy_share_distribution())) |
---|
461 | return d |
---|
462 | |
---|
463 | |
---|
464 | hunk ./src/allmydata/test/test_upload.py 1082 |
---|
465 | d.addCallback(_reset_encoding_parameters) |
---|
466 | d.addCallback(lambda client: |
---|
467 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
468 | + d.addCallback(lambda ign: |
---|
469 | + self.failUnless(self._has_happy_share_distribution())) |
---|
470 | |
---|
471 | |
---|
472 | # This scenario is basically comment:53, but changed so that the |
---|
473 | hunk ./src/allmydata/test/test_upload.py 1122 |
---|
474 | d.addCallback(_reset_encoding_parameters) |
---|
475 | d.addCallback(lambda client: |
---|
476 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
477 | + d.addCallback(lambda ign: |
---|
478 | + self.failUnless(self._has_happy_share_distribution())) |
---|
479 | |
---|
480 | |
---|
481 | # Try the same thing, but with empty servers after the first one |
---|
482 | hunk ./src/allmydata/test/test_upload.py 1155 |
---|
483 | # servers of happiness were pushed. |
---|
484 | d.addCallback(lambda results: |
---|
485 | self.failUnlessEqual(results.pushed_shares, 3)) |
---|
486 | + d.addCallback(lambda ign: |
---|
487 | + self.failUnless(self._has_happy_share_distribution())) |
---|
488 | return d |
---|
489 | |
---|
490 | def test_problem_layout_ticket1124(self): |
---|
491 | hunk ./src/allmydata/test/test_upload.py 1182 |
---|
492 | d.addCallback(_setup) |
---|
493 | d.addCallback(lambda client: |
---|
494 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
495 | + d.addCallback(lambda ign: |
---|
496 | + self.failUnless(self._has_happy_share_distribution())) |
---|
497 | return d |
---|
498 | test_problem_layout_ticket1124.todo = "Fix this after 1.7.1 release." |
---|
499 | |
---|
500 | hunk ./src/allmydata/test/test_upload.py 1221 |
---|
501 | d.addCallback(_reset_encoding_parameters) |
---|
502 | d.addCallback(lambda client: |
---|
503 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
504 | + d.addCallback(lambda ign: |
---|
505 | + self.failUnless(self._has_happy_share_distribution())) |
---|
506 | return d |
---|
507 | |
---|
508 | |
---|
509 | hunk ./src/allmydata/test/test_upload.py 1260 |
---|
510 | d.addCallback(_reset_encoding_parameters) |
---|
511 | d.addCallback(lambda client: |
---|
512 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
513 | + d.addCallback(lambda ign: |
---|
514 | + self.failUnless(self._has_happy_share_distribution())) |
---|
515 | return d |
---|
516 | |
---|
517 | |
---|
518 | hunk ./src/allmydata/test/test_upload.py 1571 |
---|
519 | d.addCallback(_prepare_client) |
---|
520 | d.addCallback(lambda client: |
---|
521 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
522 | + d.addCallback(lambda ign: |
---|
523 | + self.failUnless(self._has_happy_share_distribution())) |
---|
524 | return d |
---|
525 | |
---|
526 | |
---|
527 | hunk ./src/allmydata/test/test_upload.py 1867 |
---|
528 | d.addCallback(_setup) |
---|
529 | d.addCallback(lambda client: |
---|
530 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
531 | + d.addCallback(lambda ign: |
---|
532 | + self.failUnless(self._has_happy_share_distribution())) |
---|
533 | return d |
---|
534 | test_problem_layout_comment_187.todo = "this isn't fixed yet" |
---|
535 | |
---|
536 | hunk ./src/allmydata/test/test_upload.py 1902 |
---|
537 | d.addCallback(_setup) |
---|
538 | d.addCallback(lambda client: |
---|
539 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
540 | + d.addCallback(lambda ign: |
---|
541 | + self.failUnless(self._has_happy_share_distribution())) |
---|
542 | return d |
---|
543 | |
---|
544 | def test_upload_succeeds_with_some_homeless_shares(self): |
---|
545 | hunk ./src/allmydata/test/test_upload.py 1940 |
---|
546 | d.addCallback(_server_setup) |
---|
547 | d.addCallback(lambda client: |
---|
548 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
549 | + d.addCallback(lambda ign: |
---|
550 | + self.failUnless(self._has_happy_share_distribution())) |
---|
551 | return d |
---|
552 | |
---|
553 | |
---|
554 | hunk ./src/allmydata/test/test_upload.py 1969 |
---|
555 | d.addCallback(_server_setup) |
---|
556 | d.addCallback(lambda client: |
---|
557 | client.upload(upload.Data("data" * 10000, convergence=""))) |
---|
558 | + d.addCallback(lambda ign: |
---|
559 | + self.failUnless(self._has_happy_share_distribution())) |
---|
560 | return d |
---|
561 | |
---|
562 | |
---|
563 | hunk ./src/allmydata/test/test_web.py 3198 |
---|
564 | d.addCallback(_compute_fileurls) |
---|
565 | |
---|
566 | def _clobber_shares(ignored): |
---|
567 | - good_shares = self.find_shares(self.uris["good"]) |
---|
568 | + good_shares = self.find_uri_shares(self.uris["good"]) |
---|
569 | self.failUnlessReallyEqual(len(good_shares), 10) |
---|
570 | hunk ./src/allmydata/test/test_web.py 3200 |
---|
571 | - sick_shares = self.find_shares(self.uris["sick"]) |
---|
572 | + sick_shares = self.find_uri_shares(self.uris["sick"]) |
---|
573 | os.unlink(sick_shares[0][2]) |
---|
574 | hunk ./src/allmydata/test/test_web.py 3202 |
---|
575 | - dead_shares = self.find_shares(self.uris["dead"]) |
---|
576 | + dead_shares = self.find_uri_shares(self.uris["dead"]) |
---|
577 | for i in range(1, 10): |
---|
578 | os.unlink(dead_shares[i][2]) |
---|
579 | hunk ./src/allmydata/test/test_web.py 3205 |
---|
580 | - c_shares = self.find_shares(self.uris["corrupt"]) |
---|
581 | + c_shares = self.find_uri_shares(self.uris["corrupt"]) |
---|
582 | cso = CorruptShareOptions() |
---|
583 | cso.stdout = StringIO() |
---|
584 | cso.parseOptions([c_shares[0][2]]) |
---|
585 | hunk ./src/allmydata/test/test_web.py 3339 |
---|
586 | d.addCallback(_compute_fileurls) |
---|
587 | |
---|
588 | def _clobber_shares(ignored): |
---|
589 | - good_shares = self.find_shares(self.uris["good"]) |
---|
590 | + good_shares = self.find_uri_shares(self.uris["good"]) |
---|
591 | self.failUnlessReallyEqual(len(good_shares), 10) |
---|
592 | hunk ./src/allmydata/test/test_web.py 3341 |
---|
593 | - sick_shares = self.find_shares(self.uris["sick"]) |
---|
594 | + sick_shares = self.find_uri_shares(self.uris["sick"]) |
---|
595 | os.unlink(sick_shares[0][2]) |
---|
596 | hunk ./src/allmydata/test/test_web.py 3343 |
---|
597 | - dead_shares = self.find_shares(self.uris["dead"]) |
---|
598 | + dead_shares = self.find_uri_shares(self.uris["dead"]) |
---|
599 | for i in range(1, 10): |
---|
600 | os.unlink(dead_shares[i][2]) |
---|
601 | hunk ./src/allmydata/test/test_web.py 3346 |
---|
602 | - c_shares = self.find_shares(self.uris["corrupt"]) |
---|
603 | + c_shares = self.find_uri_shares(self.uris["corrupt"]) |
---|
604 | cso = CorruptShareOptions() |
---|
605 | cso.stdout = StringIO() |
---|
606 | cso.parseOptions([c_shares[0][2]]) |
---|
607 | hunk ./src/allmydata/test/test_web.py 3407 |
---|
608 | d.addCallback(_compute_fileurls) |
---|
609 | |
---|
610 | def _clobber_shares(ignored): |
---|
611 | - sick_shares = self.find_shares(self.uris["sick"]) |
---|
612 | + sick_shares = self.find_uri_shares(self.uris["sick"]) |
---|
613 | os.unlink(sick_shares[0][2]) |
---|
614 | d.addCallback(_clobber_shares) |
---|
615 | |
---|
616 | hunk ./src/allmydata/test/test_web.py 3897 |
---|
617 | #d.addCallback(_stash_uri, "corrupt") |
---|
618 | |
---|
619 | def _clobber_shares(ignored): |
---|
620 | - good_shares = self.find_shares(self.uris["good"]) |
---|
621 | + good_shares = self.find_uri_shares(self.uris["good"]) |
---|
622 | self.failUnlessReallyEqual(len(good_shares), 10) |
---|
623 | hunk ./src/allmydata/test/test_web.py 3899 |
---|
624 | - sick_shares = self.find_shares(self.uris["sick"]) |
---|
625 | + sick_shares = self.find_uri_shares(self.uris["sick"]) |
---|
626 | os.unlink(sick_shares[0][2]) |
---|
627 | hunk ./src/allmydata/test/test_web.py 3901 |
---|
628 | - #dead_shares = self.find_shares(self.uris["dead"]) |
---|
629 | + #dead_shares = self.find_uri_shares(self.uris["dead"]) |
---|
630 | #for i in range(1, 10): |
---|
631 | # os.unlink(dead_shares[i][2]) |
---|
632 | |
---|
633 | hunk ./src/allmydata/test/test_web.py 3905 |
---|
634 | - #c_shares = self.find_shares(self.uris["corrupt"]) |
---|
635 | + #c_shares = self.find_uri_shares(self.uris["corrupt"]) |
---|
636 | #cso = CorruptShareOptions() |
---|
637 | #cso.stdout = StringIO() |
---|
638 | #cso.parseOptions([c_shares[0][2]]) |
---|
639 | hunk ./src/allmydata/test/test_web.py 3961 |
---|
640 | |
---|
641 | def _count_leases(self, ignored, which): |
---|
642 | u = self.uris[which] |
---|
643 | - shares = self.find_shares(u) |
---|
644 | + shares = self.find_uri_shares(u) |
---|
645 | lease_counts = [] |
---|
646 | for shnum, serverid, fn in shares: |
---|
647 | sf = get_share_file(fn) |
---|
648 | } |
---|
649 | [upload.py: fix #1118 by aborting newly-homeless buckets when reassignment runs. This makes a previously failing assert correct. This version refactors 'abort' into two methods, rather than using a default argument. |
---|
650 | david-sarah@jacaranda.org**20100719044655 |
---|
651 | Ignore-this: 142d182c0739986812140bb8387077d5 |
---|
652 | ] { |
---|
653 | hunk ./src/allmydata/immutable/upload.py 140 |
---|
654 | |
---|
655 | def abort(self): |
---|
656 | """ |
---|
657 | - I abort the remote bucket writers for the share numbers in |
---|
658 | - sharenums. This is a good idea to conserve space on the storage |
---|
659 | - server. |
---|
660 | + I abort the remote bucket writers for all shares. This is a good idea |
---|
661 | + to conserve space on the storage server. |
---|
662 | """ |
---|
663 | hunk ./src/allmydata/immutable/upload.py 143 |
---|
664 | - for writer in self.buckets.itervalues(): writer.abort() |
---|
665 | + self.abort_some_buckets(self.buckets.keys()) |
---|
666 | + |
---|
667 | + def abort_some_buckets(self, sharenums): |
---|
668 | + """ |
---|
669 | + I abort the remote bucket writers for the share numbers in sharenums. |
---|
670 | + """ |
---|
671 | + for sharenum in sharenums: |
---|
672 | + if sharenum in self.buckets: |
---|
673 | + self.buckets[sharenum].abort() |
---|
674 | + del self.buckets[sharenum] |
---|
675 | |
---|
676 | |
---|
677 | class Tahoe2PeerSelector: |
---|
678 | hunk ./src/allmydata/immutable/upload.py 367 |
---|
679 | if not self.preexisting_shares[share]: |
---|
680 | del self.preexisting_shares[share] |
---|
681 | items.append((server, sharelist)) |
---|
682 | + for writer in self.use_peers: |
---|
683 | + writer.abort_some_buckets(self.homeless_shares) |
---|
684 | return self._loop() |
---|
685 | else: |
---|
686 | # Redistribution won't help us; fail. |
---|
687 | hunk ./src/allmydata/immutable/upload.py 377 |
---|
688 | self.needed_shares, |
---|
689 | self.servers_of_happiness, |
---|
690 | effective_happiness) |
---|
691 | + log.msg("server selection unsuccessful for %r: %s (%s), merged=%r" |
---|
692 | + % (self, msg, self._get_progress_message(), merged), level=log.INFREQUENT) |
---|
693 | return self._failed("%s (%s)" % (msg, self._get_progress_message())) |
---|
694 | |
---|
695 | if self.uncontacted_peers: |
---|
696 | } |
---|
697 | [immutable: use PrefixingLogMixin to organize logging in Tahoe2PeerSelector and add more detailed messages about peer |
---|
698 | zooko@zooko.com**20100719065629 |
---|
699 | Ignore-this: dce3e763dc628abf1604d1bfb9bdc829 |
---|
700 | ] { |
---|
701 | hunk ./src/allmydata/immutable/upload.py 77 |
---|
702 | # TODO: actual extensions are closer to 419 bytes, so we can probably lower |
---|
703 | # this. |
---|
704 | |
---|
705 | +def pretty_print_shnum_to_servers(s): |
---|
706 | + return ', '.join([ "sh%s: %s" % (k, '+'.join([idlib.shortnodeid_b2a(x) for x in v])) for k, v in s.iteritems() ]) |
---|
707 | + |
---|
708 | class PeerTracker: |
---|
709 | def __init__(self, peerid, storage_server, |
---|
710 | sharesize, blocksize, num_segments, num_share_hashes, |
---|
711 | hunk ./src/allmydata/immutable/upload.py 158 |
---|
712 | del self.buckets[sharenum] |
---|
713 | |
---|
714 | |
---|
715 | -class Tahoe2PeerSelector: |
---|
716 | +class Tahoe2PeerSelector(log.PrefixingLogMixin): |
---|
717 | |
---|
718 | def __init__(self, upload_id, logparent=None, upload_status=None): |
---|
719 | self.upload_id = upload_id |
---|
720 | hunk ./src/allmydata/immutable/upload.py 169 |
---|
721 | self.num_peers_contacted = 0 |
---|
722 | self.last_failure_msg = None |
---|
723 | self._status = IUploadStatus(upload_status) |
---|
724 | - self._log_parent = log.msg("%s starting" % self, parent=logparent) |
---|
725 | + log.PrefixingLogMixin.__init__(self, 'tahoe.immutable.upload', logparent, prefix=upload_id) |
---|
726 | + self.log("starting", level=log.OPERATIONAL) |
---|
727 | |
---|
728 | def __repr__(self): |
---|
729 | return "<Tahoe2PeerSelector for upload %s>" % self.upload_id |
---|
730 | hunk ./src/allmydata/immutable/upload.py 275 |
---|
731 | ds.append(d) |
---|
732 | self.num_peers_contacted += 1 |
---|
733 | self.query_count += 1 |
---|
734 | - log.msg("asking peer %s for any existing shares for " |
---|
735 | - "upload id %s" |
---|
736 | - % (idlib.shortnodeid_b2a(peer.peerid), self.upload_id), |
---|
737 | - level=log.NOISY, parent=self._log_parent) |
---|
738 | + self.log("asking peer %s for any existing shares" % |
---|
739 | + (idlib.shortnodeid_b2a(peer.peerid),), |
---|
740 | + level=log.NOISY) |
---|
741 | dl = defer.DeferredList(ds) |
---|
742 | dl.addCallback(lambda ign: self._loop()) |
---|
743 | return dl |
---|
744 | hunk ./src/allmydata/immutable/upload.py 289 |
---|
745 | Tahoe2PeerSelector._existing_shares. |
---|
746 | """ |
---|
747 | if isinstance(res, failure.Failure): |
---|
748 | - log.msg("%s got error during existing shares check: %s" |
---|
749 | + self.log("%s got error during existing shares check: %s" |
---|
750 | % (idlib.shortnodeid_b2a(peer), res), |
---|
751 | hunk ./src/allmydata/immutable/upload.py 291 |
---|
752 | - level=log.UNUSUAL, parent=self._log_parent) |
---|
753 | + level=log.UNUSUAL) |
---|
754 | self.error_count += 1 |
---|
755 | self.bad_query_count += 1 |
---|
756 | else: |
---|
757 | hunk ./src/allmydata/immutable/upload.py 298 |
---|
758 | buckets = res |
---|
759 | if buckets: |
---|
760 | self.peers_with_shares.add(peer) |
---|
761 | - log.msg("response from peer %s: alreadygot=%s" |
---|
762 | + self.log("response to get_buckets() from peer %s: alreadygot=%s" |
---|
763 | % (idlib.shortnodeid_b2a(peer), tuple(sorted(buckets))), |
---|
764 | hunk ./src/allmydata/immutable/upload.py 300 |
---|
765 | - level=log.NOISY, parent=self._log_parent) |
---|
766 | + level=log.NOISY) |
---|
767 | for bucket in buckets: |
---|
768 | self.preexisting_shares.setdefault(bucket, set()).add(peer) |
---|
769 | if self.homeless_shares and bucket in self.homeless_shares: |
---|
770 | hunk ./src/allmydata/immutable/upload.py 334 |
---|
771 | merged = merge_peers(self.preexisting_shares, self.use_peers) |
---|
772 | effective_happiness = servers_of_happiness(merged) |
---|
773 | if self.servers_of_happiness <= effective_happiness: |
---|
774 | - msg = ("peer selection successful for %s: %s" % (self, |
---|
775 | - self._get_progress_message())) |
---|
776 | - log.msg(msg, parent=self._log_parent) |
---|
777 | + msg = ("server selection successful for %s: %s: %s" % (self, |
---|
778 | + self._get_progress_message(), pretty_print_shnum_to_servers(merged))) |
---|
779 | + self.log(msg, level=log.OPERATIONAL) |
---|
780 | return (self.use_peers, self.preexisting_shares) |
---|
781 | else: |
---|
782 | # We're not okay right now, but maybe we can fix it by |
---|
783 | hunk ./src/allmydata/immutable/upload.py 380 |
---|
784 | self.needed_shares, |
---|
785 | self.servers_of_happiness, |
---|
786 | effective_happiness) |
---|
787 | - log.msg("server selection unsuccessful for %r: %s (%s), merged=%r" |
---|
788 | - % (self, msg, self._get_progress_message(), merged), level=log.INFREQUENT) |
---|
789 | + self.log("server selection unsuccessful for %r: %s (%s), merged=%s" % (self, msg, self._get_progress_message(), pretty_print_shnum_to_servers(merged)), level=log.INFREQUENT) |
---|
790 | return self._failed("%s (%s)" % (msg, self._get_progress_message())) |
---|
791 | |
---|
792 | if self.uncontacted_peers: |
---|
793 | hunk ./src/allmydata/immutable/upload.py 403 |
---|
794 | elif self.contacted_peers: |
---|
795 | # ask a peer that we've already asked. |
---|
796 | if not self._started_second_pass: |
---|
797 | - log.msg("starting second pass", parent=self._log_parent, |
---|
798 | + self.log("starting second pass", |
---|
799 | level=log.NOISY) |
---|
800 | self._started_second_pass = True |
---|
801 | num_shares = mathutil.div_ceil(len(self.homeless_shares), |
---|
802 | hunk ./src/allmydata/immutable/upload.py 441 |
---|
803 | self._get_progress_message())) |
---|
804 | if self.last_failure_msg: |
---|
805 | msg += " (%s)" % (self.last_failure_msg,) |
---|
806 | - log.msg(msg, level=log.UNUSUAL, parent=self._log_parent) |
---|
807 | + self.log(msg, level=log.UNUSUAL) |
---|
808 | return self._failed(msg) |
---|
809 | else: |
---|
810 | # we placed enough to be happy, so we're done |
---|
811 | hunk ./src/allmydata/immutable/upload.py 447 |
---|
812 | if self._status: |
---|
813 | self._status.set_status("Placed all shares") |
---|
814 | + msg = ("server selection successful (no more servers) for %s: %s: %s" % (self, |
---|
815 | + self._get_progress_message(), pretty_print_shnum_to_servers(merged))) |
---|
816 | + self.log(msg, level=log.OPERATIONAL) |
---|
817 | return (self.use_peers, self.preexisting_shares) |
---|
818 | |
---|
819 | def _got_response(self, res, peer, shares_to_ask, put_peer_here): |
---|
820 | hunk ./src/allmydata/immutable/upload.py 456 |
---|
821 | if isinstance(res, failure.Failure): |
---|
822 | # This is unusual, and probably indicates a bug or a network |
---|
823 | # problem. |
---|
824 | - log.msg("%s got error during peer selection: %s" % (peer, res), |
---|
825 | - level=log.UNUSUAL, parent=self._log_parent) |
---|
826 | + self.log("%s got error during peer selection: %s" % (peer, res), |
---|
827 | + level=log.UNUSUAL) |
---|
828 | self.error_count += 1 |
---|
829 | self.bad_query_count += 1 |
---|
830 | self.homeless_shares = list(shares_to_ask) + self.homeless_shares |
---|
831 | hunk ./src/allmydata/immutable/upload.py 476 |
---|
832 | self.last_failure_msg = msg |
---|
833 | else: |
---|
834 | (alreadygot, allocated) = res |
---|
835 | - log.msg("response from peer %s: alreadygot=%s, allocated=%s" |
---|
836 | + self.log("response to allocate_buckets() from peer %s: alreadygot=%s, allocated=%s" |
---|
837 | % (idlib.shortnodeid_b2a(peer.peerid), |
---|
838 | tuple(sorted(alreadygot)), tuple(sorted(allocated))), |
---|
839 | hunk ./src/allmydata/immutable/upload.py 479 |
---|
840 | - level=log.NOISY, parent=self._log_parent) |
---|
841 | + level=log.NOISY) |
---|
842 | progress = False |
---|
843 | for s in alreadygot: |
---|
844 | self.preexisting_shares.setdefault(s, set()).add(peer.peerid) |
---|
845 | hunk ./src/allmydata/immutable/upload.py 922 |
---|
846 | @paran already_peers: a dict mapping sharenum to a set of peerids |
---|
847 | that claim to already have this share |
---|
848 | """ |
---|
849 | - self.log("_send_shares, used_peers is %s" % (used_peers,)) |
---|
850 | + self.log("set_shareholders; used_peers is %s, already_peers is %s" % ([p.buckets for p in used_peers], already_peers)) |
---|
851 | # record already-present shares in self._results |
---|
852 | self._results.preexisting_shares = len(already_peers) |
---|
853 | |
---|
854 | hunk ./src/allmydata/immutable/upload.py 936 |
---|
855 | for shnum in peer.buckets: |
---|
856 | self._peer_trackers[shnum] = peer |
---|
857 | servermap.setdefault(shnum, set()).add(peer.peerid) |
---|
858 | + self.log("set_shareholders; %s (%s) == %s (%s)" % (len(buckets), buckets, sum([len(peer.buckets) for peer in used_peers]), [(p.buckets, p.peerid) for p in used_peers])) |
---|
859 | assert len(buckets) == sum([len(peer.buckets) for peer in used_peers]), "%s (%s) != %s (%s)" % (len(buckets), buckets, sum([len(peer.buckets) for peer in used_peers]), [(p.buckets, p.peerid) for p in used_peers]) |
---|
860 | encoder.set_shareholders(buckets, servermap) |
---|
861 | |
---|
862 | hunk ./src/allmydata/storage/server.py 8 |
---|
863 | |
---|
864 | from zope.interface import implements |
---|
865 | from allmydata.interfaces import RIStorageServer, IStatsProducer |
---|
866 | -from allmydata.util import fileutil, log, time_format |
---|
867 | +from allmydata.util import fileutil, idlib, log, time_format |
---|
868 | import allmydata # for __full_version__ |
---|
869 | |
---|
870 | from allmydata.storage.common import si_b2a, si_a2b, storage_index_to_dir |
---|
871 | hunk ./src/allmydata/storage/server.py 109 |
---|
872 | expiration_sharetypes) |
---|
873 | self.lease_checker.setServiceParent(self) |
---|
874 | |
---|
875 | + def __repr__(self): |
---|
876 | + return "<StorageServer %s>" % (idlib.shortnodeid_b2a(self.my_nodeid),) |
---|
877 | + |
---|
878 | def add_bucket_counter(self): |
---|
879 | statefile = os.path.join(self.storedir, "bucket_counter.state") |
---|
880 | self.bucket_counter = BucketCountingCrawler(self, statefile) |
---|
881 | hunk ./src/allmydata/test/test_upload.py 14 |
---|
882 | from allmydata import uri, monitor, client |
---|
883 | from allmydata.immutable import upload, encode |
---|
884 | from allmydata.interfaces import FileTooLargeError, UploadUnhappinessError |
---|
885 | +from allmydata.util import log |
---|
886 | from allmydata.util.assertutil import precondition |
---|
887 | from allmydata.util.deferredutil import DeferredListShouldSucceed |
---|
888 | from allmydata.test.no_network import GridTestMixin |
---|
889 | hunk ./src/allmydata/test/test_upload.py 714 |
---|
890 | |
---|
891 | def is_happy_enough(servertoshnums, h, k): |
---|
892 | """ I calculate whether servertoshnums achieves happiness level h. I do this with a naïve "brute force search" approach. (See src/allmydata/util/happinessutil.py for a better algorithm.) """ |
---|
893 | + print "servertoshnums: ", servertoshnums, "h: ", h, "k: ", k |
---|
894 | if len(servertoshnums) < h: |
---|
895 | return False |
---|
896 | # print "servertoshnums: ", servertoshnums, h, k |
---|
897 | hunk ./src/allmydata/test/test_upload.py 803 |
---|
898 | def _add_server(self, server_number, readonly=False): |
---|
899 | assert self.g, "I tried to find a grid at self.g, but failed" |
---|
900 | ss = self.g.make_server(server_number, readonly) |
---|
901 | + log.msg("just created a server, number: %s => %s" % (server_number, ss,)) |
---|
902 | self.g.add_server(server_number, ss) |
---|
903 | |
---|
904 | hunk ./src/allmydata/test/test_upload.py 806 |
---|
905 | - |
---|
906 | def _add_server_with_share(self, server_number, share_number=None, |
---|
907 | readonly=False): |
---|
908 | self._add_server(server_number, readonly) |
---|
909 | hunk ./src/allmydata/test/test_upload.py 866 |
---|
910 | d.addCallback(_store_shares) |
---|
911 | return d |
---|
912 | |
---|
913 | - |
---|
914 | def test_configure_parameters(self): |
---|
915 | self.basedir = self.mktemp() |
---|
916 | hooks = {0: self._set_up_nodes_extra_config} |
---|
917 | } |
---|
918 | |
---|
919 | Context: |
---|
920 | |
---|
921 | [docs/logging.txt: document that _trial_temp/test.log does not receive messages below level=OPERATIONAL, due to <http://foolscap.lothar.com/trac/ticket/154>. |
---|
922 | david-sarah@jacaranda.org**20100718230420 |
---|
923 | Ignore-this: aef40f2e74ddeabee5e122e8d80893a1 |
---|
924 | ] |
---|
925 | [immutable: test for #1124 |
---|
926 | zooko@zooko.com**20100718222907 |
---|
927 | Ignore-this: 1766e3cbab92ea2a9e246f40eb6e770b |
---|
928 | ] |
---|
929 | [trivial: fix unused import (sorry about that, pyflakes) |
---|
930 | zooko@zooko.com**20100718215133 |
---|
931 | Ignore-this: c2414e443405072b51d552295f2c0e8c |
---|
932 | ] |
---|
933 | [tests, NEWS, CREDITS re: #1117 |
---|
934 | zooko@zooko.com**20100718203225 |
---|
935 | Ignore-this: 1f08be2c692fb72cc0dd023259f11354 |
---|
936 | Give Brian and Kevan promotions, move release date in NEWS to the 18th, commit Brian's test for #1117. |
---|
937 | fixes #1117 |
---|
938 | ] |
---|
939 | [test/test_upload.py: test to see that aborted buckets are ignored by the storage server |
---|
940 | Kevan Carstensen <kevan@isnotajoke.com>**20100716001046 |
---|
941 | Ignore-this: cc075c24b1c86d737f3199af894cc780 |
---|
942 | ] |
---|
943 | [test/test_storage.py: test for the new remote_abort semantics. |
---|
944 | Kevan Carstensen <kevan@isnotajoke.com>**20100715232148 |
---|
945 | Ignore-this: d3d6491f17bf670e770ca4b385007515 |
---|
946 | ] |
---|
947 | [storage/immutable.py: make remote_abort btell the storage server about aborted buckets. |
---|
948 | Kevan Carstensen <kevan@isnotajoke.com>**20100715232105 |
---|
949 | Ignore-this: 16ab0090676355abdd5600ed44ff19c9 |
---|
950 | ] |
---|
951 | [test/test_upload.py: changes to test plumbing for #1117 tests |
---|
952 | Kevan Carstensen <kevan@isnotajoke.com>**20100715231820 |
---|
953 | Ignore-this: 78a6d359d7bf8529d283e2815bf1e2de |
---|
954 | |
---|
955 | - Add a callRemoteOnly method to FakeBucketWriter. |
---|
956 | - Change the abort method in FakeBucketWriter to not return a |
---|
957 | RuntimeError. |
---|
958 | ] |
---|
959 | [immutable/upload.py: abort buckets if peer selection fails |
---|
960 | Kevan Carstensen <kevan@isnotajoke.com>**20100715231714 |
---|
961 | Ignore-this: 2a0b643a22284df292d8ed9d91b1fd37 |
---|
962 | ] |
---|
963 | [test_encodingutil: correct an error in the previous patch to StdlibUnicode.test_open_representable. |
---|
964 | david-sarah@jacaranda.org**20100718151420 |
---|
965 | Ignore-this: af050955f623fbc0e4d78e15a0a8a144 |
---|
966 | ] |
---|
967 | [NEWS: Forward-compatibility improvements for non-ASCII caps (#1051). |
---|
968 | david-sarah@jacaranda.org**20100718143622 |
---|
969 | Ignore-this: 1edfebc4bd38a3b5c35e75c99588153f |
---|
970 | ] |
---|
971 | [test_dirnode and test_web: don't use failUnlessReallyEqual in cases where the return type from simplejson.loads can vary between unicode and str. Use to_str when comparing URIs parsed from JSON. |
---|
972 | david-sarah@jacaranda.org**20100718142915 |
---|
973 | Ignore-this: c4e78ef4b1478dd400da71cf077ffa4a |
---|
974 | ] |
---|
975 | [test_encodingutil: StdlibUnicode.test_open_representable no longer uses a mock. |
---|
976 | david-sarah@jacaranda.org**20100718125412 |
---|
977 | Ignore-this: 4bf373a5e2dfe4209e5e364124af29a3 |
---|
978 | ] |
---|
979 | [docs: add comment clarifying #1051 |
---|
980 | zooko@zooko.com**20100718053250 |
---|
981 | Ignore-this: 6cfc0930434cbdbbc262dabb58f1505d |
---|
982 | ] |
---|
983 | [docs: update NEWS |
---|
984 | zooko@zooko.com**20100718053225 |
---|
985 | Ignore-this: 63d5c782ef84812e6d010f0590866831 |
---|
986 | ] |
---|
987 | [Add tests of caps from the future that have non-ASCII characters in them (encoded as UTF-8). The changes to test_uri.py, test_client.py, and test_dirnode.py add tests of non-ASCII future caps in addition to the current tests. The changes to test_web.py just replace the tests of all-ASCII future caps with tests of non-ASCII future caps. We also change uses of failUnlessEqual to failUnlessReallyEqual, in order to catch cases where the type of a string is not as expected. |
---|
988 | david-sarah@jacaranda.org**20100711200252 |
---|
989 | Ignore-this: c2f193352369d32e06865f8f3e951894 |
---|
990 | ] |
---|
991 | [Debian documentation update |
---|
992 | jacob@appelbaum.net**20100305003004] |
---|
993 | [debian-docs-patch-final |
---|
994 | jacob@appelbaum.net**20100304085955] |
---|
995 | [M-x whitespace-cleanup |
---|
996 | zooko@zooko.com**20100718032739 |
---|
997 | Ignore-this: babfd4af6ad2fc885c957fd5c8b10c3f |
---|
998 | ] |
---|
999 | [docs: tidy up NEWS a little |
---|
1000 | zooko@zooko.com**20100718032434 |
---|
1001 | Ignore-this: 54f2820fd1a37c8967609f6bfc4e5e18 |
---|
1002 | ] |
---|
1003 | [benchmarking: update bench_dirnode.py to reflect the new directory interfaces |
---|
1004 | zooko@zooko.com**20100718031710 |
---|
1005 | Ignore-this: 368ba523dd3de80d9da29cd58afbe827 |
---|
1006 | ] |
---|
1007 | [test_encodingutil: fix test_open_representable, which is only valid when run on a platform for which we know an unrepresentable filename. |
---|
1008 | david-sarah@jacaranda.org**20100718030333 |
---|
1009 | Ignore-this: c114d92c17714a5d4ae005c15267d60c |
---|
1010 | ] |
---|
1011 | [iputil.py: Add support for FreeBSD 7,8 and 9 |
---|
1012 | francois@ctrlaltdel.ch**20100718022832 |
---|
1013 | Ignore-this: 1829b4cf4b91107f4cf87841e6167e99 |
---|
1014 | committed by: zooko@zooko.com |
---|
1015 | date: 2010-07-17 |
---|
1016 | and I also patched: NEWS and CREDITS |
---|
1017 | ] |
---|
1018 | [NEWS: add snippet about #1083 |
---|
1019 | zooko@zooko.com**20100718020653 |
---|
1020 | Ignore-this: d353a9d93cbc5a5e6ba4671f78d1e22b |
---|
1021 | ] |
---|
1022 | [fileutil: docstrings for non-obvious usage restrictions on methods of EncryptedTemporaryFile. |
---|
1023 | david-sarah@jacaranda.org**20100717054647 |
---|
1024 | Ignore-this: 46d8fc10782fa8ec2b6c5b168c841943 |
---|
1025 | ] |
---|
1026 | [Move EncryptedTemporaryFile from SFTP frontend to allmydata.util.fileutil, and make the FTP frontend also use it (fixing #1083). |
---|
1027 | david-sarah@jacaranda.org**20100711213721 |
---|
1028 | Ignore-this: e452e8ca66391aa2a1a49afe0114f317 |
---|
1029 | ] |
---|
1030 | [NEWS: reorder NEWS snippets to be in descending order of interestingness |
---|
1031 | zooko@zooko.com**20100718015929 |
---|
1032 | Ignore-this: 146c42e88a9555a868a04a69dd0e5326 |
---|
1033 | ] |
---|
1034 | [Correct stringutils->encodingutil patch to be the newer version, rather than the old version that was committed in error. |
---|
1035 | david-sarah@jacaranda.org**20100718013435 |
---|
1036 | Ignore-this: c8940c4e1aa2e9acc80cd4fe54753cd8 |
---|
1037 | ] |
---|
1038 | [test_cli.py: fix error that crept in when rebasing the patch for #1072. |
---|
1039 | david-sarah@jacaranda.org**20100718000123 |
---|
1040 | Ignore-this: 3e8f6cc3a27b747c708221dd581934f4 |
---|
1041 | ] |
---|
1042 | [stringutils: add test for when sys.stdout has no encoding attribute (fixes #1099). |
---|
1043 | david-sarah@jacaranda.org**20100717045816 |
---|
1044 | Ignore-this: f28dce6940e909f12f354086d17db54f |
---|
1045 | ] |
---|
1046 | [CLI: add 'tahoe unlink' as an alias to 'tahoe rm', for forward-compatibility. |
---|
1047 | david-sarah@jacaranda.org**20100717220411 |
---|
1048 | Ignore-this: 3ecdde7f2d0498514cef32e118e0b855 |
---|
1049 | ] |
---|
1050 | [minor code clean-up in dirnode.py |
---|
1051 | zooko@zooko.com**20100714060255 |
---|
1052 | Ignore-this: bb0ab2783203e605024b3e2f798256a1 |
---|
1053 | Impose micro-POLA by passing only the writekey instead of the whole node object to {{{_encrypt_rw_uri()}}}. Remove DummyImmutableFileNode in nodemaker.py, which is obviated by this. Add micro-optimization by precomputing the netstring of the empty string and branching on whether the writekey is present or not outside of {{{_encrypt_rw_uri()}}}. Add doc about writekey to docstring. |
---|
1054 | fixes #967 |
---|
1055 | ] |
---|
1056 | [Rename stringutils to encodingutil, and drop listdir_unicode and open_unicode (since the Python stdlib functions work fine with Unicode paths). Also move some utility functions to fileutil. |
---|
1057 | david-sarah@jacaranda.org**20100712003015 |
---|
1058 | Ignore-this: 103b809d180df17a7283077c3104c7be |
---|
1059 | ] |
---|
1060 | [Allow URIs passed in the initial JSON for t=mkdir-with-children, t=mkdir-immutable to be Unicode. Also pass the name of each child into nodemaker.create_from_cap for error reporting. |
---|
1061 | david-sarah@jacaranda.org**20100711195525 |
---|
1062 | Ignore-this: deac32d8b91ba26ede18905d3f7d2b93 |
---|
1063 | ] |
---|
1064 | [docs: CREDITS and NEWS |
---|
1065 | zooko@zooko.com**20100714060150 |
---|
1066 | Ignore-this: dc83e612f77d69e50ee975f07f6b16fe |
---|
1067 | ] |
---|
1068 | [CREDITS: more creds for Kevan, plus utf-8 BOM |
---|
1069 | zooko@zooko.com**20100619045503 |
---|
1070 | Ignore-this: 72d02bdd7a0f324f1cee8cd399c7c6de |
---|
1071 | ] |
---|
1072 | [cli.py: make command descriptions consistently end with a full stop. |
---|
1073 | david-sarah@jacaranda.org**20100714014538 |
---|
1074 | Ignore-this: 9ee7fa29ca2d1631db4049c2a389a97a |
---|
1075 | ] |
---|
1076 | [SFTP: address some of the comments in zooko's review (#1106). |
---|
1077 | david-sarah@jacaranda.org**20100712025537 |
---|
1078 | Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e |
---|
1079 | ] |
---|
1080 | [docs/logging.txt: note that setting flogging vars might affect tests with race conditions. |
---|
1081 | david-sarah@jacaranda.org**20100712050721 |
---|
1082 | Ignore-this: fc1609d215fcd5561a57fd1226206f27 |
---|
1083 | ] |
---|
1084 | [test_storage.py: potential fix for failures when logging is enabled. |
---|
1085 | david-sarah@jacaranda.org**19700713040546 |
---|
1086 | Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7 |
---|
1087 | ] |
---|
1088 | [upcase_since_on_welcome |
---|
1089 | terrellrussell@gmail.com**20100708193903] |
---|
1090 | [server_version_on_welcome_page.dpatch.txt |
---|
1091 | freestorm77@gmail.com**20100605191721 |
---|
1092 | Ignore-this: b450c76dc875f5ac8cca229a666cbd0a |
---|
1093 | |
---|
1094 | |
---|
1095 | - The storage server version is 0 for all storage nodes in the Welcome Page |
---|
1096 | |
---|
1097 | |
---|
1098 | ] |
---|
1099 | [NEWS: add NEWS snippets about two recent patches |
---|
1100 | zooko@zooko.com**20100708162058 |
---|
1101 | Ignore-this: 6c9da6a0ad7351a960bdd60f81532899 |
---|
1102 | ] |
---|
1103 | [directory_html_top_banner.dpatch |
---|
1104 | freestorm77@gmail.com**20100622205301 |
---|
1105 | Ignore-this: 1d770d975e0c414c996564774f049bca |
---|
1106 | |
---|
1107 | The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct |
---|
1108 | |
---|
1109 | ] |
---|
1110 | [tahoe_css_toolbar.dpatch |
---|
1111 | freestorm77@gmail.com**20100622210046 |
---|
1112 | Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0 |
---|
1113 | |
---|
1114 | CSS modification to be correctly diplayed with Internet Explorer 8 |
---|
1115 | |
---|
1116 | The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox. |
---|
1117 | |
---|
1118 | ] |
---|
1119 | [runnin_test_tahoe_css.dpatch |
---|
1120 | freestorm77@gmail.com**20100622214714 |
---|
1121 | Ignore-this: e0db73d68740aad09a7b9ae60a08c05c |
---|
1122 | |
---|
1123 | Runnin test for changes in tahoe.css file |
---|
1124 | |
---|
1125 | ] |
---|
1126 | [runnin_test_directory_xhtml.dpatch |
---|
1127 | freestorm77@gmail.com**20100622201403 |
---|
1128 | Ignore-this: f8962463fce50b9466405cb59fe11d43 |
---|
1129 | |
---|
1130 | Runnin test for diretory.xhtml top banner |
---|
1131 | |
---|
1132 | ] |
---|
1133 | [stringutils.py: tolerate sys.stdout having no 'encoding' attribute. |
---|
1134 | david-sarah@jacaranda.org**20100626040817 |
---|
1135 | Ignore-this: f42cad81cef645ee38ac1df4660cc850 |
---|
1136 | ] |
---|
1137 | [quickstart.html: python 2.5 -> 2.6 as recommended version |
---|
1138 | david-sarah@jacaranda.org**20100705175858 |
---|
1139 | Ignore-this: bc3a14645ea1d5435002966ae903199f |
---|
1140 | ] |
---|
1141 | [SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader). |
---|
1142 | david-sarah@jacaranda.org**20100628231926 |
---|
1143 | Ignore-this: 131b7a5787bc85a9a356b5740d9d996f |
---|
1144 | ] |
---|
1145 | [docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html. |
---|
1146 | david-sarah@jacaranda.org**20100625223929 |
---|
1147 | Ignore-this: 99a5459cac51bd867cc11ad06927ff30 |
---|
1148 | ] |
---|
1149 | [setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk" |
---|
1150 | zooko@zooko.com**20100619034928 |
---|
1151 | Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6 |
---|
1152 | ] |
---|
1153 | [trivial: tiny update to in-line comment |
---|
1154 | zooko@zooko.com**20100614045715 |
---|
1155 | Ignore-this: 10851b0ed2abfed542c97749e5d280bc |
---|
1156 | (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.) |
---|
1157 | ] |
---|
1158 | [docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around |
---|
1159 | zooko@zooko.com**20100619065318 |
---|
1160 | Ignore-this: dc6db03f696e5b6d2848699e754d8053 |
---|
1161 | ] |
---|
1162 | [docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates" |
---|
1163 | zooko@zooko.com**20100619065124 |
---|
1164 | Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c |
---|
1165 | ] |
---|
1166 | [TAG allmydata-tahoe-1.7.0 |
---|
1167 | zooko@zooko.com**20100619052631 |
---|
1168 | Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1 |
---|
1169 | ] |
---|
1170 | Patch bundle hash: |
---|
1171 | ea9a6b59d0dd8cf99124dfef11c54425f5dd0ad4 |
---|