diff -rN -u old-1.9.2/src/allmydata/mutable/common.py new-1.9.2/src/allmydata/mutable/common.py --- old-1.9.2/src/allmydata/mutable/common.py 2012-06-21 00:35:33.535864570 -0300 +++ new-1.9.2/src/allmydata/mutable/common.py 2012-06-21 00:35:33.709197226 -0300 @@ -1,6 +1,35 @@ from allmydata.util.spans import DataSpans +from allmydata.util import hashutil + +def is_salt(x): + return isinstance(x, str) and len(x) == 16 + +def check_is_verinfo(x): + if not isinstance(x, tuple): + raise TypeError("This isn't a verinfo because its type is %s instead of tuple. %r" % (type(x), x,)) + if len(x) != 9: + raise TypeError("This isn't a verinfo because its length is %s instead of 9. %r :: %s" % (len(x), x, type(x),)) + if not isinstance(x[0], (int, long)): + raise TypeError("This isn't a verinfo because the type of its 0 element is %s instead of int/long. %r :: %s" % (type(x[0]), x, type(x),)) + if not hashutil.is_hash(x[1]): + raise TypeError("This isn't a verinfo because its 1 element (%r :: %s) is not a hash. %r :: %s" % (x[1], type(x[1]), x, type(x),)) + if not hashutil.is_hash(x[2]) and not is_salt(x[2]) and x[2] is not None: + raise TypeError("This isn't a verinfo because its 2 element (%r :: %s) is neither a hash nor a salt nor None. %r :: %s" % (x[2], type(x[2]), x, type(x),)) + if not isinstance(x[3], (int, long)): + raise TypeError("This isn't a verinfo because the type of its 3 element is %s instead of int/long. %r :: %s" % (type(x[3]), x, type(x),)) + if not isinstance(x[4], (int, long)): + raise TypeError("This isn't a verinfo because the type of its 4 element is %s instead of int/long. %r :: %s" % (type(x[4]), x, type(x),)) + if not isinstance(x[5], (int, long)): + raise TypeError("This isn't a verinfo because the type of its 5 element is %s instead of int/long. %r :: %s" % (type(x[5]), x, type(x),)) + if not isinstance(x[6], (int, long)): + raise TypeError("This isn't a verinfo because the type of its 6 element is %s instead of int/long. %r :: %s" % (type(x[6]), x, type(x),)) + if not isinstance(x[7], str): + raise TypeError("This isn't a verinfo because the type of its 7 element is %s instead of str. %r :: %s" % (type(x[7]), x, type(x),)) + if not isinstance(x[8], tuple): + raise TypeError("This isn't a verinfo because the type of its 8 element is %s instead of tuple. %r :: %s" % (type(x[8]), x, type(x),)) + MODE_CHECK = "MODE_CHECK" # query all peers MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial diff -rN -u old-1.9.2/src/allmydata/mutable/layout.py new-1.9.2/src/allmydata/mutable/layout.py --- old-1.9.2/src/allmydata/mutable/layout.py 2012-06-21 00:35:33.535864570 -0300 +++ new-1.9.2/src/allmydata/mutable/layout.py 2012-06-21 00:35:33.709197226 -0300 @@ -1,10 +1,11 @@ import struct from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError, \ - BadShareError + BadShareError, check_is_verinfo from allmydata.interfaces import HASH_SIZE, SALT_SIZE, SDMF_VERSION, \ MDMF_VERSION, IMutableSlotWriter from allmydata.util import mathutil +from allmydata.util.assertutil import precondition from twisted.python import failure from twisted.internet import defer from zope.interface import implements @@ -432,13 +433,12 @@ def get_verinfo(self): """ - I return my verinfo tuple. This is used by the ServermapUpdater - to keep track of versions of mutable files. + I return a Deferred that eventually fires with my verinfo tuple. - The verinfo tuple for MDMF files contains: + The verinfo tuple contains: - seqnum - root hash - - a blank (nothing) + - a salt - segsize - datalen - k @@ -447,12 +447,9 @@ - a tuple of offsets We include the nonce in MDMF to simplify processing of version - information tuples. - - The verinfo tuple for SDMF files is the same, but contains a - 16-byte IV instead of a hash of salts. + information tuples. XXX what nonce? """ - return (self._seqnum, + x = (self._seqnum, self._share_pieces['root_hash'], self._share_pieces['salt'], self._segment_size, @@ -461,6 +458,8 @@ self._total_shares, self.get_signable(), self._get_offsets_tuple()) + check_is_verinfo(x) + return defer.succeed(x) def _get_offsets_dict(self): post_offset = HEADER_LENGTH @@ -604,10 +603,10 @@ # which prempetively read a big part of the share -- possible. # # The checkstring is the first three fields -- the version number, - # sequence number, root hash and root salt hash. This is consistent + # sequence number, root hash and root salt hash. This is consistent # XXX first *four* fields? And should root hash salt be inserted into the table of fields layout above? And should it be called "root hash of salts" or something instead of "root salt hash"? # in meaning to what we have with SDMF files, except now instead of # using the literal salt, we use a value derived from all of the - # salts -- the share hash root. + # salts -- the share hash root. # XXX the root hash of salts? # # The salt is stored before the block for each segment. The block # hash tree is computed over the combination of block and salt for @@ -1100,14 +1099,34 @@ def get_verinfo(self): - return (self._seqnum, + """ + I return a Deferred that eventually fires with my verinfo tuple. + + The verinfo tuple contains: + - seqnum + - root hash + - None + - segsize + - datalen + - k + - n + - prefix (the thing that you sign) + - a tuple of offsets + + We include the nonce in MDMF to simplify processing of version + information tuples. XXX what nonce? + """ + x = (self._seqnum, self._root_hash, - self._required_shares, - self._total_shares, + None, self._segment_size, self._data_length, + self._required_shares, + self._total_shares, self.get_signable(), self._get_offsets_tuple()) + check_is_verinfo(x) + return defer.succeed(x) def finish_publishing(self): @@ -1177,7 +1196,7 @@ f.trap(struct.error) raise BadShareError(f.value.args[0]) -class MDMFSlotReadProxy: +class MutableReadProxy: """ I read from a mutable slot filled with data written in the MDMF data format (which is described above). @@ -1242,7 +1261,7 @@ # MDMF, though we'll be left with 4 more bytes than we # need if this ends up being MDMF. This is probably less # expensive than the cost of a second roundtrip. - readvs = [(0, 123)] + readvs = [(0, 123)] # XXX 123 != 107 d = self._read(readvs, force_remote) d.addCallback(self._process_encoding_parameters) d.addCallback(self._process_offsets) @@ -1686,18 +1705,17 @@ # The offsets tuple is another component of the version # information tuple. It is basically our offsets dictionary, # itemized and in a tuple. - return self._offsets.copy() + return tuple([(key, value) for key, value in self._offsets.items()]) def get_verinfo(self): """ - I return my verinfo tuple. This is used by the ServermapUpdater - to keep track of versions of mutable files. + I return a Deferred that eventually fires with my verinfo tuple. The verinfo tuple for MDMF files contains: - seqnum - root hash - - a blank (nothing) + - salt (SDMF) or None (MDMF) - segsize - datalen - k @@ -1713,11 +1731,12 @@ """ d = self._maybe_fetch_offsets_and_header() def _build_verinfo(ignored): + precondition(self._version_number in (SDMF_VERSION, MDMF_VERSION), self._version_number) if self._version_number == SDMF_VERSION: salt_to_use = self._salt else: salt_to_use = None - return (self._sequence_number, + x = (self._sequence_number, self._root_hash, salt_to_use, self._segment_size, @@ -1726,6 +1745,8 @@ self._total_shares, self._build_prefix(), self._get_offsets_tuple()) + check_is_verinfo(x) + return x d.addCallback(_build_verinfo) return d @@ -1747,15 +1768,6 @@ readvs) - def is_sdmf(self): - """I tell my caller whether or not my remote file is SDMF or MDMF - """ - d = self._maybe_fetch_offsets_and_header() - d.addCallback(lambda ignored: - self._version_number == 0) - return d - - class LayoutInvalid(BadShareError): """ This isn't a valid MDMF mutable file diff -rN -u old-1.9.2/src/allmydata/mutable/publish.py new-1.9.2/src/allmydata/mutable/publish.py --- old-1.9.2/src/allmydata/mutable/publish.py 2012-06-21 00:35:33.539197890 -0300 +++ new-1.9.2/src/allmydata/mutable/publish.py 2012-06-21 00:35:33.709197226 -0300 @@ -16,7 +16,7 @@ from foolscap.api import eventually, fireEventually from allmydata.mutable.common import MODE_WRITE, MODE_CHECK, MODE_REPAIR, \ - UncoordinatedWriteError, NotEnoughServersError + UncoordinatedWriteError, NotEnoughServersError, check_is_verinfo from allmydata.mutable.servermap import ServerMap from allmydata.mutable.layout import get_version_from_checkstring,\ unpack_mdmf_checkstring, \ @@ -246,7 +246,7 @@ self.bad_share_checkstrings = {} # This is set at the last step of the publishing process. - self.versioninfo = "" + self.versioninfo = None # we use the servermap to populate the initial goal: this way we will # try to update each existing share in place. Since we're @@ -439,7 +439,7 @@ self.bad_share_checkstrings = {} # This is set at the last step of the publishing process. - self.versioninfo = "" + self.versioninfo = None # we use the servermap to populate the initial goal: this way we will # try to update each existing share in place. @@ -881,13 +881,21 @@ d.addErrback(self._connection_problem, writer) d.addCallback(self._got_write_answer, writer, started) ds.append(d) - self._record_verinfo() - self._status.timings['pack'] = time.time() - started - return defer.DeferredList(ds) - - - def _record_verinfo(self): - self.versioninfo = self._get_some_writer().get_verinfo() + alld = defer.DeferredList(ds) + alld.addCallback(self._record_verinfo) + def _record_time(o): + self._status.timings['pack'] = time.time() - started + return o + alld.addCallback(_record_time) + return alld + + def _record_verinfo(self, ignored): + d = self._get_some_writer().get_verinfo() + def _record(versioninfo): + check_is_verinfo(versioninfo) + self.versioninfo = versioninfo + d.addCallback(_record) + return d def _connection_problem(self, f, writer): @@ -1146,7 +1154,7 @@ # self.versioninfo is set during the last phase of publishing. # If we get there, we know that responses correspond to placed # shares, and can safely execute these statements. - if self.versioninfo: + if self.versioninfo is not None: self.log("wrote successfully: adding new share to servermap") self._servermap.add_new_share(server, writer.shnum, self.versioninfo, started) diff -rN -u old-1.9.2/src/allmydata/mutable/retrieve.py new-1.9.2/src/allmydata/mutable/retrieve.py --- old-1.9.2/src/allmydata/mutable/retrieve.py 2012-06-21 00:35:33.542531210 -0300 +++ new-1.9.2/src/allmydata/mutable/retrieve.py 2012-06-21 00:35:33.709197226 -0300 @@ -18,7 +18,7 @@ from allmydata.mutable.common import CorruptShareError, BadShareError, \ UncoordinatedWriteError -from allmydata.mutable.layout import MDMFSlotReadProxy +from allmydata.mutable.layout import MutableReadProxy class RetrieveStatus: implements(IRetrieveStatus) @@ -145,7 +145,7 @@ self._status.set_helper(False) self._status.set_progress(0.0) self._status.set_active(True) - (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, + (seqnum, root_hash, salt_or_none, segsize, datalength, k, N, prefix, offsets_tuple) = self.verinfo self._status.set_size(datalength) self._status.set_encoding(k, N) @@ -271,7 +271,7 @@ # how many shares do we need? (seqnum, root_hash, - IV, + salt_or_none, segsize, datalength, k, @@ -292,7 +292,7 @@ # data that they have, then change this method to do that. any_cache = self._node._read_from_cache(self.verinfo, shnum, 0, 1000) - reader = MDMFSlotReadProxy(server.get_rref(), + reader = MutableReadProxy(server.get_rref(), self._storage_index, shnum, any_cache) @@ -339,7 +339,7 @@ """ (seqnum, root_hash, - IV, + salt_or_none, segsize, datalength, k, @@ -351,7 +351,7 @@ self._segment_size = segsize self._data_length = datalength - if not IV: + if salt_or_none is None: self._version = MDMF_VERSION else: self._version = SDMF_VERSION @@ -516,7 +516,7 @@ """ (seqnum, root_hash, - IV, + salt_or_none, segsize, datalength, k, @@ -862,7 +862,7 @@ blocks_and_salts.update(d) # All of these blocks should have the same salt; in SDMF, it is - # the file-wide IV, while in MDMF it is the per-segment salt. In + # the file-wide salt, while in MDMF it is the per-segment salt. In # either case, we just need to get one of them and use it. # # d.items()[0] is like (shnum, (block, salt)) @@ -975,7 +975,7 @@ self._status.set_progress(1.0) # remember the encoding parameters, use them again next time - (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, + (seqnum, root_hash, salt_or_none, segsize, datalength, k, N, prefix, offsets_tuple) = self.verinfo self._node._populate_required_shares(k) self._node._populate_total_shares(N) diff -rN -u old-1.9.2/src/allmydata/mutable/servermap.py new-1.9.2/src/allmydata/mutable/servermap.py --- old-1.9.2/src/allmydata/mutable/servermap.py 2012-06-21 00:35:33.542531210 -0300 +++ new-1.9.2/src/allmydata/mutable/servermap.py 2012-06-21 00:35:33.709197226 -0300 @@ -13,8 +13,8 @@ from pycryptopp.publickey import rsa from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \ - MODE_READ, MODE_REPAIR, CorruptShareError -from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy + MODE_READ, MODE_REPAIR, CorruptShareError, check_is_verinfo +from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MutableReadProxy class UpdateStatus: implements(IServermapUpdaterStatus) @@ -121,7 +121,7 @@ self._last_update_time = 0 self.update_data = {} # shnum -> [(verinfo,(blockhashes,start,end)),..] # where blockhashes is a list of bytestrings (the result of - # layout.MDMFSlotReadProxy.get_blockhashes), and start/end are both + # layout.MutableReadProxy.get_blockhashes), and start/end are both # (block,salt) tuple-of-bytestrings from get_block_and_salt() def copy(self): @@ -640,9 +640,8 @@ to occur when the file is downloaded, or when the file is updated. """ - if verinfo: - self._node._add_to_cache(verinfo, shnum, 0, data) - + check_is_verinfo(verinfo) + self._node._add_to_cache(verinfo, shnum, 0, data) def _got_results(self, datavs, server, readsize, storage_index, started): lp = self.log(format="got result from [%(name)s], %(numshares)d shares", @@ -668,11 +667,11 @@ else: self._empty_servers.add(server) - ds = [] - + # a list of deferreds, one for each share that we found on this server. + sds = [] for shnum,datav in datavs.items(): data = datav[0] - reader = MDMFSlotReadProxy(ss, + reader = MutableReadProxy(ss, storage_index, shnum, data) @@ -698,13 +697,6 @@ # consequence, so the first entry in our deferredlist will # be None. - # - Next, we need the version information. We almost - # certainly got this by reading the first thousand or so - # bytes of the share on the storage server, so we - # shouldn't need to fetch anything at this step. - d2 = reader.get_verinfo() - d2.addErrback(lambda error, shnum=shnum, data=data: - self._got_corrupt_share(error, shnum, server, data, lp)) # - Next, we need the signature. For an SDMF share, it is # likely that we fetched this when doing our initial fetch # to get the version information. In MDMF, this lives at @@ -727,39 +719,45 @@ else: d4 = defer.succeed(None) + # - Next, we need the version information. We almost + # certainly got this by reading the first thousand or so + # bytes of the share on the storage server, so we + # shouldn't need to fetch anything at this step. + d2 = reader.get_verinfo() + d2.addErrback(lambda error, shnum=shnum, data=data: + self._got_corrupt_share(error, shnum, server, data, lp)) if self.fetch_update_data: # fetch the block hash tree and first + last segment, as # configured earlier. # Then set them in wherever we happen to want to set # them. - ds = [] - # XXX: We do this above, too. Is there a good way to - # make the two routines share the value without - # introducing more roundtrips? - ds.append(reader.get_verinfo()) - ds.append(reader.get_blockhashes()) - ds.append(reader.get_block_and_salt(self.start_segment)) - ds.append(reader.get_block_and_salt(self.end_segment)) - d5 = deferredutil.gatherResults(ds) + # a list of deferreds, one for each of four parts of this share that we want to process + pds = [] + pds.append(d2) + pds.append(reader.get_blockhashes()) + pds.append(reader.get_block_and_salt(self.start_segment)) + pds.append(reader.get_block_and_salt(self.end_segment)) + d5 = deferredutil.gatherResults(pds) d5.addCallback(self._got_update_results_one_share, shnum) else: d5 = defer.succeed(None) - dl = defer.DeferredList([d, d2, d3, d4, d5]) - dl.addBoth(self._turn_barrier) - dl.addCallback(lambda results, shnum=shnum: + pdl = defer.DeferredList([d, d2, d3, d4, d5]) + pdl.addBoth(self._turn_barrier) + pdl.addCallback(lambda results, shnum=shnum: self._got_signature_one_share(results, shnum, server, lp)) - dl.addErrback(lambda error, shnum=shnum, data=data: - self._got_corrupt_share(error, shnum, server, data, lp)) - dl.addCallback(lambda verinfo, shnum=shnum, data=data: + pdl.addCallback(lambda verinfo, shnum=shnum, data=data: self._cache_good_sharedata(verinfo, shnum, now, data)) - ds.append(dl) - # dl is a deferred list that will fire when all of the shares - # that we found on this server are done processing. When dl fires, + pdl.addErrback(lambda error, shnum=shnum, data=data: + self._got_corrupt_share(error, shnum, server, data, lp)) + sds.append(pdl) + + # sdl is a deferred list that will fire when all of the shares + # that we found on this server are done processing. When sdl fires, # we know that processing is done, so we can decrement the # semaphore-like thing that we incremented earlier. - dl = defer.DeferredList(ds, fireOnOneErrback=True) + sdl = defer.DeferredList(sds, fireOnOneErrback=True) # Are we done? Done means that there are no more queries to # send, that there are no outstanding queries, and that we # haven't received any queries that are still processing. If we @@ -767,12 +765,12 @@ # that we returned to our caller to fire, which tells them that # they have a complete servermap, and that we won't be touching # the servermap anymore. - dl.addCallback(_done_processing) - dl.addCallback(self._check_for_done) - dl.addErrback(self._fatal_error) + sdl.addCallback(_done_processing) + sdl.addCallback(self._check_for_done) + sdl.addErrback(self._fatal_error) # all done! self.log("_got_results done", parent=lp, level=log.NOISY) - return dl + return sdl def _turn_barrier(self, result): @@ -816,7 +814,8 @@ self.log("but we're not running anymore.") return None - _, verinfo, signature, __, ___ = results + _, (__, verinfo), signature, ___, ____ = results + check_is_verinfo(verinfo) (seqnum, root_hash, saltish, @@ -825,8 +824,7 @@ k, n, prefix, - offsets) = verinfo[1] - offsets_tuple = tuple( [(key,value) for key,value in offsets.items()] ) + offsets) = verinfo # XXX: This should be done for us in the method, so # presumably you can go in there and fix it. @@ -838,7 +836,8 @@ k, n, prefix, - offsets_tuple) + offsets) + check_is_verinfo(verinfo) # This tuple uniquely identifies a share on the grid; we use it # to keep track of the ones that we've already seen. @@ -884,30 +883,9 @@ """ I record the update results in results. """ - assert len(results) == 4 + assert len(results) == 4, len(results) verinfo, blockhashes, start, end = results - (seqnum, - root_hash, - saltish, - segsize, - datalen, - k, - n, - prefix, - offsets) = verinfo - offsets_tuple = tuple( [(key,value) for key,value in offsets.items()] ) - - # XXX: This should be done for us in the method, so - # presumably you can go in there and fix it. - verinfo = (seqnum, - root_hash, - saltish, - segsize, - datalen, - k, - n, - prefix, - offsets_tuple) + check_is_verinfo(verinfo) update_data = (blockhashes, start, end) self._servermap.set_update_data_for_share_and_verinfo(share, diff -rN -u old-1.9.2/src/allmydata/scripts/debug.py new-1.9.2/src/allmydata/scripts/debug.py --- old-1.9.2/src/allmydata/scripts/debug.py 2012-06-21 00:35:33.549197850 -0300 +++ new-1.9.2/src/allmydata/scripts/debug.py 2012-06-21 00:35:33.712530547 -0300 @@ -306,7 +306,7 @@ print >>out def dump_MDMF_share(m, length, options): - from allmydata.mutable.layout import MDMFSlotReadProxy + from allmydata.mutable.layout import MutableReadProxy from allmydata.util import base32, hashutil from allmydata.uri import MDMFVerifierURI from allmydata.util.encodingutil import quote_output, to_str @@ -317,7 +317,7 @@ f = open(options['filename'], "rb") storage_index = None; shnum = 0 - class ShareDumper(MDMFSlotReadProxy): + class ShareDumper(MutableReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where,length) in readvs: @@ -755,10 +755,10 @@ seqnum, base32.b2a(root_hash), expiration, quote_output(abs_sharefile)) elif share_type == "MDMF": - from allmydata.mutable.layout import MDMFSlotReadProxy + from allmydata.mutable.layout import MutableReadProxy fake_shnum = 0 # TODO: factor this out with dump_MDMF_share() - class ShareDumper(MDMFSlotReadProxy): + class ShareDumper(MutableReadProxy): def _read(self, readvs, force_remote=False, queue=False): data = [] for (where,length) in readvs: diff -rN -u old-1.9.2/src/allmydata/test/test_checker.py new-1.9.2/src/allmydata/test/test_checker.py --- old-1.9.2/src/allmydata/test/test_checker.py 2012-06-21 00:35:33.562531132 -0300 +++ new-1.9.2/src/allmydata/test/test_checker.py 2012-06-21 00:35:33.715863868 -0300 @@ -405,7 +405,7 @@ d.addCallback(_stash_mutable) def _check_cr(cr, which): - self.failUnless(cr.is_healthy(), which) + self.failUnless(cr.is_healthy(), (cr, which)) # these two should work normally d.addCallback(lambda ign: self.imm.check(Monitor(), add_lease=True)) diff -rN -u old-1.9.2/src/allmydata/test/test_mutable.py new-1.9.2/src/allmydata/test/test_mutable.py --- old-1.9.2/src/allmydata/test/test_mutable.py 2012-06-21 00:35:33.582531053 -0300 +++ new-1.9.2/src/allmydata/test/test_mutable.py 2012-06-21 00:35:33.715863868 -0300 @@ -30,7 +30,7 @@ MutableData, \ DEFAULT_MAX_SEGMENT_SIZE from allmydata.mutable.servermap import ServerMap, ServermapUpdater -from allmydata.mutable.layout import unpack_header, MDMFSlotReadProxy +from allmydata.mutable.layout import unpack_header, MutableReadProxy from allmydata.mutable.repairer import MustForceRepairError import allmydata.test.common_util as testutil @@ -192,7 +192,7 @@ # won't need to use the rref that we didn't provide, nor the # storage index that we didn't provide. We do this because # the reader will work for both MDMF and SDMF. - reader = MDMFSlotReadProxy(None, None, shnum, data) + reader = MutableReadProxy(None, None, shnum, data) # We need to get the offsets for the next part. d = reader.get_verinfo() def _do_corruption(verinfo, data, shnum, shares): @@ -202,6 +202,7 @@ segsize, datalen, k, n, prefix, o) = verinfo + o = dict(o) if isinstance(offset, tuple): offset1, offset2 = offset else: @@ -1936,7 +1937,7 @@ initial_shares = self.old_shares[0] new_shares = self.old_shares[1] # TODO: this really shouldn't change anything. When we implement - # a "minimal-bandwidth" repairer", change this test to assert: + # a "minimal-bandwidth" repairer, change this test to assert: #self.failUnlessEqual(new_shares, initial_shares) # all shares should be in the same place as before @@ -2033,13 +2034,13 @@ def test_repairable_5shares(self): d = self.publish_mdmf() - def _delete_all_shares(ign): + def _delete_shares_after_first_5(ign): shares = self._storage._peers for peerid in shares: for shnum in list(shares[peerid]): if shnum > 4: del shares[peerid][shnum] - d.addCallback(_delete_all_shares) + d.addCallback(_delete_shares_after_first_5) d.addCallback(lambda ign: self._fn.check(Monitor())) d.addCallback(lambda check_results: self._fn.repair(check_results)) def _check(crr): diff -rN -u old-1.9.2/src/allmydata/test/test_storage.py new-1.9.2/src/allmydata/test/test_storage.py --- old-1.9.2/src/allmydata/test/test_storage.py 2012-06-21 00:35:33.589197694 -0300 +++ new-1.9.2/src/allmydata/test/test_storage.py 2012-06-21 00:35:33.719197188 -0300 @@ -20,7 +20,7 @@ from allmydata.storage.expirer import LeaseCheckingCrawler from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \ ReadBucketProxy -from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ +from allmydata.mutable.layout import MDMFSlotWriteProxy, MutableReadProxy, \ LayoutInvalid, MDMFSIGNABLEHEADER, \ SIGNED_PREFIX, MDMFHEADER, \ MDMFOFFSETS, SDMFSlotWriteProxy, \ @@ -1352,11 +1352,11 @@ self.failIf(os.path.exists(bucketdir), bucketdir) -class MDMFProxies(unittest.TestCase, ShouldFailMixin): +class MutableProxies(unittest.TestCase, ShouldFailMixin): def setUp(self): self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() - self.ss = self.create("MDMFProxies storage test server") + self.ss = self.create("MutableProxies storage test server") self.rref = RemoteBucket() self.rref.target = self.ss self.secrets = (self.write_enabler("we_secret"), @@ -1385,7 +1385,7 @@ def tearDown(self): self.sparent.stopService() - shutil.rmtree(self.workdir("MDMFProxies storage test server")) + shutil.rmtree(self.workdir("MutableProxies storage test server")) def write_enabler(self, we_tag): @@ -1601,7 +1601,7 @@ def test_read(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) # Check that every method equals what we expect it to. d = defer.succeed(None) def _check_block_and_salt((block, salt)): @@ -1671,7 +1671,7 @@ def test_read_with_different_tail_segment_size(self): self.write_test_share_to_server("si1", tail_segment=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = mr.get_block_and_salt(5) def _check_tail_segment(results): block, salt = results @@ -1683,7 +1683,7 @@ def test_get_block_with_invalid_segnum(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test invalid segnum", @@ -1694,7 +1694,7 @@ def test_get_encoding_parameters_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = mr.get_encoding_parameters() def _check_encoding_parameters((k, n, segment_size, datalen)): self.failUnlessEqual(k, 3) @@ -1707,7 +1707,7 @@ def test_get_seqnum_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = mr.get_seqnum() d.addCallback(lambda seqnum: self.failUnlessEqual(seqnum, 0)) @@ -1716,7 +1716,7 @@ def test_get_root_hash_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = mr.get_root_hash() d.addCallback(lambda root_hash: self.failUnlessEqual(root_hash, self.root_hash)) @@ -1725,7 +1725,7 @@ def test_get_checkstring_first(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = mr.get_checkstring() d.addCallback(lambda checkstring: self.failUnlessEqual(checkstring, self.checkstring)) @@ -2243,7 +2243,7 @@ d.addCallback(lambda ignored: mw.finish_publishing()) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) def _check_block_and_salt((block, salt)): self.failUnlessEqual(block, self.block) self.failUnlessEqual(salt, self.salt) @@ -2304,28 +2304,12 @@ return d - def test_is_sdmf(self): - # The MDMFSlotReadProxy should also know how to read SDMF files, - # since it will encounter them on the grid. Callers use the - # is_sdmf method to test this. - self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) - d = mr.is_sdmf() - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) - return d - - def test_reads_sdmf(self): # The slot read proxy should, naturally, know how to tell us # about data in the SDMF format self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = defer.succeed(None) - d.addCallback(lambda ignored: - mr.is_sdmf()) - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) # What do we need to read? # - The sharedata @@ -2391,13 +2375,9 @@ # read more segments than that. The reader should know this and # complain if we try to do that. self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: - mr.is_sdmf()) - d.addCallback(lambda issdmf: - self.failUnless(issdmf)) - d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test bad segment", None, mr.get_block_and_salt, 1)) @@ -2405,7 +2385,7 @@ def test_read_with_prefetched_mdmf_data(self): - # The MDMFSlotReadProxy will prefill certain fields if you pass + # The MutableReadProxy will prefill certain fields if you pass # it data that you have already fetched. This is useful for # cases like the Servermap, which prefetches ~2kb of data while # finding out which shares are on the remote peer so that it @@ -2413,7 +2393,7 @@ mdmf_data = self.build_test_mdmf_share() self.write_test_share_to_server("si1") def _make_mr(ignored, length): - mr = MDMFSlotReadProxy(self.rref, "si1", 0, mdmf_data[:length]) + mr = MutableReadProxy(self.rref, "si1", 0, mdmf_data[:length]) return mr d = defer.succeed(None) @@ -2473,7 +2453,7 @@ sdmf_data = self.build_test_sdmf_share() self.write_sdmf_share_to_server("si1") def _make_mr(ignored, length): - mr = MDMFSlotReadProxy(self.rref, "si1", 0, sdmf_data[:length]) + mr = MutableReadProxy(self.rref, "si1", 0, sdmf_data[:length]) return mr d = defer.succeed(None) @@ -2538,7 +2518,7 @@ # unrelated to the actual handling of the content of the file. # The reader should behave intelligently in these cases. self.write_test_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) # We should be able to get the encoding parameters, and they # should be correct. d = defer.succeed(None) @@ -2564,7 +2544,7 @@ def test_read_with_empty_sdmf_file(self): self.write_sdmf_share_to_server("si1", empty=True) - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) # We should be able to get the encoding parameters, and they # should be correct d = defer.succeed(None) @@ -2590,7 +2570,7 @@ def test_verinfo_with_sdmf_file(self): self.write_sdmf_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) # We should be able to get the version information. d = defer.succeed(None) d.addCallback(lambda ignored: @@ -2600,7 +2580,7 @@ self.failUnlessEqual(len(verinfo), 9) (seqnum, root_hash, - salt, + unused, segsize, datalen, k, @@ -2609,7 +2589,6 @@ offsets) = verinfo self.failUnlessEqual(seqnum, 0) self.failUnlessEqual(root_hash, self.root_hash) - self.failUnlessEqual(salt, self.salt) self.failUnlessEqual(segsize, 36) self.failUnlessEqual(datalen, 36) self.failUnlessEqual(k, 3) @@ -2618,20 +2597,20 @@ 0, seqnum, root_hash, - salt, + self.salt, k, n, segsize, datalen) self.failUnlessEqual(prefix, expected_prefix) - self.failUnlessEqual(offsets, self.offsets) + self.failUnlessEqual(offsets, tuple([(key, value) for key, value in self.offsets.items()])) d.addCallback(_check_verinfo) return d def test_verinfo_with_mdmf_file(self): self.write_test_share_to_server("si1") - mr = MDMFSlotReadProxy(self.rref, "si1", 0) + mr = MutableReadProxy(self.rref, "si1", 0) d = defer.succeed(None) d.addCallback(lambda ignored: mr.get_verinfo()) @@ -2663,7 +2642,7 @@ segsize, datalen) self.failUnlessEqual(prefix, expected_prefix) - self.failUnlessEqual(offsets, self.offsets) + self.failUnlessEqual(offsets, tuple([(key, value) for key, value in self.offsets.items()])) d.addCallback(_check_verinfo) return d diff -rN -u old-1.9.2/src/allmydata/util/hashutil.py new-1.9.2/src/allmydata/util/hashutil.py --- old-1.9.2/src/allmydata/util/hashutil.py 2012-06-21 00:35:33.605864296 -0300 +++ new-1.9.2/src/allmydata/util/hashutil.py 2012-06-21 00:35:33.722530508 -0300 @@ -209,3 +209,7 @@ BACKUPDB_DIRHASH_TAG = "allmydata_backupdb_dirhash_v1" def backupdb_dirhash(contents): return tagged_hash(BACKUPDB_DIRHASH_TAG, contents) + +def is_hash(x): + return isinstance(x, str) and len(x) == CRYPTO_VAL_SIZE +