Mon Feb 22 02:45:04 GMT Standard Time 2010 david-sarah@jacaranda.org * Change direct accesses to an_uri.storage_index to calls to .get_storage_index() (fixes #948) Mon Feb 22 02:53:52 GMT Standard Time 2010 david-sarah@jacaranda.org * Additions to test_web.py for #948 Mon Feb 22 03:36:52 GMT Standard Time 2010 david-sarah@jacaranda.org * Additional fix for abbrev_si, with test Wed Feb 24 04:33:45 GMT Standard Time 2010 david-sarah@jacaranda.org * dirnode: add tests of literal dirnodes (current and fix for #948) Wed Feb 24 07:54:33 GMT Standard Time 2010 david-sarah@jacaranda.org * directories: add DIR2-LIT directories to test_deepcheck.py (#948) Wed Feb 24 08:02:20 GMT Standard Time 2010 david-sarah@jacaranda.org * Additional fixes for DIR2-LIT More Info page and deep-check/manifest operations (#948) Wed Feb 24 08:15:42 GMT Standard Time 2010 david-sarah@jacaranda.org * Updates to NEWS for 1.6.1 Thu Feb 25 04:18:24 GMT Standard Time 2010 david-sarah@jacaranda.org * Additional test for DIR2-LIT directories in test_web.py, fixed version (#948) Sat Feb 27 06:55:51 GMT Standard Time 2010 david-sarah@jacaranda.org * Change code that gives a base32 SI or an empty string to be more straightforward. (#948) New patches: [Change direct accesses to an_uri.storage_index to calls to .get_storage_index() (fixes #948) david-sarah@jacaranda.org**20100222024504 Ignore-this: 91f6fccb5fd9456aa0e02d312f902928 ] { hunk ./src/allmydata/dirnode.py 349 return self._uri def get_storage_index(self): - return self._uri._filenode_uri.storage_index + return self._uri.get_storage_index() def check(self, monitor, verify=False, add_lease=False): """Perform a file check. See IChecker.check for details.""" hunk ./src/allmydata/immutable/checker.py 42 for (serverid, serverrref) in servers: assert precondition(isinstance(serverid, str)) - prefix = "%s" % base32.b2a_l(verifycap.storage_index[:8], 60) + prefix = "%s" % base32.b2a_l(verifycap.get_storage_index()[:8], 60) log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.checker", prefix=prefix) self._verifycap = verifycap hunk ./src/allmydata/immutable/checker.py 53 self._add_lease = add_lease frs = file_renewal_secret_hash(secret_holder.get_renewal_secret(), - self._verifycap.storage_index) + self._verifycap.get_storage_index()) self.file_renewal_secret = frs fcs = file_cancel_secret_hash(secret_holder.get_cancel_secret(), hunk ./src/allmydata/immutable/checker.py 56 - self._verifycap.storage_index) + self._verifycap.get_storage_index()) self.file_cancel_secret = fcs def _get_renewal_secret(self, peerid): hunk ./src/allmydata/immutable/checker.py 148 results.""" vcap = self._verifycap - b = layout.ReadBucketProxy(bucket, serverid, vcap.storage_index) + b = layout.ReadBucketProxy(bucket, serverid, vcap.get_storage_index()) veup = download.ValidatedExtendedURIProxy(b, vcap) d = veup.start() hunk ./src/allmydata/immutable/checker.py 252 then disconnected and ceased responding, or returned a failure, it is still marked with the True flag for 'success'. """ - d = self._get_buckets(ss, self._verifycap.storage_index, serverid) + d = self._get_buckets(ss, self._verifycap.get_storage_index(), serverid) def _got_buckets(result): bucketdict, serverid, success = result hunk ./src/allmydata/immutable/checker.py 299 def _curry_empty_corrupted(res): buckets, serverid, responded = res return (set(buckets), serverid, set(), set(), responded) - d = self._get_buckets(ss, self._verifycap.storage_index, serverid) + d = self._get_buckets(ss, self._verifycap.get_storage_index(), serverid) d.addCallback(_curry_empty_corrupted) return d hunk ./src/allmydata/immutable/checker.py 304 def _format_results(self, results): - cr = CheckResults(self._verifycap, self._verifycap.storage_index) + cr = CheckResults(self._verifycap, self._verifycap.get_storage_index()) d = {} d['count-shares-needed'] = self._verifycap.needed_shares d['count-shares-expected'] = self._verifycap.total_shares hunk ./src/allmydata/immutable/checker.py 319 for sharenum in theseverifiedshares: verifiedshares.setdefault(sharenum, set()).add(thisserverid) for sharenum in thesecorruptshares: - corruptsharelocators.append((thisserverid, self._verifycap.storage_index, sharenum)) + corruptsharelocators.append((thisserverid, self._verifycap.get_storage_index(), sharenum)) for sharenum in theseincompatibleshares: hunk ./src/allmydata/immutable/checker.py 321 - incompatiblesharelocators.append((thisserverid, self._verifycap.storage_index, sharenum)) + incompatiblesharelocators.append((thisserverid, self._verifycap.get_storage_index(), sharenum)) d['count-shares-good'] = len(verifiedshares) d['count-good-share-hosts'] = len([s for s in servers.keys() if servers[s]]) hunk ./src/allmydata/immutable/download.py 754 precondition(IVerifierURI.providedBy(v), v) precondition(IDownloadTarget.providedBy(target), target) - prefix=base32.b2a_l(v.storage_index[:8], 60) - log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.download", prefix=prefix) self._storage_broker = storage_broker hunk ./src/allmydata/immutable/download.py 755 - self._verifycap = v hunk ./src/allmydata/immutable/download.py 756 - self._storage_index = v.storage_index + self._storage_index = v.get_storage_index() self._uri_extension_hash = v.uri_extension_hash hunk ./src/allmydata/immutable/download.py 759 + prefix=base32.b2a_l(self._storage_index[:8], 60) + log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.download", prefix=prefix) + self._started = time.time() self._status = s = DownloadStatus() s.set_status("Starting") hunk ./src/allmydata/immutable/filenode.py 228 return self.u.to_string() def get_storage_index(self): - return self.u.storage_index + return self.u.get_storage_index() def check_and_repair(self, monitor, verify=False, add_lease=False): verifycap = self.get_verify_cap() hunk ./src/allmydata/immutable/filenode.py 241 monitor=monitor) d = c.start() def _maybe_repair(cr): - crr = CheckAndRepairResults(self.u.storage_index) + crr = CheckAndRepairResults(self.u.get_storage_index()) crr.pre_repair_results = cr if cr.is_healthy(): crr.post_repair_results = cr hunk ./src/allmydata/immutable/repairer.py 49 def __init__(self, storage_broker, secret_holder, verifycap, monitor): assert precondition(isinstance(verifycap, CHKFileVerifierURI)) - logprefix = si_b2a(verifycap.storage_index)[:5] + logprefix = si_b2a(verifycap.get_storage_index())[:5] log.PrefixingLogMixin.__init__(self, "allmydata.immutable.repairer", prefix=logprefix) hunk ./src/allmydata/scripts/debug.py 392 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash) print >>out, " size:", u.size print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares) - print >>out, " storage index:", si_b2a(u.storage_index) - _dump_secrets(u.storage_index, secret, nodeid, out) + print >>out, " storage index:", si_b2a(u.get_storage_index()) + _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.CHKFileVerifierURI): if show_header: print >>out, "CHK Verifier URI:" hunk ./src/allmydata/scripts/debug.py 400 print >>out, " UEB hash:", base32.b2a(u.uri_extension_hash) print >>out, " size:", u.size print >>out, " k/N: %d/%d" % (u.needed_shares, u.total_shares) - print >>out, " storage index:", si_b2a(u.storage_index) + print >>out, " storage index:", si_b2a(u.get_storage_index()) elif isinstance(u, uri.LiteralFileURI): if show_header: hunk ./src/allmydata/scripts/debug.py 412 print >>out, "SSK Writeable URI:" print >>out, " writekey:", base32.b2a(u.writekey) print >>out, " readkey:", base32.b2a(u.readkey) - print >>out, " storage index:", si_b2a(u.storage_index) + print >>out, " storage index:", si_b2a(u.get_storage_index()) print >>out, " fingerprint:", base32.b2a(u.fingerprint) print >>out if nodeid: hunk ./src/allmydata/scripts/debug.py 419 we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) print >>out, " write_enabler:", base32.b2a(we) print >>out - _dump_secrets(u.storage_index, secret, nodeid, out) + _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlySSKFileURI): if show_header: hunk ./src/allmydata/scripts/debug.py 425 print >>out, "SSK Read-only URI:" print >>out, " readkey:", base32.b2a(u.readkey) - print >>out, " storage index:", si_b2a(u.storage_index) + print >>out, " storage index:", si_b2a(u.get_storage_index()) print >>out, " fingerprint:", base32.b2a(u.fingerprint) elif isinstance(u, uri.SSKVerifierURI): if show_header: hunk ./src/allmydata/scripts/debug.py 430 print >>out, "SSK Verifier URI:" - print >>out, " storage index:", si_b2a(u.storage_index) + print >>out, " storage index:", si_b2a(u.get_storage_index()) print >>out, " fingerprint:", base32.b2a(u.fingerprint) elif isinstance(u, uri.DirectoryURI): hunk ./src/allmydata/test/common.py 48 bad_shares = {} def __init__(self, filecap): - precondition(isinstance(filecap, uri.CHKFileURI), filecap) + precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap) self.my_uri = filecap hunk ./src/allmydata/test/common.py 50 - self.storage_index = self.my_uri.storage_index + self.storage_index = self.my_uri.get_storage_index() def get_uri(self): return self.my_uri.to_string() hunk ./src/allmydata/test/common.py 193 assert isinstance(filecap, (uri.WriteableSSKFileURI, uri.ReadonlySSKFileURI)) self.my_uri = filecap - self.storage_index = self.my_uri.storage_index + self.storage_index = self.my_uri.get_storage_index() return self def get_cap(self): return self.my_uri hunk ./src/allmydata/test/common.py 1005 else: k = random.choice(ks) del shares[k] - self.replace_shares(shares, storage_index=self.uri.storage_index) + self.replace_shares(shares, storage_index=self.uri.get_storage_index()) return unused hunk ./src/allmydata/test/common.py 1015 assert ks, (shares.keys(), sharenum) k = ks[0] shares[k] = corruptor_func(shares[k]) - self.replace_shares(shares, storage_index=self.uri.storage_index) + self.replace_shares(shares, storage_index=self.uri.get_storage_index()) return corruptor_func def _corrupt_all_shares(self, unused, corruptor_func): hunk ./src/allmydata/test/test_checker.py 71 serverid_1 = "\x00"*20 serverid_f = "\xff"*20 u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234) - cr = check_results.CheckResults(u, u.storage_index) + cr = check_results.CheckResults(u, u.get_storage_index()) cr.set_healthy(True) cr.set_needs_rebalancing(False) cr.set_summary("groovy") hunk ./src/allmydata/test/test_checker.py 111 cr.set_healthy(False) cr.set_recoverable(False) cr.set_summary("rather dead") - data["list-corrupt-shares"] = [(serverid_1, u.storage_index, 2)] + data["list-corrupt-shares"] = [(serverid_1, u.get_storage_index(), 2)] cr.set_data(data) html = self.render2(w) s = self.remove_tags(html) hunk ./src/allmydata/test/test_checker.py 170 serverid_f = "\xff"*20 u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234) - pre_cr = check_results.CheckResults(u, u.storage_index) + pre_cr = check_results.CheckResults(u, u.get_storage_index()) pre_cr.set_healthy(False) pre_cr.set_recoverable(True) pre_cr.set_needs_rebalancing(False) hunk ./src/allmydata/test/test_checker.py 188 } pre_cr.set_data(data) - post_cr = check_results.CheckResults(u, u.storage_index) + post_cr = check_results.CheckResults(u, u.get_storage_index()) post_cr.set_healthy(True) post_cr.set_recoverable(True) post_cr.set_needs_rebalancing(False) hunk ./src/allmydata/test/test_checker.py 206 } post_cr.set_data(data) - crr = check_results.CheckAndRepairResults(u.storage_index) + crr = check_results.CheckAndRepairResults(u.get_storage_index()) crr.pre_repair_results = pre_cr crr.post_repair_results = post_cr crr.repair_attempted = False hunk ./src/allmydata/test/test_dirnode.py 552 self.expected_verifycaps.add(u_v) si = n.get_storage_index() self.expected_storage_indexes.add(base32.b2a(si)) - expected_si = n._uri._filenode_uri.storage_index + expected_si = n._uri.get_storage_index() self.failUnlessEqual(si, expected_si) d = n.list() hunk ./src/allmydata/test/test_filenode.py 49 self.failUnlessEqual(fn1.get_write_uri(), None) self.failUnlessEqual(fn1.get_readonly_uri(), u.to_string()) self.failUnlessEqual(fn1.get_size(), 1000) - self.failUnlessEqual(fn1.get_storage_index(), u.storage_index) + self.failUnlessEqual(fn1.get_storage_index(), u.get_storage_index()) fn1.raise_error() fn2.raise_error() d = {} hunk ./src/allmydata/test/test_immutable.py 24 # The following process of deleting 8 of the shares and asserting that you can't # download it is more to test this test code than to test the Tahoe code... def _then_delete_8(unused=None): - self.replace_shares(stash[0], storage_index=self.uri.storage_index) + self.replace_shares(stash[0], storage_index=self.uri.get_storage_index()) for i in range(8): self._delete_a_share() d.addCallback(_then_delete_8) hunk ./src/allmydata/test/test_mutable.py 1926 privkey_s = privkey.serialize() u = uri.WriteableSSKFileURI(ssk_writekey_hash(privkey_s), ssk_pubkey_fingerprint_hash(pubkey_s)) - self._storage_index = u.storage_index + self._storage_index = u.get_storage_index() d.addCallback(_got_key) def _break_peer0(res): si = self._storage_index hunk ./src/allmydata/test/test_repairer.py 660 return res d.addCallback(_stash_it) def _put_it_all_back(ignored): - self.replace_shares(stash[0], storage_index=self.uri.storage_index) + self.replace_shares(stash[0], storage_index=self.uri.get_storage_index()) return ignored def _repair_from_corruption(shnum, corruptor_func): hunk ./src/allmydata/test/test_upload.py 260 def _check_large(self, newuri, size): u = uri.from_string(newuri) self.failUnless(isinstance(u, uri.CHKFileURI)) - self.failUnless(isinstance(u.storage_index, str)) - self.failUnlessEqual(len(u.storage_index), 16) + self.failUnless(isinstance(u.get_storage_index(), str)) + self.failUnlessEqual(len(u.get_storage_index()), 16) self.failUnless(isinstance(u.key, str)) self.failUnlessEqual(len(u.key), 16) self.failUnlessEqual(u.size, size) hunk ./src/allmydata/test/test_upload.py 385 def _check_large(self, newuri, size): u = uri.from_string(newuri) self.failUnless(isinstance(u, uri.CHKFileURI)) - self.failUnless(isinstance(u.storage_index, str)) - self.failUnlessEqual(len(u.storage_index), 16) + self.failUnless(isinstance(u.get_storage_index(), str)) + self.failUnlessEqual(len(u.get_storage_index()), 16) self.failUnless(isinstance(u.key, str)) self.failUnlessEqual(len(u.key), 16) self.failUnlessEqual(u.size, size) hunk ./src/allmydata/test/test_upload.py 473 def _check_large(self, newuri, size): u = uri.from_string(newuri) self.failUnless(isinstance(u, uri.CHKFileURI)) - self.failUnless(isinstance(u.storage_index, str)) - self.failUnlessEqual(len(u.storage_index), 16) + self.failUnless(isinstance(u.get_storage_index(), str)) + self.failUnlessEqual(len(u.get_storage_index()), 16) self.failUnless(isinstance(u.key, str)) self.failUnlessEqual(len(u.key), 16) self.failUnlessEqual(u.size, size) hunk ./src/allmydata/test/test_uri.py 105 needed_shares=needed_shares, total_shares=total_shares, size=size) - self.failUnlessEqual(u.storage_index, storage_index) + self.failUnlessEqual(u.get_storage_index(), storage_index) self.failUnlessEqual(u.key, key) self.failUnlessEqual(u.uri_extension_hash, uri_extension_hash) self.failUnlessEqual(u.needed_shares, needed_shares) hunk ./src/allmydata/test/test_uri.py 125 self.failUnlessEqual(uri.CHKFileURI.init_from_human_encoding(he), u) u2 = uri.from_string(u.to_string()) - self.failUnlessEqual(u2.storage_index, storage_index) + self.failUnlessEqual(u2.get_storage_index(), storage_index) self.failUnlessEqual(u2.key, key) self.failUnlessEqual(u2.uri_extension_hash, uri_extension_hash) self.failUnlessEqual(u2.needed_shares, needed_shares) hunk ./src/allmydata/test/test_uri.py 322 u5 = u4.get_verify_cap() self.failUnless(IVerifierURI.providedBy(u5)) - self.failUnlessEqual(u5.storage_index, u.storage_index) + self.failUnlessEqual(u5.get_storage_index(), u.get_storage_index()) u7 = u.get_verify_cap() self.failUnless(IVerifierURI.providedBy(u7)) hunk ./src/allmydata/test/test_uri.py 325 - self.failUnlessEqual(u7.storage_index, u.storage_index) + self.failUnlessEqual(u7.get_storage_index(), u.get_storage_index()) he = u5.to_human_encoding() u5_h = uri.SSKVerifierURI.init_from_human_encoding(he) hunk ./src/allmydata/test/test_web.py 1532 self.failUnless(filecap.startswith("URI:SSK:"), filecap) self.filecap = filecap u = uri.WriteableSSKFileURI.init_from_string(filecap) - self.failUnless(u.storage_index in FakeMutableFileNode.all_contents) + self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents) n = self.s.create_node_from_uri(filecap) return n.download_best_version() d.addCallback(_check) hunk ./src/allmydata/test/test_web.py 2784 self.failUnless(filecap.startswith("URI:SSK:"), filecap) self.filecap = filecap u = uri.WriteableSSKFileURI.init_from_string(filecap) - self.failUnless(u.storage_index in FakeMutableFileNode.all_contents) + self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents) n = self.s.create_node_from_uri(filecap) return n.download_best_version() d.addCallback(_check1) hunk ./src/allmydata/uri.py 437 return self._filenode_uri.to_string().split(':')[2][:5] def abbrev_si(self): - return base32.b2a(self._filenode_uri.storage_index)[:5] + return base32.b2a(self._filenode_uri.get_storage_index())[:5] def is_mutable(self): return True } [Additions to test_web.py for #948 david-sarah@jacaranda.org**20100222025352 Ignore-this: b99be703923efc75db75894a05e6a527 ] { hunk ./src/allmydata/test/test_web.py 1213 d2.addCallback(lambda ign: self.failUnlessRWChildURIIs(n, u"dirchild", caps['dircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-lit", + caps['litdircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-empty", + caps['emptydircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: hunk ./src/allmydata/test/test_web.py 1245 d2.addCallback(lambda ign: self.failUnlessROChildURIIs(n, u"dirchild-imm", caps['immdircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-lit", + caps['litdircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-empty", + caps['emptydircap'])) return d2 d.addCallback(_check) d.addCallback(lambda res: hunk ./src/allmydata/test/test_web.py 1263 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap']) d.addErrback(self.explain_web_error) return d hunk ./src/allmydata/test/test_web.py 2086 d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap']) d.addCallback(lambda res: self._foo_node.get(u"newdir")) d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap']) + d.addCallback(lambda res: self._foo_node.get(u"newdir")) + d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap']) return d def test_POST_mkdir_immutable_bad(self): hunk ./src/allmydata/test/test_web.py 2172 unknown_immcap = "imm.lafs://immutable_from_the_future" node4 = self.s.create_node_from_uri(make_mutable_file_uri()) dircap = DirectoryNode(node4, None, None).get_uri() + litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm" + emptydircap = "URI:DIR2-LIT:" newkids = {u"child-imm": ["filenode", {"rw_uri": filecap1, "ro_uri": self._make_readonly(filecap1), "metadata": md1, }], hunk ./src/allmydata/test/test_web.py 2186 u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], u"dirchild": ["dirnode", {"rw_uri": dircap, "ro_uri": self._make_readonly(dircap)}], + u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}], + u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}], } return newkids, {'filecap1': filecap1, 'filecap2': filecap2, hunk ./src/allmydata/test/test_web.py 2195 'unknown_rwcap': unknown_rwcap, 'unknown_rocap': unknown_rocap, 'unknown_immcap': unknown_immcap, - 'dircap': dircap} + 'dircap': dircap, + 'litdircap': litdircap, + 'emptydircap': emptydircap} def _create_immutable_children(self): contents, n, filecap1 = self.makefile(12) hunk ./src/allmydata/test/test_web.py 2207 assert not dnode.is_mutable() unknown_immcap = "imm.lafs://immutable_from_the_future" immdircap = dnode.get_uri() + litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm" + emptydircap = "URI:DIR2-LIT:" newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1, "metadata": md1, }], u"unknownchild-imm": ["unknown", {"ro_uri": unknown_immcap}], hunk ./src/allmydata/test/test_web.py 2213 u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}], + u"dirchild-lit": ["dirnode", {"ro_uri": litdircap}], + u"dirchild-empty": ["dirnode", {"ro_uri": emptydircap}], } return newkids, {'filecap1': filecap1, 'unknown_immcap': unknown_immcap, hunk ./src/allmydata/test/test_web.py 2218 - 'immdircap': immdircap} + 'immdircap': immdircap, + 'litdircap': litdircap, + 'emptydircap': emptydircap} def test_POST_mkdir_no_parentdir_initial_children(self): (newkids, caps) = self._create_initial_children() hunk ./src/allmydata/test/test_web.py 2289 d2.addCallback(lambda ign: self.failUnlessROChildURIIs(n, u"dirchild-imm", caps['immdircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-lit", + caps['litdircap'])) + d2.addCallback(lambda ign: + self.failUnlessROChildURIIs(n, u"dirchild-empty", + caps['emptydircap'])) return d2 d.addCallback(_after_mkdir) return d } [Additional fix for abbrev_si, with test david-sarah@jacaranda.org**20100222033652 Ignore-this: 7dc1c7031cd395fb4ec0a5aa96e69a10 ] { hunk ./src/allmydata/test/test_uri.py 473 self.failIf(IFileURI.providedBy(u1)) self.failUnless(IDirnodeURI.providedBy(u1)) self.failUnlessEqual(u1.get_verify_cap(), None) + self.failUnlessEqual(u1.get_storage_index(), None) + self.failUnlessEqual(u1.abbrev_si(), "") hunk ./src/allmydata/uri.py 437 return self._filenode_uri.to_string().split(':')[2][:5] def abbrev_si(self): - return base32.b2a(self._filenode_uri.get_storage_index())[:5] + si = self._filenode_uri.get_storage_index() + if si is None: + return "" + return base32.b2a(si)[:5] def is_mutable(self): return True } [dirnode: add tests of literal dirnodes (current and fix for #948) david-sarah@jacaranda.org**20100224043345 Ignore-this: f18cd17d72ed2495a646fa6c3af42aa1 ] { hunk ./src/allmydata/test/test_dirnode.py 6 from zope.interface import implements from twisted.trial import unittest from twisted.internet import defer +from twisted.internet.interfaces import IConsumer from allmydata import uri, dirnode from allmydata.client import Client from allmydata.immutable import upload hunk ./src/allmydata/test/test_dirnode.py 27 from base64 import b32decode import common_util as testutil +class MemAccum: + implements(IConsumer) + def registerProducer(self, producer, streaming): + self.producer = producer + self.producer.resumeProducing() + pass + def unregisterProducer(self): + pass + def write(self, data): + assert not hasattr(self, 'data') + self.data = data + self.producer.resumeProducing() + +setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" +one_uri = "URI:LIT:n5xgk" # LIT for "one" +mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" +empty_litdir_uri = "URI:DIR2-LIT:" +tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT +mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" +future_write_uri = "x-tahoe-crazy://I_am_from_the_future." +future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." + class Dirnode(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin, ErrorMixin): timeout = 240 # It takes longer than 120 seconds on Francois's arm box. hunk ./src/allmydata/test/test_dirnode.py 76 c = self.g.clients[0] nm = c.nodemaker - setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" - one_uri = "URI:LIT:n5xgk" # LIT for "one" - mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" - mut_read_uri = "URI:SSK-RO:jf6wkflosyvntwxqcdo7a54jvm:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" - future_write_uri = "x-tahoe-crazy://I_am_from_the_future." - future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), hunk ./src/allmydata/test/test_dirnode.py 82 u"mut": (nm.create_from_cap(mut_write_uri, mut_read_uri), {}), u"fut": (nm.create_from_cap(future_write_uri, future_read_uri), {}), u"fro": (nm.create_from_cap(None, future_read_uri), {}), + u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}), + u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}), } d = c.create_dirnode(kids) hunk ./src/allmydata/test/test_dirnode.py 100 d.addCallback(_created) def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), - [u"fro", u"fut", u"mut", u"one", u"two"]) + self.failUnlessEqual(set(children.keys()), + set([u"one", u"two", u"mut", u"fut", u"fro", u"empty_litdir", u"tiny_litdir"])) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] mut_node, mut_metadata = children[u"mut"] hunk ./src/allmydata/test/test_dirnode.py 107 fut_node, fut_metadata = children[u"fut"] fro_node, fro_metadata = children[u"fro"] + emptylit_node, emptylit_metadata = children[u"empty_litdir"] + tinylit_node, tinylit_metadata = children[u"tiny_litdir"] self.failUnlessEqual(one_node.get_size(), 3) self.failUnlessEqual(one_node.get_uri(), one_uri) hunk ./src/allmydata/test/test_dirnode.py 133 self.failUnlessEqual(fro_node.get_uri(), "ro." + future_read_uri) self.failUnlessEqual(fut_node.get_readonly_uri(), "ro." + future_read_uri) self.failUnless(isinstance(fro_metadata, dict), fro_metadata) + + self.failIf(emptylit_node.is_unknown()) + self.failUnlessEqual(emptylit_node.get_storage_index(), None) + self.failIf(tinylit_node.is_unknown()) + self.failUnlessEqual(tinylit_node.get_storage_index(), None) + + d2 = defer.succeed(None) + d2.addCallback(lambda ignored: emptylit_node.list()) + d2.addCallback(lambda children: self.failUnlessEqual(children, {})) + d2.addCallback(lambda ignored: tinylit_node.list()) + d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()), + set([u"short"]))) + d2.addCallback(lambda ignored: tinylit_node.list()) + d2.addCallback(lambda children: children[u"short"][0].read(MemAccum())) + d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end.")) + return d2 + d.addCallback(_check_kids) d.addCallback(lambda ign: nm.create_new_mutable_directory(kids)) hunk ./src/allmydata/test/test_dirnode.py 179 c = self.g.clients[0] nm = c.nodemaker - setup_py_uri = "URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861" - one_uri = "URI:LIT:n5xgk" # LIT for "one" - mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" - mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" - future_write_uri = "x-tahoe-crazy://I_am_from_the_future." - future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future." kids = {u"one": (nm.create_from_cap(one_uri), {}), u"two": (nm.create_from_cap(setup_py_uri), {"metakey": "metavalue"}), hunk ./src/allmydata/test/test_dirnode.py 183 u"fut": (nm.create_from_cap(None, future_read_uri), {}), + u"empty_litdir": (nm.create_from_cap(empty_litdir_uri), {}), + u"tiny_litdir": (nm.create_from_cap(tiny_litdir_uri), {}), } d = c.create_immutable_dirnode(kids) hunk ./src/allmydata/test/test_dirnode.py 204 d.addCallback(_created) def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"fut", u"one", u"two"]) + self.failUnlessEqual(set(children.keys()), + set([u"one", u"two", u"fut", u"empty_litdir", u"tiny_litdir"])) one_node, one_metadata = children[u"one"] two_node, two_metadata = children[u"two"] fut_node, fut_metadata = children[u"fut"] hunk ./src/allmydata/test/test_dirnode.py 209 + emptylit_node, emptylit_metadata = children[u"empty_litdir"] + tinylit_node, tinylit_metadata = children[u"tiny_litdir"] self.failUnlessEqual(one_node.get_size(), 3) self.failUnlessEqual(one_node.get_uri(), one_uri) hunk ./src/allmydata/test/test_dirnode.py 226 self.failUnlessEqual(fut_node.get_uri(), "imm." + future_read_uri) self.failUnlessEqual(fut_node.get_readonly_uri(), "imm." + future_read_uri) self.failUnless(isinstance(fut_metadata, dict), fut_metadata) + + self.failIf(emptylit_node.is_unknown()) + self.failUnlessEqual(emptylit_node.get_storage_index(), None) + self.failIf(tinylit_node.is_unknown()) + self.failUnlessEqual(tinylit_node.get_storage_index(), None) + + d2 = defer.succeed(None) + d2.addCallback(lambda ignored: emptylit_node.list()) + d2.addCallback(lambda children: self.failUnlessEqual(children, {})) + d2.addCallback(lambda ignored: tinylit_node.list()) + d2.addCallback(lambda children: self.failUnlessEqual(set(children.keys()), + set([u"short"]))) + d2.addCallback(lambda ignored: tinylit_node.list()) + d2.addCallback(lambda children: children[u"short"][0].read(MemAccum())) + d2.addCallback(lambda accum: self.failUnlessEqual(accum.data, "The end.")) + return d2 + d.addCallback(_check_kids) d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string())) hunk ./src/allmydata/test/test_dirnode.py 399 # should have been stripped (and "ro." should have been prepended to the # ro_uri, since it's unknown). def _check_kids(children): - self.failUnlessEqual(sorted(children.keys()), [u"child"]) + self.failUnlessEqual(set(children.keys()), set([u"child"])) child_node, child_metadata = children[u"child"] self.failUnlessEqual(child_node.get_write_uri(), stripped_write_uri) hunk ./src/allmydata/test/test_dirnode.py 656 d.addCallback(lambda res: n.list()) d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted([u"child", u"subdir"]))) + self.failUnlessEqual(set(children.keys()), + set([u"child", u"subdir"]))) d.addCallback(lambda res: n.start_deep_stats().when_done()) def _check_deepstats(stats): hunk ./src/allmydata/test/test_dirnode.py 729 child, metadata = res self.failUnless(isinstance(child, dirnode.DirectoryNode)) # edge-metadata needs at least one path segment - self.failUnlessEqual(sorted(metadata.keys()), []) + self.failUnlessEqual(set(metadata.keys()), set([])) d.addCallback(_check_child_and_metadata1) d.addCallback(lambda res: n.get_child_and_metadata_at_path(u"child")) hunk ./src/allmydata/test/test_dirnode.py 965 d.addCallback(lambda res: n.list()) d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted([u"child"]))) + self.failUnlessEqual(set(children.keys()), + set([u"child"]))) uploadable1 = upload.Data("some data", convergence="converge") d.addCallback(lambda res: n.add_file(u"newfile", uploadable1)) hunk ./src/allmydata/test/test_dirnode.py 981 overwrite=False)) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted([u"child", u"newfile"]))) + self.failUnlessEqual(set(children.keys()), + set([u"child", u"newfile"]))) d.addCallback(lambda res: n.get_metadata_for(u"newfile")) d.addCallback(lambda metadata: self.failUnlessEqual(set(metadata.keys()), hunk ./src/allmydata/test/test_dirnode.py 1012 n.move_child_to(u"child", self.subdir2)) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted([u"newfile", u"subdir2"]))) + self.failUnlessEqual(set(children.keys()), + set([u"newfile", u"subdir2"]))) d.addCallback(lambda res: self.subdir2.list()) d.addCallback(lambda children: hunk ./src/allmydata/test/test_dirnode.py 1016 - self.failUnlessEqual(sorted(children.keys()), - sorted([u"child"]))) + self.failUnlessEqual(set(children.keys()), + set([u"child"]))) d.addCallback(lambda res: self.subdir2.get(u"child")) d.addCallback(lambda child: self.failUnlessEqual(child.get_uri(), hunk ./src/allmydata/test/test_dirnode.py 1028 self.subdir2.move_child_to(u"child", n, u"newchild")) d.addCallback(lambda res: n.list()) d.addCallback(lambda children: - self.failUnlessEqual(sorted(children.keys()), - sorted([u"newchild", u"newfile", - u"subdir2"]))) + self.failUnlessEqual(set(children.keys()), + set([u"newchild", u"newfile", + u"subdir2"]))) d.addCallback(lambda res: self.subdir2.list()) d.addCallback(lambda children: hunk ./src/allmydata/test/test_dirnode.py 1033 - self.failUnlessEqual(sorted(children.keys()), [])) + self.failUnlessEqual(set(children.keys()), set([]))) # now make sure that we honor overwrite=False d.addCallback(lambda res: hunk ./src/allmydata/test/test_dirnode.py 1083 return d d.addCallback(_check) def _check_kids(kids2): - self.failUnlessEqual(sorted(kids.keys()), sorted(kids2.keys())) + self.failUnlessEqual(set(kids.keys()), set(kids2.keys())) self.failUnlessEqual(kids2[u"kid2"][1]["metakey"], "metavalue") d.addCallback(_check_kids) return d } [directories: add DIR2-LIT directories to test_deepcheck.py (#948) david-sarah@jacaranda.org**20100224075433 Ignore-this: ed1dcbe45870f5efae0ebbcdff677a4b ] { hunk ./src/allmydata/test/test_deepcheck.py 149 if not unit: # stream should end with a newline, so split returns "" continue - yield simplejson.loads(unit) + try: + yield simplejson.loads(unit) + except ValueError, le: + le.args = tuple(le.args + (unit,)) + raise def web(self, n, method="GET", **kwargs): # returns (data, url) hunk ./src/allmydata/test/test_deepcheck.py 207 class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): # construct a small directory tree (with one dir, one immutable file, one - # mutable file, one LIT file, and a loop), and then check/examine it in - # various ways. + # mutable file, two LIT files, one DIR2:LIT empty dir, one DIR2:LIT tiny + # dir, and a loop), and then check/examine it in various ways. def set_up_tree(self): # 2.9s hunk ./src/allmydata/test/test_deepcheck.py 213 - # root - # mutable - # large - # small - # small2 - # loop -> root c0 = self.g.clients[0] d = c0.create_dirnode() def _created_root(n): hunk ./src/allmydata/test/test_deepcheck.py 247 self.small2_uri = n.get_uri() d.addCallback(_created_small2) + empty_litdir_uri = "URI:DIR2-LIT:" + tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir")) + def _created_empty_lit_dir(n): + self.empty_lit_dir = n + self.empty_lit_dir_uri = n.get_uri() + self.root.set_node(u"empty_lit_dir", n) + d.addCallback(_created_empty_lit_dir) + + d.addCallback(lambda ign: self.root._create_and_validate_node(None, tiny_litdir_uri, name=u"test_deepcheck tiny_lit_dir")) + def _created_tiny_lit_dir(n): + self.tiny_lit_dir = n + self.tiny_lit_dir_uri = n.get_uri() + self.root.set_node(u"tiny_lit_dir", n) + d.addCallback(_created_tiny_lit_dir) + d.addCallback(lambda ign: self.root.set_node(u"loop", self.root)) return d hunk ./src/allmydata/test/test_deepcheck.py 348 return d def check_stats_good(self, s): - self.failUnlessEqual(s["count-directories"], 1) - self.failUnlessEqual(s["count-files"], 4) + self.failUnlessEqual(s["count-directories"], 3) + self.failUnlessEqual(s["count-files"], 5) self.failUnlessEqual(s["count-immutable-files"], 1) hunk ./src/allmydata/test/test_deepcheck.py 351 - self.failUnlessEqual(s["count-literal-files"], 2) + self.failUnlessEqual(s["count-literal-files"], 3) self.failUnlessEqual(s["count-mutable-files"], 1) # don't check directories: their size will vary # s["largest-directory"] hunk ./src/allmydata/test/test_deepcheck.py 356 # s["size-directories"] - self.failUnlessEqual(s["largest-directory-children"], 5) + self.failUnlessEqual(s["largest-directory-children"], 7) self.failUnlessEqual(s["largest-immutable-file"], 13000) # to re-use this function for both the local # dirnode.start_deep_stats() and the webapi t=start-deep-stats, we hunk ./src/allmydata/test/test_deepcheck.py 364 # returns a list of tuples, but JSON only knows about lists., so # t=start-deep-stats returns a list of lists. histogram = [tuple(stuff) for stuff in s["size-files-histogram"]] - self.failUnlessEqual(histogram, [(11, 31, 2), + self.failUnlessEqual(histogram, [(4, 10, 1), (11, 31, 2), (10001, 31622, 1), ]) self.failUnlessEqual(s["size-immutable-files"], 13000) hunk ./src/allmydata/test/test_deepcheck.py 368 - self.failUnlessEqual(s["size-literal-files"], 48) + self.failUnlessEqual(s["size-literal-files"], 56) def do_web_stream_manifest(self, ignored): d = self.web(self.root, method="POST", t="stream-manifest") hunk ./src/allmydata/test/test_deepcheck.py 381 files = [u for u in units if u["type"] in ("file", "directory")] assert units[-1]["type"] == "stats" stats = units[-1]["stats"] - self.failUnlessEqual(len(files), 5) - # [root,mutable,large] are distributed, [small,small2] are not + self.failUnlessEqual(len(files), 8) + # [root,mutable,large] are distributed, [small,small2,empty_litdir,tiny_litdir] are not self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 384 - if f["verifycap"] is not None]), 3) + if f["verifycap"] != ""]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 386 - if f["verifycap"] is None]), 2) + if f["verifycap"] == ""]), 5) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 388 - if f["repaircap"] is not None]), 3) + if f["repaircap"] != ""]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 390 - if f["repaircap"] is None]), 2) + if f["repaircap"] == ""]), 5) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 392 - if f["storage-index"] is not None]), 3) + if f["storage-index"] != ""]), 3) self.failUnlessEqual(len([f for f in files hunk ./src/allmydata/test/test_deepcheck.py 394 - if f["storage-index"] is None]), 2) + if f["storage-index"] == ""]), 5) # make sure that a mutable file has filecap==repaircap!=verifycap mutable = [f for f in files if f["cap"] is not None hunk ./src/allmydata/test/test_deepcheck.py 437 d.addCallback(self.failUnlessEqual, None, "small") d.addCallback(lambda ign: self.small2.check(Monitor())) d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor())) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor())) + d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir") # and again with verify=True d.addCallback(lambda ign: self.root.check(Monitor(), verify=True)) hunk ./src/allmydata/test/test_deepcheck.py 453 d.addCallback(self.failUnlessEqual, None, "small") d.addCallback(lambda ign: self.small2.check(Monitor(), verify=True)) d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "tiny_lit_dir") # and check_and_repair(), which should be a nop d.addCallback(lambda ign: self.root.check_and_repair(Monitor())) hunk ./src/allmydata/test/test_deepcheck.py 463 d.addCallback(self.check_and_repair_is_healthy, self.root, "root") d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor())) d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable") - #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor())) - #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large") - #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor())) - #TODO d.addCallback(self.failUnlessEqual, None, "small") - #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor())) - #TODO d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.large.check_and_repair(Monitor())) + d.addCallback(self.check_and_repair_is_healthy, self.large, "large") + d.addCallback(lambda ign: self.small.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "small") + d.addCallback(lambda ign: self.small2.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor())) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor())) # check_and_repair(verify=True) d.addCallback(lambda ign: self.root.check_and_repair(Monitor(), verify=True)) hunk ./src/allmydata/test/test_deepcheck.py 478 d.addCallback(self.check_and_repair_is_healthy, self.root, "root") d.addCallback(lambda ign: self.mutable.check_and_repair(Monitor(), verify=True)) d.addCallback(self.check_and_repair_is_healthy, self.mutable, "mutable") - #TODO d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.check_and_repair_is_healthy, self.large, "large", - #TODO incomplete=True) - #TODO d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.failUnlessEqual, None, "small") - #TODO d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True)) - #TODO d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.large.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.check_and_repair_is_healthy, self.large, "large", incomplete=True) + d.addCallback(lambda ign: self.small.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "small") + d.addCallback(lambda ign: self.small2.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(self.failUnlessEqual, None, "small2") + d.addCallback(lambda ign: self.empty_lit_dir.check_and_repair(Monitor(), verify=True)) + d.addCallback(self.failUnlessEqual, None, "empty_lit_dir") + d.addCallback(lambda ign: self.tiny_lit_dir.check_and_repair(Monitor(), verify=True)) # now deep-check the root, with various verify= and repair= options hunk ./src/allmydata/test/test_deepcheck.py 631 d.addCallback(self.json_check_lit, self.small, "small") d.addCallback(lambda ign: self.web_json(self.small2, t="check")) d.addCallback(self.json_check_lit, self.small2, "small2") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir") # check and verify d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 653 d.addCallback(lambda ign: self.web_json(self.small2, t="check", verify="true")) d.addCallback(self.json_check_lit, self.small2, "small2+v") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", verify="true")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+v") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", verify="true")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+v") # check and repair, no verify d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 674 d.addCallback(lambda ign: self.web_json(self.small2, t="check", repair="true")) d.addCallback(self.json_check_lit, self.small2, "small2+r") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true")) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+r") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true")) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+r") # check+verify+repair d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 695 d.addCallback(lambda ign: self.web_json(self.small2, t="check", repair="true", verify="true")) d.addCallback(self.json_check_lit, self.small2, "small2+vr") + d.addCallback(lambda ign: self.web_json(self.empty_lit_dir, t="check", repair="true", verify=True)) + d.addCallback(self.json_check_lit, self.empty_lit_dir, "empty_lit_dir+vr") + d.addCallback(lambda ign: self.web_json(self.tiny_lit_dir, t="check", repair="true", verify=True)) + d.addCallback(self.json_check_lit, self.tiny_lit_dir, "tiny_lit_dir+vr") # now run a deep-check, with various verify= and repair= flags d.addCallback(lambda ign: hunk ./src/allmydata/test/test_deepcheck.py 723 d.addCallback(lambda ign: self.web(self.large, t="info")) d.addCallback(lambda ign: self.web(self.small, t="info")) d.addCallback(lambda ign: self.web(self.small2, t="info")) + d.addCallback(lambda ign: self.web(self.empty_lit_dir, t="info")) + d.addCallback(lambda ign: self.web(self.tiny_lit_dir, t="info")) return d hunk ./src/allmydata/test/test_deepcheck.py 765 def _check((out,err)): self.failUnlessEqual(err, "") lines = [l for l in out.split("\n") if l] - self.failUnlessEqual(len(lines), 5) + self.failUnlessEqual(len(lines), 8) caps = {} for l in lines: try: hunk ./src/allmydata/test/test_deepcheck.py 780 self.failUnlessEqual(caps[self.large.get_uri()], "large") self.failUnlessEqual(caps[self.small.get_uri()], "small") self.failUnlessEqual(caps[self.small2.get_uri()], "small2") + self.failUnlessEqual(caps[self.empty_lit_dir.get_uri()], "empty_lit_dir") + self.failUnlessEqual(caps[self.tiny_lit_dir.get_uri()], "tiny_lit_dir") d.addCallback(_check) return d hunk ./src/allmydata/test/test_deepcheck.py 851 lines = [l.strip() for l in out.split("\n") if l] self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines) - self.failUnless("count-literal-files: 2" in lines) - self.failUnless("count-files: 4" in lines) - self.failUnless("count-directories: 1" in lines) + self.failUnless("count-literal-files: 3" in lines) + self.failUnless("count-files: 5" in lines) + self.failUnless("count-directories: 3" in lines) self.failUnless("size-immutable-files: 13000 (13.00 kB, 12.70 kiB)" in lines, lines) hunk ./src/allmydata/test/test_deepcheck.py 855 - self.failUnless("size-literal-files: 48" in lines) - self.failUnless(" 11-31 : 2 (31 B, 31 B)".strip() in lines) - self.failUnless("10001-31622 : 1 (31.62 kB, 30.88 kiB)".strip() in lines) + self.failUnless("size-literal-files: 56" in lines, lines) + self.failUnless(" 4-10 : 1 (10 B, 10 B)".strip() in lines, lines) + self.failUnless(" 11-31 : 2 (31 B, 31 B)".strip() in lines, lines) + self.failUnless("10001-31622 : 1 (31.62 kB, 30.88 kiB)".strip() in lines, lines) d.addCallback(_check3) return d hunk ./src/allmydata/test/test_deepcheck.py 873 self.failUnlessEqual(data["count-immutable-files"], 1) self.failUnlessEqual(data["count-immutable-files"], 1) self.failUnlessEqual(data["count-mutable-files"], 1) - self.failUnlessEqual(data["count-literal-files"], 2) - self.failUnlessEqual(data["count-files"], 4) - self.failUnlessEqual(data["count-directories"], 1) + self.failUnlessEqual(data["count-literal-files"], 3) + self.failUnlessEqual(data["count-files"], 5) + self.failUnlessEqual(data["count-directories"], 3) self.failUnlessEqual(data["size-immutable-files"], 13000) hunk ./src/allmydata/test/test_deepcheck.py 877 - self.failUnlessEqual(data["size-literal-files"], 48) + self.failUnlessEqual(data["size-literal-files"], 56) + self.failUnless([4,10,1] in data["size-files-histogram"]) self.failUnless([11,31,2] in data["size-files-histogram"]) self.failUnless([10001,31622,1] in data["size-files-histogram"]) d.addCallback(_check4) } [Additional fixes for DIR2-LIT More Info page and deep-check/manifest operations (#948) david-sarah@jacaranda.org**20100224080220 Ignore-this: 3b431b712f380b5476231ebd99648a7f ] { hunk ./src/allmydata/check_results.py 102 def __init__(self, root_storage_index): self.root_storage_index = root_storage_index if root_storage_index is None: - self.root_storage_index_s = "" + self.root_storage_index_s = "" # is this correct? else: self.root_storage_index_s = base32.b2a(root_storage_index) hunk ./src/allmydata/dirnode.py 359 def list(self): """I return a Deferred that fires with a dictionary mapping child - name to a tuple of (IFileNode or IDirectoryNode, metadata).""" + name to a tuple of (IFilesystemNode, metadata).""" return self._read() def has_child(self, name): hunk ./src/allmydata/dirnode.py 384 def get(self, name): """I return a Deferred that fires with the named child node, - which is either an IFileNode or an IDirectoryNode.""" + which is an IFilesystemNode.""" assert isinstance(name, unicode) d = self._read() d.addCallback(self._get, name) hunk ./src/allmydata/dirnode.py 392 def get_child_and_metadata(self, name): """I return a Deferred that fires with the (node, metadata) pair for - the named child. The node is either an IFileNode or an - IDirectoryNode, and the metadata is a dictionary.""" + the named child. The node is an IFilesystemNode, and the metadata + is a dictionary.""" assert isinstance(name, unicode) d = self._read() d.addCallback(self._get_with_metadata, name) hunk ./src/allmydata/dirnode.py 416 return d def get_child_at_path(self, path): - """Transform a child path into an IDirectoryNode or IFileNode. + """Transform a child path into an IFilesystemNode. I perform a recursive series of 'get' operations to find the named descendant node. I return a Deferred that fires with the node, or hunk ./src/allmydata/dirnode.py 430 return d def get_child_and_metadata_at_path(self, path): - """Transform a child path into an IDirectoryNode or IFileNode and + """Transform a child path into an IFilesystemNode and a metadata dictionary from the last edge that was traversed. """ hunk ./src/allmydata/dirnode.py 835 root_si = root.get_storage_index() self._lp = log.msg(format="deep-check starting (%(si)s)," " verify=%(verify)s, repair=%(repair)s", - si=base32.b2a(root_si), verify=verify, repair=repair) + si=base32.b2a(root_si or ""), verify=verify, repair=repair) self._verify = verify self._repair = repair self._add_lease = add_lease hunk ./src/allmydata/web/directory.py 928 status = { "stats": s["stats"], "finished": m.is_finished(), - "origin": base32.b2a(m.origin_si), + "origin": base32.b2a(m.origin_si or ""), } if m.is_finished(): # don't return manifest/verifycaps/SIs unless the operation is hunk ./src/allmydata/web/directory.py 949 return simplejson.dumps(status, indent=1) def _si_abbrev(self): - return base32.b2a(self.monitor.origin_si)[:6] + si = self.monitor.origin_si + if not si: + return "" + return base32.b2a(si)[:6] def render_title(self, ctx): return T.title["Manifest of SI=%s" % self._si_abbrev()] hunk ./src/allmydata/web/directory.py 1050 v = node.get_verify_cap() if v: v = v.to_string() - d["verifycap"] = v + d["verifycap"] = v or "" r = node.get_repair_cap() if r: hunk ./src/allmydata/web/directory.py 1055 r = r.to_string() - d["repaircap"] = r + d["repaircap"] = r or "" si = node.get_storage_index() if si: hunk ./src/allmydata/web/directory.py 1060 si = base32.b2a(si) - d["storage-index"] = si + d["storage-index"] = si or "" j = simplejson.dumps(d, ensure_ascii=True) assert "\n" not in j hunk ./src/allmydata/web/directory.py 1110 v = node.get_verify_cap() if v: v = v.to_string() - data["verifycap"] = v + data["verifycap"] = v or "" r = node.get_repair_cap() if r: hunk ./src/allmydata/web/directory.py 1115 r = r.to_string() - data["repaircap"] = r + data["repaircap"] = r or "" si = node.get_storage_index() if si: hunk ./src/allmydata/web/directory.py 1120 si = base32.b2a(si) - data["storage-index"] = si + data["storage-index"] = si or "" if self.repair: d = node.check_and_repair(self.monitor, self.verify, self.add_lease) hunk ./src/allmydata/web/info.py 89 node = self.original if not IDirectoryNode.providedBy(node): return "" - return ctx.tag[node.get_verify_cap().to_string()] + verifier = node.get_verify_cap() + if verifier: + return ctx.tag[node.get_verify_cap().to_string()] + return "" def render_file_writecap(self, ctx, data): node = self.original } [Updates to NEWS for 1.6.1 david-sarah@jacaranda.org**20100224081542 Ignore-this: ae1ca1892d7013bcb5f54f201459632 ] { hunk ./NEWS 7 ** Bugfixes -*** Correct handling of Immutable Directories +*** Correct handling of Small Immutable Directories hunk ./NEWS 9 -Immutable directories can now be deep-checked (in v1.6.0 running deep-check on -a system which included immutable directories would result in an exception -which caused the deep-check operation to abort). (#948) +Immutable directories can now be deep-checked and listed in the web UI in +all cases. (In v1.6.0, some operations, such as deep-check, on a directory +graph that included very small immutable directories, would result in an +exception causing the whole operation to abort.) (#948) ** Usability Improvements hunk ./NEWS 16 -Improved user interface messages (#681, #837, #939). +Improved user interface messages and error reporting. (#681, #837, #939) + +The timeouts for operation handles have been greatly increased, so that +you can view the results of an operation up to 4 days after it has +completed. After viewing them for the first time, the results are +retained for a further day. (#577) * Release 1.6.0 (2010-02-01) } [Additional test for DIR2-LIT directories in test_web.py, fixed version (#948) david-sarah@jacaranda.org**20100225041824 Ignore-this: 86d710f438439f27aa372b84411af011 ] { hunk ./src/allmydata/test/common.py 116 pass def get_size(self): + if isinstance(self.my_uri, uri.LiteralFileURI): + return self.my_uri.get_size() try: data = self.all_contents[self.my_uri.to_string()] except KeyError, le: hunk ./src/allmydata/test/common.py 136 return d def _read(self, ignored, consumer, offset, size): - if self.my_uri.to_string() not in self.all_contents: - raise NotEnoughSharesError(None, 0, 3) - data = self.all_contents[self.my_uri.to_string()] + if isinstance(self.my_uri, uri.LiteralFileURI): + data = self.my_uri.data + else: + if self.my_uri.to_string() not in self.all_contents: + raise NotEnoughSharesError(None, 0, 3) + data = self.all_contents[self.my_uri.to_string()] start = offset if size is not None: end = offset + size hunk ./src/allmydata/test/common.py 296 return d def download_best_version(self): + if isinstance(self.my_uri, uri.LiteralFileURI): + return defer.succeed(self.my_uri.data) if self.storage_index not in self.all_contents: return defer.fail(NotEnoughSharesError(None, 0, 3)) return defer.succeed(self.all_contents[self.storage_index]) hunk ./src/allmydata/test/common.py 301 + def overwrite(self, new_contents): if len(new_contents) > self.MUTABLE_SIZELIMIT: raise FileTooLargeError("SDMF is limited to one segment, and " hunk ./src/allmydata/test/common.py 475 if self.stats_gatherer_furl: write("stats_gatherer.furl", self.stats_gatherer_furl) - # give subclasses a chance to append liens to the node's tahoe.cfg + # give subclasses a chance to append lines to the node's tahoe.cfg # files before they are launched. self._set_up_nodes_extra_config() hunk ./src/allmydata/test/test_web.py 997 self.failUnless(MKDIR_BUTTON_RE.search(res), res) d.addCallback(_check4) + # and at a literal directory + tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT + d.addCallback(lambda res: + self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True)) + def _check5(res): + self.failUnless('(immutable)' in res, res) + self.failUnless(re.search('FILE' + r'\s+short', res), res) + d.addCallback(_check5) return d def test_GET_DIRURL_badtype(self): } [Change code that gives a base32 SI or an empty string to be more straightforward. (#948) david-sarah@jacaranda.org**20100227065551 Ignore-this: ba2b0eb430635fcfb09faeca5046ed21 ] { hunk ./src/allmydata/dirnode.py 833 class DeepChecker: def __init__(self, root, verify, repair, add_lease): root_si = root.get_storage_index() + if root_si: + root_si_base32 = base32.b2a(root_si) + else: + root_si_base32 = "" self._lp = log.msg(format="deep-check starting (%(si)s)," " verify=%(verify)s, repair=%(repair)s", hunk ./src/allmydata/dirnode.py 839 - si=base32.b2a(root_si or ""), verify=verify, repair=repair) + si=root_si_base32, verify=verify, repair=repair) self._verify = verify self._repair = repair self._add_lease = add_lease hunk ./src/allmydata/web/directory.py 926 m = self.monitor s = m.get_status() + if m.origin_si: + origin_base32 = base32.b2a(m.origin_si) + else: + origin_base32 = "" status = { "stats": s["stats"], "finished": m.is_finished(), hunk ./src/allmydata/web/directory.py 932 - "origin": base32.b2a(m.origin_si or ""), + "origin": origin_base32, } if m.is_finished(): # don't return manifest/verifycaps/SIs unless the operation is } Context: [docs: update relnotes.txt for v1.6.1 zooko@zooko.com**20100224065755 Ignore-this: 6d078e94425462ac8d074e3e7c82da28 ] [docs: NEWS and relnotes-short.txt and CREDITS for v1.6.1 zooko@zooko.com**20100224065231 Ignore-this: 41c056ae48c639e5a934d4c1983bc118 ] [misc/coverage.el: improve filename matching Brian Warner **20100224044757 Ignore-this: 8d9fb1d2a71e01370da006a2fef04346 ] [test_util.py: improve coverage of util.time_format Brian Warner **20100224044637 Ignore-this: bd93495132fe73a9c117d35c1a4e2d72 ] [docs/performance.txt: split out CPU from network, expand on mutable costs Brian Warner **20100224043813 Ignore-this: 4779e78ca0eed1dcbd1652e6287219f1 ] [Tweak to 'tahoe ls --help' output (#837) david-sarah@jacaranda.org**20100224030231 Ignore-this: 9c86ff8ee1f2c9b8a4f6e205a58905f ] [Test behaviour of 'tahoe ls' for unknown objects (#837) david-sarah@jacaranda.org**20100224025913 Ignore-this: b999f6239796a90cadb41e8650aa3782 ] [docs/FTP: the Twisted patch (t3462) has landed, will be in the next release Brian Warner **20100223210402 Ignore-this: ddc5c8da8c95d8c19380d8c7ecbaf18 ] [Change OphandleTable to use a deterministic clock, so we can test it Kevan Carstensen **20100220210713 Ignore-this: a7437f4eda359bdfa243bd534f23bf52 To test the changes for #577, we need a deterministic way to simulate the passage of long periods of time. twisted.internet.task.Clock seems, from my Googling, to be the way to go for this functionality. I changed a few things so that OphandleTable would use twisted.internet.task.Clock when testing: * WebishServer.__init___ now takes an optional 'clock' parameter, * which it passes to the root.Root instance it creates. * root.Root.__init__ now takes an optional 'clock' parameter, which it passes to the OphandleTable.__init__ method. * OphandleTable.__init__ now takes an optional 'clock' parameter. If it is provided, and it isn't None, its callLater method will be used to schedule ophandle expirations (as opposed to using reactor.callLater, which is what OphandleTable does normally). * The WebMixin object in test_web.py now sets a self.clock parameter, which is a twisted.internet.task.Clock that it feeds to the WebishServer it creates. Tests using the WebMixin can control the passage of time in OphandleTable by accessing self.clock. ] [Add tests for the ophandle expiration behavior in #577 Kevan Carstensen **20100221010455 Ignore-this: 87a435108999c24920354b58fd78353f ] [Update docs/frontends/webapi.txt to reflect the new expiration times in #577 Kevan Carstensen **20100221010716 Ignore-this: cefee2ba800c285ae4148fe2dff39a3b ] [Increase ophandle expiration times, per #577 Kevan Carstensen **20100221010512 Ignore-this: 247f61fe8855a0c76fef3777a957f495 ] [setup: comment-out the dependency on pycrypto, see #953 zooko@zooko.com**20100215050844 Ignore-this: 2751120921ff35b8189d8fcd896da149 ] [Add tests for #939 Kevan Carstensen **20100212062137 Ignore-this: 5459e8c64ba76cca70aa720e68549637 ] [Alter CLI utilities to handle nonexistent aliases better Kevan Carstensen **20100211024318 Ignore-this: e698ea4a57f5fe27c24336581ca0cf65 ] [web/storage.py: display total-seen on the last-complete-cycle line. For #940. Brian Warner **20100208002010 Ignore-this: c0ed860f3e9628d3171d2b055d96c5aa ] [adding pycrypto to the auto dependencies secorp@allmydata.com**20100206054314 Ignore-this: b873fc00a6a5b001d30d479e6053cf2f ] [Improve behaviour of 'tahoe ls' for unknown objects, addressing kevan's comments david-sarah@jacaranda.org**20100220061313 Ignore-this: 6205025c477f1c999473a4ae67e1c83 ] [docs running.html - "tahoe run ." does not work with the current installation, replaced with "tahoe start ." secorp@allmydata.com**20100206165320 Ignore-this: fdb2dcb0e417d303cd43b1951a4f8c03 ] [code coverage: replace figleaf with coverage.py, should work on py2.6 now. Brian Warner **20100203165421 Ignore-this: 46ab590360be6a385cb4fc4e68b6b42c It still lacks the right HTML report (the builtin report is very pretty, but lacks the "lines uncovered" numbers that I want), and the half-finished delta-from-last-run measurements. ] [More cleanups to test_cli using new utilities for reading and writing files. david-sarah@jacaranda.org**20100206013855 Ignore-this: 9fd2294406b346bfe9144fff6a61f789 ] [Fix race conditions and missing callback in allmydata.test.test_cli.Cp.test_copy_using_filecap, add utilities for one-liner reading and writing of files, and fix cases in test_cli where files were not being closed after writing. david-sarah@jacaranda.org**20100206013727 Ignore-this: 49da6c33190d526a4ae84c472f04d5f4 ] [docs: install.html: link into Python 2.5.5 download page zooko@zooko.com**20100202065852 Ignore-this: 1a9471b8175b7de5741d8445a7ede29d ] [TAG allmydata-tahoe-1.6.0 zooko@zooko.com**20100202061125 Ignore-this: dee6ade7ac1452cf5d1d9c69a8146d84 ] [docs: install.html: recommend Python 2.5 (because I can build extension modules for it with mingw), architecture.txt: point out that our Proof of Retrievability feature is client-side-only zooko@zooko.com**20100202053842 Ignore-this: e33fd413a91771c77b17d7de0f215bea ] [architecture.txt: remove trailing whitespace, wrap lines: no content changes Brian Warner **20100202055304 Ignore-this: 1662f37d1162858ac2619db27bcc411f ] [docs: a couple of small edits to release notes (thanks Peter) zooko@zooko.com**20100202054832 Ignore-this: 1d0963c43ff19c92775b124c49c8a88a ] [docs: CREDITS: where due zooko@zooko.com**20100202053831 Ignore-this: 11646dd603ac715ae8277a4bb9562215 ] [docs: a few small edits to performance.txt and README zooko@zooko.com**20100202052750 Ignore-this: bf8b1b7438e8fb6da09eec9713c78533 ] [More comprehensive changes and ticket references for NEWS david-sarah@jacaranda.org**20100202061256 Ignore-this: 696cf0106e8a7fd388afc5b55fba8a1b ] [docs: a few edits to architecture.txt, most significantly highlighting "future work" to avoid confusing it with the current version, and adding a "future work" about a random-sampling Proof of Retrievability verifier zooko@zooko.com**20100202045117 Ignore-this: 81122b3042ea9ee6bc12e795c2386d59 ] [docs: a few edits and updates to relnotes.txt, relnotes-short.txt, and NEWS in preparation for v1.6.0 zooko@zooko.com**20100202043222 Ignore-this: d90c644fa61d78e33cbdf0be428bb07a ] [Fill in 'docs/performance.txt' with some performance information Kevan Carstensen **20100202005914 Ignore-this: c66b255b2bd2e7e11f5707b25e7b38be ] [Document leakage of cap URLs via phishing filters in known_issues.txt david-sarah@jacaranda.org**20100202015238 Ignore-this: 78e668dbca77c0e3a73e10c0b74cf024 ] [docs: updates to relnotes.txt, NEWS, architecture, historical_known_issues, install.html, etc. zooko@zooko.com**20100201181809 Ignore-this: f4fc924652af746862c8ee4d9ba97bf6 ] [immutable: downloader accepts notifications of buckets even if those notifications arrive after he has begun downloading shares. zooko@zooko.com**20100201061610 Ignore-this: 5b09709f27603a3157eba7ba70028955 This can be useful if one of the ones that he has already begun downloading fails. See #287 for discussion. This fixes part of #287 which part was a regression caused by #928, namely this fixes fail-over in case a share is corrupted (or the server returns an error or disconnects). This does not fix the related issue mentioned in #287 if a server hangs and doesn't reply to requests for blocks. ] [tests: don't require tahoe to run with no noise if we are using an old twisted that emits DeprecationWarnings zooko@zooko.com**20100201052323 Ignore-this: 69668c772cce612a0c6936a2195ebd2a ] [tests: stop being surprised that Nevow no longer prints out warnings when it tries to find its static files zooko@zooko.com**20100201041144 Ignore-this: 77b4ac383165d98dfe2a9008ce794742 Unless we are using a sufficiently new version of Nevow, in which case if it prints out warnings then this is a hard test failure. :-) ] [cli: suppress DeprecationWarnings emitted from importing nevow and twisted. Fixes #859 david-sarah@jacaranda.org**20100201004429 Ignore-this: 22d7216921cd5f04381c0194ed501bbe ] [Improvements to test_hung_server, and fix for status updates in download.py david-sarah@jacaranda.org**20100130064303 Ignore-this: dd889c643afdcf0f86d55855aafda6ad ] [Improvements to test_unknownnode to cover invalid cap URIs with known prefixes david-sarah@jacaranda.org**20100130063908 Ignore-this: e1a298942c21207473e418ea5efd6276 ] [immutable: fix bug in tests, change line-endings to unix style, add comment zooko@zooko.com**20100129184237 Ignore-this: f6bd875fe974c55c881e05eddf8d3436 ] [Fix invalid trailing commas in JSON example david-sarah@jacaranda.org**20100129201742 Ignore-this: d99e0a8ead4fafabf39a1daf11ec450b ] [docs: update relnotes.txt for Tahoe-LAFS v1.6 zooko@zooko.com**20100128171257 Ignore-this: 920df92152aead69ef861b9b2e8ff218 ] [Address comments by Kevan on 833 and add test for stripping spaces david-sarah@jacaranda.org**20100127230642 Ignore-this: de36aeaf4afb3ba05dbeb49a5e9a6b26 ] [New tests for #928 david-sarah@jacaranda.org**20100129123845 Ignore-this: 5c520f40141f0d9c000ffb05a4698995 ] [immutable: download from the first servers which provide at least K buckets instead of waiting for all servers to reply zooko@zooko.com**20100127233417 Ignore-this: c855355a40d96827e1d0c469a8d8ab3f This should put an end to the phenomenon I've been seeing that a single hung server can cause all downloads on a grid to hang. Also it should speed up all downloads by (a) not-waiting for responses to queries that it doesn't need, and (b) downloading shares from the servers which answered the initial query the fastest. Also, do not count how many buckets you've gotten when deciding whether the download has enough shares or not -- instead count how many buckets to *unique* shares that you've gotten. This appears to improve a slightly weird behavior in the current download code in which receiving >= K different buckets all to the same sharenumber would make it think it had enough to download the file when in fact it hadn't. This patch needs tests before it is actually ready for trunk. ] [Eliminate 'foo if test else bar' syntax that isn't supported by Python 2.4 david-sarah@jacaranda.org**20100129035210 Ignore-this: 70eafd487b4b6299beedd63b4a54a0c ] [Fix example JSON in webapi.txt that cannot occur in practice david-sarah@jacaranda.org**20100129032742 Ignore-this: 361a1ba663d77169aeef93caef870097 ] [Add mutable field to t=json output for unknown nodes, when mutability is known david-sarah@jacaranda.org**20100129031424 Ignore-this: 1516d63559bdfeb6355485dff0f5c04e ] [Show -IMM and -RO suffixes for types of immutable and read-only unknown nodes in directory listings david-sarah@jacaranda.org**20100128220800 Ignore-this: dc5c17c0a566398f88e4303c41321e66 ] [Fix inaccurate comment in test_mutant_dirnodes_are_omitted david-sarah@jacaranda.org**20100128202456 Ignore-this: 9fa17ed7feac9e4d084f1b2338c76fca ] [test_runner: cleanup, refactor common code into a non-executable method Brian Warner **20100127224040 Ignore-this: 4cb4aada87777771f688edfd8129ffca Having both test_node() and test_client() (one of which calls the other) felt confusing to me, so I changed it to have test_node(), test_client(), and a common do_create() helper method. ] [scripts/runner.py: simplify David-Sarah's clever grouped-commands usage trick Brian Warner **20100127223758 Ignore-this: 70877ebf06ae59f32960b0aa4ce1d1ae ] [tahoe backup: skip all symlinks, with warning. Fixes #850, addresses #641. Brian Warner **20100127223517 Ignore-this: ab5cf05158d32a575ca8efc0f650033f ] [NEWS: update with all recent user-visible changes Brian Warner **20100127222209 Ignore-this: 277d24568018bf4f3fb7736fda64eceb ] ["tahoe backup": fix --exclude-vcs docs to include Git Brian Warner **20100127201044 Ignore-this: 756a58dde21bdc65aa62b81803605b5 ] [docs: fix references to --no-storage, explanation of [storage] section Brian Warner **20100127200956 Ignore-this: f4be1763a585e1ac6299a4f1b94a59e0 ] [cli: merge the better version of David-Sarah's split-usage-and-help patch with the earlier version that I mistakenly committed zooko@zooko.com**20100126044559 Ignore-this: 284d188e13b7901013cbb650168e6447 ] [Split tahoe --help options into groups. david-sarah@jacaranda.org**20100112043935 Ignore-this: 610f9c41b00e6863e3cd047379733e3a ] [Miscellaneous documentation, test, and code formatting tweaks. david-sarah@jacaranda.org**20100127070309 Ignore-this: 84ca7e4bb7c64221ae2c61144ef5edef ] [Prevent mutable objects from being retrieved from an immutable directory, and associated forward-compatibility improvements. david-sarah@jacaranda.org**20100127064430 Ignore-this: 5ef6a3554cf6bef0bf0712cc7d6c0252 ] [Use if instead of assert to check for twisted ftp patch david-sarah@jacaranda.org**20100127015529 Ignore-this: 66959d946bd1a835ece6f074e75086b2 ] [docs: further CREDITS level-ups for Nils, Kevan, David-Sarah zooko@zooko.com**20100126170021 Ignore-this: 1e513e85cf7b7abf57f056e6d7544b38 ] [ftpd: clearer error message if Twisted needs a patch (by Nils Durner) zooko@zooko.com**20100126143411 Ignore-this: 440e6831ae6da5135c1edd081c93871f ] [Add 'docs/performance.txt', which (for the moment) describes mutable file performance issues Kevan Carstensen **20100115204500 Ignore-this: ade4e500217db2509aee35aacc8c5dbf ] [docs: more CREDITS for François, Kevan, and David-Sarah zooko@zooko.com**20100126132133 Ignore-this: f37d4977c13066fcac088ba98a31b02e ] [tahoe_backup.py: display warnings on errors instead of stopping the whole backup. Fix #729. francois@ctrlaltdel.ch**20100120094249 Ignore-this: 7006ea4b0910b6d29af6ab4a3997a8f9 This patch displays a warning to the user in two cases: 1. When special files like symlinks, fifos, devices, etc. are found in the local source. 2. If files or directories are not readables by the user running the 'tahoe backup' command. In verbose mode, the number of skipped files and directories is printed at the end of the backup. Exit status returned by 'tahoe backup': - 0 everything went fine - 1 the backup failed - 2 files were skipped during the backup ] [Message saying that we couldn't find bin/tahoe should say where we looked david-sarah@jacaranda.org**20100116204556 Ignore-this: 1068576fd59ea470f1e19196315d1bb ] [Change running.html to describe 'tahoe run' david-sarah@jacaranda.org**20100112044409 Ignore-this: 23ad0114643ce31b56e19bb14e011e4f ] [cli: split usage strings into groups (patch by David-Sarah Hopwood) zooko@zooko.com**20100126043921 Ignore-this: 51928d266a7292b873f87f7d53c9a01e ] [Add create-node CLI command, and make create-client equivalent to create-node --no-storage (fixes #760) david-sarah@jacaranda.org**20100116052055 Ignore-this: 47d08b18c69738685e13ff365738d5a ] [contrib/fuse/runtests.py: Fix #888, configure settings in tahoe.cfg and don't treat warnings as failure francois@ctrlaltdel.ch**20100109123010 Ignore-this: 2590d44044acd7dfa3690c416cae945c Fix a few bitrotten pieces in the FUSE test script. It now configures tahoe node settings by editing tahoe.cfg which is the new supported method. It alos tolerate warnings issued by the mount command, the cause of these warnings is the same as in #876 (contrib/fuse/runtests.py doesn't tolerate deprecations warnings). ] [Fix webapi t=mkdir with multpart/form-data, as on the Welcome page. Closes #919. Brian Warner **20100121065052 Ignore-this: 1f20ea0a0f1f6d6c1e8e14f193a92c87 ] [Fix boodlegrid use of set_children david-sarah@jacaranda.org**20100126063414 Ignore-this: 3aa2d4836f76303b2bacecd09611f999 ] [Remove replace= parameter to mkdir-immutable and mkdir-with-children david-sarah@jacaranda.org**20100124224325 Ignore-this: 25207bcc946c0c43d9528718e76ba7b ] [Warn about test failures due to setting FLOG* env vars david-sarah@jacaranda.org**20100124220629 Ignore-this: 1c25247ca0f0840390a1b7259a9f4a3c ] [Patch to accept t=set-children as well as t=set_children david-sarah@jacaranda.org**20100124030020 Ignore-this: 2c061f12af817cdf77feeeb64098ec3a ] [tahoe_add_alias.py: minor refactoring Brian Warner **20100115064220 Ignore-this: 29910e81ad11209c9e493d65fd2dab9b ] [test_dirnode.py: reduce scope of a Client instance, suggested by Kevan. Brian Warner **20100115062713 Ignore-this: b35efd9e6027e43de6c6f509bfb4ccaa ] [test_provisioning: STAN is not always a list. Fix by David-Sarah Hopwood. Brian Warner **20100115014632 Ignore-this: 9989de7f1e00907706d2b63153138219 ] [web/directory.py mkdir-immutable: hush pyflakes, add TODO for #903 behavior Brian Warner **20100114222804 Ignore-this: 717cd3b9a1c8aeee76938c9641db7356 ] [hush pyflakes-0.4.0 warnings: slightly less-trivial fixes. Closes #900. Brian Warner **20100114221719 Ignore-this: f774f4637e256ad55502659413a811a8 This includes one fix (in test_web) which was testing the wrong thing. ] [hush pyflakes-0.4.0 warnings: remove trivial unused variables. For #900. Brian Warner **20100114221529 Ignore-this: e96106c8f1a99fbf93306fbfe9a294cf ] [tahoe add-alias/create-alias: don't corrupt non-newline-terminated alias Brian Warner **20100114210246 Ignore-this: 9c994792e53a85159d708760a9b1b000 file. Closes #741. ] [change docs and --help to use "grid" instead of "virtual drive": closes #892. Brian Warner **20100114201119 Ignore-this: a20d4a4dcc4de4e3b404ff72d40fc29b Thanks to David-Sarah Hopwood for the patch. ] [backupdb.txt: fix ST_CTIME reference Brian Warner **20100114194052 Ignore-this: 5a189c7a1181b07dd87f0a08ea31b6d3 ] [client.py: fix/update comments on KeyGenerator Brian Warner **20100113004226 Ignore-this: 2208adbb3fd6a911c9f44e814583cabd ] [Clean up log.err calls, for one of the issues in #889. Brian Warner **20100112013343 Ignore-this: f58455ce15f1fda647c5fb25d234d2db allmydata.util.log.err() either takes a Failure as the first positional argument, or takes no positional arguments and must be invoked in an exception handler. Fixed its signature to match both foolscap.logging.log.err and twisted.python.log.err . Included a brief unit test. ] [tidy up DeadReferenceError handling, ignore them in add_lease calls Brian Warner **20100112000723 Ignore-this: 72f1444e826fd0b9db6d318f89603c38 Stop checking separately for ConnectionDone/ConnectionLost, since those have been folded into DeadReferenceError since foolscap-0.3.1 . Write rrefutil.trap_deadref() in terms of rrefutil.trap_and_discard() to improve code coverage. ] [NEWS: improve "tahoe backup" notes, mention first-backup-after-upgrade duration Brian Warner **20100111190132 Ignore-this: 10347c590b3375964579ba6c2b0edb4f Thanks to Francois Deppierraz for the suggestion. ] [test_repairer: add (commented-out) test_each_byte, to see exactly what the Brian Warner **20100110203552 Ignore-this: 8e84277d5304752edeff052b97821815 Verifier misses The results (described in #819) match our expectations: it misses corruption in unused share fields and in most container fields (which are only visible to the storage server, not the client). 1265 bytes of a 2753 byte share (hosting a 56-byte file with an artifically small segment size) are unused, mostly in the unused tail of the overallocated UEB space (765 bytes), and the allocated-but-unwritten plaintext_hash_tree (480 bytes). ] [repairer: fix some wrong offsets in the randomized verifier tests, debugged by Brian zooko@zooko.com**20100110203721 Ignore-this: 20604a609db8706555578612c1c12feb fixes #819 ] [test_repairer: fix colliding basedir names, which caused test inconsistencies Brian Warner **20100110084619 Ignore-this: b1d56dd27e6ab99a7730f74ba10abd23 ] [repairer: add deterministic test for #819, mark as TODO zooko@zooko.com**20100110013619 Ignore-this: 4cb8bb30b25246de58ed2b96fa447d68 ] [contrib/fuse/runtests.py: Tolerate the tahoe CLI returning deprecation warnings francois@ctrlaltdel.ch**20100109175946 Ignore-this: 419c354d9f2f6eaec03deb9b83752aee Depending on the versions of external libraries such as Twisted of Foolscap, the tahoe CLI can display deprecation warnings on stdout. The tests should not interpret those warnings as a failure if the node is in fact correctly started. See http://allmydata.org/trac/tahoe/ticket/859 for an example of deprecation warnings. fixes #876 ] [contrib: fix fuse_impl_c to use new Python API zooko@zooko.com**20100109174956 Ignore-this: 51ca1ec7c2a92a0862e9b99e52542179 original patch by Thomas Delaet, fixed by François, reviewed by Brian, committed by me ] [docs: CREDITS: add David-Sarah to the CREDITS file zooko@zooko.com**20100109060435 Ignore-this: 896062396ad85f9d2d4806762632f25a ] [mutable/publish: don't loop() right away upon DeadReferenceError. Closes #877 Brian Warner **20100102220841 Ignore-this: b200e707b3f13aa8251981362b8a3e61 The bug was that a disconnected server could cause us to re-enter the initial loop() call, sending multiple queries to a single server, provoking an incorrect UCWE. To fix it, stall the loop() with an eventual.fireEventually() ] [immutable/checker.py: oops, forgot some imports. Also hush pyflakes. Brian Warner **20091229233909 Ignore-this: 4d61bd3f8113015a4773fd4768176e51 ] [mutable repair: return successful=False when numshares**20091229233746 Ignore-this: d881c3275ff8c8bee42f6a80ca48441e instead of weird errors. Closes #874 and #786. Previously, if the file had 0 shares, this would raise TypeError as it tried to call download_version(None). If the file had some shares but fewer than 'k', it would incorrectly raise MustForceRepairError. Added get_successful() to the IRepairResults API, to give repair() a place to report non-code-bug problems like this. ] [node.py/interfaces.py: minor docs fixes Brian Warner **20091229230409 Ignore-this: c86ad6342ef0f95d50639b4f99cd4ddf ] [NEWS: fix 1.4.1 announcement w.r.t. add-lease behavior in older releases Brian Warner **20091229230310 Ignore-this: bbbbb9c961f3bbcc6e5dbe0b1594822 ] [checker: don't let failures in add-lease affect checker results. Closes #875. Brian Warner **20091229230108 Ignore-this: ef1a367b93e4d01298c2b1e6ca59c492 Mutable servermap updates and the immutable checker, when run with add_lease=True, send both the do-you-have-block and add-lease commands in parallel, to avoid an extra round trip time. Many older servers have problems with add-lease and raise various exceptions, which don't generally matter. The client-side code was catching+ignoring some of them, but unrecognized exceptions were passed through to the DYHB code, concealing the DYHB results from the checker, making it think the server had no shares. The fix is to separate the code paths. Both commands are sent at the same time, but the errback path from add-lease is handled separately. Known exceptions are ignored, the others (both unknown-remote and all-local) are logged (log.WEIRD, which will trigger an Incident), but neither will affect the DYHB results. The add-lease message is sent first, and we know that the server handles them synchronously. So when the checker is done, we can be sure that all the add-lease messages have been retired. This makes life easier for unit tests. ] [test_cli: verify fix for "tahoe get" not creating empty file on error (#121) Brian Warner **20091227235444 Ignore-this: 6444d52413b68eb7c11bc3dfdc69c55f ] [addendum to "Fix 'tahoe ls' on files (#771)" Brian Warner **20091227232149 Ignore-this: 6dd5e25f8072a3153ba200b7fdd49491 tahoe_ls.py: tolerate missing metadata web/filenode.py: minor cleanups test_cli.py: test 'tahoe ls FILECAP' ] [Fix 'tahoe ls' on files (#771). Patch adapted from Kevan Carstensen. Brian Warner **20091227225443 Ignore-this: 8bf8c7b1cd14ea4b0ebd453434f4fe07 web/filenode.py: also serve edge metadata when using t=json on a DIRCAP/childname object. tahoe_ls.py: list file objects as if we were listing one-entry directories. Show edge metadata if we have it, which will be true when doing 'tahoe ls DIRCAP/filename' and false when doing 'tahoe ls FILECAP' ] [tahoe_get: don't create the output file on error. Closes #121. Brian Warner **20091227220404 Ignore-this: 58d5e793a77ec6e87d9394ade074b926 ] [webapi: don't accept zero-length childnames during traversal. Closes #358, #676. Brian Warner **20091227201043 Ignore-this: a9119dec89e1c7741f2289b0cad6497b This forbids operations that would implicitly create a directory with a zero-length (empty string) name, like what you'd get if you did "tahoe put local /oops/blah" (#358) or "POST /uri/CAP//?t=mkdir" (#676). The error message is fairly friendly too. Also added code to "tahoe put" to catch this error beforehand and suggest the correct syntax (i.e. without the leading slash). ] [CLI: send 'Accept:' header to ask for text/plain tracebacks. Closes #646. Brian Warner **20091227195828 Ignore-this: 44c258d4d4c7dac0ed58adb22f73331 The webapi has been looking for an Accept header since 1.4.0, but it treats a missing header as equal to */* (to honor RFC2616). This change finally modifies our CLI tools to ask for "text/plain, application/octet-stream", which seems roughly correct (we either want a plain-text traceback or error message, or an uninterpreted chunk of binary data to save to disk). Some day we'll figure out how JSON fits into this scheme. ] [Makefile: upload-tarballs: switch from xfer-client to flappclient, closes #350 Brian Warner **20091227163703 Ignore-this: 3beeecdf2ad9c2438ab57f0e33dcb357 I've also set up a new flappserver on source@allmydata.org to receive the tarballs. We still need to replace the gutsy buildslave (which is where the tarballs used to be generated+uploaded) and give it the new FURL. ] [misc/ringsim.py: make it deterministic, more detail about grid-is-full behavior Brian Warner **20091227024832 Ignore-this: a691cc763fb2e98a4ce1767c36e8e73f ] [misc/ringsim.py: tool to discuss #302 Brian Warner **20091226060339 Ignore-this: fc171369b8f0d97afeeb8213e29d10ed ] [docs: fix helper.txt to describe new config style zooko@zooko.com**20091224223522 Ignore-this: 102e7692dc414a4b466307f7d78601fe ] [docs/stats.txt: add TOC, notes about controlling gatherer's listening port Brian Warner **20091224202133 Ignore-this: 8eef63b0e18db5aa8249c2eafde02c05 Thanks to Jody Harris for the suggestions. ] [Add docs/stats.py, explaining Tahoe stats, the gatherer, and the munin plugins. Brian Warner **20091223052400 Ignore-this: 7c9eeb6e5644eceda98b59a67730ccd5 ] [more #859: avoid deprecation warning for unit tests too, hush pyflakes Brian Warner **20091215000147 Ignore-this: 193622e24d31077da825a11ed2325fd3 * factor maybe-import-sha logic into util.hashutil ] [use hashlib module if available, thus avoiding a DeprecationWarning for importing the old sha module; fixes #859 zooko@zooko.com**20091214212703 Ignore-this: 8d0f230a4bf8581dbc1b07389d76029c ] [docs: reflow architecture.txt to 78-char lines zooko@zooko.com**20091208232943 Ignore-this: 88f55166415f15192e39407815141f77 ] [docs: update the about.html a little zooko@zooko.com**20091208212737 Ignore-this: 3fe2d9653c6de0727d3e82bd70f2a8ed ] [docs: remove obsolete doc file "codemap.txt" zooko@zooko.com**20091113163033 Ignore-this: 16bc21a1835546e71d1b344c06c61ebb I started to update this to reflect the current codebase, but then I thought (a) nobody seemed to notice that it hasn't been updated since December 2007, and (b) it will just bit-rot again, so I'm removing it. ] [mutable/retrieve.py: stop reaching into private MutableFileNode attributes Brian Warner **20091208172921 Ignore-this: 61e548798c1105aed66a792bf26ceef7 ] [mutable/servermap.py: stop reaching into private MutableFileNode attributes Brian Warner **20091208172608 Ignore-this: b40a6b62f623f9285ad96fda139c2ef2 ] [mutable/servermap.py: oops, query N+e servers in MODE_WRITE, not k+e Brian Warner **20091208171156 Ignore-this: 3497f4ab70dae906759007c3cfa43bc under normal conditions, this wouldn't cause any problems, but if the shares are really sparse (perhaps because new servers were added), then file-modifies might stop looking too early and leave old shares in place ] [control.py: fix speedtest: use download_best_version (not read) on mutable nodes Brian Warner **20091207060512 Ignore-this: 7125eabfe74837e05f9291dd6414f917 ] [FTP-and-SFTP.txt: fix ssh-keygen pointer Brian Warner **20091207052803 Ignore-this: bc2a70ee8c58ec314e79c1262ccb22f7 ] [setup: ignore _darcs in the "test-clean" test and make the "clean" step remove all .egg's in the root dir zooko@zooko.com**20091206184835 Ignore-this: 6066bd160f0db36d7bf60aba405558d2 ] [remove MutableFileNode.download(), prefer download_best_version() instead Brian Warner **20091201225438 Ignore-this: 5733eb373a902063e09fd52cc858dec0 ] [Simplify immutable download API: use just filenode.read(consumer, offset, size) Brian Warner **20091201225330 Ignore-this: bdedfb488ac23738bf52ae6d4ab3a3fb * remove Downloader.download_to_data/download_to_filename/download_to_filehandle * remove download.Data/FileName/FileHandle targets * remove filenode.download/download_to_data/download_to_filename methods * leave Downloader.download (the whole Downloader will go away eventually) * add util.consumer.MemoryConsumer/download_to_data, for convenience (this is mostly used by unit tests, but it gets used by enough non-test code to warrant putting it in allmydata.util) * update tests * removes about 180 lines of code. Yay negative code days! Overall plan is to rewrite immutable/download.py and leave filenode.read() as the sole read-side API. ] [server.py: undo my bogus 'correction' of David-Sarah's comment fix Brian Warner **20091201024607 Ignore-this: ff4bb58f6a9e045b900ac3a89d6f506a and move it to a better line ] [Implement more coherent behavior when copying with dircaps/filecaps (closes #761). Patch by Kevan Carstensen. "Brian Warner "**20091130211009] [storage.py: update comment "Brian Warner "**20091130195913] [storage server: detect disk space usage on Windows too (fixes #637) david-sarah@jacaranda.org**20091121055644 Ignore-this: 20fb30498174ce997befac7701fab056 ] [make status of finished operations consistently "Finished" david-sarah@jacaranda.org**20091121061543 Ignore-this: 97d483e8536ccfc2934549ceff7055a3 ] [NEWS: update with all user-visible changes since the last release Brian Warner **20091127224217 Ignore-this: 741da6cd928e939fb6d21a61ea3daf0b ] [update "tahoe backup" docs, and webapi.txt's mkdir-with-children Brian Warner **20091127055900 Ignore-this: defac1fb9a2335b0af3ef9dbbcc67b7e ] [Add dirnodes to backupdb and "tahoe backup", closes #606. Brian Warner **20091126234257 Ignore-this: fa88796fcad1763c6a2bf81f56103223 * backups now share dirnodes with any previous backup, in any location, so renames and moves are handled very efficiently * "tahoe backup" no longer bothers reading the previous snapshot * if you switch grids, you should delete ~/.tahoe/private/backupdb.sqlite, to force new uploads of all files and directories ] [webapi: fix t=check for DIR2-LIT (i.e. empty immutable directories) Brian Warner **20091126232731 Ignore-this: 8513c890525c69c1eca0e80d53a231f8 ] [PipelineError: fix str() on python2.4 . Closes #842. Brian Warner **20091124212512 Ignore-this: e62c92ea9ede2ab7d11fe63f43b9c942 ] [test_uri.py: s/NewDirnode/Dirnode/ , now that they aren't "new" anymore Brian Warner **20091120075553 Ignore-this: 61c8ef5e45a9d966873a610d8349b830 ] [interface name cleanups: IFileNode, IImmutableFileNode, IMutableFileNode Brian Warner **20091120075255 Ignore-this: e3d193c229e2463e1d0b0c92306de27f The proper hierarchy is: IFilesystemNode +IFileNode ++IMutableFileNode ++IImmutableFileNode +IDirectoryNode Also expand test_client.py (NodeMaker) to hit all IFilesystemNode types. ] [class name cleanups: s/FileNode/ImmutableFileNode/ Brian Warner **20091120072239 Ignore-this: 4b3218f2d0e585c62827e14ad8ed8ac1 also fix test/bench_dirnode.py for recent dirnode changes ] [Use DIR-IMM and t=mkdir-immutable for "tahoe backup", for #828 Brian Warner **20091118192813 Ignore-this: a4720529c9bc6bc8b22a3d3265925491 ] [web/directory.py: use "DIR-IMM" to describe immutable directories, not DIR-RO Brian Warner **20091118191832 Ignore-this: aceafd6ab4bf1cc0c2a719ef7319ac03 ] [web/info.py: hush pyflakes Brian Warner **20091118191736 Ignore-this: edc5f128a2b8095fb20686a75747c8 ] [make get_size/get_current_size consistent for all IFilesystemNode classes Brian Warner **20091118191624 Ignore-this: bd3449cf96e4827abaaf962672c1665a * stop caching most_recent_size in dirnode, rely upon backing filenode for it * start caching most_recent_size in MutableFileNode * return None when you don't know, not "?" * only render None as "?" in the web "more info" page * add get_size/get_current_size to UnknownNode ] [ImmutableDirectoryURIVerifier: fix verifycap handling Brian Warner **20091118164238 Ignore-this: 6bba5c717b54352262eabca6e805d590 ] [Add t=mkdir-immutable to the webapi. Closes #607. Brian Warner **20091118070900 Ignore-this: 311e5fab9a5f28b9e8a28d3d08f3c0d * change t=mkdir-with-children to not use multipart/form encoding. Instead, the request body is all JSON. t=mkdir-immutable uses this format too. * make nodemaker.create_immutable_dirnode() get convergence from SecretHolder, but let callers override it * raise NotDeepImmutableError instead of using assert() * add mutable= argument to DirectoryNode.create_subdirectory(), default True ] [move convergence secret into SecretHolder, next to lease secret Brian Warner **20091118015444 Ignore-this: 312f85978a339f2d04deb5bcb8f511bc ] [nodemaker: implement immutable directories (internal interface), for #607 Brian Warner **20091112002233 Ignore-this: d09fccf41813fdf7e0db177ed9e5e130 * nodemaker.create_from_cap() now handles DIR2-CHK and DIR2-LIT * client.create_immutable_dirnode() is used to create them * no webapi yet ] [stop using IURI()/etc as an adapter Brian Warner **20091111224542 Ignore-this: 9611da7ea6a4696de2a3b8c08776e6e0 ] [clean up uri-vs-cap terminology, emphasize cap instances instead of URI strings Brian Warner **20091111222619 Ignore-this: 93626385f6e7f039ada71f54feefe267 * "cap" means a python instance which encapsulates a filecap/dircap (uri.py) * "uri" means a string with a "URI:" prefix * FileNode instances are created with (and retain) a cap instance, and generate uri strings on demand * .get_cap/get_readcap/get_verifycap/get_repaircap return cap instances * .get_uri/get_readonly_uri return uri strings * add filenode.download_to_filename() for control.py, should find a better way * use MutableFileNode.init_from_cap, not .init_from_uri * directory URI instances: use get_filenode_cap, not get_filenode_uri * update/cleanup bench_dirnode.py to match, add Makefile target to run it ] [add parser for immutable directory caps: DIR2-CHK, DIR2-LIT, DIR2-CHK-Verifier Brian Warner **20091104181351 Ignore-this: 854398cc7a75bada57fa97c367b67518 ] [wui: s/TahoeLAFS/Tahoe-LAFS/ zooko@zooko.com**20091029035050 Ignore-this: 901e64cd862e492ed3132bd298583c26 ] [tests: bump up the timeout on test_repairer to see if 120 seconds was too short for François's ARM box to do the test even when it was doing it right. zooko@zooko.com**20091027224800 Ignore-this: 95e93dc2e018b9948253c2045d506f56 ] [dirnode.pack_children(): add deep_immutable= argument Brian Warner **20091026162809 Ignore-this: d5a2371e47662c4bc6eff273e8181b00 This will be used by DIR2:CHK to enforce the deep-immutability requirement. ] [webapi: use t=mkdir-with-children instead of a children= arg to t=mkdir . Brian Warner **20091026011321 Ignore-this: 769cab30b6ab50db95000b6c5a524916 This is safer: in the earlier API, an old webapi server would silently ignore the initial children, and clients trying to set them would have to fetch the newly-created directory to discover the incompatibility. In the new API, clients using t=mkdir-with-children against an old webapi server will get a clear error. ] [nodemaker.create_new_mutable_directory: pack_children() in initial_contents= Brian Warner **20091020005118 Ignore-this: bd43c4eefe06fd32b7492bcb0a55d07e instead of creating an empty file and then adding the children later. This should speed up mkdir(initial_children) considerably, removing two roundtrips and an entire read-modify-write cycle, probably bringing it down to a single roundtrip. A quick test (against the volunteergrid) suggests a 30% speedup. test_dirnode: add new tests to enforce the restrictions that interfaces.py claims for create_new_mutable_directory(): no UnknownNodes, metadata dicts ] [test_dirnode.py: add tests of initial_children= args to client.create_dirnode Brian Warner **20091017194159 Ignore-this: 2e2da28323a4d5d815466387914abc1b and nodemaker.create_new_mutable_directory ] [update many dirnode interfaces to accept dict-of-nodes instead of dict-of-caps Brian Warner **20091017192829 Ignore-this: b35472285143862a856bf4b361d692f0 interfaces.py: define INodeMaker, document argument values, change create_new_mutable_directory() to take dict-of-nodes. Change dirnode.set_nodes() and dirnode.create_subdirectory() too. nodemaker.py: use INodeMaker, update create_new_mutable_directory() client.py: have create_dirnode() delegate initial_children= to nodemaker dirnode.py (Adder): take dict-of-nodes instead of list-of-nodes, which updates set_nodes() and create_subdirectory() web/common.py (convert_initial_children_json): create dict-of-nodes web/directory.py: same web/unlinked.py: same test_dirnode.py: update tests to match ] [dirnode.py: move pack_children() out to a function, for eventual use by others Brian Warner **20091017180707 Ignore-this: 6a823fb61f2c180fd38d6742d3196a7a ] [move dirnode.CachingDict to dictutil.AuxValueDict, generalize method names, Brian Warner **20091017180005 Ignore-this: b086933cf429df0fcea16a308d2640dd improve tests. Let dirnode _pack_children accept either dict or AuxValueDict. ] [test/common.py: update FakeMutableFileNode to new contents= callable scheme Brian Warner **20091013052154 Ignore-this: 62f00a76454a2190d1c8641c5993632f ] [The initial_children= argument to nodemaker.create_new_mutable_directory is Brian Warner **20091013031922 Ignore-this: 72e45317c21f9eb9ec3bd79bd4311f48 now enabled. ] [client.create_mutable_file(contents=) now accepts a callable, which is Brian Warner **20091013031232 Ignore-this: 3c89d2f50c1e652b83f20bd3f4f27c4b invoked with the new MutableFileNode and is supposed to return the initial contents. This can be used by e.g. a new dirnode which needs the filenode's writekey to encrypt its initial children. create_mutable_file() still accepts a bytestring too, or None for an empty file. ] [webapi: t=mkdir now accepts initial children, using the same JSON that t=json Brian Warner **20091013023444 Ignore-this: 574a46ed46af4251abf8c9580fd31ef7 emits. client.create_dirnode(initial_children=) now works. ] [replace dirnode.create_empty_directory() with create_subdirectory(), which Brian Warner **20091013021520 Ignore-this: 6b57cb51bcfcc6058d0df569fdc8a9cf takes an initial_children= argument ] [dirnode.set_children: change return value: fire with self instead of None Brian Warner **20091013015026 Ignore-this: f1d14e67e084e4b2a4e25fa849b0e753 ] [dirnode.set_nodes: change return value: fire with self instead of None Brian Warner **20091013014546 Ignore-this: b75b3829fb53f7399693f1c1a39aacae ] [dirnode.set_children: take a dict, not a list Brian Warner **20091013002440 Ignore-this: 540ce72ce2727ee053afaae1ff124e21 ] [dirnode.set_uri/set_children: change signature to take writecap+readcap Brian Warner **20091012235126 Ignore-this: 5df617b2d379a51c79148a857e6026b1 instead of a single cap. The webapi t=set_children call benefits too. ] [replace Client.create_empty_dirnode() with create_dirnode(), in anticipation Brian Warner **20091012224506 Ignore-this: cbdaa4266ecb3c6496ffceab4f95709d of adding initial_children= argument. Includes stubbed-out initial_children= support. ] [test_web.py: use a less-fake client, making test harness smaller Brian Warner **20091012222808 Ignore-this: 29e95147f8c94282885c65b411d100bb ] [webapi.txt: document t=set_children, other small edits Brian Warner **20091009200446 Ignore-this: 4d7e76b04a7b8eaa0a981879f778ea5d ] [Verifier: check the full cryptext-hash tree on each share. Removed .todos Brian Warner **20091005221849 Ignore-this: 6fb039c5584812017d91725e687323a5 from the last few test_repairer tests that were waiting on this. ] [Verifier: check the full block-hash-tree on each share Brian Warner **20091005214844 Ignore-this: 3f7ccf6d253f32340f1bf1da27803eee Removed the .todo from two test_repairer tests that check this. The only remaining .todos are on the three crypttext-hash-tree tests. ] [Verifier: check the full share-hash chain on each share Brian Warner **20091005213443 Ignore-this: 3d30111904158bec06a4eac22fd39d17 Removed the .todo from two test_repairer tests that check this. ] [test_repairer: rename Verifier test cases to be more precise and less verbose Brian Warner **20091005201115 Ignore-this: 64be7094e33338c7c2aea9387e138771 ] [immutable/checker.py: rearrange code a little bit, make it easier to follow Brian Warner **20091005200252 Ignore-this: 91cc303fab66faf717433a709f785fb5 ] [test/common.py: wrap docstrings to 80cols so I can read them more easily Brian Warner **20091005200143 Ignore-this: b180a3a0235cbe309c87bd5e873cbbb3 ] [immutable/download.py: wrap to 80cols, no functional changes Brian Warner **20091005192542 Ignore-this: 6b05fe3dc6d78832323e708b9e6a1fe ] [CHK-hashes.svg: cross out plaintext hashes, since we don't include Brian Warner **20091005010803 Ignore-this: bea2e953b65ec7359363aa20de8cb603 them (until we finish #453) ] [docs: a few licensing clarifications requested by Ubuntu zooko@zooko.com**20090927033226 Ignore-this: 749fc8c9aeb6dc643669854a3e81baa7 ] [setup: remove binary WinFUSE modules zooko@zooko.com**20090924211436 Ignore-this: 8aefc571d2ae22b9405fc650f2c2062 I would prefer to have just source code, or indications of what 3rd-party packages are required, under revision control, and have the build process generate o r acquire the binaries as needed. Also, having these in our release tarballs is interfering with getting Tahoe-LAFS uploaded into Ubuntu Karmic. (Technicall y, they would accept binary modules as long as they came with the accompanying source so that they could satisfy their obligations under GPL2+ and TGPPL1+, bu t it is easier for now to remove the binaries from the source tree.) In this case, the binaries are from the tahoe-w32-client project: http://allmydata.org/trac/tahoe-w32-client , from which you can also get the source. ] [setup: remove binary _fusemodule.so 's zooko@zooko.com**20090924211130 Ignore-this: 74487bbe27d280762ac5dd5f51e24186 I would prefer to have just source code, or indications of what 3rd-party packages are required, under revision control, and have the build process generate or acquire the binaries as needed. Also, having these in our release tarballs is interfering with getting Tahoe-LAFS uploaded into Ubuntu Karmic. (Technically, they would accept binary modules as long as they came with the accompanying source so that they could satisfy their obligations under GPL2+ and TGPPL1+, but it is easier for now to remove the binaries from the source tree.) In this case, these modules come from the MacFUSE project: http://code.google.com/p/macfuse/ ] [doc: add a copy of LGPL2 for documentation purposes for ubuntu zooko@zooko.com**20090924054218 Ignore-this: 6a073b48678a7c84dc4fbcef9292ab5b ] [setup: remove a convenience copy of figleaf, to ease inclusion into Ubuntu Karmic Koala zooko@zooko.com**20090924053215 Ignore-this: a0b0c990d6e2ee65c53a24391365ac8d We need to carefully document the licence of figleaf in order to get Tahoe-LAFS into Ubuntu Karmic Koala. However, figleaf isn't really a part of Tahoe-LAFS per se -- this is just a "convenience copy" of a development tool. The quickest way to make Tahoe-LAFS acceptable for Karmic then, is to remove figleaf from the Tahoe-LAFS tarball itself. People who want to run figleaf on Tahoe-LAFS (as everyone should want) can install figleaf themselves. I haven't tested this -- there may be incompatibilities between upstream figleaf and the copy that we had here... ] [setup: shebang for misc/build-deb.py to fail quickly zooko@zooko.com**20090819135626 Ignore-this: 5a1b893234d2d0bb7b7346e84b0a6b4d Without this patch, when I ran "chmod +x ./misc/build-deb.py && ./misc/build-deb.py" then it hung indefinitely. (I wonder what it was doing.) ] [docs: Shawn Willden grants permission for his contributions under GPL2+|TGPPL1+ zooko@zooko.com**20090921164651 Ignore-this: ef1912010d07ff2ffd9678e7abfd0d57 ] [docs: Csaba Henk granted permission to license fuse.py under the same terms as Tahoe-LAFS itself zooko@zooko.com**20090921154659 Ignore-this: c61ba48dcb7206a89a57ca18a0450c53 ] [setup: mark setup.py as having utf-8 encoding in it zooko@zooko.com**20090920180343 Ignore-this: 9d3850733700a44ba7291e9c5e36bb91 ] [doc: licensing cleanups zooko@zooko.com**20090920171631 Ignore-this: 7654f2854bf3c13e6f4d4597633a6630 Use nice utf-8 © instead of "(c)". Remove licensing statements on utility modules that have been assigned to allmydata.com by their original authors. (Nattraverso was not assigned to allmydata.com -- it was LGPL'ed -- but I checked and src/allmydata/util/iputil.py was completely rewritten and doesn't contain any line of code from nattraverso.) Add notes to misc/debian/copyright about licensing on files that aren't just allmydata.com-licensed. ] [build-deb.py: run darcsver early, otherwise we get the wrong version later on Brian Warner **20090918033620 Ignore-this: 6635c5b85e84f8aed0d8390490c5392a ] [new approach for debian packaging, sharing pieces across distributions. Still experimental, still only works for sid. warner@lothar.com**20090818190527 Ignore-this: a75eb63db9106b3269badbfcdd7f5ce1 ] [new experimental deb-packaging rules. Only works for sid so far. Brian Warner **20090818014052 Ignore-this: 3a26ad188668098f8f3cc10a7c0c2f27 ] [setup.py: read _version.py and pass to setup(version=), so more commands work Brian Warner **20090818010057 Ignore-this: b290eb50216938e19f72db211f82147e like "setup.py --version" and "setup.py --fullname" ] [test/check_speed.py: fix shbang line Brian Warner **20090818005948 Ignore-this: 7f3a37caf349c4c4de704d0feb561f8d ] [setup: remove bundled version of darcsver-1.2.1 zooko@zooko.com**20090816233432 Ignore-this: 5357f26d2803db2d39159125dddb963a That version of darcsver emits a scary error message when the darcs executable or the _darcs subdirectory is not found. This error is hidden (unless the --loud option is passed) in darcsver >= 1.3.1. Fixes #788. ] [de-Service-ify Helper, pass in storage_broker and secret_holder directly. Brian Warner **20090815201737 Ignore-this: 86b8ac0f90f77a1036cd604dd1304d8b This makes it more obvious that the Helper currently generates leases with the Helper's own secrets, rather than getting values from the client, which is arguably a bug that will likely be resolved with the Accounting project. ] [immutable.Downloader: pass StorageBroker to constructor, stop being a Service Brian Warner **20090815192543 Ignore-this: af5ab12dbf75377640a670c689838479 child of the client, access with client.downloader instead of client.getServiceNamed("downloader"). The single "Downloader" instance is scheduled for demolition anyways, to be replaced by individual filenode.download calls. ] [tests: double the timeout on test_runner.RunNode.test_introducer since feisty hit a timeout zooko@zooko.com**20090815160512 Ignore-this: ca7358bce4bdabe8eea75dedc39c0e67 I'm not sure if this is an actual timing issue (feisty is running on an overloaded VM if I recall correctly), or it there is a deeper bug. ] [stop making History be a Service, it wasn't necessary Brian Warner **20090815114415 Ignore-this: b60449231557f1934a751c7effa93cfe ] [Overhaul IFilesystemNode handling, to simplify tests and use POLA internally. Brian Warner **20090815112846 Ignore-this: 1db1b9c149a60a310228aba04c5c8e5f * stop using IURI as an adapter * pass cap strings around instead of URI instances * move filenode/dirnode creation duties from Client to new NodeMaker class * move other Client duties to KeyGenerator, SecretHolder, History classes * stop passing Client reference to dirnode/filenode constructors - pass less-powerful references instead, like StorageBroker or Uploader * always create DirectoryNodes by wrapping a filenode (mutable for now) * remove some specialized mock classes from unit tests Detailed list of changes (done one at a time, then merged together) always pass a string to create_node_from_uri(), not an IURI instance always pass a string to IFilesystemNode constructors, not an IURI instance stop using IURI() as an adapter, switch on cap prefix in create_node_from_uri() client.py: move SecretHolder code out to a separate class test_web.py: hush pyflakes client.py: move NodeMaker functionality out into a separate object LiteralFileNode: stop storing a Client reference immutable Checker: remove Client reference, it only needs a SecretHolder immutable Upload: remove Client reference, leave SecretHolder and StorageBroker immutable Repairer: replace Client reference with StorageBroker and SecretHolder immutable FileNode: remove Client reference mutable.Publish: stop passing Client mutable.ServermapUpdater: get StorageBroker in constructor, not by peeking into Client reference MutableChecker: reference StorageBroker and History directly, not through Client mutable.FileNode: removed unused indirection to checker classes mutable.FileNode: remove Client reference client.py: move RSA key generation into a separate class, so it can be passed to the nodemaker move create_mutable_file() into NodeMaker test_dirnode.py: stop using FakeClient mockups, use NoNetworkGrid instead. This simplifies the code, but takes longer to run (17s instead of 6s). This should come down later when other cleanups make it possible to use simpler (non-RSA) fake mutable files for dirnode tests. test_mutable.py: clean up basedir names client.py: move create_empty_dirnode() into NodeMaker dirnode.py: get rid of DirectoryNode.create remove DirectoryNode.init_from_uri, refactor NodeMaker for customization, simplify test_web's mock Client to match stop passing Client to DirectoryNode, make DirectoryNode.create_with_mutablefile the normal DirectoryNode constructor, start removing client from NodeMaker remove Client from NodeMaker move helper status into History, pass History to web.Status instead of Client test_mutable.py: fix minor typo ] [docs: edits for docs/running.html from Sam Mason zooko@zooko.com**20090809201416 Ignore-this: 2207e80449943ebd4ed50cea57c43143 ] [docs: install.html: instruct Debian users to use this document and not to go find the DownloadDebianPackages page, ignore the warning at the top of it, and try it zooko@zooko.com**20090804123840 Ignore-this: 49da654f19d377ffc5a1eff0c820e026 http://allmydata.org/pipermail/tahoe-dev/2009-August/002507.html ] [docs: relnotes.txt: reflow to 63 chars wide because google groups and some web forms seem to wrap to that zooko@zooko.com**20090802135016 Ignore-this: 53b1493a0491bc30fb2935fad283caeb ] [docs: about.html: fix English usage noticed by Amber zooko@zooko.com**20090802050533 Ignore-this: 89965c4650f9bd100a615c401181a956 ] [docs: fix mis-spelled word in about.html zooko@zooko.com**20090802050320 Ignore-this: fdfd0397bc7cef9edfde425dddeb67e5 ] [TAG allmydata-tahoe-1.5.0 zooko@zooko.com**20090802031303 Ignore-this: 94e5558e7225c39a86aae666ea00f166 ] Patch bundle hash: df5b24ed869925e246ca06bacd5b041d47b82eaf