diff -rN -u old-timestamps/docs/frontends/webapi.txt new-timestamps/docs/frontends/webapi.txt --- old-timestamps/docs/frontends/webapi.txt 2009-04-08 21:33:40.000000000 -0600 +++ new-timestamps/docs/frontends/webapi.txt 2009-04-08 21:33:41.000000000 -0600 @@ -381,28 +381,44 @@ GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json This returns a machine-parseable JSON-encoded description of the given - object. The JSON always contains a list, and the first element of the list - is always a flag that indicates whether the referenced object is a file or a - directory. If it is a file, then the information includes file size and URI, - like this: + object. The JSON always contains a list, and the first element of the list is + always a flag that indicates whether the referenced object is a file or a + directory. If it is a capability to a file, then the information includes + file size and URI, like this: GET /uri/$FILECAP?t=json : + + [ "filenode", { "ro_uri": file_uri, + "verify_uri": verify_uri, + "size": bytes, + "mutable": false, + } ] + + If it is a capability to a directory followed by a path from that directory + to a file, then the information also includes metadata from the link to the + file in the parent directory, like this: + GET /uri/$DIRCAP/[SUBDIRS../]FILENAME?t=json : [ "filenode", { "ro_uri": file_uri, "verify_uri": verify_uri, "size": bytes, "mutable": false, - "metadata": {"ctime": 1202777696.7564139, - "mtime": 1202777696.7564139 + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "__sys": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139, } + } } ] If it is a directory, then it includes information about the children of this directory, as a mapping from child name to a set of data about the child (the same data that would appear in a corresponding GET?t=json of the child itself). The child entries also include metadata about each child, - including creation- and modification- timestamps. The output looks like + including link-creation- and link-change- timestamps. The output looks like this: GET /uri/$DIRCAP?t=json : @@ -418,13 +434,21 @@ "metadata": { "ctime": 1202777696.7564139, "mtime": 1202777696.7564139 + "__sys": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139, } + } } ], "subdir": [ "dirnode", { "rw_uri": rwuri, "ro_uri": rouri, "metadata": { "ctime": 1202778102.7589991, "mtime": 1202778111.2160511, + "__sys": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139, + } } } ] } } ] diff -rN -u old-timestamps/docs/specifications/dirnodes.txt new-timestamps/docs/specifications/dirnodes.txt --- old-timestamps/docs/specifications/dirnodes.txt 2009-04-08 21:33:40.000000000 -0600 +++ new-timestamps/docs/specifications/dirnodes.txt 2009-04-08 21:33:41.000000000 -0600 @@ -176,30 +176,29 @@ netstring(cap) = 4+len(cap) encrypted(cap) = 16+cap+32 JSON({}) = 2 - JSON({ctime=float,mtime=float}): 57 - netstring(metadata) = 4+57 = 61 + JSON({ctime=float,mtime=float,'__sys':{linkcrtime=float,linkmotime=float}}): 137 + netstring(metadata) = 4+137 = 141 so a CHK entry is: - 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+57 -And a 15-byte filename gives a 336-byte entry. When the entry points at a + 5+ 4+len(name) + 4+97 + 5+16+97+32 + 4+137 +And a 15-byte filename gives a 416-byte entry. When the entry points at a subdirectory instead of a file, the entry is a little bit smaller. So an -empty directory uses 0 bytes, a directory with one child uses about 336 -bytes, a directory with two children uses about 672, etc. +empty directory uses 0 bytes, a directory with one child uses about 416 +bytes, a directory with two children uses about 832, etc. When the dirnode data is encoding using our default 3-of-10, that means we -get 112ish bytes of data in each share per child. +get 139ish bytes of data in each share per child. The pubkey, signature, and hashes form the first 935ish bytes of the container, then comes our data, then about 1216 bytes of encprivkey. So if we read the first: 1kB: we get 65bytes of dirnode data : only empty directories - 1kiB: 89bytes of dirnode data : maybe one short-named subdir - 2kB: 1065bytes: about 9 entries - 3kB: 2065bytes: about 18 entries, or 7.5 entries plus the encprivkey - 4kB: 3065bytes: about 27 entries, or about 16.5 plus the encprivkey + 2kB: 1065bytes: about 8 + 3kB: 2065bytes: about 15 entries, or 6 entries plus the encprivkey + 4kB: 3065bytes: about 22 entries, or about 13 plus the encprivkey -So we've written the code to do an initial read of 2kB from each share when +So we've written the code to do an initial read of 4kB from each share when we read the mutable file, which should give good performance (one RTT) for small directories. diff -rN -u old-timestamps/src/allmydata/dirnode.py new-timestamps/src/allmydata/dirnode.py --- old-timestamps/src/allmydata/dirnode.py 2009-04-08 21:33:40.000000000 -0600 +++ new-timestamps/src/allmydata/dirnode.py 2009-04-08 21:33:42.000000000 -0600 @@ -83,15 +83,41 @@ metadata = children[name][1].copy() else: metadata = {"ctime": now, - "mtime": now} - if new_metadata is None: - # update timestamps + "mtime": now, + "__sys": { + "linkcrtime": now, + "linkmotime": now, + } + } + + if new_metadata is not None: + # Overwrite all metadata. + newmd = new_metadata.copy() + + # Except '__sys'. + if newmd.has_key('__sys'): + del newmd['__sys'] + if metadata.has_key('__sys'): + newmd['__sys'] = metadata['__sys'] + + metadata = newmd + else: + # For backwards compatibility with Tahoe < 1.4.0: if "ctime" not in metadata: metadata["ctime"] = now metadata["mtime"] = now - else: - # just replace it - metadata = new_metadata.copy() + + # update timestamps + sysmd = metadata.get('__sys', {}) + if not 'linkcrtime' in sysmd: + if "ctime" in metadata: + # In Tahoe < 1.4.0 we used the word "ctime" to mean what Tahoe >= 1.4.0 + # calls "linkcrtime". + sysmd["linkcrtime"] = metadata["ctime"] + else: + sysmd["linkcrtime"] = now + sysmd["linkmotime"] = now + children[name] = (child, metadata) new_contents = self.node._pack_contents(children) return new_contents diff -rN -u old-timestamps/src/allmydata/mutable/servermap.py new-timestamps/src/allmydata/mutable/servermap.py --- old-timestamps/src/allmydata/mutable/servermap.py 2009-04-08 21:33:40.000000000 -0600 +++ new-timestamps/src/allmydata/mutable/servermap.py 2009-04-08 21:33:42.000000000 -0600 @@ -374,7 +374,7 @@ # fixed-size slots so we can retrieve less data. For now, we'll just # read 2000 bytes, which also happens to read enough actual data to # pre-fetch a 9-entry dirnode. - self._read_size = 2000 + self._read_size = 4000 if mode == MODE_CHECK: # we use unpack_prefix_and_signature, so we need 1k self._read_size = 1000 diff -rN -u old-timestamps/src/allmydata/scripts/tahoe_ls.py new-timestamps/src/allmydata/scripts/tahoe_ls.py --- old-timestamps/src/allmydata/scripts/tahoe_ls.py 2009-04-08 21:33:40.000000000 -0600 +++ new-timestamps/src/allmydata/scripts/tahoe_ls.py 2009-04-08 21:33:42.000000000 -0600 @@ -65,8 +65,20 @@ name = unicode(name) child = children[name] childtype = child[0] - ctime = child[1]["metadata"].get("ctime") - mtime = child[1]["metadata"].get("mtime") + + # linkcrtime is not really what unix filesystems mean by "ctime", but + # it *is* apparently what many or even most unix programmers and users + # think that a unix filesystem means by "ctime"... + ctime = child[1].get("metadata", {}).get('__sys', {}).get("linkcrtime") + if not ctime: + ctime = child[1]["metadata"].get("ctime") + + # linkmotime is not really what unix filesystems mean by "mtime", + # because linkmotime is a property of the link and mtime is a property + # of the file contents... + mtime = child[1].get("metadata", {}).get('__sys', {}).get("linkmotime") + if not mtime: + mtime = child[1]["metadata"].get("mtime") rw_uri = child[1].get("rw_uri") ro_uri = child[1].get("ro_uri") if ctime: diff -rN -u old-timestamps/src/allmydata/test/test_dirnode.py new-timestamps/src/allmydata/test/test_dirnode.py --- old-timestamps/src/allmydata/test/test_dirnode.py 2009-04-08 21:33:41.000000000 -0600 +++ new-timestamps/src/allmydata/test/test_dirnode.py 2009-04-08 21:33:42.000000000 -0600 @@ -416,7 +416,7 @@ d.addCallback(lambda res: n.get_metadata_for(u"child")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) d.addCallback(lambda res: self.shouldFail(NoSuchChildError, "gcamap-no", @@ -439,7 +439,7 @@ self.failUnlessEqual(child.get_uri(), fake_file_uri.to_string()) self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"]) + ["__sys", "ctime", "mtime"]) d.addCallback(_check_child_and_metadata2) d.addCallback(lambda res: @@ -448,21 +448,21 @@ child, metadata = res self.failUnless(isinstance(child, FakeDirectoryNode)) self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"]) + ["__sys", "ctime", "mtime"]) d.addCallback(_check_child_and_metadata3) # set_uri + metadata # it should be possible to add a child without any metadata d.addCallback(lambda res: n.set_uri(u"c2", fake_file_uri.to_string(), {})) d.addCallback(lambda res: n.get_metadata_for(u"c2")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['__sys'])) # if we don't set any defaults, the child should get timestamps d.addCallback(lambda res: n.set_uri(u"c3", fake_file_uri.to_string())) d.addCallback(lambda res: n.get_metadata_for(u"c3")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) # or we can add specific metadata at set_uri() time, which # overrides the timestamps @@ -470,7 +470,7 @@ {"key": "value"})) d.addCallback(lambda res: n.get_metadata_for(u"c4")) d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + self.failUnless((frozenset(metadata.keys()) == frozenset(["key", "__sys"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"c2")) d.addCallback(lambda res: n.delete(u"c3")) @@ -486,14 +486,14 @@ n.set_node, u"d2", n2, overwrite=False)) d.addCallback(lambda res: n.get_metadata_for(u"d2")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + d.addCallback(lambda metadata: self.failUnlessEqual(metadata.keys(), ['__sys'])) # if we don't set any defaults, the child should get timestamps d.addCallback(lambda res: n.set_node(u"d3", n)) d.addCallback(lambda res: n.get_metadata_for(u"d3")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) # or we can add specific metadata at set_node() time, which # overrides the timestamps @@ -501,7 +501,7 @@ {"key": "value"})) d.addCallback(lambda res: n.get_metadata_for(u"d4")) d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + self.failUnless((frozenset(metadata.keys()) == frozenset(["key", "__sys"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"d2")) d.addCallback(lambda res: n.delete(u"d3")) @@ -526,12 +526,12 @@ d.addCallback(lambda res: n.get_metadata_for(u"e1")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) d.addCallback(lambda res: n.get_metadata_for(u"e2")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + d.addCallback(lambda metadata: self.failUnlessEqual(frozenset(metadata.keys()), frozenset(['__sys']))) d.addCallback(lambda res: n.get_metadata_for(u"e3")) d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + self.failUnless((frozenset(metadata.keys()) == frozenset(["key", "__sys"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"e1")) d.addCallback(lambda res: n.delete(u"e2")) @@ -556,12 +556,12 @@ d.addCallback(lambda res: n.get_metadata_for(u"f1")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) d.addCallback(lambda res: n.get_metadata_for(u"f2")) - d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {})) + d.addCallback(lambda metadata: self.failUnlessEqual(frozenset(metadata.keys()), frozenset(['__sys']))) d.addCallback(lambda res: n.get_metadata_for(u"f3")) d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + self.failUnless((frozenset(metadata.keys()) == frozenset(["key", "__sys"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"f1")) d.addCallback(lambda res: n.delete(u"f2")) @@ -628,7 +628,7 @@ d.addCallback(lambda res: n.get_metadata_for(u"no_timestamps")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) d.addCallback(lambda res: n.delete(u"no_timestamps")) d.addCallback(lambda res: n.delete(u"subdir")) @@ -659,7 +659,7 @@ d.addCallback(lambda res: n.get_metadata_for(u"newfile")) d.addCallback(lambda metadata: self.failUnlessEqual(sorted(metadata.keys()), - ["ctime", "mtime"])) + ["__sys", "ctime", "mtime"])) d.addCallback(lambda res: n.add_file(u"newfile-metadata", uploadable, @@ -668,7 +668,7 @@ self.failUnless(IFileNode.providedBy(newnode))) d.addCallback(lambda res: n.get_metadata_for(u"newfile-metadata")) d.addCallback(lambda metadata: - self.failUnlessEqual(metadata, {"key": "value"})) + self.failUnless((frozenset(metadata.keys()) == frozenset(["key", "__sys"])) and (metadata['key'] == "value"), metadata)) d.addCallback(lambda res: n.delete(u"newfile-metadata")) d.addCallback(lambda res: n.create_empty_directory(u"subdir2")) diff -rN -u old-timestamps/src/allmydata/util/time_format.py new-timestamps/src/allmydata/util/time_format.py --- old-timestamps/src/allmydata/util/time_format.py 2009-04-08 21:33:41.000000000 -0600 +++ new-timestamps/src/allmydata/util/time_format.py 2009-04-08 21:33:42.000000000 -0600 @@ -19,6 +19,11 @@ now = t() return datetime.datetime.utcfromtimestamp(now).isoformat(sep) +def iso_local(now=None, sep='_', t=time.time): + if now is None: + now = t() + return datetime.datetime.fromtimestamp(now).isoformat(sep) + def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})[T_ ](?P\d{2}):(?P\d{2}):(?P\d{2})(?P\.\d+)?")): """ The inverse of iso_utc(). diff -rN -u old-timestamps/src/allmydata/web/directory.py new-timestamps/src/allmydata/web/directory.py --- old-timestamps/src/allmydata/web/directory.py 2009-04-08 21:33:41.000000000 -0600 +++ new-timestamps/src/allmydata/web/directory.py 2009-04-08 21:33:42.000000000 -0600 @@ -13,7 +13,7 @@ from foolscap.eventual import fireEventually -from allmydata.util import base32 +from allmydata.util import base32, time_format from allmydata.uri import from_string_dirnode from allmydata.interfaces import IDirectoryNode, IFileNode, IMutableFileNode, \ ExistingChildError, NoSuchChildError @@ -592,16 +592,25 @@ ctx.fillSlots("rename", rename) times = [] - TIME_FORMAT = "%H:%M:%S %d-%b-%Y" - if "ctime" in metadata: - ctime = time.strftime(TIME_FORMAT, - time.localtime(metadata["ctime"])) - times.append("c: " + ctime) - if "mtime" in metadata: - mtime = time.strftime(TIME_FORMAT, - time.localtime(metadata["mtime"])) + linkcrtime = metadata.get('__sys', {}).get("linkcrtime") + if linkcrtime is not None: + times.append("lcr: " + time_format.iso_local(linkcrtime)) + else: + # For backwards-compatibility with links last modified by Tahoe < 1.4.0: + if "ctime" in metadata: + ctime = time_format.iso_local(metadata["ctime"]) + times.append("c: " + ctime) + linkmotime = metadata.get('__sys', {}).get("linkmotime") + if linkmotime is not None: if times: times.append(T.br()) + times.append("lmo: " + time_format.iso_local(linkmotime)) + else: + # For backwards-compatibility with links last modified by Tahoe < 1.4.0: + if "mtime" in metadata: + mtime = time_format.iso_local(metadata["mtime"]) + if times: + times.append(T.br()) times.append("m: " + mtime) ctx.fillSlots("times", times)