Changeset ea373de in trunk for src/allmydata

Timestamp:

2009-10-17T18:00:05Z (16 years ago)

Author:

Brian Warner <warner@…>

Branches:

master

Children:

8a7c980

Parents:

e63f59f

Message:

move dirnode.CachingDict? to dictutil.AuxValueDict?, generalize method names,
improve tests. Let dirnode _pack_children accept either dict or AuxValueDict?.

Location:

src/allmydata

Files:

• dirnode.py (modified) (3 diffs)
• test/test_dirnode.py (modified) (1 diff)
• test/test_util.py (modified) (1 diff)
• util/dictutil.py (modified) (1 diff)

src/allmydata/dirnode.py ¶

@@ -22 +22 @@
      LiteralFileURI, from_string
 from pycryptopp.cipher.aes import AES
-
-class CachingDict(dict):
-    def __init__(self, *args):
-        super(CachingDict, self).__init__(*args)
-        self.serialized = {}
-
-    def __setitem__(self, k, v):
-        super(CachingDict, self).__setitem__(k, v)
-        self.serialized[k] = None
-
-    def get_both_items(self, k):
-        return (self.serialized.setdefault(k, None),
-                super(CachingDict, self).__getitem__(k))
-
-    def set_both_items(self, key, serialized, t):
-        self.serialized[key] = serialized
-        super(CachingDict, self).__setitem__(key, t)
+from allmydata.util.dictutil import AuxValueDict
 
 class Deleter:
@@ -210 +194 @@
         # an empty directory is serialized as an empty string
         if data == "":
-            return CachingDict()
+            return AuxValueDict()
         writeable = not self.is_readonly()
-        children = CachingDict()
+        children = AuxValueDict()
         position = 0
         while position < len(data):
@@ -229 +213 @@
             metadata = simplejson.loads(metadata_s)
             assert isinstance(metadata, dict)
-            children.set_both_items(name, entry, (child, metadata))
+            children.set_with_aux(name, (child, metadata), auxilliary=entry)
         return children
 
     def _pack_contents(self, children):
         # expects children in the same format as _unpack_contents
-        assert isinstance(children, CachingDict)
+        has_aux = isinstance(children, AuxValueDict)
         entries = []
         for name in sorted(children.keys()):
-            entry, metadata = children.get_both_items(name)
-            if entry == None:
-                child, metadata = metadata
-                assert isinstance(name, unicode)
+            assert isinstance(name, unicode)
+            entry = None
+            if has_aux:
+                entry = children.get_aux(name)
+            if not entry:
+                child, metadata = children.get(name)
                 assert IFilesystemNode.providedBy(child), (name,child)
                 assert isinstance(metadata, dict)

src/allmydata/test/test_dirnode.py ¶

@@ -776 +776 @@
         self.failUnlessEqual(file1_rwcap,
                              children[u'file1'][0].get_uri())
-
-    def test_caching_dict(self):
-        d = dirnode.CachingDict()
-        d.set_both_items("test", "test2", ("test3", "test4"))
-        cached, value = d.get_both_items("test")
-
-        self.failUnlessEqual(cached, "test2")
-        self.failUnlessEqual(value, ("test3", "test4"))
-
-        d['test'] = ("test3", "test2")
-
-        cached, value = d.get_both_items("test")
-
-        self.failUnlessEqual(cached, None)
-        self.failUnlessEqual(value, ("test3", "test2"))
 
 class FakeMutableFile:

src/allmydata/test/test_util.py ¶

@@ -1275 +1275 @@
         self.failUnlessEqual(d.items(), [("c", 1), ("a", 3)])
 
+    def test_auxdict(self):
+        d = dictutil.AuxValueDict()
+        # we put the serialized form in the auxdata
+        d.set_with_aux("key", ("filecap", "metadata"), "serialized")
+
+        self.failUnlessEqual(d.keys(), ["key"])
+        self.failUnlessEqual(d["key"], ("filecap", "metadata"))
+        self.failUnlessEqual(d.get_aux("key"), "serialized")
+        def _get_missing(key):
+            return d[key]
+        self.failUnlessRaises(KeyError, _get_missing, "nonkey")
+        self.failUnlessEqual(d.get("nonkey"), None)
+        self.failUnlessEqual(d.get("nonkey", "nonvalue"), "nonvalue")
+        self.failUnlessEqual(d.get_aux("nonkey"), None)
+        self.failUnlessEqual(d.get_aux("nonkey", "nonvalue"), "nonvalue")
+
+        d["key"] = ("filecap2", "metadata2")
+        self.failUnlessEqual(d["key"], ("filecap2", "metadata2"))
+        self.failUnlessEqual(d.get_aux("key"), None)
+
+        d.set_with_aux("key2", "value2", "aux2")
+        self.failUnlessEqual(sorted(d.keys()), ["key", "key2"])
+        del d["key2"]
+        self.failUnlessEqual(d.keys(), ["key"])
+        self.failIf("key2" in d)
+        self.failUnlessRaises(KeyError, _get_missing, "key2")
+        self.failUnlessEqual(d.get("key2"), None)
+        self.failUnlessEqual(d.get_aux("key2"), None)
+        d["key2"] = "newvalue2"
+        self.failUnlessEqual(d.get("key2"), "newvalue2")
+        self.failUnlessEqual(d.get_aux("key2"), None)
+
+        d = dictutil.AuxValueDict({1:2,3:4})
+        self.failUnlessEqual(sorted(d.keys()), [1,3])
+        self.failUnlessEqual(d[1], 2)
+        self.failUnlessEqual(d.get_aux(1), None)
+
+        d = dictutil.AuxValueDict([ (1,2), (3,4) ])
+        self.failUnlessEqual(sorted(d.keys()), [1,3])
+        self.failUnlessEqual(d[1], 2)
+        self.failUnlessEqual(d.get_aux(1), None)
+
+        d = dictutil.AuxValueDict(one=1, two=2)
+        self.failUnlessEqual(sorted(d.keys()), ["one","two"])
+        self.failUnlessEqual(d["one"], 1)
+        self.failUnlessEqual(d.get_aux("one"), None)
+
 class Pipeline(unittest.TestCase):
     def pause(self, *args, **kwargs):

src/allmydata/util/dictutil.py ¶

@@ -606 +606 @@
         del self.d[le[1]]
         return le[1]
+
+class AuxValueDict(dict):
+    """I behave like a regular dict, but each key is associated with two
+    values: the main value, and an auxilliary one. Setting the main value
+    (with the usual d[key]=value) clears the auxvalue. You can set both main
+    and auxvalue at the same time, and can retrieve the values separately.
+
+    The main use case is a dictionary that represents unpacked child values
+    for a directory node, where a common pattern is to modify one or more
+    children and then pass the dict back to a packing function. The original
+    packed representation can be cached in the auxvalue, and the packing
+    function can use it directly on all unmodified children. On large
+    directories with a complex packing function, this can save considerable
+    time."""
+
+    def __init__(self, *args, **kwargs):
+        super(AuxValueDict, self).__init__(*args, **kwargs)
+        self.auxilliary = {}
+
+    def __setitem__(self, key, value):
+        super(AuxValueDict, self).__setitem__(key, value)
+        self.auxilliary[key] = None # clear the auxvalue
+
+    def __delitem__(self, key):
+        super(AuxValueDict, self).__delitem__(key)
+        self.auxilliary.pop(key)
+
+    def get_aux(self, key, default=None):
+        """Retrieve the auxilliary value. There is no way to distinguish
+        between an auxvalue of 'None' and a key that does not have an
+        auxvalue, and get_aux() will not raise KeyError when called with a
+        missing key."""
+        return self.auxilliary.get(key, default)
+
+    def set_with_aux(self, key, value, auxilliary):
+        """Set both the main value and the auxilliary value. There is no way
+        to distinguish between an auxvalue of 'None' and a key that does not
+        have an auxvalue."""
+        super(AuxValueDict, self).__setitem__(key, value)
+        self.auxilliary[key] = auxilliary