Changeset 8844655 in trunk

Timestamp:

2010-08-04T18:45:49Z (15 years ago)

Author:

Brian Warner <warner@…>

Branches:

master

Children:

cd8d415

Parents:

2bd8749

Message:

One fix for bug #1154: webapi GETs with a 'Range' header broke new-downloader.

The Range header causes n.read() to be called with an offset= of type 'long',
which eventually got used in a Spans/DataSpans? object's len method.
Apparently python doesn't permit len() to return longs, only ints.
Rewrote Spans/DataSpans? to use s.len() instead of len(s) aka s.len() .
Added a test in test_download. Note that test_web didn't catch this because
it uses mock FileNodes? for speed: it's probably time to rewrite that.

There is still an unresolved error-recovery problem in #1154, so I'm not
closing the ticket quite yet.

Location:

src/allmydata

Files:

• immutable/downloader/share.py (modified) (1 diff)
• test/test_download.py (modified) (1 diff)
• test/test_util.py (modified) (12 diffs)
• util/spans.py (modified) (4 diffs)

src/allmydata/immutable/downloader/share.py ¶

@@ -260 +260 @@
         # and sometimes you can't even get what you need
         disappointment = needed & self._unavailable
-        if len(disappointment):
+        if disappointment.len():
             self.had_corruption = True
             raise DataUnavailable("need %s but will never get it" %

src/allmydata/test/test_download.py ¶

@@ -320 +320 @@
             self.failIfEqual(shares, [0,1,2])
         d.addCallback(_check_failover)
+        return d
+
+    def test_long_offset(self):
+        # bug #1154: mplayer doing a seek-to-end results in an offset of type
+        # 'long', rather than 'int', and apparently __len__ is required to
+        # return an int. Rewrote Spans/DataSpans to provide s.len() instead
+        # of len(s) .
+        self.basedir = self.mktemp()
+        self.set_up_grid()
+        self.c0 = self.g.clients[0]
+        self.load_shares()
+        n = self.c0.create_node_from_uri(immutable_uri)
+
+        c = MemoryConsumer()
+        d = n.read(c, 0L, 10L)
+        d.addCallback(lambda c: len("".join(c.chunks)))
+        d.addCallback(lambda size: self.failUnlessEqual(size, 10))
         return d
 

src/allmydata/test/test_util.py ¶

@@ -1615 +1615 @@
             yield (prevstart, prevend-prevstart+1)
 
-    def __len__(self):
-        # this also gets us bool(s)
+    def __nonzero__(self): # this gets us bool()
+        return self.len()
+
+    def len(self):
         return len(self._have)
 
@@ -1660 +1662 @@
         self.failIf(s)
         self.failIf((0,1) in s)
-        self.failUnlessEqual(len(s), 0)
+        self.failUnlessEqual(s.len(), 0)
 
         s1 = Spans(3, 4) # 3,4,5,6
@@ -1673 +1675 @@
         self.failIf((10,1) in s1)
         self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11])
-        self.failUnlessEqual(len(s2), 6)
+        self.failUnlessEqual(s2.len(), 6)
 
         s2.add(15,2).add(20,2)
         self.failUnlessEqual(list(s2.each()), [3,4,5,6,10,11,15,16,20,21])
-        self.failUnlessEqual(len(s2), 10)
+        self.failUnlessEqual(s2.len(), 10)
 
         s2.remove(4,3).remove(15,1)
         self.failUnlessEqual(list(s2.each()), [3,10,11,16,20,21])
-        self.failUnlessEqual(len(s2), 6)
+        self.failUnlessEqual(s2.len(), 6)
 
         s1 = SimpleSpans(3, 4) # 3 4 5 6
@@ -1691 +1693 @@
         self.failUnlessEqual(list(s), [(3,4)])
         self.failUnless(s)
-        self.failUnlessEqual(len(s), 4)
+        self.failUnlessEqual(s.len(), 4)
         self.failIf((0,1) in s)
         self.failUnless((3,4) in s)
@@ -1818 +1820 @@
             #print "s2 now %s" % s2.dump()
             self.failUnlessEqual(list(s1.each()), list(s2.each()))
-            self.failUnlessEqual(len(s1), len(s2))
+            self.failUnlessEqual(s1.len(), s2.len())
             self.failUnlessEqual(bool(s1), bool(s2))
             self.failUnlessEqual(list(s1), list(s2))
@@ -1838 +1840 @@
     # (start,length) in s -> True if (start..start+length-1) are all members
     #  NOT equivalent to x in list(s)
-    # len(s) -> number of bytes, for testing, bool(), and accounting/limiting
-    # bool(s)  (__len__)
+    # s.len() -> number of bytes, for testing, bool(), and accounting/limiting
+    # bool(s)  (__nonzeron__)
     # s = s1+s2, s1-s2, +=s1, -=s1
 
@@ -1894 +1896 @@
                 self.add(start, data)
 
-    def __len__(self):
+    def __nonzero__(self): # this gets us bool()
+        return self.len()
+    def len(self):
         return len(self.missing.replace("1", ""))
     def _dump(self):
@@ -1931 +1935 @@
     def do_basic(self, klass):
         ds = klass()
-        self.failUnlessEqual(len(ds), 0)
+        self.failUnlessEqual(ds.len(), 0)
         self.failUnlessEqual(list(ds._dump()), [])
         self.failUnlessEqual(sum([len(d) for (s,d) in ds.get_chunks()]), 0)
@@ -1940 +1944 @@
 
         ds.add(2, "four")
-        self.failUnlessEqual(len(ds), 4)
+        self.failUnlessEqual(ds.len(), 4)
         self.failUnlessEqual(list(ds._dump()), [2,3,4,5])
         self.failUnlessEqual(sum([len(d) for (s,d) in ds.get_chunks()]), 4)
@@ -1950 +1954 @@
 
         ds2 = klass(ds)
-        self.failUnlessEqual(len(ds2), 4)
+        self.failUnlessEqual(ds2.len(), 4)
         self.failUnlessEqual(list(ds2._dump()), [2,3,4,5])
         self.failUnlessEqual(sum([len(d) for (s,d) in ds2.get_chunks()]), 4)
@@ -1963 +1967 @@
 
         ds.add(0, "23")
-        self.failUnlessEqual(len(ds), 6)
+        self.failUnlessEqual(ds.len(), 6)
         self.failUnlessEqual(list(ds._dump()), [0,1,2,3,4,5])
         self.failUnlessEqual(sum([len(d) for (s,d) in ds.get_chunks()]), 6)
@@ -2126 +2130 @@
             #print "s1 now %s" % list(s1._dump())
             #print "s2 now %s" % list(s2._dump())
-            self.failUnlessEqual(len(s1), len(s2))
+            self.failUnlessEqual(s1.len(), s2.len())
             self.failUnlessEqual(list(s1._dump()), list(s2._dump()))
             for j in range(100):

src/allmydata/util/spans.py ¶

@@ -141 +141 @@
 
     def dump(self):
-        return "len=%d: %s" % (len(self),
+        return "len=%d: %s" % (self.len(),
                                ",".join(["[%d-%d]" % (start,start+l-1)
                                          for (start,l) in self._spans]) )
@@ -154 +154 @@
             yield s
 
-    def __len__(self):
-        # this also gets us bool(s)
+    def __nonzero__(self): # this gets us bool()
+        return self.len()
+
+    def len(self):
+        # guess what! python doesn't allow __len__ to return a long, only an
+        # int. So we stop using len(spans), use spans.len() instead.
         return sum([length for start,length in self._spans])
 
@@ -229 +233 @@
                 self.add(start, data)
 
-    def __len__(self):
+    def __nonzero__(self): # this gets us bool()
+        return self.len()
+
+    def len(self):
         # return number of bytes we're holding
         return sum([len(data) for (start,data) in self.spans])
@@ -240 +247 @@
 
     def dump(self):
-        return "len=%d: %s" % (len(self),
+        return "len=%d: %s" % (self.len(),
                                ",".join(["[%d-%d]" % (start,start+len(data)-1)
                                          for (start,data) in self.spans]) )