Ticket #393: code-cleanup-and-datalength-fixes.dpatch

File code-cleanup-and-datalength-fixes.dpatch, 10.6 KB (added by kevan, at 2011-08-26T22:19:03Z)

fixes described in comment:139 and comment:152

Line 
1Sun Aug 21 11:49:45 PDT 2011  Kevan Carstensen <kevan@isnotajoke.com>
2  * interfaces: fix typo
3
4Sun Aug 21 11:50:04 PDT 2011  Kevan Carstensen <kevan@isnotajoke.com>
5  * retrieve: code cleanup, use .callback and .errback appropriately
6
7Fri Aug 26 14:39:37 PDT 2011  Kevan Carstensen <kevan@isnotajoke.com>
8  * mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation
9
10Fri Aug 26 14:44:19 PDT 2011  Kevan Carstensen <kevan@isnotajoke.com>
11  * mutable/filenode: clean up stale and misleading comments
12
13New patches:
14
15[interfaces: fix typo
16Kevan Carstensen <kevan@isnotajoke.com>**20110821184945
17 Ignore-this: 10a587147aefc63738469bf2e855351b
18] hunk ./src/allmydata/interfaces.py 452
19         Add a block and salt to the share.
20         """
21 
22-    def put_encprivey(encprivkey):
23+    def put_encprivkey(encprivkey):
24         """
25         Add the encrypted private key to the share.
26         """
27[retrieve: code cleanup, use .callback and .errback appropriately
28Kevan Carstensen <kevan@isnotajoke.com>**20110821185004
29 Ignore-this: e9228a6562e8450b2df0b27d169a9fd5
30] {
31hunk ./src/allmydata/mutable/retrieve.py 901
32         # hash tree, and is a leaf in the share hash tree. This is fine,
33         # since any share corruption will be detected in the share hash
34         # tree.
35-        #needed.discard(0)
36         self.log("getting blockhashes for segment %d, share %d: %s" % \
37                  (segnum, reader.shnum, str(needed)))
38         d1 = reader.get_blockhashes(needed, queue=True, force_remote=True)
39hunk ./src/allmydata/mutable/retrieve.py 1108
40 
41         if self._verify:
42             ret = list(self._bad_shares)
43+            eventually(self.done_deferred.callback, ret)
44         else:
45             format = ("ran out of peers: "
46                       "have %(have)d of %(total)d segments "
47hunk ./src/allmydata/mutable/retrieve.py 1124
48                                      (format % args, str(self._last_failure)))
49             f = failure.Failure(e)
50             ret = f
51-        eventually(self._done_deferred.callback, ret)
52+            eventually(self._done_deferred.errback, ret)
53}
54[mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation
55Kevan Carstensen <kevan@isnotajoke.com>**20110826213937
56 Ignore-this: f0b9077417a28a5333e161b6d527e3b9
57] {
58hunk ./src/allmydata/mutable/publish.py 158
59         self.data = data
60 
61         # XXX: Use the MutableFileVersion instead.
62-        self.datalength = self._node.get_size()
63-        if data.get_size() > self.datalength:
64-            self.datalength = data.get_size()
65+        self.new_length = max(self._node.get_size(), offset + data.get_size())
66 
67         self.log("starting update")
68         self.log("adding new data of length %d at offset %d" % \
69hunk ./src/allmydata/mutable/publish.py 163
70                     (data.get_size(), offset))
71-        self.log("new data length is %d" % self.datalength)
72-        self._status.set_size(self.datalength)
73+        self.log("new data length is %d" % self.new_length)
74+        self._status.set_size(self.new_length)
75         self._status.set_status("Started")
76         self._started = time.time()
77 
78hunk ./src/allmydata/mutable/publish.py 275
79                                                 self.required_shares,
80                                                 self.total_shares,
81                                                 self.segment_size,
82-                                                self.datalength)
83+                                                self.new_length)
84             self.writers[shnum].peerid = peerid
85             assert (peerid, shnum) in self._servermap.servermap
86             old_versionid, old_timestamp = self._servermap.servermap[key]
87hunk ./src/allmydata/mutable/publish.py 349
88         assert IMutableUploadable.providedBy(newdata)
89 
90         self.data = newdata
91-        self.datalength = newdata.get_size()
92+        self.new_length = newdata.get_size()
93         #if self.datalength >= DEFAULT_MAX_SEGMENT_SIZE:
94         #    self._version = MDMF_VERSION
95         #else:
96hunk ./src/allmydata/mutable/publish.py 355
97         #    self._version = SDMF_VERSION
98 
99-        self.log("starting publish, datalen is %s" % self.datalength)
100-        self._status.set_size(self.datalength)
101+        self.log("starting publish, datalen is %s" % self.new_length)
102+        self._status.set_size(self.new_length)
103         self._status.set_status("Started")
104         self._started = time.time()
105 
106hunk ./src/allmydata/mutable/publish.py 480
107                                                 self.required_shares,
108                                                 self.total_shares,
109                                                 self.segment_size,
110-                                                self.datalength)
111+                                                self.new_length)
112             self.writers[shnum].peerid = peerid
113             if (peerid, shnum) in self._servermap.servermap:
114                 old_versionid, old_timestamp = self._servermap.servermap[key]
115hunk ./src/allmydata/mutable/publish.py 538
116         if self._version == MDMF_VERSION:
117             segment_size = DEFAULT_MAX_SEGMENT_SIZE # 128 KiB by default
118         else:
119-            segment_size = self.datalength # SDMF is only one segment
120+            segment_size = self.new_length # SDMF is only one segment
121         # this must be a multiple of self.required_shares
122         segment_size = mathutil.next_multiple(segment_size,
123                                               self.required_shares)
124hunk ./src/allmydata/mutable/publish.py 548
125         if segment_size:
126             # We use div_ceil instead of integer division here because
127             # it is semantically correct.
128-            # If datalength isn't an even multiple of segment_size, but
129-            # is larger than segment_size, datalength // segment_size
130-            # will be the largest number such that num <= datalength and
131+            # If new_length isn't an even multiple of segment_size, but
132+            # is larger than segment_size, new_length // segment_size
133+            # will be the largest number such that num <= new_length and
134             # num % segment_size == 0. But that's not what we want,
135             # because it ignores the extra data. div_ceil will give us
136             # the right number of segments for the data that we're
137hunk ./src/allmydata/mutable/publish.py 555
138             # given.
139-            self.num_segments = mathutil.div_ceil(self.datalength,
140+            self.num_segments = mathutil.div_ceil(self.new_length,
141                                                   segment_size)
142 
143             self.starting_segment = offset // segment_size
144hunk ./src/allmydata/mutable/publish.py 573
145             assert self.num_segments in (0, 1) # SDMF
146         # calculate the tail segment size.
147 
148-        if segment_size and self.datalength:
149-            self.tail_segment_size = self.datalength % segment_size
150+        if segment_size and self.new_length:
151+            self.tail_segment_size = self.new_length % segment_size
152             self.log("got tail segment size %d" % self.tail_segment_size)
153         else:
154             self.tail_segment_size = 0
155hunk ./src/allmydata/mutable/publish.py 602
156         self._current_segment = self.starting_segment
157         self.end_segment = self.num_segments - 1
158         # Now figure out where the last segment should be.
159-        if self.data.get_size() != self.datalength:
160+        if offset + self.data.get_size() != self.new_length:
161             # We're updating a few segments in the middle of a mutable
162             # file, so we don't want to republish the whole thing.
163             # (we don't have enough data to do that even if we wanted
164hunk ./src/allmydata/mutable/publish.py 607
165             # to)
166-            end = self.data.get_size()
167+            end = offset + self.data.get_size()
168             self.end_segment = end // segment_size
169             if end % segment_size == 0:
170                 self.end_segment -= 1
171hunk ./src/allmydata/mutable/publish.py 1295
172 
173 
174     def get_size(self):
175-        return self._offset + self._newdata.get_size()
176+        return self._newdata.get_size()
177 
178 
179     def read(self, length):
180}
181[mutable/filenode: clean up stale and misleading comments
182Kevan Carstensen <kevan@isnotajoke.com>**20110826214419
183 Ignore-this: 9d79831b814b4ddfd46023f61c5f4930
184] {
185hunk ./src/allmydata/mutable/filenode.py 549
186         new_contents as an argument. I return a Deferred that eventually
187         fires with the results of my replacement process.
188         """
189-        # TODO: Update downloader hints.
190         return self._do_serialized(self._overwrite, new_contents)
191 
192 
193hunk ./src/allmydata/mutable/filenode.py 569
194         creating/updating our own servermap. I return a Deferred that
195         fires with the results of my upload.
196         """
197-        # TODO: Update downloader hints
198         return self._do_serialized(self._upload, new_contents, servermap)
199 
200 
201hunk ./src/allmydata/mutable/filenode.py 580
202         Deferred that eventually fires with an UploadResults instance
203         describing this process.
204         """
205-        # TODO: Update downloader hints.
206         return self._do_serialized(self._modify, modifier, backoffer)
207 
208 
209hunk ./src/allmydata/mutable/filenode.py 647
210         return u.update()
211 
212 
213-    #def set_version(self, version):
214-        # I can be set in two ways:
215-        #  1. When the node is created.
216-        #  2. (for an existing share) when the Servermap is updated
217-        #     before I am read.
218-    #    assert version in (MDMF_VERSION, SDMF_VERSION)
219-    #    self._protocol_version = version
220-
221-
222     def get_version(self):
223         return self._protocol_version
224 
225hunk ./src/allmydata/mutable/filenode.py 671
226 
227 
228     def _upload(self, new_contents, servermap):
229-        """
230-        A MutableFileNode still has to have some way of getting
231-        published initially, which is what I am here for. After that,
232-        all publishing, updating, modifying and so on happens through
233-        MutableFileVersions.
234-        """
235         assert self._pubkey, "update_servermap must be called before publish"
236 
237         # Define IPublishInvoker with a set_downloader_hints method?
238}
239
240Context:
241
242[test_mutable.Update: increase timeout from 120s to 400s, slaves are failing
243Brian Warner <warner@lothar.com>**20110825230140
244 Ignore-this: 101b1924a30cdbda9b2e419e95ca15ec
245] 
246[tests: fix check_memory test
247zooko@zooko.com**20110825201116
248 Ignore-this: 4d66299fa8cb61d2ca04b3f45344d835
249 fixes #1503
250] 
251[TAG allmydata-tahoe-1.9.0a1
252warner@lothar.com**20110825161122
253 Ignore-this: 3cbf49f00dbda58189f893c427f65605
254] 
255Patch bundle hash:
2567add88d8b5da41b74f86c083972ed743f5757083