1 | Sun Aug 21 11:49:45 PDT 2011 Kevan Carstensen <kevan@isnotajoke.com> |
---|
2 | * interfaces: fix typo |
---|
3 | |
---|
4 | Sun Aug 21 11:50:04 PDT 2011 Kevan Carstensen <kevan@isnotajoke.com> |
---|
5 | * retrieve: code cleanup, use .callback and .errback appropriately |
---|
6 | |
---|
7 | Fri Aug 26 14:39:37 PDT 2011 Kevan Carstensen <kevan@isnotajoke.com> |
---|
8 | * mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation |
---|
9 | |
---|
10 | Fri Aug 26 14:44:19 PDT 2011 Kevan Carstensen <kevan@isnotajoke.com> |
---|
11 | * mutable/filenode: clean up stale and misleading comments |
---|
12 | |
---|
13 | New patches: |
---|
14 | |
---|
15 | [interfaces: fix typo |
---|
16 | Kevan Carstensen <kevan@isnotajoke.com>**20110821184945 |
---|
17 | Ignore-this: 10a587147aefc63738469bf2e855351b |
---|
18 | ] hunk ./src/allmydata/interfaces.py 452 |
---|
19 | Add a block and salt to the share. |
---|
20 | """ |
---|
21 | |
---|
22 | - def put_encprivey(encprivkey): |
---|
23 | + def put_encprivkey(encprivkey): |
---|
24 | """ |
---|
25 | Add the encrypted private key to the share. |
---|
26 | """ |
---|
27 | [retrieve: code cleanup, use .callback and .errback appropriately |
---|
28 | Kevan Carstensen <kevan@isnotajoke.com>**20110821185004 |
---|
29 | Ignore-this: e9228a6562e8450b2df0b27d169a9fd5 |
---|
30 | ] { |
---|
31 | hunk ./src/allmydata/mutable/retrieve.py 901 |
---|
32 | # hash tree, and is a leaf in the share hash tree. This is fine, |
---|
33 | # since any share corruption will be detected in the share hash |
---|
34 | # tree. |
---|
35 | - #needed.discard(0) |
---|
36 | self.log("getting blockhashes for segment %d, share %d: %s" % \ |
---|
37 | (segnum, reader.shnum, str(needed))) |
---|
38 | d1 = reader.get_blockhashes(needed, queue=True, force_remote=True) |
---|
39 | hunk ./src/allmydata/mutable/retrieve.py 1108 |
---|
40 | |
---|
41 | if self._verify: |
---|
42 | ret = list(self._bad_shares) |
---|
43 | + eventually(self.done_deferred.callback, ret) |
---|
44 | else: |
---|
45 | format = ("ran out of peers: " |
---|
46 | "have %(have)d of %(total)d segments " |
---|
47 | hunk ./src/allmydata/mutable/retrieve.py 1124 |
---|
48 | (format % args, str(self._last_failure))) |
---|
49 | f = failure.Failure(e) |
---|
50 | ret = f |
---|
51 | - eventually(self._done_deferred.callback, ret) |
---|
52 | + eventually(self._done_deferred.errback, ret) |
---|
53 | } |
---|
54 | [mutable/publish: fix buggy get_size implementation in TransformingUploadable, clarify self.datalength calculation |
---|
55 | Kevan Carstensen <kevan@isnotajoke.com>**20110826213937 |
---|
56 | Ignore-this: f0b9077417a28a5333e161b6d527e3b9 |
---|
57 | ] { |
---|
58 | hunk ./src/allmydata/mutable/publish.py 158 |
---|
59 | self.data = data |
---|
60 | |
---|
61 | # XXX: Use the MutableFileVersion instead. |
---|
62 | - self.datalength = self._node.get_size() |
---|
63 | - if data.get_size() > self.datalength: |
---|
64 | - self.datalength = data.get_size() |
---|
65 | + self.new_length = max(self._node.get_size(), offset + data.get_size()) |
---|
66 | |
---|
67 | self.log("starting update") |
---|
68 | self.log("adding new data of length %d at offset %d" % \ |
---|
69 | hunk ./src/allmydata/mutable/publish.py 163 |
---|
70 | (data.get_size(), offset)) |
---|
71 | - self.log("new data length is %d" % self.datalength) |
---|
72 | - self._status.set_size(self.datalength) |
---|
73 | + self.log("new data length is %d" % self.new_length) |
---|
74 | + self._status.set_size(self.new_length) |
---|
75 | self._status.set_status("Started") |
---|
76 | self._started = time.time() |
---|
77 | |
---|
78 | hunk ./src/allmydata/mutable/publish.py 275 |
---|
79 | self.required_shares, |
---|
80 | self.total_shares, |
---|
81 | self.segment_size, |
---|
82 | - self.datalength) |
---|
83 | + self.new_length) |
---|
84 | self.writers[shnum].peerid = peerid |
---|
85 | assert (peerid, shnum) in self._servermap.servermap |
---|
86 | old_versionid, old_timestamp = self._servermap.servermap[key] |
---|
87 | hunk ./src/allmydata/mutable/publish.py 349 |
---|
88 | assert IMutableUploadable.providedBy(newdata) |
---|
89 | |
---|
90 | self.data = newdata |
---|
91 | - self.datalength = newdata.get_size() |
---|
92 | + self.new_length = newdata.get_size() |
---|
93 | #if self.datalength >= DEFAULT_MAX_SEGMENT_SIZE: |
---|
94 | # self._version = MDMF_VERSION |
---|
95 | #else: |
---|
96 | hunk ./src/allmydata/mutable/publish.py 355 |
---|
97 | # self._version = SDMF_VERSION |
---|
98 | |
---|
99 | - self.log("starting publish, datalen is %s" % self.datalength) |
---|
100 | - self._status.set_size(self.datalength) |
---|
101 | + self.log("starting publish, datalen is %s" % self.new_length) |
---|
102 | + self._status.set_size(self.new_length) |
---|
103 | self._status.set_status("Started") |
---|
104 | self._started = time.time() |
---|
105 | |
---|
106 | hunk ./src/allmydata/mutable/publish.py 480 |
---|
107 | self.required_shares, |
---|
108 | self.total_shares, |
---|
109 | self.segment_size, |
---|
110 | - self.datalength) |
---|
111 | + self.new_length) |
---|
112 | self.writers[shnum].peerid = peerid |
---|
113 | if (peerid, shnum) in self._servermap.servermap: |
---|
114 | old_versionid, old_timestamp = self._servermap.servermap[key] |
---|
115 | hunk ./src/allmydata/mutable/publish.py 538 |
---|
116 | if self._version == MDMF_VERSION: |
---|
117 | segment_size = DEFAULT_MAX_SEGMENT_SIZE # 128 KiB by default |
---|
118 | else: |
---|
119 | - segment_size = self.datalength # SDMF is only one segment |
---|
120 | + segment_size = self.new_length # SDMF is only one segment |
---|
121 | # this must be a multiple of self.required_shares |
---|
122 | segment_size = mathutil.next_multiple(segment_size, |
---|
123 | self.required_shares) |
---|
124 | hunk ./src/allmydata/mutable/publish.py 548 |
---|
125 | if segment_size: |
---|
126 | # We use div_ceil instead of integer division here because |
---|
127 | # it is semantically correct. |
---|
128 | - # If datalength isn't an even multiple of segment_size, but |
---|
129 | - # is larger than segment_size, datalength // segment_size |
---|
130 | - # will be the largest number such that num <= datalength and |
---|
131 | + # If new_length isn't an even multiple of segment_size, but |
---|
132 | + # is larger than segment_size, new_length // segment_size |
---|
133 | + # will be the largest number such that num <= new_length and |
---|
134 | # num % segment_size == 0. But that's not what we want, |
---|
135 | # because it ignores the extra data. div_ceil will give us |
---|
136 | # the right number of segments for the data that we're |
---|
137 | hunk ./src/allmydata/mutable/publish.py 555 |
---|
138 | # given. |
---|
139 | - self.num_segments = mathutil.div_ceil(self.datalength, |
---|
140 | + self.num_segments = mathutil.div_ceil(self.new_length, |
---|
141 | segment_size) |
---|
142 | |
---|
143 | self.starting_segment = offset // segment_size |
---|
144 | hunk ./src/allmydata/mutable/publish.py 573 |
---|
145 | assert self.num_segments in (0, 1) # SDMF |
---|
146 | # calculate the tail segment size. |
---|
147 | |
---|
148 | - if segment_size and self.datalength: |
---|
149 | - self.tail_segment_size = self.datalength % segment_size |
---|
150 | + if segment_size and self.new_length: |
---|
151 | + self.tail_segment_size = self.new_length % segment_size |
---|
152 | self.log("got tail segment size %d" % self.tail_segment_size) |
---|
153 | else: |
---|
154 | self.tail_segment_size = 0 |
---|
155 | hunk ./src/allmydata/mutable/publish.py 602 |
---|
156 | self._current_segment = self.starting_segment |
---|
157 | self.end_segment = self.num_segments - 1 |
---|
158 | # Now figure out where the last segment should be. |
---|
159 | - if self.data.get_size() != self.datalength: |
---|
160 | + if offset + self.data.get_size() != self.new_length: |
---|
161 | # We're updating a few segments in the middle of a mutable |
---|
162 | # file, so we don't want to republish the whole thing. |
---|
163 | # (we don't have enough data to do that even if we wanted |
---|
164 | hunk ./src/allmydata/mutable/publish.py 607 |
---|
165 | # to) |
---|
166 | - end = self.data.get_size() |
---|
167 | + end = offset + self.data.get_size() |
---|
168 | self.end_segment = end // segment_size |
---|
169 | if end % segment_size == 0: |
---|
170 | self.end_segment -= 1 |
---|
171 | hunk ./src/allmydata/mutable/publish.py 1295 |
---|
172 | |
---|
173 | |
---|
174 | def get_size(self): |
---|
175 | - return self._offset + self._newdata.get_size() |
---|
176 | + return self._newdata.get_size() |
---|
177 | |
---|
178 | |
---|
179 | def read(self, length): |
---|
180 | } |
---|
181 | [mutable/filenode: clean up stale and misleading comments |
---|
182 | Kevan Carstensen <kevan@isnotajoke.com>**20110826214419 |
---|
183 | Ignore-this: 9d79831b814b4ddfd46023f61c5f4930 |
---|
184 | ] { |
---|
185 | hunk ./src/allmydata/mutable/filenode.py 549 |
---|
186 | new_contents as an argument. I return a Deferred that eventually |
---|
187 | fires with the results of my replacement process. |
---|
188 | """ |
---|
189 | - # TODO: Update downloader hints. |
---|
190 | return self._do_serialized(self._overwrite, new_contents) |
---|
191 | |
---|
192 | |
---|
193 | hunk ./src/allmydata/mutable/filenode.py 569 |
---|
194 | creating/updating our own servermap. I return a Deferred that |
---|
195 | fires with the results of my upload. |
---|
196 | """ |
---|
197 | - # TODO: Update downloader hints |
---|
198 | return self._do_serialized(self._upload, new_contents, servermap) |
---|
199 | |
---|
200 | |
---|
201 | hunk ./src/allmydata/mutable/filenode.py 580 |
---|
202 | Deferred that eventually fires with an UploadResults instance |
---|
203 | describing this process. |
---|
204 | """ |
---|
205 | - # TODO: Update downloader hints. |
---|
206 | return self._do_serialized(self._modify, modifier, backoffer) |
---|
207 | |
---|
208 | |
---|
209 | hunk ./src/allmydata/mutable/filenode.py 647 |
---|
210 | return u.update() |
---|
211 | |
---|
212 | |
---|
213 | - #def set_version(self, version): |
---|
214 | - # I can be set in two ways: |
---|
215 | - # 1. When the node is created. |
---|
216 | - # 2. (for an existing share) when the Servermap is updated |
---|
217 | - # before I am read. |
---|
218 | - # assert version in (MDMF_VERSION, SDMF_VERSION) |
---|
219 | - # self._protocol_version = version |
---|
220 | - |
---|
221 | - |
---|
222 | def get_version(self): |
---|
223 | return self._protocol_version |
---|
224 | |
---|
225 | hunk ./src/allmydata/mutable/filenode.py 671 |
---|
226 | |
---|
227 | |
---|
228 | def _upload(self, new_contents, servermap): |
---|
229 | - """ |
---|
230 | - A MutableFileNode still has to have some way of getting |
---|
231 | - published initially, which is what I am here for. After that, |
---|
232 | - all publishing, updating, modifying and so on happens through |
---|
233 | - MutableFileVersions. |
---|
234 | - """ |
---|
235 | assert self._pubkey, "update_servermap must be called before publish" |
---|
236 | |
---|
237 | # Define IPublishInvoker with a set_downloader_hints method? |
---|
238 | } |
---|
239 | |
---|
240 | Context: |
---|
241 | |
---|
242 | [test_mutable.Update: increase timeout from 120s to 400s, slaves are failing |
---|
243 | Brian Warner <warner@lothar.com>**20110825230140 |
---|
244 | Ignore-this: 101b1924a30cdbda9b2e419e95ca15ec |
---|
245 | ] |
---|
246 | [tests: fix check_memory test |
---|
247 | zooko@zooko.com**20110825201116 |
---|
248 | Ignore-this: 4d66299fa8cb61d2ca04b3f45344d835 |
---|
249 | fixes #1503 |
---|
250 | ] |
---|
251 | [TAG allmydata-tahoe-1.9.0a1 |
---|
252 | warner@lothar.com**20110825161122 |
---|
253 | Ignore-this: 3cbf49f00dbda58189f893c427f65605 |
---|
254 | ] |
---|
255 | Patch bundle hash: |
---|
256 | 7add88d8b5da41b74f86c083972ed743f5757083 |
---|