Ticket #999: split_s3share_classes_and_prune_unused_methods.dpatch

File split_s3share_classes_and_prune_unused_methods.dpatch, 20.2 KB (added by zooko, at 2011-09-29T05:53:00Z)
Line 
11 patch for repository /home/zooko/playground/tahoe-lafs/999/dsv16:
2
3Wed Sep 28 23:50:38 MDT 2011  zooko@zooko.com
4  * split Immutable S3 Share into for-reading and for-writing classes, remove unused (as far as I can tell) methods, use cStringIO for buffering the writes
5  TODO: define the interfaces that the new classes claim to implement
6
7New patches:
8
9[split Immutable S3 Share into for-reading and for-writing classes, remove unused (as far as I can tell) methods, use cStringIO for buffering the writes
10zooko@zooko.com**20110929055038
11 Ignore-this: 82d8c4488a8548936285a975ef5a1559
12 TODO: define the interfaces that the new classes claim to implement
13] {
14hunk ./src/allmydata/interfaces.py 503
15 
16     def get_used_space():
17         """
18-        Returns the amount of backend storage including overhead, in bytes, used
19-        by this share.
20+        Returns the amount of backend storage including overhead (which may
21+        have to be estimated), in bytes, used by this share.
22         """
23 
24     def unlink():
25hunk ./src/allmydata/storage/backends/s3/immutable.py 3
26 
27 import struct
28+from cStringIO import StringIO
29 
30 from twisted.internet import defer
31 
32hunk ./src/allmydata/storage/backends/s3/immutable.py 27
33 #  data_length+0x0c: first lease. Each lease record is 72 bytes.
34 
35 
36-class ImmutableS3Share(object):
37-    implements(IStoredShare)
38+class ImmutableS3ShareBase(object):
39+    implements(IShareBase) # XXX
40 
41     sharetype = "immutable"
42     LEASE_SIZE = struct.calcsize(">L32s32sL")  # for compatibility
43hunk ./src/allmydata/storage/backends/s3/immutable.py 35
44     HEADER = ">LLL"
45     HEADER_SIZE = struct.calcsize(HEADER)
46 
47-    def __init__(self, s3bucket, storageindex, shnum, max_size=None, data=None):
48-        """
49-        If max_size is not None then I won't allow more than max_size to be written to me.
50-
51-        Clients should use the load_immutable_s3_share and create_immutable_s3_share
52-        factory functions rather than creating instances directly.
53-        """
54+    def __init__(self, s3bucket, storageindex, shnum):
55         self._s3bucket = s3bucket
56         self._storageindex = storageindex
57         self._shnum = shnum
58hunk ./src/allmydata/storage/backends/s3/immutable.py 39
59-        self._max_size = max_size
60-        self._data = data
61         self._key = get_s3_share_key(storageindex, shnum)
62hunk ./src/allmydata/storage/backends/s3/immutable.py 40
63-        self._data_offset = self.HEADER_SIZE
64-        self._loaded = False
65 
66     def __repr__(self):
67hunk ./src/allmydata/storage/backends/s3/immutable.py 42
68-        return ("<ImmutableS3Share at %r>" % (self._key,))
69-
70-    def load(self):
71-        if self._max_size is not None:  # creating share
72-            # The second field, which was the four-byte share data length in
73-            # Tahoe-LAFS versions prior to 1.3.0, is not used; we always write 0.
74-            # We also write 0 for the number of leases.
75-            self._home.setContent(struct.pack(self.HEADER, 1, 0, 0) )
76-            self._end_offset = self.HEADER_SIZE + self._max_size
77-            self._size = self.HEADER_SIZE
78-            self._writes = []
79-            self._loaded = True
80-            return defer.succeed(None)
81-
82-        if self._data is None:
83-            # If we don't already have the data, get it from S3.
84-            d = self._s3bucket.get_object(self._key)
85-        else:
86-            d = defer.succeed(self._data)
87-
88-        def _got_data(data):
89-            self._data = data
90-            header = self._data[:self.HEADER_SIZE]
91-            (version, unused, num_leases) = struct.unpack(self.HEADER, header)
92-
93-            if version != 1:
94-                msg = "%r had version %d but we wanted 1" % (self, version)
95-                raise UnknownImmutableContainerVersionError(msg)
96-
97-            # We cannot write leases in share files, but allow them to be present
98-            # in case a share file is copied from a disk backend, or in case we
99-            # need them in future.
100-            self._size = len(self._data)
101-            self._end_offset = self._size - (num_leases * self.LEASE_SIZE)
102-            self._loaded = True
103-        d.addCallback(_got_data)
104-        return d
105-
106-    def close(self):
107-        # This will briefly use memory equal to double the share size.
108-        # We really want to stream writes to S3, but I don't think txaws supports that yet
109-        # (and neither does IS3Bucket, since that's a thin wrapper over the txaws S3 API).
110-
111-        self._data = "".join(self._writes)
112-        del self._writes
113-        self._s3bucket.put_object(self._key, self._data)
114-        return defer.succeed(None)
115-
116-    def get_used_space(self):
117-        return self._size
118+        return ("<%s at %r>" % (self.__class__.__name__, self._key,))
119 
120     def get_storage_index(self):
121         return self._storageindex
122hunk ./src/allmydata/storage/backends/s3/immutable.py 53
123     def get_shnum(self):
124         return self._shnum
125 
126-    def unlink(self):
127-        self._data = None
128-        self._writes = None
129-        return self._s3bucket.delete_object(self._key)
130+class ImmutableS3ShareForWriting(ImmutableS3ShareBase):
131+    implements(IShareForWriting) # XXX
132+
133+    def __init__(self, s3bucket, storageindex, shnum, max_size):
134+        """
135+        I won't allow more than max_size to be written to me.
136+        """
137+        precondition(isinstance(max_size, (int, long)), max_size)
138+        ImmutableS3ShareBase.__init__(self, s3bucket, storageindex, shnum)
139+        self._max_size = max_size
140+        self._end_offset = self.HEADER_SIZE + self._max_size
141+
142+        self._buf = StringIO()
143+        # The second field, which was the four-byte share data length in
144+        # Tahoe-LAFS versions prior to 1.3.0, is not used; we always write 0.
145+        # We also write 0 for the number of leases.
146+        self._buf.write(struct.pack(self.HEADER, 1, 0, 0) )
147+
148+    def close(self):
149+        # We really want to stream writes to S3, but txaws doesn't support
150+        # that yet (and neither does IS3Bucket, since that's a thin wrapper
151+        # over the txaws S3 API).  See
152+        # https://bugs.launchpad.net/txaws/+bug/767205 and
153+        # https://bugs.launchpad.net/txaws/+bug/783801
154+        return self._s3bucket.put_object(self._key, self._buf.getvalue())
155 
156     def get_allocated_size(self):
157         return self._max_size
158hunk ./src/allmydata/storage/backends/s3/immutable.py 82
159 
160-    def get_size(self):
161-        return self._size
162+    def write_share_data(self, offset, data):
163+        self._buf.seek(offset)
164+        self._buf.write(data)
165+        if self._buf.tell() > self._max_size:
166+            raise DataTooLargeError(self._max_size, offset, len(data))
167+        return defer.succeed(None)
168+
169+class ImmutableS3ShareForReading(object):
170+    implements(IStoredShareForReading) # XXX
171+
172+    def __init__(self, s3bucket, storageindex, shnum, data):
173+        ImmutableS3ShareBase.__init__(self, s3bucket, storageindex, shnum)
174+        self._data = data
175+
176+        header = self._data[:self.HEADER_SIZE]
177+        (version, unused, num_leases) = struct.unpack(self.HEADER, header)
178 
179hunk ./src/allmydata/storage/backends/s3/immutable.py 99
180-    def get_data_length(self):
181-        return self._end_offset - self._data_offset
182+        if version != 1:
183+            msg = "%r had version %d but we wanted 1" % (self, version)
184+            raise UnknownImmutableContainerVersionError(msg)
185+
186+        # We cannot write leases in share files, but allow them to be present
187+        # in case a share file is copied from a disk backend, or in case we
188+        # need them in future.
189+        self._end_offset = len(self._data) - (num_leases * self.LEASE_SIZE)
190 
191     def readv(self, readv):
192         datav = []
193hunk ./src/allmydata/storage/backends/s3/immutable.py 119
194 
195         # Reads beyond the end of the data are truncated. Reads that start
196         # beyond the end of the data return an empty string.
197-        seekpos = self._data_offset+offset
198+        seekpos = self.HEADER_SIZE+offset
199         actuallength = max(0, min(length, self._end_offset-seekpos))
200         if actuallength == 0:
201             return defer.succeed("")
202hunk ./src/allmydata/storage/backends/s3/immutable.py 124
203         return defer.succeed(self._data[offset:offset+actuallength])
204-
205-    def write_share_data(self, offset, data):
206-        length = len(data)
207-        precondition(offset >= self._size, "offset = %r, size = %r" % (offset, self._size))
208-        if self._max_size is not None and offset+length > self._max_size:
209-            raise DataTooLargeError(self._max_size, offset, length)
210-
211-        if offset > self._size:
212-            self._writes.append("\x00" * (offset - self._size))
213-        self._writes.append(data)
214-        self._size = offset + len(data)
215-        return defer.succeed(None)
216-
217-    def add_lease(self, lease_info):
218-        pass
219-
220-
221-def load_immutable_s3_share(s3bucket, storageindex, shnum, data=None):
222-    return ImmutableS3Share(s3bucket, storageindex, shnum, data=data).load()
223-
224-def create_immutable_s3_share(s3bucket, storageindex, shnum, max_size):
225-    return ImmutableS3Share(s3bucket, storageindex, shnum, max_size=max_size).load()
226hunk ./src/allmydata/storage/backends/s3/s3_backend.py 9
227 from allmydata.storage.common import si_a2b
228 from allmydata.storage.bucket import BucketWriter
229 from allmydata.storage.backends.base import Backend, ShareSet
230-from allmydata.storage.backends.s3.immutable import load_immutable_s3_share, create_immutable_s3_share
231+from allmydata.storage.backends.s3.immutable import ImmutableS3ShareForReading, ImmutableS3ShareForWriting
232 from allmydata.storage.backends.s3.mutable import load_mutable_s3_share, create_mutable_s3_share
233 from allmydata.storage.backends.s3.s3_common import get_s3_share_key, NUM_RE
234 from allmydata.mutable.layout import MUTABLE_MAGIC
235hunk ./src/allmydata/storage/backends/s3/s3_backend.py 107
236                 return load_mutable_s3_share(self._s3bucket, self._storageindex, shnum, data=data)
237             else:
238                 # assume it's immutable
239-                return load_immutable_s3_share(self._s3bucket, self._storageindex, shnum, data=data)
240+                return ImmutableS3ShareForReading(self._s3bucket, self._storageindex, shnum, data=data)
241         d.addCallback(_make_share)
242         return d
243 
244hunk ./src/allmydata/storage/backends/s3/s3_backend.py 116
245         return False
246 
247     def make_bucket_writer(self, storageserver, shnum, max_space_per_bucket, lease_info, canary):
248-        d = create_immutable_s3_share(self._s3bucket, self.get_storage_index(), shnum,
249+        immsh = ImmutableS3ShareForWriting(self._s3bucket, self.get_storage_index(), shnum,
250                                       max_size=max_space_per_bucket)
251hunk ./src/allmydata/storage/backends/s3/s3_backend.py 118
252-        def _created(immsh):
253-            return BucketWriter(storageserver, immsh, lease_info, canary)
254-        d.addCallback(_created)
255-        return d
256+        return defer.succeed(BucketWriter(storageserver, immsh, lease_info, canary))
257 
258     def _create_mutable_share(self, storageserver, shnum, write_enabler):
259         serverid = storageserver.get_serverid()
260}
261
262Context:
263
264[Comment out an assertion that was causing all mutable tests to fail. THIS IS PROBABLY WRONG. refs #999
265david-sarah@jacaranda.org**20110929041110
266 Ignore-this: 1e402d51ec021405b191757a37b35a94
267] 
268[Fix some incorrect or incomplete asyncifications. refs #999
269david-sarah@jacaranda.org**20110929040800
270 Ignore-this: ed70e9af2190217c84fd2e8c41de4c7e
271] 
272[Add some debugging assertions that share objects are not Deferred. refs #999
273david-sarah@jacaranda.org**20110929040657
274 Ignore-this: 5c7f56a146f5a3c353c6fe5b090a7dc5
275] 
276[scripts/debug.py: take account of some API changes. refs #999
277david-sarah@jacaranda.org**20110929040539
278 Ignore-this: 933c3d44b993c041105038c7d4514386
279] 
280[Make get_sharesets_for_prefix synchronous for the time being (returning a Deferred breaks crawlers). refs #999
281david-sarah@jacaranda.org**20110929040136
282 Ignore-this: e94b93d4f3f6173d9de80c4121b68748
283] 
284[More asyncification of tests. refs #999
285david-sarah@jacaranda.org**20110929035644
286 Ignore-this: 28b650a9ef593b3fd7524f6cb562ad71
287] 
288[no_network.py: add some assertions that the things we wrap using LocalWrapper are not Deferred (which is not supported and causes hard-to-debug failures). refs #999
289david-sarah@jacaranda.org**20110929035537
290 Ignore-this: fd103fbbb54fbbc17b9517c78313120e
291] 
292[Add some debugging code (switched off) to no_network.py. When switched on (PRINT_TRACEBACKS = True), this prints the stack trace associated with the caller of a remote method, mitigating the problem that the traceback normally gets lost at that point. TODO: think of a better way to preserve the traceback that can be enabled by default. refs #999
293david-sarah@jacaranda.org**20110929035341
294 Ignore-this: 2a593ec3ee450719b241ea8d60a0f320
295] 
296[Use factory functions to create share objects rather than their constructors, to allow the factory to return a Deferred. Also change some methods on IShareSet and IStoredShare to return Deferreds. Refactor some constants associated with mutable shares. refs #999
297david-sarah@jacaranda.org**20110928052324
298 Ignore-this: bce0ac02f475bcf31b0e3b340cd91198
299] 
300[Work in progress for asyncifying the backend interface (necessary to call txaws methods that return Deferreds). This is incomplete so lots of tests fail. refs #999
301david-sarah@jacaranda.org**20110927073903
302 Ignore-this: ebdc6c06c3baa9460af128ec8f5b418b
303] 
304[mutable/publish.py: don't crash if there are no writers in _report_verinfo. refs #999
305david-sarah@jacaranda.org**20110928014126
306 Ignore-this: 9999c82bb3057f755a6e86baeafb8a39
307] 
308[scripts/debug.py: fix incorrect arguments to dump_immutable_share. refs #999
309david-sarah@jacaranda.org**20110928014049
310 Ignore-this: 1078ee3f06a2f36b29e0cf694d2851cd
311] 
312[test_system.py: more debug output for a failing check in test_filesystem. refs #999
313david-sarah@jacaranda.org**20110928014019
314 Ignore-this: e8bb77b8f7db12db7cd69efb6e0ed130
315] 
316[test_system.py: incorrect arguments were being passed to the constructor for MutableDiskShare. refs #999
317david-sarah@jacaranda.org**20110928013857
318 Ignore-this: e9719f74e7e073e37537f9a71614b8a0
319] 
320[Undo an incompatible change to RIStorageServer. refs #999
321david-sarah@jacaranda.org**20110928013729
322 Ignore-this: bea4c0f6cb71202fab942cd846eab693
323] 
324[mutable/publish.py: resolve conflicting patches. refs #999
325david-sarah@jacaranda.org**20110927073530
326 Ignore-this: 6154a113723dc93148151288bd032439
327] 
328[test_storage.py: fix test_no_st_blocks. refs #999
329david-sarah@jacaranda.org**20110927072848
330 Ignore-this: 5f12b784920f87d09c97c676d0afa6f8
331] 
332[Cleanups to S3 backend (not including Deferred changes). refs #999
333david-sarah@jacaranda.org**20110927071855
334 Ignore-this: f0dca788190d92b1edb1ee1498fb34dc
335] 
336[Cleanups to disk backend. refs #999
337david-sarah@jacaranda.org**20110927071544
338 Ignore-this: e9d3fd0e85aaf301c04342fffdc8f26
339] 
340[test_storage.py: fix test_status_bad_disk_stats. refs #999
341david-sarah@jacaranda.org**20110927071403
342 Ignore-this: 6108fee69a60962be2df2ad11b483a11
343] 
344[util/deferredutil.py: add some utilities for asynchronous iteration. refs #999
345david-sarah@jacaranda.org**20110927070947
346 Ignore-this: ac4946c1e5779ea64b85a1a420d34c9e
347] 
348[Add 'has-immutable-readv' to server version information. refs #999
349david-sarah@jacaranda.org**20110923220935
350 Ignore-this: c3c4358f2ab8ac503f99c968ace8efcf
351] 
352[Minor cleanup to disk backend. refs #999
353david-sarah@jacaranda.org**20110923205510
354 Ignore-this: 79f92d7c2edb14cfedb167247c3f0d08
355] 
356[Update the S3 backend. refs #999
357david-sarah@jacaranda.org**20110923205345
358 Ignore-this: 5ca623a17e09ddad4cab2f51b49aec0a
359] 
360[Update the null backend to take into account interface changes. Also, it now records which shares are present, but not their contents. refs #999
361david-sarah@jacaranda.org**20110923205219
362 Ignore-this: 42a23d7e253255003dc63facea783251
363] 
364[Make EmptyShare.check_testv a simple function. refs #999
365david-sarah@jacaranda.org**20110923204945
366 Ignore-this: d0132c085f40c39815fa920b77fc39ab
367] 
368[The cancel secret needs to be unique, even if it isn't explicitly provided. refs #999
369david-sarah@jacaranda.org**20110923204914
370 Ignore-this: 6c44bb908dd4c0cdc59506b2d87a47b0
371] 
372[Implement readv for immutable shares. refs #999
373david-sarah@jacaranda.org**20110923204611
374 Ignore-this: 24f14b663051169d66293020e40c5a05
375] 
376[Remove redundant si_s argument from check_write_enabler. refs #999
377david-sarah@jacaranda.org**20110923204425
378 Ignore-this: 25be760118dbce2eb661137f7d46dd20
379] 
380[interfaces.py: add fill_in_space_stats method to IStorageBackend. refs #999
381david-sarah@jacaranda.org**20110923203723
382 Ignore-this: 59371c150532055939794fed6c77dcb6
383] 
384[Add incomplete S3 backend. refs #999
385david-sarah@jacaranda.org**20110923041314
386 Ignore-this: b48df65699e3926dcbb87b5f755cdbf1
387] 
388[Move advise_corrupt_share to allmydata/storage/backends/base.py, since it will be common to the disk and S3 backends. refs #999
389david-sarah@jacaranda.org**20110923041115
390 Ignore-this: 782b49f243bd98fcb6c249f8e40fd9f
391] 
392[A few comment cleanups. refs #999
393david-sarah@jacaranda.org**20110923041003
394 Ignore-this: f574b4a3954b6946016646011ad15edf
395] 
396[mutable/publish.py: elements should not be removed from a dictionary while it is being iterated over. refs #393
397david-sarah@jacaranda.org**20110923040825
398 Ignore-this: 135da94bd344db6ccd59a576b54901c1
399] 
400[Blank line cleanups.
401david-sarah@jacaranda.org**20110923012044
402 Ignore-this: 8e1c4ecb5b0c65673af35872876a8591
403] 
404[Reinstate the cancel_lease methods of ImmutableDiskShare and MutableDiskShare, since they are needed for lease expiry. refs #999
405david-sarah@jacaranda.org**20110922183323
406 Ignore-this: a11fb0dd0078ff627cb727fc769ec848
407] 
408[Fix most of the crawler tests. refs #999
409david-sarah@jacaranda.org**20110922183008
410 Ignore-this: 116c0848008f3989ba78d87c07ec783c
411] 
412[Fix some more test failures. refs #999
413david-sarah@jacaranda.org**20110922045451
414 Ignore-this: b726193cbd03a7c3d343f6e4a0f33ee7
415] 
416[uri.py: resolve a conflict between trunk and the pluggable-backends patches. refs #999
417david-sarah@jacaranda.org**20110921222038
418 Ignore-this: ffeeab60d8e71a6a29a002d024d76fcf
419] 
420[Fix more shallow bugs, mainly FilePathification. Also, remove the max_space_per_bucket parameter from BucketWriter since it can be obtained from the _max_size attribute of the share (via a new get_allocated_size() accessor). refs #999
421david-sarah@jacaranda.org**20110921221421
422 Ignore-this: 600e3ccef8533aa43442fa576c7d88cf
423] 
424[More fixes to tests needed for pluggable backends. refs #999
425david-sarah@jacaranda.org**20110921184649
426 Ignore-this: 9be0d3a98e350fd4e17a07d2c00bb4ca
427] 
428[docs/backends/S3.rst, disk.rst: describe type of space settings as 'quantity of space', not 'str'. refs #999
429david-sarah@jacaranda.org**20110921031705
430 Ignore-this: a74ed8e01b0a1ab5f07a1487d7bf138
431] 
432[docs/backends/S3.rst: remove Issues section. refs #999
433david-sarah@jacaranda.org**20110921031625
434 Ignore-this: c83d8f52b790bc32488869e6ee1df8c2
435] 
436[Fix some incorrect attribute accesses. refs #999
437david-sarah@jacaranda.org**20110921031207
438 Ignore-this: f1ea4c3ea191f6d4b719afaebd2b2bcd
439] 
440[docs/backends: document the configuration options for the pluggable backends scheme. refs #999
441david-sarah@jacaranda.org**20110920171737
442 Ignore-this: 5947e864682a43cb04e557334cda7c19
443] 
444[Work-in-progress, includes fix to bug involving BucketWriter. refs #999
445david-sarah@jacaranda.org**20110920033803
446 Ignore-this: 64e9e019421454e4d08141d10b6e4eed
447] 
448[Pluggable backends -- all other changes. refs #999
449david-sarah@jacaranda.org**20110919233256
450 Ignore-this: 1a77b6b5d178b32a9b914b699ba7e957
451] 
452[Pluggable backends -- new and moved files, changes to moved files. refs #999
453david-sarah@jacaranda.org**20110919232926
454 Ignore-this: ec5d2d1362a092d919e84327d3092424
455] 
456[interfaces.py: 'which -> that' grammar cleanup.
457david-sarah@jacaranda.org**20110825003217
458 Ignore-this: a3e15f3676de1b346ad78aabdfb8cac6
459] 
460[test/test_runner.py: BinTahoe.test_path has rare nondeterministic failures; this patch probably fixes a problem where the actual cause of failure is masked by a string conversion error.
461david-sarah@jacaranda.org**20110927225336
462 Ignore-this: 6f1ad68004194cc9cea55ace3745e4af
463] 
464[docs/configuration.rst: add section about the types of node, and clarify when setting web.port enables web-API service. fixes #1444
465zooko@zooko.com**20110926203801
466 Ignore-this: ab94d470c68e720101a7ff3c207a719e
467] 
468[TAG allmydata-tahoe-1.9.0a2
469warner@lothar.com**20110925234811
470 Ignore-this: e9649c58f9c9017a7d55008938dba64f
471] 
472Patch bundle hash:
4733819636e5b64e4cb9d41f1961b1036ad97f1d021