Ticket #1117: 1117patch.dpatch

Thu Jul 15 16:17:14 PDT 2010  Kevan Carstensen <kevan@isnotajoke.com>
  * immutable/upload.py: abort buckets if peer selection fails

Thu Jul 15 16:18:20 PDT 2010  Kevan Carstensen <kevan@isnotajoke.com>
  * test/test_upload.py: changes to test plumbing for #1117 tests
  
      - Add a callRemoteOnly method to FakeBucketWriter.
      - Change the abort method in FakeBucketWriter to not return a
        RuntimeError.

Thu Jul 15 16:21:05 PDT 2010  Kevan Carstensen <kevan@isnotajoke.com>
  * storage/immutable.py: make remote_abort btell the storage server about aborted buckets.

Thu Jul 15 16:21:48 PDT 2010  Kevan Carstensen <kevan@isnotajoke.com>
  * test/test_storage.py: test for the new remote_abort semantics.

Thu Jul 15 17:10:46 PDT 2010  Kevan Carstensen <kevan@isnotajoke.com>
  * test/test_upload.py: test to see that aborted buckets are ignored by the storage server

New patches:

[immutable/upload.py: abort buckets if peer selection fails
Kevan Carstensen <kevan@isnotajoke.com>**20100715231714
 Ignore-this: 2a0b643a22284df292d8ed9d91b1fd37
] {
hunk ./src/allmydata/immutable/upload.py 138
         return (alreadygot, set(b.keys()))
 
 
+    def abort(self):
+        """
+        I abort the remote bucket writers for the share numbers in
+        sharenums. This is a good idea to conserve space on the storage
+        server.
+        """
+        for writer in self.buckets.itervalues(): writer.abort()
+
+
 class Tahoe2PeerSelector:
 
     def __init__(self, upload_id, logparent=None, upload_status=None):
hunk ./src/allmydata/immutable/upload.py 367
                                           self.needed_shares,
                                           self.servers_of_happiness,
                                           effective_happiness)
-                    raise UploadUnhappinessError("%s (%s)" % (msg,
-                                                 self._get_progress_message()))
+                    return self._failed("%s (%s)" % (msg, self._get_progress_message()))
 
         if self.uncontacted_peers:
             peer = self.uncontacted_peers.pop(0)
hunk ./src/allmydata/immutable/upload.py 428
                 if self.last_failure_msg:
                     msg += " (%s)" % (self.last_failure_msg,)
                 log.msg(msg, level=log.UNUSUAL, parent=self._log_parent)
-                raise UploadUnhappinessError(msg)
+                return self._failed(msg)
             else:
                 # we placed enough to be happy, so we're done
                 if self._status:
hunk ./src/allmydata/immutable/upload.py 517
         return self._loop()
 
 
+    def _failed(self, msg):
+        """
+        I am called when peer selection fails. I first abort all of the
+        remote buckets that I allocated during my unsuccessful attempt to
+        place shares for this file. I then raise an
+        UploadUnhappinessError with my msg argument.
+        """
+        for peer in self.use_peers:
+            assert isinstance(peer, PeerTracker)
+
+            peer.abort()
+
+        raise UploadUnhappinessError(msg)
+
+
 class EncryptAnUploadable:
     """This is a wrapper that takes an IUploadable and provides
     IEncryptedUploadable."""
}
[test/test_upload.py: changes to test plumbing for #1117 tests
Kevan Carstensen <kevan@isnotajoke.com>**20100715231820
 Ignore-this: 78a6d359d7bf8529d283e2815bf1e2de
 
     - Add a callRemoteOnly method to FakeBucketWriter.
     - Change the abort method in FakeBucketWriter to not return a
       RuntimeError.
] {
hunk ./src/allmydata/test/test_upload.py 162
         d.addCallback(lambda res: _call())
         return d
 
+
+    def callRemoteOnly(self, methname, *args, **kwargs):
+        d = self.callRemote(methname, *args, **kwargs)
+        del d # callRemoteOnly ignores this
+        return None
+
+
     def remote_write(self, offset, data):
         precondition(not self.closed)
         precondition(offset >= 0)
hunk ./src/allmydata/test/test_upload.py 183
         self.closed = True
 
     def remote_abort(self):
-        log.err(RuntimeError("uh oh, I was asked to abort"))
+        pass
 
 class FakeClient:
     DEFAULT_ENCODING_PARAMETERS = {"k":25,
}
[storage/immutable.py: make remote_abort btell the storage server about aborted buckets.
Kevan Carstensen <kevan@isnotajoke.com>**20100715232105
 Ignore-this: 16ab0090676355abdd5600ed44ff19c9
] {
hunk ./src/allmydata/storage/immutable.py 282
     def _abort(self):
         if self.closed:
             return
+
         os.remove(self.incominghome)
         # if we were the last share to be moved, remove the incoming/
         # directory that was our parent
hunk ./src/allmydata/storage/immutable.py 289
         parentdir = os.path.split(self.incominghome)[0]
         if not os.listdir(parentdir):
             os.rmdir(parentdir)
+        self._sharefile = None
 
hunk ./src/allmydata/storage/immutable.py 291
+        # We are now considered closed for further writing. We must tell
+        # the storage server about this so that it stops expecting us to
+        # use the space it allocated for us earlier.
+        self.closed = True
+        self.ss.bucket_writer_closed(self, 0)
 
 
 class BucketReader(Referenceable):
}
[test/test_storage.py: test for the new remote_abort semantics.
Kevan Carstensen <kevan@isnotajoke.com>**20100715232148
 Ignore-this: d3d6491f17bf670e770ca4b385007515
] hunk ./src/allmydata/test/test_storage.py 324
         self.failIf(os.path.exists(incoming_prefix_dir), incoming_prefix_dir)
         self.failUnless(os.path.exists(incoming_dir), incoming_dir)
 
+    def test_abort(self):
+        # remote_abort, when called on a writer, should make sure that
+        # the allocated size of the bucket is not counted by the storage
+        # server when accounting for space.
+        ss = self.create("test_abort")
+        already, writers = self.allocate(ss, "allocate", [0, 1, 2], 150)
+        self.failIfEqual(ss.allocated_size(), 0)
+
+        # Now abort the writers.
+        for writer in writers.itervalues():
+            writer.remote_abort()
+        self.failUnlessEqual(ss.allocated_size(), 0)
+
+
     def test_allocate(self):
         ss = self.create("test_allocate")
 
[test/test_upload.py: test to see that aborted buckets are ignored by the storage server
Kevan Carstensen <kevan@isnotajoke.com>**20100716001046
 Ignore-this: cc075c24b1c86d737f3199af894cc780
] hunk ./src/allmydata/test/test_upload.py 1809
         return d
 
 
+    def test_peer_selector_bucket_abort(self):
+        # If peer selection for an upload fails due to an unhappy
+        # layout, the peer selection process should abort the buckets it
+        # allocates before failing, so that the space can be re-used.
+        self.basedir = self.mktemp()
+        self.set_up_grid(num_servers=5)
+
+        # Try to upload a file with happy=7, which is unsatisfiable with
+        # the current grid. This will fail, but should not take up any
+        # space on the storage servers after it fails.
+        client = self.g.clients[0]
+        client.DEFAULT_ENCODING_PARAMETERS['happy'] = 7
+        d = defer.succeed(None)
+        d.addCallback(lambda ignored:
+            self.shouldFail(UploadUnhappinessError,
+                            "test_peer_selection_bucket_abort",
+                            "",
+                            client.upload, upload.Data("data" * 10000,
+                                                       convergence="")))
+        # wait for the abort messages to get there.
+        def _turn_barrier(res):
+            return fireEventually(res)
+        d.addCallback(_turn_barrier)
+        def _then(ignored):
+            for server in self.g.servers_by_number.values():
+                self.failUnlessEqual(server.allocated_size(), 0)
+        d.addCallback(_then)
+        return d
+
+
+    def test_encoder_bucket_abort(self):
+        # If enough servers die in the process of encoding and uploading
+        # a file to make the layout unhappy, we should cancel the
+        # newly-allocated buckets before dying.
+        self.basedir = self.mktemp()
+        self.set_up_grid(num_servers=4)
+
+        client = self.g.clients[0]
+        client.DEFAULT_ENCODING_PARAMETERS['happy'] = 7
+
+        d = defer.succeed(None)
+        d.addCallback(lambda ignored:
+            self.shouldFail(UploadUnhappinessError,
+                            "test_encoder_bucket_abort",
+                            "",
+                            self._do_upload_with_broken_servers, 1))
+        def _turn_barrier(res):
+            return fireEventually(res)
+        d.addCallback(_turn_barrier)
+        def _then(ignored):
+            for server in self.g.servers_by_number.values():
+                self.failUnlessEqual(server.allocated_size(), 0)
+        d.addCallback(_then)
+        return d
+
+
     def _set_up_nodes_extra_config(self, clientdir):
         cfgfn = os.path.join(clientdir, "tahoe.cfg")
         oldcfg = open(cfgfn, "r").read()

Context:

[SFTP: address some of the comments in zooko's review (#1106).
david-sarah@jacaranda.org**20100712025537
 Ignore-this: c3921638a2d4f1de2a776ae78e4dc37e
] 
[docs/logging.txt: note that setting flogging vars might affect tests with race conditions.
david-sarah@jacaranda.org**20100712050721
 Ignore-this: fc1609d215fcd5561a57fd1226206f27
] 
[test_storage.py: potential fix for failures when logging is enabled.
david-sarah@jacaranda.org**19700713040546
 Ignore-this: 5815693a0df3e64c52c3c6b7be2846c7
] 
[upcase_since_on_welcome
terrellrussell@gmail.com**20100708193903] 
[server_version_on_welcome_page.dpatch.txt
freestorm77@gmail.com**20100605191721
 Ignore-this: b450c76dc875f5ac8cca229a666cbd0a
 
 
 - The storage server version is 0 for all storage nodes in the Welcome Page
 
 
] 
[NEWS: add NEWS snippets about two recent patches
zooko@zooko.com**20100708162058
 Ignore-this: 6c9da6a0ad7351a960bdd60f81532899
] 
[directory_html_top_banner.dpatch
freestorm77@gmail.com**20100622205301
 Ignore-this: 1d770d975e0c414c996564774f049bca
 
 The div tag with the link "Return to Welcome page" on the directory.xhtml page is not correct
 
] 
[tahoe_css_toolbar.dpatch
freestorm77@gmail.com**20100622210046
 Ignore-this: 5b3ebb2e0f52bbba718a932f80c246c0
 
 CSS modification to be correctly diplayed with Internet Explorer 8
 
 The links on the top of page directory.xhtml are not diplayed in the same line as display with Firefox.
 
] 
[runnin_test_tahoe_css.dpatch
freestorm77@gmail.com**20100622214714
 Ignore-this: e0db73d68740aad09a7b9ae60a08c05c
 
 Runnin test for changes in tahoe.css file
 
] 
[runnin_test_directory_xhtml.dpatch
freestorm77@gmail.com**20100622201403
 Ignore-this: f8962463fce50b9466405cb59fe11d43
 
 Runnin test for diretory.xhtml top banner
 
] 
[stringutils.py: tolerate sys.stdout having no 'encoding' attribute.
david-sarah@jacaranda.org**20100626040817
 Ignore-this: f42cad81cef645ee38ac1df4660cc850
] 
[quickstart.html: python 2.5 -> 2.6 as recommended version
david-sarah@jacaranda.org**20100705175858
 Ignore-this: bc3a14645ea1d5435002966ae903199f
] 
[SFTP: don't call .stopProducing on the producer registered with OverwriteableFileConsumer (which breaks with warner's new downloader).
david-sarah@jacaranda.org**20100628231926
 Ignore-this: 131b7a5787bc85a9a356b5740d9d996f
] 
[docs/how_to_make_a_tahoe-lafs_release.txt: trivial correction, install.html should now be quickstart.html.
david-sarah@jacaranda.org**20100625223929
 Ignore-this: 99a5459cac51bd867cc11ad06927ff30
] 
[setup: in the Makefile, refuse to upload tarballs unless someone has passed the environment variable "BB_BRANCH" with value "trunk"
zooko@zooko.com**20100619034928
 Ignore-this: 276ddf9b6ad7ec79e27474862e0f7d6
] 
[trivial: tiny update to in-line comment
zooko@zooko.com**20100614045715
 Ignore-this: 10851b0ed2abfed542c97749e5d280bc
 (I'm actually committing this patch as a test of the new eager-annotation-computation of trac-darcs.)
] 
[docs: about.html link to home page early on, and be decentralized storage instead of cloud storage this time around
zooko@zooko.com**20100619065318
 Ignore-this: dc6db03f696e5b6d2848699e754d8053
] 
[docs: update about.html, especially to have a non-broken link to quickstart.html, and also to comment out the broken links to "for Paranoids" and "for Corporates"
zooko@zooko.com**20100619065124
 Ignore-this: e292c7f51c337a84ebfeb366fbd24d6c
] 
[TAG allmydata-tahoe-1.7.0
zooko@zooko.com**20100619052631
 Ignore-this: d21e27afe6d85e2e3ba6a3292ba2be1
] 
Patch bundle hash:
00056b67380e13933aba18bc9d0dca86cfcf6566