Ticket #694: 694.txt

Sat Jun 20 14:28:22 PDT 2009  kevan@isnotajoke.com
  * change max filesize limit tests
  
  Instead of testing to see that the previous SDMF filesize limit was being
  obeyed, we now test to make sure that we can insert files larger than that
  limit.

Sat Jun 20 14:31:30 PDT 2009  kevan@isnotajoke.com
  * remove upper limit on SDMF filesize

New patches:

[change max filesize limit tests
kevan@isnotajoke.com**20090620212822
 Ignore-this: 38e7c62a308c3c93e79df4bf72f4f675
 
 Instead of testing to see that the previous SDMF filesize limit was being
 obeyed, we now test to make sure that we can insert files larger than that
 limit.
] {
hunk ./src/allmydata/test/test_mutable.py 268
     return res
 
 class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
+    # this used to be in Publish, but we removed the limit. Some of 
+    # these tests test whether the new code correctly allows files 
+    # larger than the limit.
+    OLD_MAX_SEGMENT_SIZE = 3500000
     def setUp(self):
         self.client = FakeClient()
 
hunk ./src/allmydata/test/test_mutable.py 362
         return d
 
     def test_create_with_too_large_contents(self):
-        BIG = "a" * (Publish.MAX_SEGMENT_SIZE+1)
-        d = self.shouldFail(FileTooLargeError, "too_large",
-                            "SDMF is limited to one segment, and %d > %d" %
-                            (len(BIG), Publish.MAX_SEGMENT_SIZE),
-                            self.client.create_mutable_file, BIG)
-        d.addCallback(lambda res: self.client.create_mutable_file("small"))
+        BIG = "a" * (self.OLD_MAX_SEGMENT_SIZE + 1)
+        d = self.client.create_mutable_file(BIG)
         def _created(n):
hunk ./src/allmydata/test/test_mutable.py 365
-            return self.shouldFail(FileTooLargeError, "too_large_2",
-                                   "SDMF is limited to one segment, and %d > %d" %
-                                   (len(BIG), Publish.MAX_SEGMENT_SIZE),
-                                   n.overwrite, BIG)
+            d = n.overwrite(BIG)
+            return d
         d.addCallback(_created)
         return d
 
hunk ./src/allmydata/test/test_mutable.py 387
         def _error_modifier(old_contents, servermap, first_time):
             raise ValueError("oops")
         def _toobig_modifier(old_contents, servermap, first_time):
-            return "b" * (Publish.MAX_SEGMENT_SIZE+1)
+            return "b" * (self.OLD_MAX_SEGMENT_SIZE+1)
         calls = []
         def _ucw_error_modifier(old_contents, servermap, first_time):
             # simulate an UncoordinatedWriteError once
hunk ./src/allmydata/test/test_mutable.py 427
             d.addCallback(lambda res: self.failUnlessEqual(res, "line1line2"))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "err"))
 
-            d.addCallback(lambda res:
-                          self.shouldFail(FileTooLargeError, "toobig_modifier",
-                                          "SDMF is limited to one segment",
-                                          n.modify, _toobig_modifier))
+
             d.addCallback(lambda res: n.download_best_version())
             d.addCallback(lambda res: self.failUnlessEqual(res, "line1line2"))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "big"))
hunk ./src/allmydata/test/test_mutable.py 456
             d.addCallback(lambda res: self.failUnlessEqual(res,
                                                            "line1line2line3"))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 4, "ucw"))
-
+            d.addCallback(lambda res: n.modify(_toobig_modifier))
             return d
         d.addCallback(_created)
         return d
}
[remove upper limit on SDMF filesize
kevan@isnotajoke.com**20090620213130
 Ignore-this: 5bc48c7421c73827909a17e651799d0c
] {
hunk ./src/allmydata/mutable/publish.py 93
     To make the initial publish, set servermap to None.
     """
 
-    # we limit the segment size as usual to constrain our memory footprint.
-    # The max segsize is higher for mutable files, because we want to support
-    # dirnodes with up to 10k children, and each child uses about 330 bytes.
-    # If you actually put that much into a directory you'll be using a
-    # footprint of around 14MB, which is higher than we'd like, but it is
-    # more important right now to support large directories than to make
-    # memory usage small when you use them. Once we implement MDMF (with
-    # multiple segments), we will drop this back down, probably to 128KiB.
-    MAX_SEGMENT_SIZE = 3500000
-
     def __init__(self, filenode, servermap):
         self._node = filenode
         self._servermap = servermap
hunk ./src/allmydata/mutable/publish.py 136
         # 5: when enough responses are back, we're done
 
         self.log("starting publish, datalen is %s" % len(newdata))
-        if len(newdata) > self.MAX_SEGMENT_SIZE:
-            raise FileTooLargeError("SDMF is limited to one segment, and "
-                                    "%d > %d" % (len(newdata),
-                                                 self.MAX_SEGMENT_SIZE))
         self._status.set_size(len(newdata))
         self._status.set_status("Started")
         self._started = time.time()
hunk ./src/allmydata/mutable/publish.py 251
         return self.done_deferred
 
     def setup_encoding_parameters(self):
-        segment_size = min(self.MAX_SEGMENT_SIZE, len(self.newdata))
+        segment_size = len(self.newdata)
         # this must be a multiple of self.required_shares
         segment_size = mathutil.next_multiple(segment_size,
                                               self.required_shares)
}

Context:

[Makefile: add jaunty support, rearrange debian sections in order of release
warner@lothar.com**20090618050502] 
[test_util: add known-answer tests for hashutil tags
warner@lothar.com**20090618045709] 
[tests: bump up a timeout that expired on Zandr's box
zooko@zooko.com**20090613195703
 Ignore-this: 853000f43665396b3734a99f72bd472d
] 
[util: hooray!  A clean implementation of this simple utility!  Black Dew pointed out that the inverse of time.gmtime() is hidden in the "calendar" module.
zooko@zooko.com**20090613160112
 Ignore-this: 8359cc48ca1b8e2793e8b2afe2050cf4
] 
[util: Brian's horrible hack to figure out how much localtime and utctime differ.  Now we'll see if it works on Windows.
zooko@zooko.com**20090612204556
 Ignore-this: 8c36431da4707da76472956c7750ecbd
] 
[util: oops, time.tzset() doesn't work on Windows -- hopefully the new "London" unit test passes on Windows when we skip tzset() on platforms that don't have it
zooko@zooko.com**20090612000920
 Ignore-this: 7d314b8334cfa3f65f9635e3d3eb727e
] 
[setup: edit install.html to warn Windows users away from Python v2.6
zooko@zooko.com**20090611225506
 Ignore-this: 89ad63eab49ede883ef92f2de5b5fc54
] 
[util: fix time_format.iso_utc_time_to_seconds() so that it works even in London
zooko@zooko.com**20090611221129
 Ignore-this: 14dbb9840587797f848226fdb3645c08
] 
[setup: run the same "make quicktest" on Windows as on non-Windows
zooko@zooko.com**20090611193214
 Ignore-this: d0c78377d3892373ec3d91e9e98bd8bd
 I checked and it behaves about as well on Windows as the previous version did.
] 
[tests: significantly increase timeouts that triggered on Zandr's ARM box
zooko@zooko.com**20090610161043
 Ignore-this: 2b3c556c5166a8267b4b15664d3aadfb
] 
[test: multiple by 10 or so all timeouts that Zandr's ARM box just overran
zooko@zooko.com**20090610125639
 Ignore-this: bf62f063ab46814fd78de55a5fbc9d84
] 
[tests: bump up timeout on a test that timed out on draco
zooko@zooko.com**20090610044628
 Ignore-this: f598b98cbae44dc947937c6ca54c10cb
] 
[tests: raise the timeout for test_cli since Zandr's ARM machine totally burst through the old one
zooko@zooko.com**20090609210509] 
[test_cli.Backup: increase timeout massively, it takes 1200s on zandr's ARM linkstation
warner@lothar.com**20090609052801] 
[tests: double the timeouts on some tests which time-out on Francois's box
zooko@zooko.com**20090609021753
 Ignore-this: b2727b04402f24a9b9123d2f84068106
] 
[tests: bump up timeouts so that the tests can finish before timeout on Francois's little arm box
zooko@zooko.com**20090608225557
 Ignore-this: fb83698338b2f12546cd3e1dcb896d34
] 
[tests: increase timeouts on some other tests that timed-out on Francois's arm box
zooko@zooko.com**20090605143437
 Ignore-this: 2903cc20d914fc074c8d7a6c47740ba6
] 
[tests: bump up the timeout on a bunch of tests that took longer than the default timeout (120s) on François Lenny-armv5tel
zooko@zooko.com**20090605031444
 Ignore-this: 84d67849b1f8edc88bf7001e31b5f7f3
] 
[backup: remove the --no-backupdb command, the handling of "can't import sqlite", and the related tests, and change an error message to more correctly indicate failure to load the database from disk rather than failure to import sqlite module
zooko@zooko.com**20090604173131
 Ignore-this: 8200a9fdfc49243c280ecd1d0c44fa19
 Fixes #728.
] 
[setup: require pysqlite >= v2.0.5. if we are running on Python < 2.5
zooko@zooko.com**20090604154548
 Ignore-this: cf04f46079821df209d01dad2e24b40b
] 
[setup: add pysqlite and sqlite to get_package_versions()
zooko@zooko.com**20090604153728
 Ignore-this: a1dea7fabeab2b08fb0d8d462facdb4d
] 
[more refactoring: move get_all_serverids() and get_nickname_for_serverid() from Client to storage_broker
warner@lothar.com**20090602030750] 
[more storage_broker refactoring: downloader gets a broker instead of a client,
warner@lothar.com**20090602022511
 use Client.get_storage_broker() accessor instead of direct attribute access.
] 
[test_runner.py: remove test_client_no_noise: the issue in question is
warner@lothar.com**20090601225007
 ticketed in http://divmod.org/trac/ticket/2830 and doesn't need a Tahoe-side
 change, plus this test fails on win32 for unrelated reasons (and test_client
 is the place to think about the win32 issue).
] 
[remove plaintext-hashing code from the helper interface, to close #722
warner@lothar.com**20090601224916
 and deny the Helper the ability to mount a partial-information-guessing
 attack. This will probably break compatibility between new clients and very
 old (pre-1.0) helpers.
] 
[start to factor server-connection-management into a distinct 'StorageServerFarmBroker' object, separate from the client and the introducer. This is the starting point for #467: static server selection
warner@lothar.com**20090601210604] 
[mutable: catch and display first error, so code bugs which break all servers get displayed better
warner@lothar.com**20090601210407] 
[misc/run-with-pythonpath.py: exec() the child (on unix), to remove the intermediate process
warner@lothar.com**20090601210137] 
[docs: small edit to about.html
zooko@zooko.com**20090528233422
 Ignore-this: 1cfbb1f8426ed6d63b2d3952e4464ddc
] 
[docs: add links to Tahoe-LAFS for Paranoids and Tahoe-LAFS for Corporates in about.html
zooko@zooko.com**20090528232717
 Ignore-this: 7b70baa700d6b6f6e9ceec4132efe5
] 
[docs: edit about.html and include network-and-reliance-topology.png (loaded from http://allmydata.org )
zooko@zooko.com**20090527150916
 Ignore-this: 44adc61cde8ced8be2f0a7dfc7d95dad
] 
[docs: a few more edits to network-and-reliance-topology.svg
zooko@zooko.com**20090527150458
 Ignore-this: 2eac8c33fe71be25ff809b399c6193c1
] 
[docs: update network-and-reliance-topology.svg for beauty and clarity
zooko@zooko.com**20090527031123
 Ignore-this: 5510914849771900ac29b4312470d84
] 
[docs: update NEWS, relnotes.txt, CREDITS to mention WUI Style
zooko@zooko.com**20090526233654
 Ignore-this: 72d16ec833bc4a22af23d29ea1d5ff8b
] 
[Modify markup of Tahoe web pages to be more amenable to styling; some minor changes of wording.
Kevin Reid <kpreid@mac.com>**20090526232545
 Ignore-this: 8845937f0df6c7ddc07abe3211428a6f
] 
[Tweak wording in directory page: not-read-only is "modifiable", mention creating a directory _in this directory_.
Kevin Reid <kpreid@mac.com>**20090526232414
 Ignore-this: f006ec52ba2051802e025a60bcface56
] 
[Comment on duplication of code/markup found during styling project.
Kevin Reid <kpreid@mac.com>**20090503203442
 Ignore-this: a4b7f9f0ab57d2c03be9ba761be8d854
] 
[Add CSS styles to spiff up the Tahoe WUI's appearance, particularly the welcome page and directories.
Kevin Reid <kpreid@mac.com>**20090503203142
 Ignore-this: 5c50af241c1a958b5180ef2b6a49f626
] 
[Link all Tahoe web pages to the /tahoe_css stylesheet which already exists.
Kevin Reid <kpreid@mac.com>**20090503202533
 Ignore-this: 2ea8d14d3168b9502cf39d5ea3f2f2a8
] 
[Fix broken link from Provisioning to Reliability page.
Kevin Reid <kpreid@mac.com>**20090501191050
 Ignore-this: 56dc1a5e659b70cc02dc4df7b5d518cd
] 
[docs: network-and-reliance-topology.svg: nicer server icons, mv out of the "specifications" subdir
zooko@zooko.com**20090526165842
 Ignore-this: 8f47ab3a0ab782c1f0d46e10bcaebe5b
] 
[docs: update network-and-reliance-topology.svg
zooko@zooko.com**20090526163105
 Ignore-this: 2b864b4ed8743d4a15dfbb7eff3fa561
] 
[accounting-overview.txt: more edits
warner@lothar.com**20090523190359] 
[accounting-overview.txt: small edits
warner@lothar.com**20090523184011] 
[_auto_deps.py: require foolscap-0.4.1, which adds an important fix for py2.4
warner@lothar.com**20090523011103] 
[immutable/encode.py: tolerate immediate _remove_shareholder by copying the
warner@lothar.com**20090522184424
 landlord list before iterating over it. This can probably only happen in unit
 tests, but cleaning it up makes certain test failures easier to analyze.
] 
[switch to using RemoteException instead of 'wrapped' RemoteReferences. Should fix #653, the rref-EQ problem
warner@lothar.com**20090522004632] 
[switch all foolscap imports to use foolscap.api or foolscap.logging
warner@lothar.com**20090522003823] 
[_auto_deps.py: bump our foolscap dependency to 0.4.0, since I'm about to start using its new features
warner@lothar.com**20090522002100] 
[test_runner.py: fix minor typo
warner@lothar.com**20090520033620] 
[setup: fix bug (wrong import) in error message, as noticed by pyflakes
zooko@zooko.com**20090519195642
 Ignore-this: f1b9f8c00b46c1b5f2f20e5fc424f341
] 
[setup: fix trivial bug in recent patch to test base64.py at startup
zooko@zooko.com**20090519195129
 Ignore-this: f6be038f74b53ca69e7109fe34adfbc
] 
[setup: make Tahoe exit at startup with a useful error message if the base64.py module is buggy (fixes part of #710)
zooko@zooko.com**20090519194555
 Ignore-this: aa4d398235ddca8d417d61c9688e154
] 
[test_introducer.py: add a test for the python2.4.0/2.4.1 bug in base64.b32decode
warner@lothar.com**20090519034101] 
[immutable WriteBucketProxy: use pipeline to speed up uploads by overlapping roundtrips, for #392
warner@lothar.com**20090518234422] 
[util/pipeline.py: new utility class to manage size-limited work pipelines, for #392
warner@lothar.com**20090518234326] 
[docs: add a diagram that I'm about to show to the Boulder Linux Users Group: network-and-reliance-topology.svg
zooko@zooko.com**20090514232059
 Ignore-this: 2420c0a7c254c9f0f2349d9130490d33
] 
[tests: mark test_runner as coded in utf-8 instead of ascii
zooko@zooko.com**20090507223151
 Ignore-this: ccf1ba9e5a9b53602701a36f9fdb545e
] 
[tests: raise timeout on test_runner.RunNode.test_introducer from 120s to 240s, since it hit the 120s time-out on François Lenny-armv5tel
zooko@zooko.com**20090507215012
 Ignore-this: ba18fe6832ba255d4971e8f623ed7da5
] 
[setup: fix comment in setup.py
zooko@zooko.com**20090507215003
 Ignore-this: c46ef664630d52733138ef7fbc551c1c
] 
[docs: how_to_make_a_tahoe_release.txt: a couple of small edits
zooko@zooko.com**20090507214932
 Ignore-this: ae92aa835ad369f4b9e6e49d681957a3
] 
[.darcs-boringfile: also ignore .gitignore
warner@allmydata.com**20090415210550
 Ignore-this: d29db314a1e506f6240859559436b4c3
] 
[.darcs-boringfile: ignore .git, I'm starting to play around with it
warner@allmydata.com**20090415205929
 Ignore-this: 89234453516483c9586cd6e1351e88b5
] 
[fix quicktest: stop using setuptools, add misc/run-with-pythonpath.py, to make it run faster
warner@lothar.com**20090414201400] 
[TAG allmydata-tahoe-1.4.1
zooko@zooko.com**20090414025636
 Ignore-this: de78fc32364c83e9f4e26b5abcfdea4a
] 
Patch bundle hash:
dd7e9bb9e1eed21e3968c447dd304eb229df2f3c