Ticket #637: ticket637-tested.darcspatch.txt

diff -rN old-tahoe/_auto_deps.py new-tahoe/_auto_deps.py
40,41c40,42
< ## can be easy_install'ed and also which actually makes "import win32api" succeed.  Users have
< ## to manually install pywin32 on Windows before installing Tahoe.
---
> ## can be easy_install'ed and also which actually makes "import win32api" succeed.
> ## See http://sourceforge.net/tracker/index.php?func=detail&aid=1799934&group_id=78018&atid=551954
> ## Users have to manually install pywin32 on Windows before installing Tahoe.
47,52c48,51
< ##    # management.  Then the specification and the evolution of Twisted's reliance on pywin32 can
< ##    # be confined to the Twisted setup data, and Tahoe can remain blissfully ignorant about such
< ##    # things as if a future version of Twisted requires a different version of pywin32, or if a
< ##    # future version of Twisted implements process management without using pywin32 at all,
< ##    # etc..  That is twisted ticket #3238 -- http://twistedmatrix.com/trac/ticket/3238 .  But
< ##    # until Twisted does that, Tahoe needs to be non-ignorant of the following requirement:
---
> ##    # management.  That is twisted ticket #3238 -- http://twistedmatrix.com/trac/ticket/3238 .
> ##    # On the other hand, Tahoe also depends on pywin32 for getting free disk space statistics
> ##    # (although that is not a hard requirement: if win32api can't be imported then we don't
> ##    # rely on having the disk stats).
diff -rN old-tahoe/docs/configuration.txt new-tahoe/docs/configuration.txt
305,306c305,308
<  server will not accept any share which causes the amount of free space (as
<  measured by 'df', or more specifically statvfs(2)) to drop below this value.
---
>  server will not accept any share which causes the amount of free disk space
>  to drop below this value. (The free space is measured by a call to statvfs(2)
>  on Unix, or GetDiskFreeSpaceEx on Windows, and is the space available to the
>  user account under which the storage server runs.)
diff -rN old-tahoe/src/allmydata/storage/server.py new-tahoe/src/allmydata/storage/server.py
38a39,47
>     windows = False
> 
>     try:
>         import win32api, win32con
>         windows = True
>         # <http://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
>         win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS | win32con.SEM_NOOPENFILEERRORBOX)
>     except ImportError:
>         pass
73c82
<                 log.msg("warning: [storage]reserved_space= is set, but this platform does not support statvfs(2), so this reservation cannot be honored",
---
>                 log.msg("warning: [storage]reserved_space= is set, but this platform does not support an API to get disk statistics (statvfs(2) or GetDiskFreeSpaceEx), so this reservation cannot be honored",
150,151c159,183
<     def do_statvfs(self):
<         return os.statvfs(self.storedir)
---
>     def get_disk_stats(self):
>         """Return disk statistics for the storage disk, in the form of a dict with the following fields.
>           total:            total bytes on disk
>           free_for_root:    bytes actually free on disk
>           free_for_nonroot: bytes free for "a non-privileged user" [Unix] or the current user [Windows];
>                               might take into account quotas depending on platform
>           used:             bytes used on disk
>           avail:            bytes available excluding reserved space
>         An AttributeError can occur if the OS has no API to get disk information.
>         An EnvironmentError can occur if the OS call fails."""
> 
>         if self.windows:
>             # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx.
>             # <http://docs.activestate.com/activepython/2.5/pywin32/win32api__GetDiskFreeSpaceEx_meth.html>
>             #
>             # Although the docs say that the argument should be the root directory of a disk, GetDiskFreeSpaceEx
>             # actually accepts any path on that disk (like its Win32 equivalent).
> 
>             (free_for_nonroot, total, free_for_root) = self.win32api.GetDiskFreeSpaceEx(self.storedir)
>         else:
>             # For Unix-like systems.
>             # <http://docs.python.org/library/os.html#os.statvfs>
>             # <http://opengroup.org/onlinepubs/7990989799/xsh/fstatvfs.html>
>             # <http://opengroup.org/onlinepubs/7990989799/xsh/sysstatvfs.h.html>
>             s = os.statvfs(self.storedir)
153,165d184
<     def get_stats(self):
<         # remember: RIStatsProvider requires that our return dict
<         # contains numeric values.
<         stats = { 'storage_server.allocated': self.allocated_size(), }
<         stats["storage_server.reserved_space"] = self.reserved_space
<         for category,ld in self.get_latencies().items():
<             for name,v in ld.items():
<                 stats['storage_server.latencies.%s.%s' % (category, name)] = v
<         writeable = True
<         if self.readonly_storage:
<             writeable = False
<         try:
<             s = self.do_statvfs()
176,190c195,217
<             disk_total = s.f_frsize * s.f_blocks
<             disk_used = s.f_frsize * (s.f_blocks - s.f_bfree)
<             # spacetime predictors should look at the slope of disk_used.
<             disk_free_for_root = s.f_frsize * s.f_bfree
<             disk_free_for_nonroot = s.f_frsize * s.f_bavail
< 
<             # include our local policy here: if we stop accepting shares when
<             # the available space drops below 1GB, then include that fact in
<             # disk_avail.
<             disk_avail = disk_free_for_nonroot - self.reserved_space
<             disk_avail = max(disk_avail, 0)
<             if self.readonly_storage:
<                 disk_avail = 0
<             if disk_avail == 0:
<                 writeable = False
---
>             total = s.f_frsize * s.f_blocks
>             free_for_root = s.f_frsize * s.f_bfree
>             free_for_nonroot = s.f_frsize * s.f_bavail
> 
>         # valid for all platforms:
>         used = total - free_for_root
>         avail = max(free_for_nonroot - self.reserved_space, 0)
> 
>         return { 'total': total, 'free_for_root': free_for_root, 'free_for_nonroot': free_for_nonroot,
>                  'used': used, 'avail': avail, }
> 
>     def get_stats(self):
>         # remember: RIStatsProvider requires that our return dict
>         # contains numeric values.
>         stats = { 'storage_server.allocated': self.allocated_size(), }
>         stats['storage_server.reserved_space'] = self.reserved_space
>         for category,ld in self.get_latencies().items():
>             for name,v in ld.items():
>                 stats['storage_server.latencies.%s.%s' % (category, name)] = v
> 
>         try:
>             disk = self.get_disk_stats()
>             writeable = disk['avail'] > 0
193,197c220,224
<             stats["storage_server.disk_total"] = disk_total
<             stats["storage_server.disk_used"] = disk_used
<             stats["storage_server.disk_free_for_root"] = disk_free_for_root
<             stats["storage_server.disk_free_for_nonroot"] = disk_free_for_nonroot
<             stats["storage_server.disk_avail"] = disk_avail
---
>             stats['storage_server.disk_total'] = disk['total']
>             stats['storage_server.disk_used'] = disk['used']
>             stats['storage_server.disk_free_for_root'] = disk['free_for_root']
>             stats['storage_server.disk_free_for_nonroot'] = disk['free_for_nonroot']
>             stats['storage_server.disk_avail'] = disk['avail']
199,201c226,235
<             # os.statvfs is available only on unix
<             pass
<         stats["storage_server.accepting_immutable_shares"] = int(writeable)
---
>             writeable = True
>         except EnvironmentError:
>             log.msg("OS call to get disk statistics failed", level=log.UNUSUAL)
>             writeable = False
> 
>         if self.readonly_storage:
>             stats['storage_server.disk_avail'] = 0
>             writeable = False
> 
>         stats['storage_server.accepting_immutable_shares'] = int(writeable)
205c239
<             stats["storage_server.total_bucket_count"] = bucket_count
---
>             stats['storage_server.total_bucket_count'] = bucket_count
208,214d241
< 
<     def stat_disk(self, d):
<         s = os.statvfs(d)
<         # s.f_bavail: available to non-root users
<         disk_avail = s.f_frsize * s.f_bavail
<         return disk_avail
< 
216c243,246
<         # returns None if it cannot be measured (windows)
---
>         """Returns available space for share storage in bytes, or None if no API to get this
>            information is available."""
>         if self.readonly_storage:
>             return 0
218,219c248
<             disk_avail = self.stat_disk(self.storedir)
<             disk_avail -= self.reserved_space
---
>             return self.get_disk_stats()['avail']
221,224c250,253
<             disk_avail = None
<         if self.readonly_storage:
<             disk_avail = 0
<         return disk_avail
---
>             return None
>         except EnvironmentError:
>             log.msg("OS call to get disk statistics failed", level=log.UNUSUAL)
>             return 0
235,236c264
<             # we're on a platform that doesn't have 'df', so make a vague
<             # guess.
---
>             # We're on a platform that has no API to get disk stats.
237a266
> 
291c320
<         # self.readonly_storage causes remaining_space=0
---
>         # self.readonly_storage causes remaining_space <= 0
diff -rN old-tahoe/src/allmydata/test/test_storage.py new-tahoe/src/allmydata/test/test_storage.py
231,232c231,233
<     def stat_disk(self, d):
<         return self.DISKAVAIL
---
>     DISKAVAIL = 0
>     def get_disk_stats(self):
>         return { 'free_for_nonroot': self.DISKAVAIL, 'avail': max(self.DISKAVAIL - self.reserved_space, 0), }
415c416
<         # the FakeDiskStorageServer doesn't do real statvfs() calls
---
>         # the FakeDiskStorageServer doesn't do real calls to get_disk_stats
470a472,488
>     def test_disk_stats(self):
>         # This will spuriously fail if there is zero disk space left (but so will other tests).
>         ss = self.create("test_disk_stats", reserved_space=0)
> 
>         disk = ss.get_disk_stats()
>         self.failUnless(disk['total'] > 0, disk['total'])
>         self.failUnless(disk['used'] > 0, disk['used'])
>         self.failUnless(disk['free_for_root'] > 0, disk['free_for_root'])
>         self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot'])
>         self.failUnless(disk['avail'] > 0, disk['avail'])
> 
>     def test_disk_stats_avail_nonnegative(self):
>         ss = self.create("test_disk_stats_avail_nonnegative", reserved_space=2**64)
> 
>         disk = ss.get_disk_stats()
>         self.failUnlessEqual(disk['avail'], 0)
> 
627,628c645
<         self.failUnlessEqual(stats["storage_server.accepting_immutable_shares"],
<                              False)
---
>         self.failUnlessEqual(stats["storage_server.accepting_immutable_shares"], 0)
630,632c647,648
<             # windows does not have os.statvfs, so it doesn't give us disk
<             # stats. But if there are stats, readonly_storage means
<             # disk_avail=0
---
>             # Some platforms may not have an API to get disk stats.
>             # But if there are stats, readonly_storage means disk_avail=0
2408,2409c2424,2425
< class NoStatvfsServer(StorageServer):
<     def do_statvfs(self):
---
> class NoDiskStatsServer(StorageServer):
>     def get_disk_stats(self):
2411a2428,2431
> class BadDiskStatsServer(StorageServer):
>     def get_disk_stats(self):
>         raise OSError
> 
2453,2456c2473,2476
<     def test_status_no_statvfs(self):
<         # windows has no os.statvfs . Make sure the code handles that even on
<         # unix.
<         basedir = "storage/WebStatus/status_no_statvfs"
---
>     def test_status_no_disk_stats(self):
>         # Some platforms may have no disk stats API. Make sure the code can handle that
>         # (test runs on all platforms).
>         basedir = "storage/WebStatus/status_no_disk_stats"
2458c2478
<         ss = NoStatvfsServer(basedir, "\x00" * 20)
---
>         ss = NoDiskStatsServer(basedir, "\x00" * 20)
2465a2486,2503
>         self.failUnless("Space Available to Tahoe: ?" in s, s)
>         self.failUnless(ss.get_available_space() is None)
> 
>     def test_status_bad_disk_stats(self):
>         # If the API to get disk stats exists but a call to it fails, then the status should
>         # show that no shares will be accepted, and get_available_space() should be 0.
>         basedir = "storage/WebStatus/status_bad_disk_stats"
>         fileutil.make_dirs(basedir)
>         ss = BadDiskStatsServer(basedir, "\x00" * 20)
>         ss.setServiceParent(self.s)
>         w = StorageStatus(ss)
>         html = w.renderSynchronously()
>         self.failUnless("<h1>Storage Server Status</h1>" in html, html)
>         s = remove_tags(html)
>         self.failUnless("Accepting new shares: No" in s, s)
>         self.failUnless("Total disk space: ?" in s, s)
>         self.failUnless("Space Available to Tahoe: ?" in s, s)
>         self.failUnless(ss.get_available_space() == 0)