Changeset ba019bf in trunk


Ignore:
Timestamp:
2008-10-30T20:39:09Z (16 years ago)
Author:
Brian Warner <warner@…>
Branches:
master
Children:
845b77ae
Parents:
c205a54
Message:

#527: expire the cached files that are used to support Range: headers, every hour, when the file is unused and older than an hour

Location:
src/allmydata
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • TabularUnified src/allmydata/client.py

    rc205a54 rba019bf  
    1919from allmydata.control import ControlServer
    2020from allmydata.introducer.client import IntroducerClient
    21 from allmydata.util import hashutil, base32, pollmixin, fileutil
     21from allmydata.util import hashutil, base32, pollmixin, cachedir
    2222from allmydata.uri import LiteralFileURI
    2323from allmydata.dirnode import NewDirectoryNode
     
    189189        self._node_cache = weakref.WeakValueDictionary() # uri -> node
    190190        self.add_service(Uploader(helper_furl, self.stats_provider))
    191         self.download_cachedir = os.path.join(self.basedir,
    192                                               "private", "cache", "download")
    193         fileutil.make_dirs(self.download_cachedir)
     191        download_cachedir = os.path.join(self.basedir,
     192                                         "private", "cache", "download")
     193        self.download_cache = cachedir.CacheDirectoryManager(download_cachedir)
     194        self.download_cache.setServiceParent(self)
    194195        self.add_service(Downloader(self.stats_provider))
    195196        self.add_service(MutableWatcher(self.stats_provider))
     
    340341                    node = LiteralFileNode(u, self) # LIT
    341342                else:
    342                     cachefile = os.path.join(self.download_cachedir,
    343                                              base32.b2a(u.storage_index))
    344                     # TODO: cachefile manager, weakref, expire policy
     343                    key = base32.b2a(u.storage_index)
     344                    cachefile = self.download_cache.get_file(key)
    345345                    node = FileNode(u, self, cachefile) # CHK
    346346            else:
  • TabularUnified src/allmydata/immutable/filenode.py

    rc205a54 rba019bf  
    6363        return data
    6464
    65 class CacheFile:
     65class DownloadCache:
    6666    implements(IDownloadTarget)
    6767
    68     def __init__(self, node, filename):
    69         self.node = node
     68    def __init__(self, node, cachefile):
     69        self._downloader = node._client.getServiceNamed("downloader")
     70        self._uri = node.get_uri()
     71        self._storage_index = node.get_storage_index()
    7072        self.milestones = set() # of (offset,size,Deferred)
    71         self.cachefilename = filename
     73        self.cachefile = cachefile
    7274        self.download_in_progress = False
    7375        # five states:
     
    8991            log.msg(format=("immutable filenode read [%(si)s]: " +
    9092                            "starting download"),
    91                     si=base32.b2a(self.node.u.storage_index),
     93                    si=base32.b2a(self._storage_index),
    9294                    umid="h26Heg", level=log.OPERATIONAL)
    93             downloader = self.node._client.getServiceNamed("downloader")
    94             d2 = downloader.download(self.node.get_uri(), self)
     95            d2 = self._downloader.download(self._uri, self)
    9596            d2.addBoth(self._download_done)
    9697            d2.addErrback(self._download_failed)
     
    100101    def read(self, consumer, offset, size):
    101102        assert offset+size <= self.get_filesize()
    102         f = PortionOfFile(self.cachefilename, offset, size)
     103        f = PortionOfFile(self.cachefile.get_filename(), offset, size)
    103104        d = basic.FileSender().beginFileTransfer(f, consumer)
    104105        d.addCallback(lambda lastSent: consumer)
     
    125126                                "%(offset)d+%(size)d vs %(filesize)d: " +
    126127                                "done"),
    127                         si=base32.b2a(self.node.u.storage_index),
     128                        si=base32.b2a(self._storage_index),
    128129                        offset=offset, size=size, filesize=current_size,
    129130                        umid="nuedUg", level=log.NOISY)
     
    134135                                "%(offset)d+%(size)d vs %(filesize)d: " +
    135136                                "still waiting"),
    136                         si=base32.b2a(self.node.u.storage_index),
     137                        si=base32.b2a(self._storage_index),
    137138                        offset=offset, size=size, filesize=current_size,
    138139                        umid="8PKOhg", level=log.NOISY)
     
    140141    def get_filesize(self):
    141142        try:
    142             filesize = os.stat(self.cachefilename)[stat.ST_SIZE]
     143            filesize = os.stat(self.cachefile.get_filename())[stat.ST_SIZE]
    143144        except OSError:
    144145            filesize = 0
     
    147148
    148149    def open(self, size):
    149         self.f = open(self.cachefilename, "wb")
     150        self.f = open(self.cachefile.get_filename(), "wb")
    150151
    151152    def write(self, data):
     
    172173    def __init__(self, uri, client, cachefile):
    173174        _ImmutableFileNodeBase.__init__(self, uri, client)
    174         self.cachefile = CacheFile(self, cachefile)
     175        self.download_cache = DownloadCache(self, cachefile)
    175176
    176177    def get_uri(self):
     
    231232            return self.download(download.ConsumerAdapter(consumer))
    232233
    233         d = self.cachefile.when_range_available(offset, size)
    234         d.addCallback(lambda res: self.cachefile.read(consumer, offset, size))
     234        d = self.download_cache.when_range_available(offset, size)
     235        d.addCallback(lambda res:
     236                      self.download_cache.read(consumer, offset, size))
    235237        return d
    236238
  • TabularUnified src/allmydata/test/test_filenode.py

    rc205a54 rba019bf  
    55from allmydata.immutable import filenode, download
    66from allmydata.mutable.node import MutableFileNode
    7 from allmydata.util import hashutil
     7from allmydata.util import hashutil, cachedir
    88from allmydata.test.common import download_to_data
    99
    1010class NotANode:
    1111    pass
     12
     13class FakeClient:
     14    # just enough to let the node acquire a downloader (which it won't use)
     15    def getServiceNamed(self, name):
     16        return None
    1217
    1318class Node(unittest.TestCase):
     
    1823                           total_shares=10,
    1924                           size=1000)
    20         c = None
    21         fn1 = filenode.FileNode(u, c, "cachefile")
    22         fn2 = filenode.FileNode(u.to_string(), c, "cachefile")
     25        c = FakeClient()
     26        cf = cachedir.CacheFile("none")
     27        fn1 = filenode.FileNode(u, c, cf)
     28        fn2 = filenode.FileNode(u.to_string(), c, cf)
    2329        self.failUnlessEqual(fn1, fn2)
    2430        self.failIfEqual(fn1, "I am not a filenode")
Note: See TracChangeset for help on using the changeset viewer.