Changeset 7b7b0c9 in trunk


Ignore:
Timestamp:
2010-08-04T07:27:02Z (15 years ago)
Author:
Brian Warner <warner@…>
Branches:
master
Children:
63b61ce
Parents:
797828f
Message:

Rewrite immutable downloader (#798). This patch includes higher-level
integration into the NodeMaker?, and updates the web-status display to handle
the new download events.

Location:
src/allmydata
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • TabularUnified src/allmydata/client.py

    r797828f r7b7b0c9  
    1 import os, stat, time
     1import os, stat, time, weakref
    22from allmydata.interfaces import RIStorageServer
    33from allmydata import node
     
    55from zope.interface import implements
    66from twisted.internet import reactor, defer
     7from twisted.application import service
    78from twisted.application.internet import TimerService
    89from foolscap.api import Referenceable
     
    1314from allmydata import storage_client
    1415from allmydata.immutable.upload import Uploader
    15 from allmydata.immutable.download import Downloader
    1616from allmydata.immutable.offloaded import Helper
    1717from allmydata.control import ControlServer
    1818from allmydata.introducer.client import IntroducerClient
    19 from allmydata.util import hashutil, base32, pollmixin, cachedir, log
     19from allmydata.util import hashutil, base32, pollmixin, log
    2020from allmydata.util.encodingutil import get_filesystem_encoding
    2121from allmydata.util.abbreviate import parse_abbreviated_size
     
    9696            return defer.succeed( (verifier, signer) )
    9797
     98class Terminator(service.Service):
     99    def __init__(self):
     100        self._clients = weakref.WeakKeyDictionary()
     101    def register(self, c):
     102        self._clients[c] = None
     103    def stopService(self):
     104        for c in self._clients:
     105            c.stop()
     106        return service.Service.stopService(self)
     107
    98108
    99109class Client(node.Node, pollmixin.PollMixin):
     
    280290        self.init_client_storage_broker()
    281291        self.history = History(self.stats_provider)
     292        self.terminator = Terminator()
     293        self.terminator.setServiceParent(self)
    282294        self.add_service(Uploader(helper_furl, self.stats_provider))
    283         download_cachedir = os.path.join(self.basedir,
    284                                          "private", "cache", "download")
    285         self.download_cache_dirman = cachedir.CacheDirectoryManager(download_cachedir)
    286         self.download_cache_dirman.setServiceParent(self)
    287         self.downloader = Downloader(self.storage_broker, self.stats_provider)
    288295        self.init_stub_client()
    289296        self.init_nodemaker()
     
    344351                                   self.get_history(),
    345352                                   self.getServiceNamed("uploader"),
    346                                    self.downloader,
    347                                    self.download_cache_dirman,
     353                                   self.terminator,
    348354                                   self.get_encoding_parameters(),
    349355                                   self._key_generator)
  • TabularUnified src/allmydata/interfaces.py

    r797828f r7b7b0c9  
    2424LeaseRenewSecret = Hash # used to protect bucket lease renewal requests
    2525LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests
     26
     27KiB = 1024
     28DEFAULT_MAX_SEGMENT_SIZE = 128*KiB
    2629
    2730class RIStubClient(RemoteInterface):
  • TabularUnified src/allmydata/nodemaker.py

    r797828f r7b7b0c9  
    22from zope.interface import implements
    33from allmydata.interfaces import INodeMaker
    4 from allmydata.immutable.filenode import ImmutableFileNode, LiteralFileNode
     4from allmydata.immutable.literal import LiteralFileNode
     5from allmydata.immutable.filenode import ImmutableFileNode, CiphertextFileNode
    56from allmydata.immutable.upload import Data
    67from allmydata.mutable.filenode import MutableFileNode
     
    1314
    1415    def __init__(self, storage_broker, secret_holder, history,
    15                  uploader, downloader, download_cache_dirman,
     16                 uploader, terminator,
    1617                 default_encoding_parameters, key_generator):
    1718        self.storage_broker = storage_broker
     
    1920        self.history = history
    2021        self.uploader = uploader
    21         self.downloader = downloader
    22         self.download_cache_dirman = download_cache_dirman
     22        self.terminator = terminator
    2323        self.default_encoding_parameters = default_encoding_parameters
    2424        self.key_generator = key_generator
     
    3030    def _create_immutable(self, cap):
    3131        return ImmutableFileNode(cap, self.storage_broker, self.secret_holder,
    32                                  self.downloader, self.history,
    33                                  self.download_cache_dirman)
     32                                 self.terminator, self.history)
     33    def _create_immutable_verifier(self, cap):
     34        return CiphertextFileNode(cap, self.storage_broker, self.secret_holder,
     35                                  self.terminator, self.history)
    3436    def _create_mutable(self, cap):
    3537        n = MutableFileNode(self.storage_broker, self.secret_holder,
     
    7476        if isinstance(cap, uri.CHKFileURI):
    7577            return self._create_immutable(cap)
     78        if isinstance(cap, uri.CHKFileVerifierURI):
     79            return self._create_immutable_verifier(cap)
    7680        if isinstance(cap, (uri.ReadonlySSKFileURI, uri.WriteableSSKFileURI)):
    7781            return self._create_mutable(cap)
  • TabularUnified src/allmydata/web/download-status.xhtml

    r797828f r7b7b0c9  
    1919</ul>
    2020
     21<div n:render="events"></div>
    2122
    2223<div n:render="results">
  • TabularUnified src/allmydata/web/status.py

    r797828f r7b7b0c9  
    359359        return defer.maybeDeferred(self.download_status.get_results)
    360360
     361    def relative_time(self, t):
     362        if t is None:
     363            return t
     364        if self.download_status.started is not None:
     365            return t - self.download_status.started
     366        return t
     367    def short_relative_time(self, t):
     368        t = self.relative_time(t)
     369        if t is None:
     370            return ""
     371        return "+%.6fs" % t
     372
     373    def renderHTTP(self, ctx):
     374        req = inevow.IRequest(ctx)
     375        t = get_arg(req, "t")
     376        if t == "json":
     377            return self.json(req)
     378        return rend.Page.renderHTTP(self, ctx)
     379
     380    def json(self, req):
     381        req.setHeader("content-type", "text/plain")
     382        data = {}
     383        dyhb_events = []
     384        for serverid,requests in self.download_status.dyhb_requests.iteritems():
     385            for req in requests:
     386                dyhb_events.append( (base32.b2a(serverid),) + req )
     387        dyhb_events.sort(key=lambda req: req[1])
     388        data["dyhb"] = dyhb_events
     389        request_events = []
     390        for serverid,requests in self.download_status.requests.iteritems():
     391            for req in requests:
     392                request_events.append( (base32.b2a(serverid),) + req )
     393        request_events.sort(key=lambda req: (req[4],req[1]))
     394        data["requests"] = request_events
     395        data["segment"] = self.download_status.segment_events
     396        data["read"] = self.download_status.read_events
     397        return simplejson.dumps(data, indent=1) + "\n"
     398
     399    def render_events(self, ctx, data):
     400        if not self.download_status.storage_index:
     401            return
     402        srt = self.short_relative_time
     403        l = T.ul()
     404
     405        t = T.table(class_="status-download-events")
     406        t[T.tr[T.td["serverid"], T.td["sent"], T.td["received"],
     407               T.td["shnums"], T.td["RTT"]]]
     408        dyhb_events = []
     409        for serverid,requests in self.download_status.dyhb_requests.iteritems():
     410            for req in requests:
     411                dyhb_events.append( (serverid,) + req )
     412        dyhb_events.sort(key=lambda req: req[1])
     413        for d_ev in dyhb_events:
     414            (serverid, sent, shnums, received) = d_ev
     415            serverid_s = idlib.shortnodeid_b2a(serverid)
     416            rtt = received - sent
     417            t[T.tr(style="background: %s" % self.color(serverid))[
     418                [T.td[serverid_s], T.td[srt(sent)], T.td[srt(received)],
     419                 T.td[",".join([str(shnum) for shnum in shnums])],
     420                 T.td[self.render_time(None, rtt)],
     421                 ]]]
     422        l["DYHB Requests:", t]
     423
     424        t = T.table(class_="status-download-events")
     425        t[T.tr[T.td["range"], T.td["start"], T.td["finish"], T.td["got"],
     426               T.td["time"], T.td["decrypttime"], T.td["pausedtime"],
     427               T.td["speed"]]]
     428        for r_ev in self.download_status.read_events:
     429            (start, length, requesttime, finishtime, bytes, decrypt, paused) = r_ev
     430            print r_ev
     431            if finishtime is not None:
     432                rtt = finishtime - requesttime - paused
     433                speed = self.render_rate(None, 1.0 * bytes / rtt)
     434                rtt = self.render_time(None, rtt)
     435                decrypt = self.render_time(None, decrypt)
     436                paused = self.render_time(None, paused)
     437            else:
     438                speed, rtt, decrypt, paused = "","","",""
     439            t[T.tr[T.td["[%d:+%d]" % (start, length)],
     440                   T.td[srt(requesttime)], T.td[srt(finishtime)],
     441                   T.td[bytes], T.td[rtt], T.td[decrypt], T.td[paused],
     442                   T.td[speed],
     443                   ]]
     444        l["Read Events:", t]
     445
     446        t = T.table(class_="status-download-events")
     447        t[T.tr[T.td["type"], T.td["segnum"], T.td["when"], T.td["range"],
     448               T.td["decodetime"], T.td["segtime"], T.td["speed"]]]
     449        reqtime = (None, None)
     450        for s_ev in self.download_status.segment_events:
     451            (etype, segnum, when, segstart, seglen, decodetime) = s_ev
     452            if etype == "request":
     453                t[T.tr[T.td["request"], T.td["seg%d" % segnum],
     454                       T.td[srt(when)]]]
     455                reqtime = (segnum, when)
     456            elif etype == "delivery":
     457                if reqtime[0] == segnum:
     458                    segtime = when - reqtime[1]
     459                    speed = self.render_rate(None, 1.0 * seglen / segtime)
     460                    segtime = self.render_time(None, segtime)
     461                else:
     462                    segtime, speed = "", ""
     463                t[T.tr[T.td["delivery"], T.td["seg%d" % segnum],
     464                       T.td[srt(when)],
     465                       T.td["[%d:+%d]" % (segstart, seglen)],
     466                       T.td[self.render_time(None,decodetime)],
     467                       T.td[segtime], T.td[speed]]]
     468            elif etype == "error":
     469                t[T.tr[T.td["error"], T.td["seg%d" % segnum]]]
     470        l["Segment Events:", t]
     471
     472        t = T.table(border="1")
     473        t[T.tr[T.td["serverid"], T.td["shnum"], T.td["range"],
     474               T.td["txtime"], T.td["rxtime"], T.td["received"], T.td["RTT"]]]
     475        reqtime = (None, None)
     476        request_events = []
     477        for serverid,requests in self.download_status.requests.iteritems():
     478            for req in requests:
     479                request_events.append( (serverid,) + req )
     480        request_events.sort(key=lambda req: (req[4],req[1]))
     481        for r_ev in request_events:
     482            (peerid, shnum, start, length, sent, receivedlen, received) = r_ev
     483            rtt = None
     484            if received is not None:
     485                rtt = received - sent
     486            peerid_s = idlib.shortnodeid_b2a(peerid)
     487            t[T.tr(style="background: %s" % self.color(peerid))[
     488                T.td[peerid_s], T.td[shnum],
     489                T.td["[%d:+%d]" % (start, length)],
     490                T.td[srt(sent)], T.td[srt(received)], T.td[receivedlen],
     491                T.td[self.render_time(None, rtt)],
     492                ]]
     493        l["Requests:", t]
     494
     495        return l
     496
     497    def color(self, peerid):
     498        def m(c):
     499            return min(ord(c) / 2 + 0x80, 0xff)
     500        return "#%02x%02x%02x" % (m(peerid[0]), m(peerid[1]), m(peerid[2]))
     501
    361502    def render_results(self, ctx, data):
    362503        d = self.download_results()
     
    372513        started_s = time.strftime(TIME_FORMAT,
    373514                                  time.localtime(data.get_started()))
    374         return started_s
     515        return started_s + " (%s)" % data.get_started()
    375516
    376517    def render_si(self, ctx, data):
  • TabularUnified src/allmydata/web/tahoe.css

    r797828f r7b7b0c9  
    137137  padding: 0 1em;
    138138}
     139
     140/* recent upload/download status pages */
     141
     142table.status-download-events {
     143  border: 1px solid #aaa;
     144}
     145table.status-download-events td {
     146  border: 1px solid #a00;
     147  padding: 2px
     148}
Note: See TracChangeset for help on using the changeset viewer.