source: trunk/src/allmydata/web/storage.py

Last change on this file was 1cfe843d, checked in by Alexandre Detiste <alexandre.detiste@…>, at 2024-02-22T23:40:25Z

more python2 removal
  • Property mode set to 100644
File size: 12.0 KB
Line 
1"""
2Ported to Python 3.
3"""
4
5import time
6from twisted.python.filepath import FilePath
7from twisted.web.template import (
8    Element,
9    XMLFile,
10    tags as T,
11    renderer,
12    renderElement
13)
14from allmydata.web.common import (
15    abbreviate_time,
16    MultiFormatResource
17)
18from allmydata.util.abbreviate import abbreviate_space
19from allmydata.util import time_format, idlib, jsonbytes as json
20
21
22def remove_prefix(s, prefix):
23    if not s.startswith(prefix):
24        return None
25    return s[len(prefix):]
26
27
28class StorageStatusElement(Element):
29    """Class to render a storage status page."""
30
31    loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml"))
32
33    def __init__(self, storage, nickname=""):
34        """
35        :param _StorageServer storage: data about storage.
36        :param string nickname: friendly name for storage.
37        """
38        super(StorageStatusElement, self).__init__()
39        self._storage = storage
40        self._nickname = nickname
41
42    @renderer
43    def nickname(self, req, tag):
44        return tag(self._nickname)
45
46    @renderer
47    def nodeid(self, req, tag):
48        return tag(idlib.nodeid_b2a(self._storage.my_nodeid))
49
50    def _get_storage_stat(self, key):
51        """Get storage server statistics.
52
53        Storage Server keeps a dict that contains various usage and
54        latency statistics.  The dict looks like this:
55
56          {
57            'storage_server.accepting_immutable_shares': 1,
58            'storage_server.allocated': 0,
59            'storage_server.disk_avail': 106539192320,
60            'storage_server.disk_free_for_nonroot': 106539192320,
61            'storage_server.disk_free_for_root': 154415284224,
62            'storage_server.disk_total': 941088460800,
63            'storage_server.disk_used': 786673176576,
64            'storage_server.latencies.add-lease.01_0_percentile': None,
65            'storage_server.latencies.add-lease.10_0_percentile': None,
66            ...
67          }
68
69        ``StorageServer.get_stats()`` returns the above dict.  Storage
70        status page uses a subset of the items in the dict, concerning
71        disk usage.
72
73        :param str key: storage server statistic we want to know.
74        """
75        return self._storage.get_stats().get(key)
76
77    def render_abbrev_space(self, size):
78        if size is None:
79            return u"?"
80        return abbreviate_space(size)
81
82    def render_space(self, size):
83        if size is None:
84            return u"?"
85        return u"%d" % size
86
87    @renderer
88    def storage_stats(self, req, tag):
89        # Render storage status table that appears near the top of the page.
90        total = self._get_storage_stat("storage_server.disk_total")
91        used = self._get_storage_stat("storage_server.disk_used")
92        free_root = self._get_storage_stat("storage_server.disk_free_for_root")
93        free_nonroot = self._get_storage_stat("storage_server.disk_free_for_nonroot")
94        reserved = self._get_storage_stat("storage_server.reserved_space")
95        available = self._get_storage_stat("storage_server.disk_avail")
96
97        tag.fillSlots(
98            disk_total = self.render_space(total),
99            disk_total_abbrev = self.render_abbrev_space(total),
100            disk_used = self.render_space(used),
101            disk_used_abbrev = self.render_abbrev_space(used),
102            disk_free_for_root = self.render_space(free_root),
103            disk_free_for_root_abbrev = self.render_abbrev_space(free_root),
104            disk_free_for_nonroot = self.render_space(free_nonroot),
105            disk_free_for_nonroot_abbrev = self.render_abbrev_space(free_nonroot),
106            reserved_space = self.render_space(reserved),
107            reserved_space_abbrev = self.render_abbrev_space(reserved),
108            disk_avail = self.render_space(available),
109            disk_avail_abbrev = self.render_abbrev_space(available)
110        )
111        return tag
112
113    @renderer
114    def accepting_immutable_shares(self, req, tag):
115        accepting = self._get_storage_stat("storage_server.accepting_immutable_shares")
116        return tag({True: "Yes", False: "No"}[bool(accepting)])
117
118    @renderer
119    def last_complete_bucket_count(self, req, tag):
120        s = self._storage.bucket_counter.get_state()
121        count = s.get("last-complete-bucket-count")
122        if count is None:
123            return tag("Not computed yet")
124        return tag(str(count))
125
126    @renderer
127    def count_crawler_status(self, req, tag):
128        p = self._storage.bucket_counter.get_progress()
129        return tag(self.format_crawler_progress(p))
130
131    def format_crawler_progress(self, p):
132        cycletime = p["estimated-time-per-cycle"]
133        cycletime_s = ""
134        if cycletime is not None:
135            cycletime_s = " (estimated cycle time %s)" % abbreviate_time(cycletime)
136
137        if p["cycle-in-progress"]:
138            pct = p["cycle-complete-percentage"]
139            soon = p["remaining-sleep-time"]
140
141            eta = p["estimated-cycle-complete-time-left"]
142            eta_s = ""
143            if eta is not None:
144                eta_s = " (ETA %ds)" % eta
145
146            return ["Current crawl %.1f%% complete" % pct,
147                    eta_s,
148                    " (next work in %s)" % abbreviate_time(soon),
149                    cycletime_s,
150                    ]
151        else:
152            soon = p["remaining-wait-time"]
153            return ["Next crawl in %s" % abbreviate_time(soon),
154                    cycletime_s]
155
156    @renderer
157    def storage_running(self, req, tag):
158        if self._storage:
159            return tag
160        return T.h1("No Storage Server Running")
161
162    @renderer
163    def lease_expiration_enabled(self, req, tag):
164        lc = self._storage.lease_checker
165        if lc.expiration_enabled:
166            return tag("Enabled: expired leases will be removed")
167        else:
168            return tag("Disabled: scan-only mode, no leases will be removed")
169
170    @renderer
171    def lease_expiration_mode(self, req, tag):
172        lc = self._storage.lease_checker
173        if lc.mode == "age":
174            if lc.override_lease_duration is None:
175                tag("Leases will expire naturally, probably 31 days after "
176                    "creation or renewal.")
177            else:
178                tag("Leases created or last renewed more than %s ago "
179                    "will be considered expired."
180                    % abbreviate_time(lc.override_lease_duration))
181        else:
182            assert lc.mode == "cutoff-date"
183            localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date))
184            isoutcdate = time_format.iso_utc_date(lc.cutoff_date)
185            tag("Leases created or last renewed before %s (%s) UTC "
186                "will be considered expired."
187                % (isoutcdate, localizedutcdate, ))
188        if len(lc.mode) > 2:
189            tag(" The following sharetypes will be expired: ",
190                " ".join(sorted(lc.sharetypes_to_expire)), ".")
191        return tag
192
193    @renderer
194    def lease_current_cycle_progress(self, req, tag):
195        lc = self._storage.lease_checker
196        p = lc.get_progress()
197        return tag(self.format_crawler_progress(p))
198
199    @renderer
200    def lease_current_cycle_results(self, req, tag):
201        lc = self._storage.lease_checker
202        p = lc.get_progress()
203        if not p["cycle-in-progress"]:
204            return ""
205        s = lc.get_state()
206        so_far = s["cycle-to-date"]
207        sr = so_far["space-recovered"]
208        er = s["estimated-remaining-cycle"]
209        esr = er["space-recovered"]
210        ec = s["estimated-current-cycle"]
211        ecr = ec["space-recovered"]
212
213        p = T.ul()
214        def add(*pieces):
215            p(T.li(pieces))
216
217        def maybe(d):
218            if d is None:
219                return "?"
220            return "%d" % d
221        add("So far, this cycle has examined %d shares in %d buckets"
222            % (sr["examined-shares"], sr["examined-buckets"]),
223            " (%d mutable / %d immutable)"
224            % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]),
225            " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]),
226                            abbreviate_space(sr["examined-diskbytes-immutable"])),
227            )
228        add("and has recovered: ", self.format_recovered(sr, "actual"))
229        if so_far["expiration-enabled"]:
230            add("The remainder of this cycle is expected to recover: ",
231                self.format_recovered(esr, "actual"))
232            add("The whole cycle is expected to examine %s shares in %s buckets"
233                % (maybe(ecr["examined-shares"]), maybe(ecr["examined-buckets"])))
234            add("and to recover: ", self.format_recovered(ecr, "actual"))
235
236        else:
237            add("If expiration were enabled, we would have recovered: ",
238                self.format_recovered(sr, "configured"), " by now")
239            add("and the remainder of this cycle would probably recover: ",
240                self.format_recovered(esr, "configured"))
241            add("and the whole cycle would probably recover: ",
242                self.format_recovered(ecr, "configured"))
243
244        add("if we were strictly using each lease's default 31-day lease lifetime "
245            "(instead of our configured behavior), "
246            "this cycle would be expected to recover: ",
247            self.format_recovered(ecr, "original"))
248
249        if so_far["corrupt-shares"]:
250            add("Corrupt shares:",
251                T.ul( (T.li( ["SI %s shnum %d" % (si, shnum)
252                              for si, shnum in so_far["corrupt-shares"] ]
253                             ))))
254        return tag("Current cycle:", p)
255
256    @renderer
257    def lease_last_cycle_results(self, req, tag):
258        lc = self._storage.lease_checker
259        h = lc.get_state()["history"]
260        if not h:
261            return ""
262        biggest = str(max(int(k) for k in h.keys()))
263        last = h[biggest]
264
265        start, end = last["cycle-start-finish-times"]
266        tag("Last complete cycle (which took %s and finished %s ago)"
267            " recovered: " % (abbreviate_time(end-start),
268                              abbreviate_time(time.time() - end)),
269            self.format_recovered(last["space-recovered"], "actual"))
270
271        p = T.ul()
272
273        def add(*pieces):
274            p(T.li(pieces))
275
276        saw = self.format_recovered(last["space-recovered"], "examined")
277        add("and saw a total of ", saw)
278
279        if not last["expiration-enabled"]:
280            rec = self.format_recovered(last["space-recovered"], "configured")
281            add("but expiration was not enabled. If it had been, "
282                "it would have recovered: ", rec)
283
284        if last["corrupt-shares"]:
285            add("Corrupt shares:",
286                T.ul( (T.li( ["SI %s shnum %d" % (si, shnum)
287                              for si, shnum in last["corrupt-shares"] ]
288                             ))))
289
290        return tag(p)
291
292    @staticmethod
293    def format_recovered(sr, a):
294        def maybe(d):
295            if d is None:
296                return "?"
297            return "%d" % d
298        return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
299               (maybe(sr["%s-shares" % a]),
300                maybe(sr["%s-buckets" % a]),
301                maybe(sr["%s-buckets-mutable" % a]),
302                maybe(sr["%s-buckets-immutable" % a]),
303                abbreviate_space(sr["%s-diskbytes" % a]),
304                abbreviate_space(sr["%s-diskbytes-mutable" % a]),
305                abbreviate_space(sr["%s-diskbytes-immutable" % a]),
306                )
307
308class StorageStatus(MultiFormatResource):
309    def __init__(self, storage, nickname=""):
310        super(StorageStatus, self).__init__()
311        self._storage = storage
312        self._nickname = nickname
313
314    def render_HTML(self, req):
315        return renderElement(req, StorageStatusElement(self._storage, self._nickname))
316
317    def render_JSON(self, req):
318        req.setHeader("content-type", "text/plain")
319        d = {"stats": self._storage.get_stats(),
320             "bucket-counter": self._storage.bucket_counter.get_state(),
321             "lease-checker": self._storage.lease_checker.get_state(),
322             "lease-checker-progress": self._storage.lease_checker.get_progress(),
323             }
324        return json.dumps(d, indent=1) + "\n"
Note: See TracBrowser for help on using the repository browser.