source: trunk/src/allmydata/scripts/debug.py

Last change on this file was 29a66e5, checked in by Itamar Turner-Trauring <itamar@…>, at 2023-03-24T16:01:12Z

Fix lint.
  • Property mode set to 100644
File size: 41.8 KB
Line 
1"""
2Ported to Python 3.
3"""
4
5from future.utils import bchr
6
7import struct, time, os, sys
8
9from twisted.python import usage, failure
10from twisted.internet import defer
11from foolscap.logging import cli as foolscap_cli
12
13from allmydata.scripts.common import BaseOptions
14from allmydata import uri
15from allmydata.storage.mutable import MutableShareFile
16from allmydata.storage.immutable import ShareFile
17from allmydata.mutable.layout import unpack_share
18from allmydata.mutable.layout import MDMFSlotReadProxy
19from allmydata.mutable.common import NeedMoreDataError
20from allmydata.immutable.layout import ReadBucketProxy
21from allmydata.util import base32
22from allmydata.util.encodingutil import quote_output
23from allmydata.scripts.types_ import SubCommands
24
25class DumpOptions(BaseOptions):
26    def getSynopsis(self):
27        return "Usage: tahoe [global-options] debug dump-share SHARE_FILENAME"
28
29    optFlags = [
30        ["offsets", None, "Display a table of section offsets."],
31        ["leases-only", None, "Dump leases but not CHK contents."],
32        ]
33
34    description = """
35Print lots of information about the given share, by parsing the share's
36contents. This includes share type, lease information, encoding parameters,
37hash-tree roots, public keys, and segment sizes. This command also emits a
38verify-cap for the file that uses the share.
39
40 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
41"""
42
43    def parseArgs(self, filename):
44        from allmydata.util.encodingutil import argv_to_abspath
45        self['filename'] = argv_to_abspath(filename)
46
47def dump_share(options):
48    from allmydata.storage.mutable import MutableShareFile
49    from allmydata.util.encodingutil import quote_output
50
51    out = options.stdout
52
53    # check the version, to see if we have a mutable or immutable share
54    print("share filename: %s" % quote_output(options['filename']), file=out)
55
56    with open(options['filename'], "rb") as f:
57        if MutableShareFile.is_valid_header(f.read(32)):
58            return dump_mutable_share(options)
59        # otherwise assume it's immutable
60        return dump_immutable_share(options)
61
62def dump_immutable_share(options):
63    from allmydata.storage.immutable import ShareFile
64
65    out = options.stdout
66    f = ShareFile(options['filename'])
67    if not options["leases-only"]:
68        dump_immutable_chk_share(f, out, options)
69    dump_immutable_lease_info(f, out)
70    print(file=out)
71    return 0
72
73def dump_immutable_chk_share(f, out, options):
74    from allmydata import uri
75    from allmydata.util import base32
76    from allmydata.immutable.layout import ReadBucketProxy
77    from allmydata.util.encodingutil import quote_output, to_bytes
78
79    # use a ReadBucketProxy to parse the bucket and find the uri extension
80    bp = ReadBucketProxy(None, None, '')
81    offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
82    print("%20s: %d" % ("version", bp._version), file=out)
83    seek = offsets['uri_extension']
84    length = struct.unpack(bp._fieldstruct,
85                           f.read_share_data(seek, bp._fieldsize))[0]
86    seek += bp._fieldsize
87    UEB_data = f.read_share_data(seek, length)
88
89    unpacked = uri.unpack_extension_readable(UEB_data)
90    keys1 = ("size", "num_segments", "segment_size",
91             "needed_shares", "total_shares")
92    keys2 = ("codec_name", "codec_params", "tail_codec_params")
93    keys3 = ("plaintext_hash", "plaintext_root_hash",
94             "crypttext_hash", "crypttext_root_hash",
95             "share_root_hash", "UEB_hash")
96    display_keys = {"size": "file_size"}
97
98    def to_string(v):
99        if isinstance(v, bytes):
100            return str(v, "utf-8")
101        else:
102            return str(v)
103
104    for k in keys1:
105        if k in unpacked:
106            dk = display_keys.get(k, k)
107            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)
108    print(file=out)
109    for k in keys2:
110        if k in unpacked:
111            dk = display_keys.get(k, k)
112            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)
113    print(file=out)
114    for k in keys3:
115        if k in unpacked:
116            dk = display_keys.get(k, k)
117            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)
118
119    leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
120    if leftover:
121        print(file=out)
122        print("LEFTOVER:", file=out)
123        for k in sorted(leftover):
124            print("%20s: %s" % (k, to_string(unpacked[k])), file=out)
125
126    # the storage index isn't stored in the share itself, so we depend upon
127    # knowing the parent directory name to get it
128    pieces = options['filename'].split(os.sep)
129    if len(pieces) >= 2:
130        piece = to_bytes(pieces[-2])
131        if base32.could_be_base32_encoded(piece):
132            storage_index = base32.a2b(piece)
133            uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
134            u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
135                                      unpacked["needed_shares"],
136                                      unpacked["total_shares"], unpacked["size"])
137            verify_cap = u.to_string()
138            print("%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False)), file=out)
139
140    sizes = {}
141    sizes['data'] = (offsets['plaintext_hash_tree'] -
142                           offsets['data'])
143    sizes['validation'] = (offsets['uri_extension'] -
144                           offsets['plaintext_hash_tree'])
145    sizes['uri-extension'] = len(UEB_data)
146    print(file=out)
147    print(" Size of data within the share:", file=out)
148    for k in sorted(sizes):
149        print("%20s: %s" % (k, sizes[k]), file=out)
150
151    if options['offsets']:
152        print(file=out)
153        print(" Section Offsets:", file=out)
154        print("%20s: %s" % ("share data", f._data_offset), file=out)
155        for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
156                  "block_hashes", "share_hashes", "uri_extension"]:
157            name = {"data": "block data"}.get(k,k)
158            offset = f._data_offset + offsets[k]
159            print("  %20s: %s   (0x%x)" % (name, offset, offset), file=out)
160        print("%20s: %s" % ("leases", f._lease_offset), file=out)
161
162def dump_immutable_lease_info(f, out):
163    # display lease information too
164    print(file=out)
165    leases = list(f.get_leases())
166    if leases:
167        for i,lease in enumerate(leases):
168            when = format_expiration_time(lease.get_expiration_time())
169            print(" Lease #%d: owner=%d, expire in %s" \
170                  % (i, lease.owner_num, when), file=out)
171    else:
172        print(" No leases.", file=out)
173
174def format_expiration_time(expiration_time):
175    now = time.time()
176    remains = expiration_time - now
177    when = "%ds" % remains
178    if remains > 24*3600:
179        when += " (%d days)" % (remains // (24*3600))
180    elif remains > 3600:
181        when += " (%d hours)" % (remains // 3600)
182    return when
183
184
185def dump_mutable_share(options):
186    from allmydata.storage.mutable import MutableShareFile
187    from allmydata.util import base32, idlib
188    out = options.stdout
189    m = MutableShareFile(options['filename'])
190    f = open(options['filename'], "rb")
191    WE, nodeid = m._read_write_enabler_and_nodeid(f)
192    num_extra_leases = m._read_num_extra_leases(f)
193    data_length = m._read_data_length(f)
194    extra_lease_offset = m._read_extra_lease_offset(f)
195    container_size = extra_lease_offset - m.DATA_OFFSET
196    leases = list(m._enumerate_leases(f))
197
198    share_type = "unknown"
199    f.seek(m.DATA_OFFSET)
200    version = f.read(1)
201    if version == b"\x00":
202        # this slot contains an SMDF share
203        share_type = "SDMF"
204    elif version == b"\x01":
205        share_type = "MDMF"
206    f.close()
207
208    print(file=out)
209    print("Mutable slot found:", file=out)
210    print(" share_type: %s" % share_type, file=out)
211    print(" write_enabler: %s" % str(base32.b2a(WE), "utf-8"), file=out)
212    print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out)
213    print(" num_extra_leases: %d" % num_extra_leases, file=out)
214    print(" container_size: %d" % container_size, file=out)
215    print(" data_length: %d" % data_length, file=out)
216    if leases:
217        for (leasenum, lease) in leases:
218            print(file=out)
219            print(" Lease #%d:" % leasenum, file=out)
220            print("  ownerid: %d" % lease.owner_num, file=out)
221            when = format_expiration_time(lease.get_expiration_time())
222            print("  expires in %s" % when, file=out)
223            print("  renew_secret: %s" % lease.present_renew_secret(), file=out)
224            print("  cancel_secret: %s" % lease.present_cancel_secret(), file=out)
225            print("  secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out)
226    else:
227        print("No leases.", file=out)
228    print(file=out)
229
230    if share_type == "SDMF":
231        dump_SDMF_share(m, data_length, options)
232    elif share_type == "MDMF":
233        dump_MDMF_share(m, data_length, options)
234
235    return 0
236
237def dump_SDMF_share(m, length, options):
238    from allmydata.mutable.layout import unpack_share, unpack_header
239    from allmydata.mutable.common import NeedMoreDataError
240    from allmydata.util import base32, hashutil
241    from allmydata.uri import SSKVerifierURI
242    from allmydata.util.encodingutil import quote_output, to_bytes
243
244    offset = m.DATA_OFFSET
245
246    out = options.stdout
247
248    f = open(options['filename'], "rb")
249    f.seek(offset)
250    data = f.read(min(length, 2000))
251    f.close()
252
253    try:
254        pieces = unpack_share(data)
255    except NeedMoreDataError as e:
256        # retry once with the larger size
257        size = e.needed_bytes
258        f = open(options['filename'], "rb")
259        f.seek(offset)
260        data = f.read(min(length, size))
261        f.close()
262        pieces = unpack_share(data)
263
264    (seqnum, root_hash, IV, k, N, segsize, datalen,
265     pubkey, signature, share_hash_chain, block_hash_tree,
266     share_data, enc_privkey) = pieces
267    (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
268     ig_datalen, offsets) = unpack_header(data)
269
270    print(" SDMF contents:", file=out)
271    print("  seqnum: %d" % seqnum, file=out)
272    print("  root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out)
273    print("  IV: %s" % str(base32.b2a(IV), "utf-8"), file=out)
274    print("  required_shares: %d" % k, file=out)
275    print("  total_shares: %d" % N, file=out)
276    print("  segsize: %d" % segsize, file=out)
277    print("  datalen: %d" % datalen, file=out)
278    print("  enc_privkey: %d bytes" % len(enc_privkey), file=out)
279    print("  pubkey: %d bytes" % len(pubkey), file=out)
280    print("  signature: %d bytes" % len(signature), file=out)
281    share_hash_ids = ",".join(sorted([str(hid)
282                                      for hid in share_hash_chain.keys()]))
283    print("  share_hash_chain: %s" % share_hash_ids, file=out)
284    print("  block_hash_tree: %d nodes" % len(block_hash_tree), file=out)
285
286    # the storage index isn't stored in the share itself, so we depend upon
287    # knowing the parent directory name to get it
288    pieces = options['filename'].split(os.sep)
289    if len(pieces) >= 2:
290        piece = to_bytes(pieces[-2])
291        if base32.could_be_base32_encoded(piece):
292            storage_index = base32.a2b(piece)
293            fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
294            u = SSKVerifierURI(storage_index, fingerprint)
295            verify_cap = u.to_string()
296            print("  verify-cap:", quote_output(verify_cap, quotemarks=False), file=out)
297
298    if options['offsets']:
299        # NOTE: this offset-calculation code is fragile, and needs to be
300        # merged with MutableShareFile's internals.
301        print(file=out)
302        print(" Section Offsets:", file=out)
303        def printoffset(name, value, shift=0):
304            print("%s%20s: %s   (0x%x)" % (" "*shift, name, value, value), file=out)
305        printoffset("first lease", m.HEADER_SIZE)
306        printoffset("share data", m.DATA_OFFSET)
307        o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
308        printoffset("seqnum", o_seqnum, 2)
309        o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
310        printoffset("root_hash", o_root_hash, 2)
311        for k in ["signature", "share_hash_chain", "block_hash_tree",
312                  "share_data",
313                  "enc_privkey", "EOF"]:
314            name = {"share_data": "block data",
315                    "EOF": "end of share data"}.get(k,k)
316            offset = m.DATA_OFFSET + offsets[k]
317            printoffset(name, offset, 2)
318        f = open(options['filename'], "rb")
319        printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
320        f.close()
321
322    print(file=out)
323
324def dump_MDMF_share(m, length, options):
325    from allmydata.mutable.layout import MDMFSlotReadProxy
326    from allmydata.util import base32, hashutil
327    from allmydata.uri import MDMFVerifierURI
328    from allmydata.util.encodingutil import quote_output, to_bytes
329
330    offset = m.DATA_OFFSET
331    out = options.stdout
332
333    f = open(options['filename'], "rb")
334    storage_index = None; shnum = 0
335
336    class ShareDumper(MDMFSlotReadProxy):
337        def _read(self, readvs, force_remote=False, queue=False):
338            data = []
339            for (where,length) in readvs:
340                f.seek(offset+where)
341                data.append(f.read(length))
342            return defer.succeed({shnum: data})
343
344    p = ShareDumper(None, storage_index, shnum)
345    def extract(func):
346        stash = []
347        # these methods return Deferreds, but we happen to know that they run
348        # synchronously when not actually talking to a remote server
349        d = func()
350        d.addCallback(stash.append)
351        return stash[0]
352
353    verinfo = extract(p.get_verinfo)
354    encprivkey = extract(p.get_encprivkey)
355    signature = extract(p.get_signature)
356    pubkey = extract(p.get_verification_key)
357    block_hash_tree = extract(p.get_blockhashes)
358    share_hash_chain = extract(p.get_sharehashes)
359    f.close()
360
361    (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
362     offsets) = verinfo
363
364    print(" MDMF contents:", file=out)
365    print("  seqnum: %d" % seqnum, file=out)
366    print("  root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out)
367    #print("  IV: %s" % base32.b2a(IV), file=out)
368    print("  required_shares: %d" % k, file=out)
369    print("  total_shares: %d" % N, file=out)
370    print("  segsize: %d" % segsize, file=out)
371    print("  datalen: %d" % datalen, file=out)
372    print("  enc_privkey: %d bytes" % len(encprivkey), file=out)
373    print("  pubkey: %d bytes" % len(pubkey), file=out)
374    print("  signature: %d bytes" % len(signature), file=out)
375    share_hash_ids = ",".join([str(hid)
376                               for hid in sorted(share_hash_chain.keys())])
377    print("  share_hash_chain: %s" % share_hash_ids, file=out)
378    print("  block_hash_tree: %d nodes" % len(block_hash_tree), file=out)
379
380    # the storage index isn't stored in the share itself, so we depend upon
381    # knowing the parent directory name to get it
382    pieces = options['filename'].split(os.sep)
383    if len(pieces) >= 2:
384        piece = to_bytes(pieces[-2])
385        if base32.could_be_base32_encoded(piece):
386            storage_index = base32.a2b(piece)
387            fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
388            u = MDMFVerifierURI(storage_index, fingerprint)
389            verify_cap = u.to_string()
390            print("  verify-cap:", quote_output(verify_cap, quotemarks=False), file=out)
391
392    if options['offsets']:
393        # NOTE: this offset-calculation code is fragile, and needs to be
394        # merged with MutableShareFile's internals.
395
396        print(file=out)
397        print(" Section Offsets:", file=out)
398        def printoffset(name, value, shift=0):
399            print("%s%.20s: %s   (0x%x)" % (" "*shift, name, value, value), file=out)
400        printoffset("first lease", m.HEADER_SIZE, 2)
401        printoffset("share data", m.DATA_OFFSET, 2)
402        o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
403        printoffset("seqnum", o_seqnum, 4)
404        o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
405        printoffset("root_hash", o_root_hash, 4)
406        for k in ["enc_privkey", "share_hash_chain", "signature",
407                  "verification_key", "verification_key_end",
408                  "share_data", "block_hash_tree", "EOF"]:
409            name = {"share_data": "block data",
410                    "verification_key": "pubkey",
411                    "verification_key_end": "end of pubkey",
412                    "EOF": "end of share data"}.get(k,k)
413            offset = m.DATA_OFFSET + offsets[k]
414            printoffset(name, offset, 4)
415        f = open(options['filename'], "rb")
416        printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2)
417        f.close()
418
419    print(file=out)
420
421
422
423class DumpCapOptions(BaseOptions):
424    def getSynopsis(self):
425        return "Usage: tahoe [global-options] debug dump-cap [options] FILECAP"
426    optParameters = [
427        ["nodeid", "n",
428         None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
429        ["client-secret", "c", None,
430         "Specify the client's base secret (ASCII), to construct secrets."],
431        ["client-dir", "d", None,
432         "Specify the client's base directory, from which a -c secret will be read."],
433        ]
434    def parseArgs(self, cap):
435        self.cap = cap
436
437    description = """
438Print information about the given cap-string (aka: URI, file-cap, dir-cap,
439read-cap, write-cap). The URI string is parsed and unpacked. This prints the
440type of the cap, its storage index, and any derived keys.
441
442 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq
443
444This may be useful to determine if a read-cap and a write-cap refer to the
445same time, or to extract the storage-index from a file-cap (to then use with
446find-shares)
447
448If additional information is provided (storage server nodeid and/or client
449base secret), this command will compute the shared secrets used for the
450write-enabler and for lease-renewal.
451"""
452
453
454def dump_cap(options):
455    from allmydata import uri
456    from allmydata.util import base32
457    from base64 import b32decode
458    from urllib.parse import unquote, urlparse
459
460    out = options.stdout
461    cap = options.cap
462    nodeid = None
463    if options['nodeid']:
464        nodeid = b32decode(options['nodeid'].upper())
465    secret = None
466    if options['client-secret']:
467        secret = base32.a2b(options['client-secret'].encode("ascii"))
468    elif options['client-dir']:
469        secretfile = os.path.join(options['client-dir'], "private", "secret")
470        try:
471            secret = base32.a2b(open(secretfile, "rb").read().strip())
472        except EnvironmentError:
473            pass
474
475    if cap.startswith("http"):
476        scheme, netloc, path, params, query, fragment = urlparse(cap)
477        assert path.startswith("/uri/")
478        cap = unquote(path[len("/uri/"):])
479
480    u = uri.from_string(cap)
481
482    print(file=out)
483    dump_uri_instance(u, nodeid, secret, out)
484
485def _dump_secrets(storage_index, secret, nodeid, out):
486    from allmydata.util import hashutil
487    from allmydata.util import base32
488
489    if secret:
490        crs = hashutil.my_renewal_secret_hash(secret)
491        print(" client renewal secret:", str(base32.b2a(crs), "ascii"), file=out)
492        frs = hashutil.file_renewal_secret_hash(crs, storage_index)
493        print(" file renewal secret:", str(base32.b2a(frs), "ascii"), file=out)
494        if nodeid:
495            renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
496            print(" lease renewal secret:", str(base32.b2a(renew), "ascii"), file=out)
497        ccs = hashutil.my_cancel_secret_hash(secret)
498        print(" client cancel secret:", str(base32.b2a(ccs), "ascii"), file=out)
499        fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
500        print(" file cancel secret:", str(base32.b2a(fcs), "ascii"), file=out)
501        if nodeid:
502            cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
503            print(" lease cancel secret:", str(base32.b2a(cancel), "ascii"), file=out)
504
505def dump_uri_instance(u, nodeid, secret, out, show_header=True):
506    from allmydata import uri
507    from allmydata.storage.server import si_b2a
508    from allmydata.util import base32, hashutil
509    from allmydata.util.encodingutil import quote_output
510
511    if isinstance(u, uri.CHKFileURI):
512        if show_header:
513            print("CHK File:", file=out)
514        print(" key:", str(base32.b2a(u.key), "ascii"), file=out)
515        print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
516        print(" size:", u.size, file=out)
517        print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
518        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
519        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
520    elif isinstance(u, uri.CHKFileVerifierURI):
521        if show_header:
522            print("CHK Verifier URI:", file=out)
523        print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
524        print(" size:", u.size, file=out)
525        print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
526        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
527
528    elif isinstance(u, uri.LiteralFileURI):
529        if show_header:
530            print("Literal File URI:", file=out)
531        print(" data:", quote_output(u.data), file=out)
532
533    elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
534        if show_header:
535            print("SDMF Writeable URI:", file=out)
536        print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out)
537        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
538        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
539        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
540        print(file=out)
541        if nodeid:
542            we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
543            print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out)
544            print(file=out)
545        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
546    elif isinstance(u, uri.ReadonlySSKFileURI):
547        if show_header:
548            print("SDMF Read-only URI:", file=out)
549        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
550        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
551        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
552    elif isinstance(u, uri.SSKVerifierURI):
553        if show_header:
554            print("SDMF Verifier URI:", file=out)
555        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
556        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
557
558    elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
559        if show_header:
560            print("MDMF Writeable URI:", file=out)
561        print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out)
562        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
563        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
564        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
565        print(file=out)
566        if nodeid:
567            we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
568            print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out)
569            print(file=out)
570        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
571    elif isinstance(u, uri.ReadonlyMDMFFileURI):
572        if show_header:
573            print("MDMF Read-only URI:", file=out)
574        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
575        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
576        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
577    elif isinstance(u, uri.MDMFVerifierURI):
578        if show_header:
579            print("MDMF Verifier URI:", file=out)
580        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
581        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
582
583
584    elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
585        if show_header:
586            print("CHK Directory URI:", file=out)
587        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
588    elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
589        if show_header:
590            print("CHK Directory Verifier URI:", file=out)
591        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
592
593    elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
594        if show_header:
595            print("Directory Writeable URI:", file=out)
596        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
597    elif isinstance(u, uri.ReadonlyDirectoryURI):
598        if show_header:
599            print("Directory Read-only URI:", file=out)
600        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
601    elif isinstance(u, uri.DirectoryURIVerifier):
602        if show_header:
603            print("Directory Verifier URI:", file=out)
604        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
605
606    elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
607        if show_header:
608            print("Directory Writeable URI:", file=out)
609        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
610    elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
611        if show_header:
612            print("Directory Read-only URI:", file=out)
613        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
614    elif isinstance(u, uri.MDMFDirectoryURIVerifier):
615        if show_header:
616            print("Directory Verifier URI:", file=out)
617        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
618
619    else:
620        print("unknown cap type", file=out)
621
622class FindSharesOptions(BaseOptions):
623    def getSynopsis(self):
624        return "Usage: tahoe [global-options] debug find-shares STORAGE_INDEX NODEDIRS.."
625
626    def parseArgs(self, storage_index_s, *nodedirs):
627        from allmydata.util.encodingutil import argv_to_abspath
628        self.si_s = storage_index_s
629        self.nodedirs = list(map(argv_to_abspath, nodedirs))
630
631    description = """
632Locate all shares for the given storage index. This command looks through one
633or more node directories to find the shares. It returns a list of filenames,
634one per line, for each share file found.
635
636 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*
637
638It may be useful during testing, when running a test grid in which all the
639nodes are on a local disk. The share files thus located can be counted,
640examined (with dump-share), or corrupted/deleted to test checker/repairer.
641"""
642
643def find_shares(options):
644    """Given a storage index and a list of node directories, emit a list of
645    all matching shares to stdout, one per line. For example:
646
647     find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
648
649    gives:
650
651    /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
652    /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
653    /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
654    """
655    from allmydata.storage.server import si_a2b, storage_index_to_dir
656    from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
657
658    out = options.stdout
659    sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8")))
660    for d in options.nodedirs:
661        d = os.path.join(d, "storage", "shares", sharedir)
662        if os.path.exists(d):
663            for shnum in listdir_unicode(d):
664                print(quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False), file=out)
665
666    return 0
667
668
669class CatalogSharesOptions(BaseOptions):
670    def parseArgs(self, *nodedirs):
671        from allmydata.util.encodingutil import argv_to_abspath
672        self.nodedirs = list(map(argv_to_abspath, nodedirs))
673        if not nodedirs:
674            raise usage.UsageError("must specify at least one node directory")
675
676    def getSynopsis(self):
677        return "Usage: tahoe [global-options] debug catalog-shares NODEDIRS.."
678
679    description = """
680Locate all shares in the given node directories, and emit a one-line summary
681of each share. Run it like this:
682
683 tahoe debug catalog-shares testgrid/node-* >allshares.txt
684
685The lines it emits will look like the following:
686
687 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
688 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
689 UNKNOWN $abspath_sharefile
690
691This command can be used to build up a catalog of shares from many storage
692servers and then sort the results to compare all shares for the same file. If
693you see shares with the same SI but different parameters/filesize/UEB_hash,
694then something is wrong. The misc/find-share/anomalies.py script may be
695useful for purpose.
696"""
697
698def call(c, *args, **kwargs):
699    # take advantage of the fact that ImmediateReadBucketProxy returns
700    # Deferreds that are already fired
701    results = []
702    failures = []
703    d = defer.maybeDeferred(c, *args, **kwargs)
704    d.addCallbacks(results.append, failures.append)
705    if failures:
706        failures[0].raiseException()
707    return results[0]
708
709def describe_share(abs_sharefile, si_s, shnum_s, now, out):
710    with open(abs_sharefile, "rb") as f:
711        prefix = f.read(32)
712        if MutableShareFile.is_valid_header(prefix):
713            _describe_mutable_share(abs_sharefile, f, now, si_s, out)
714        elif ShareFile.is_valid_header(prefix):
715            _describe_immutable_share(abs_sharefile, now, si_s, out)
716        else:
717            print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out)
718
719def _describe_mutable_share(abs_sharefile, f, now, si_s, out):
720    # mutable share
721    m = MutableShareFile(abs_sharefile)
722    WE, nodeid = m._read_write_enabler_and_nodeid(f)
723    data_length = m._read_data_length(f)
724    expiration_time = min( [lease.get_expiration_time()
725                            for (i,lease) in m._enumerate_leases(f)] )
726    expiration = max(0, expiration_time - now)
727
728    share_type = "unknown"
729    f.seek(m.DATA_OFFSET)
730    version = f.read(1)
731    if version == b"\x00":
732        # this slot contains an SMDF share
733        share_type = "SDMF"
734    elif version == b"\x01":
735        share_type = "MDMF"
736
737    if share_type == "SDMF":
738        f.seek(m.DATA_OFFSET)
739
740        # Read at least the mutable header length, if possible.  If there's
741        # less data than that in the share, don't try to read more (we won't
742        # be able to unpack the header in this case but we surely don't want
743        # to try to unpack bytes *following* the data section as if they were
744        # header data).  Rather than 2000 we could use HEADER_LENGTH from
745        # allmydata/mutable/layout.py, probably.
746        data = f.read(min(data_length, 2000))
747
748        try:
749            pieces = unpack_share(data)
750        except NeedMoreDataError as e:
751            # retry once with the larger size
752            size = e.needed_bytes
753            f.seek(m.DATA_OFFSET)
754            data = f.read(min(data_length, size))
755            pieces = unpack_share(data)
756        (seqnum, root_hash, IV, k, N, segsize, datalen,
757         pubkey, signature, share_hash_chain, block_hash_tree,
758         share_data, enc_privkey) = pieces
759
760        print("SDMF %s %d/%d %d #%d:%s %d %s" % \
761              (si_s, k, N, datalen,
762               seqnum, str(base32.b2a(root_hash), "utf-8"),
763               expiration, quote_output(abs_sharefile)), file=out)
764    elif share_type == "MDMF":
765        fake_shnum = 0
766        # TODO: factor this out with dump_MDMF_share()
767        class ShareDumper(MDMFSlotReadProxy):
768            def _read(self, readvs, force_remote=False, queue=False):
769                data = []
770                for (where,length) in readvs:
771                    f.seek(m.DATA_OFFSET+where)
772                    data.append(f.read(length))
773                return defer.succeed({fake_shnum: data})
774
775        p = ShareDumper(None, "fake-si", fake_shnum)
776        def extract(func):
777            stash = []
778            # these methods return Deferreds, but we happen to know that
779            # they run synchronously when not actually talking to a
780            # remote server
781            d = func()
782            d.addCallback(stash.append)
783            return stash[0]
784
785        verinfo = extract(p.get_verinfo)
786        (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
787         offsets) = verinfo
788        print("MDMF %s %d/%d %d #%d:%s %d %s" % \
789              (si_s, k, N, datalen,
790               seqnum, str(base32.b2a(root_hash), "utf-8"),
791               expiration, quote_output(abs_sharefile)), file=out)
792    else:
793        print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out)
794
795
796def _describe_immutable_share(abs_sharefile, now, si_s, out):
797    class ImmediateReadBucketProxy(ReadBucketProxy):
798        def __init__(self, sf):
799            self.sf = sf
800            ReadBucketProxy.__init__(self, None, None, "")
801        def __repr__(self):
802            return "<ImmediateReadBucketProxy>"
803        def _read(self, offset, size):
804            return defer.succeed(sf.read_share_data(offset, size))
805
806    # use a ReadBucketProxy to parse the bucket and find the uri extension
807    sf = ShareFile(abs_sharefile)
808    bp = ImmediateReadBucketProxy(sf)
809
810    expiration_time = min(lease.get_expiration_time()
811                          for lease in sf.get_leases())
812    expiration = max(0, expiration_time - now)
813
814    UEB_data = call(bp.get_uri_extension)
815    unpacked = uri.unpack_extension_readable(UEB_data)
816
817    k = unpacked["needed_shares"]
818    N = unpacked["total_shares"]
819    filesize = unpacked["size"]
820    ueb_hash = unpacked["UEB_hash"]
821
822    print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
823                                        str(ueb_hash, "utf-8"), expiration,
824                                        quote_output(abs_sharefile)), file=out)
825
826
827def catalog_shares(options):
828    from allmydata.util.encodingutil import listdir_unicode, quote_output
829
830    out = options.stdout
831    err = options.stderr
832    now = time.time()
833    for d in options.nodedirs:
834        d = os.path.join(d, "storage", "shares")
835        try:
836            abbrevs = listdir_unicode(d)
837        except EnvironmentError:
838            # ignore nodes that have storage turned off altogether
839            pass
840        else:
841            for abbrevdir in sorted(abbrevs):
842                if abbrevdir == "incoming":
843                    continue
844                abbrevdir = os.path.join(d, abbrevdir)
845                # this tool may get run against bad disks, so we can't assume
846                # that listdir_unicode will always succeed. Try to catalog as much
847                # as possible.
848                try:
849                    sharedirs = listdir_unicode(abbrevdir)
850                    for si_s in sorted(sharedirs):
851                        si_dir = os.path.join(abbrevdir, si_s)
852                        catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
853                except:
854                    print("Error processing %s" % quote_output(abbrevdir), file=err)
855                    failure.Failure().printTraceback(err)
856
857    return 0
858
859def _as_number(s):
860    try:
861        return int(s)
862    except ValueError:
863        return "not int"
864
865def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
866    from allmydata.util.encodingutil import listdir_unicode, quote_output
867
868    try:
869        for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
870            abs_sharefile = os.path.join(si_dir, shnum_s)
871            assert os.path.isfile(abs_sharefile)
872            try:
873                describe_share(abs_sharefile, si_s, shnum_s, now,
874                               out)
875            except:
876                print("Error processing %s" % quote_output(abs_sharefile), file=err)
877                failure.Failure().printTraceback(err)
878    except:
879        print("Error processing %s" % quote_output(si_dir), file=err)
880        failure.Failure().printTraceback(err)
881
882class CorruptShareOptions(BaseOptions):
883    def getSynopsis(self):
884        return "Usage: tahoe [global-options] debug corrupt-share SHARE_FILENAME"
885
886    optParameters = [
887        ["offset", "o", "block-random", "Specify which bit to flip."],
888        ]
889
890    description = """
891Corrupt the given share by flipping a bit. This will cause a
892verifying/downloading client to log an integrity-check failure incident, and
893downloads will proceed with a different share.
894
895The --offset parameter controls which bit should be flipped. The default is
896to flip a single random bit of the block data.
897
898 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
899
900Obviously, this command should not be used in normal operation.
901"""
902    def parseArgs(self, filename):
903        self['filename'] = filename
904
905def corrupt_share(options):
906    import random
907    from allmydata.storage.mutable import MutableShareFile
908    from allmydata.storage.immutable import ShareFile
909    from allmydata.mutable.layout import unpack_header
910    from allmydata.immutable.layout import ReadBucketProxy
911    out = options.stdout
912    fn = options['filename']
913    assert options["offset"] == "block-random", "other offsets not implemented"
914    # first, what kind of share is it?
915
916    def flip_bit(start, end):
917        offset = random.randrange(start, end)
918        bit = random.randrange(0, 8)
919        print("[%d..%d):  %d.b%d" % (start, end, offset, bit), file=out)
920        f = open(fn, "rb+")
921        f.seek(offset)
922        d = f.read(1)
923        d = bchr(ord(d) ^ 0x01)
924        f.seek(offset)
925        f.write(d)
926        f.close()
927
928    with open(fn, "rb") as f:
929        prefix = f.read(32)
930
931        if MutableShareFile.is_valid_header(prefix):
932            # mutable
933            m = MutableShareFile(fn)
934            with open(fn, "rb") as f:
935                f.seek(m.DATA_OFFSET)
936                # Read enough data to get a mutable header to unpack.
937                data = f.read(2000)
938            # make sure this slot contains an SMDF share
939            assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported"
940            f.close()
941
942            (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
943             ig_datalen, offsets) = unpack_header(data)
944
945            assert version == 0, "we only handle v0 SDMF files"
946            start = m.DATA_OFFSET + offsets["share_data"]
947            end = m.DATA_OFFSET + offsets["enc_privkey"]
948            flip_bit(start, end)
949        else:
950            # otherwise assume it's immutable
951            f = ShareFile(fn)
952            bp = ReadBucketProxy(None, None, '')
953            offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
954            start = f._data_offset + offsets["data"]
955            end = f._data_offset + offsets["plaintext_hash_tree"]
956            flip_bit(start, end)
957
958
959
960class ReplOptions(BaseOptions):
961    def getSynopsis(self):
962        return "Usage: tahoe debug repl (OBSOLETE)"
963
964def repl(options):
965    print("'tahoe debug repl' is obsolete. Please run 'python' in a virtualenv.", file=options.stderr)
966    return 1
967
968
969DEFAULT_TESTSUITE = 'allmydata'
970
971class TrialOptions(BaseOptions):
972    def getSynopsis(self):
973        return "Usage: tahoe debug trial (OBSOLETE)"
974
975def trial(config):
976    print("'tahoe debug trial' is obsolete. Please run 'tox', or use 'trial' in a virtualenv.", file=config.stderr)
977    return 1
978
979def fixOptionsClass(args):
980    (subcmd, shortcut, OptionsClass, desc) = args
981    class FixedOptionsClass(OptionsClass):
982        def getSynopsis(self):
983            t = OptionsClass.getSynopsis(self)
984            i = t.find("Usage: flogtool ")
985            if i >= 0:
986                return "Usage: tahoe [global-options] debug flogtool " + t[i+len("Usage: flogtool "):]
987            else:
988                return "Usage: tahoe [global-options] debug flogtool %s [options]" % (subcmd,)
989    return (subcmd, shortcut, FixedOptionsClass, desc)
990
991class FlogtoolOptions(foolscap_cli.Options):
992    def __init__(self):
993        super(FlogtoolOptions, self).__init__()
994        self.subCommands = list(map(fixOptionsClass, self.subCommands))
995
996    def getSynopsis(self):
997        return "Usage: tahoe [global-options] debug flogtool COMMAND [flogtool-options]"
998
999    def parseOptions(self, all_subargs, *a, **kw):
1000        self.flogtool_args = list(all_subargs)
1001        return super(FlogtoolOptions, self).parseOptions(self.flogtool_args, *a, **kw)
1002
1003    def getUsage(self, width=None):
1004        t = super(FlogtoolOptions, self).getUsage(width)
1005        t += """
1006The 'tahoe debug flogtool' command uses the correct imports for this instance
1007of Tahoe-LAFS.
1008
1009Please run 'tahoe debug flogtool COMMAND --help' for more details on each
1010subcommand.
1011"""
1012        return t
1013
1014    def opt_help(self):
1015        print(str(self))
1016        sys.exit(0)
1017
1018def flogtool(config):
1019    sys.argv = ['flogtool'] + config.flogtool_args
1020    return foolscap_cli.run_flogtool()
1021
1022
1023class DebugCommand(BaseOptions):
1024    subCommands = [
1025        ["dump-share", None, DumpOptions,
1026         "Unpack and display the contents of a share (uri_extension and leases)."],
1027        ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
1028        ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
1029        ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
1030        ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
1031        ["repl", None, ReplOptions, "OBSOLETE"],
1032        ["trial", None, TrialOptions, "OBSOLETE"],
1033        ["flogtool", None, FlogtoolOptions, "Utilities to access log files."],
1034        ]
1035    def postOptions(self):
1036        if not hasattr(self, 'subOptions'):
1037            raise usage.UsageError("must specify a subcommand")
1038    synopsis = "COMMAND"
1039
1040    def getUsage(self, width=None):
1041        t = BaseOptions.getUsage(self, width)
1042        t += """\
1043
1044Please run e.g. 'tahoe debug dump-share --help' for more details on each
1045subcommand.
1046"""
1047        return t
1048
1049subDispatch = {
1050    "dump-share": dump_share,
1051    "dump-cap": dump_cap,
1052    "find-shares": find_shares,
1053    "catalog-shares": catalog_shares,
1054    "corrupt-share": corrupt_share,
1055    "repl": repl,
1056    "trial": trial,
1057    "flogtool": flogtool,
1058    }
1059
1060
1061def do_debug(options):
1062    so = options.subOptions
1063    so.stdout = options.stdout
1064    so.stderr = options.stderr
1065    f = subDispatch[options.subCommand]
1066    return f(so)
1067
1068
1069subCommands : SubCommands = [
1070    ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
1071    ]
1072
1073dispatch = {
1074    "debug": do_debug,
1075    }
Note: See TracBrowser for help on using the repository browser.