source: trunk/src/allmydata/web/filenode.py

Last change on this file was 3ff9c45, checked in by Jean-Paul Calderone <exarkun@…>, at 2023-01-06T20:40:48Z

expose the private-key feature in the tahoe put cli
  • Property mode set to 100644
File size: 21.0 KB
Line 
1"""
2Ported to Python 3.
3"""
4from __future__ import annotations
5
6from twisted.web import http, static
7from twisted.internet import defer
8from twisted.web.resource import (
9    Resource,
10    ErrorPage,
11)
12
13from allmydata.interfaces import ExistingChildError
14from allmydata.monitor import Monitor
15from allmydata.immutable.upload import FileHandle
16from allmydata.mutable.publish import MutableFileHandle
17from allmydata.mutable.common import MODE_READ
18from allmydata.util import log, base32
19from allmydata.util.encodingutil import quote_output
20from allmydata.blacklist import (
21    FileProhibited,
22    ProhibitedNode,
23)
24
25from allmydata.web.common import (
26    get_keypair,
27    boolean_of_arg,
28    exception_to_child,
29    get_arg,
30    get_filenode_metadata,
31    get_format,
32    get_mutable_type,
33    parse_offset_arg,
34    parse_replace_arg,
35    render_exception,
36    should_create_intermediate_directories,
37    text_plain,
38    WebError,
39    handle_when_done,
40)
41from allmydata.web.check_results import (
42    CheckResultsRenderer,
43    CheckAndRepairResultsRenderer,
44    LiteralCheckResultsRenderer,
45)
46from allmydata.web.info import MoreInfo
47from allmydata.util import jsonbytes as json
48
49class ReplaceMeMixin(object):
50    def replace_me_with_a_child(self, req, client, replace):
51        # a new file is being uploaded in our place.
52        file_format = get_format(req, "CHK")
53        mutable_type = get_mutable_type(file_format)
54        if mutable_type is not None:
55            data = MutableFileHandle(req.content)
56            keypair = get_keypair(req)
57            d = client.create_mutable_file(data, version=mutable_type, unique_keypair=keypair)
58            def _uploaded(newnode):
59                d2 = self.parentnode.set_node(self.name, newnode,
60                                              overwrite=replace)
61                d2.addCallback(lambda res: newnode)
62                return d2
63            d.addCallback(_uploaded)
64        else:
65            assert file_format == "CHK"
66            uploadable = FileHandle(req.content, convergence=client.convergence)
67            d = self.parentnode.add_file(self.name, uploadable,
68                                         overwrite=replace)
69        def _done(filenode):
70            log.msg("webish upload complete",
71                    facility="tahoe.webish", level=log.NOISY, umid="TCjBGQ")
72            if self.node:
73                # we've replaced an existing file (or modified a mutable
74                # file), so the response code is 200
75                req.setResponseCode(http.OK)
76            else:
77                # we've created a new file, so the code is 201
78                req.setResponseCode(http.CREATED)
79            return filenode.get_uri()
80        d.addCallback(_done)
81        return d
82
83    def replace_me_with_a_childcap(self, req, client, replace):
84        req.content.seek(0)
85        childcap = req.content.read()
86        childnode = client.create_node_from_uri(childcap, None, name=self.name)
87        d = self.parentnode.set_node(self.name, childnode, overwrite=replace)
88        d.addCallback(lambda res: childnode.get_uri())
89        return d
90
91
92    def replace_me_with_a_formpost(self, req, client, replace):
93        # create a new file, maybe mutable, maybe immutable
94        file_format = get_format(req, "CHK")
95        contents = req.fields["file"]
96        if file_format in ("SDMF", "MDMF"):
97            mutable_type = get_mutable_type(file_format)
98            uploadable = MutableFileHandle(contents.file)
99            keypair = get_keypair(req)
100            d = client.create_mutable_file(uploadable, version=mutable_type, unique_keypair=keypair)
101            def _uploaded(newnode):
102                d2 = self.parentnode.set_node(self.name, newnode,
103                                              overwrite=replace)
104                d2.addCallback(lambda res: newnode.get_uri())
105                return d2
106            d.addCallback(_uploaded)
107            return d
108
109        uploadable = FileHandle(contents.file, convergence=client.convergence)
110        d = self.parentnode.add_file(self.name, uploadable, overwrite=replace)
111        d.addCallback(lambda newnode: newnode.get_uri())
112        return d
113
114
115class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
116    def __init__(self, client, parentnode, name):
117        super(PlaceHolderNodeHandler, self).__init__()
118        self.client = client
119        assert parentnode
120        self.parentnode = parentnode
121        self.name = name
122        self.node = None
123
124    @render_exception
125    def render_PUT(self, req):
126        t = get_arg(req, b"t", b"").strip()
127        replace = parse_replace_arg(get_arg(req, "replace", "true"))
128
129        assert self.parentnode and self.name
130        if req.getHeader("content-range"):
131            raise WebError("Content-Range in PUT not yet supported",
132                           http.NOT_IMPLEMENTED)
133        if not t:
134            return self.replace_me_with_a_child(req, self.client, replace)
135        if t == b"uri":
136            return self.replace_me_with_a_childcap(req, self.client, replace)
137
138        raise WebError("PUT to a file: bad t=%s" % str(t, "utf-8"))
139
140    @render_exception
141    def render_POST(self, req):
142        t = get_arg(req, b"t", b"").strip()
143        replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
144        if t == b"upload":
145            # like PUT, but get the file data from an HTML form's input field.
146            # We could get here from POST /uri/mutablefilecap?t=upload,
147            # or POST /uri/path/file?t=upload, or
148            # POST /uri/path/dir?t=upload&name=foo . All have the same
149            # behavior, we just ignore any name= argument
150            d = self.replace_me_with_a_formpost(req, self.client, replace)
151        else:
152            # t=mkdir is handled in DirectoryNodeHandler._POST_mkdir, so
153            # there are no other t= values left to be handled by the
154            # placeholder.
155            raise WebError("POST to a file: bad t=%s" % str(t, "utf-8"))
156
157        return handle_when_done(req, d)
158
159
160class FileNodeHandler(Resource, ReplaceMeMixin, object):
161    def __init__(self, client, node, parentnode=None, name=None):
162        super(FileNodeHandler, self).__init__()
163        self.client = client
164        assert node
165        self.node = node
166        self.parentnode = parentnode
167        self.name = name
168
169    @exception_to_child
170    def getChild(self, name, req):
171        if isinstance(self.node, ProhibitedNode):
172            raise FileProhibited(self.node.reason)
173        if should_create_intermediate_directories(req):
174                return ErrorPage(
175                    http.CONFLICT,
176                    u"Cannot create directory %s, because its parent is a file, "
177                    u"not a directory" % quote_output(name, encoding='utf-8'),
178                    "no details"
179                )
180        return ErrorPage(
181            http.BAD_REQUEST,
182            u"Files have no children named %s" % quote_output(name, encoding='utf-8'),
183            "no details",
184        )
185
186    @render_exception
187    def render_GET(self, req):
188        t = str(get_arg(req, b"t", b"").strip(), "ascii")
189
190        # t=info contains variable ophandles, so is not allowed an ETag.
191        FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"]
192        if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
193            # if the client already has the ETag then we can
194            # short-circuit the whole process.
195            si = self.node.get_storage_index()
196            if si and req.setETag(b'%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
197                return b""
198
199        if not t:
200            # just get the contents
201            # the filename arrives as part of the URL or in a form input
202            # element, and will be sent back in a Content-Disposition header.
203            # Different browsers use various character sets for this name,
204            # sometimes depending upon how language environment is
205            # configured. Firefox sends the equivalent of
206            # urllib.quote(name.encode("utf-8")), while IE7 sometimes does
207            # latin-1. Browsers cannot agree on how to interpret the name
208            # they see in the Content-Disposition header either, despite some
209            # 11-year old standards (RFC2231) that explain how to do it
210            # properly. So we assume that at least the browser will agree
211            # with itself, and echo back the same bytes that we were given.
212            filename = get_arg(req, "filename", self.name) or "unknown"
213            d = self.node.get_best_readable_version()
214            d.addCallback(lambda dn: FileDownloader(dn, filename))
215            return d
216        if t == "json":
217            # We do this to make sure that fields like size and
218            # mutable-type (which depend on the file on the grid and not
219            # just on the cap) are filled in. The latter gets used in
220            # tests, in particular.
221            #
222            # TODO: Make it so that the servermap knows how to update in
223            # a mode specifically designed to fill in these fields, and
224            # then update it in that mode.
225            if self.node.is_mutable():
226                d = self.node.get_servermap(MODE_READ)
227            else:
228                d = defer.succeed(None)
229            if self.parentnode and self.name:
230                d.addCallback(lambda ignored:
231                    self.parentnode.get_metadata_for(self.name))
232            else:
233                d.addCallback(lambda ignored: None)
234            d.addCallback(lambda md: _file_json_metadata(req, self.node, md))
235            return d
236        if t == "info":
237            return MoreInfo(self.node)
238        if t == "uri":
239            return _file_uri(req, self.node)
240        if t == "readonly-uri":
241            return _file_read_only_uri(req, self.node)
242        raise WebError("GET file: bad t=%s" % t)
243
244    @render_exception
245    def render_HEAD(self, req):
246        t = get_arg(req, b"t", b"").strip()
247        if t:
248            raise WebError("HEAD file: bad t=%s" % t)
249        filename = get_arg(req, b"filename", self.name) or "unknown"
250        d = self.node.get_best_readable_version()
251        d.addCallback(lambda dn: FileDownloader(dn, filename))
252        return d
253
254    @render_exception
255    def render_PUT(self, req):
256        t = get_arg(req, b"t", b"").strip()
257        replace = parse_replace_arg(get_arg(req, b"replace", b"true"))
258        offset = parse_offset_arg(get_arg(req, b"offset", None))
259
260        if not t:
261            if not replace:
262                # this is the early trap: if someone else modifies the
263                # directory while we're uploading, the add_file(overwrite=)
264                # call in replace_me_with_a_child will do the late trap.
265                raise ExistingChildError()
266
267            if self.node.is_mutable():
268                # Are we a readonly filenode? We shouldn't allow callers
269                # to try to replace us if we are.
270                if self.node.is_readonly():
271                    raise WebError("PUT to a mutable file: replace or update"
272                                   " requested with read-only cap")
273                if offset is None:
274                    return self.replace_my_contents(req)
275
276                if offset >= 0:
277                    return self.update_my_contents(req, offset)
278
279                raise WebError("PUT to a mutable file: Invalid offset")
280
281            else:
282                if offset is not None:
283                    raise WebError("PUT to a file: append operation invoked "
284                                   "on an immutable cap")
285
286                assert self.parentnode and self.name
287                return self.replace_me_with_a_child(req, self.client, replace)
288
289        if t == b"uri":
290            if not replace:
291                raise ExistingChildError()
292            assert self.parentnode and self.name
293            return self.replace_me_with_a_childcap(req, self.client, replace)
294
295        raise WebError("PUT to a file: bad t=%s" % str(t, "utf-8"))
296
297    @render_exception
298    def render_POST(self, req):
299        t = get_arg(req, b"t", b"").strip()
300        replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
301        if t == b"check":
302            d = self._POST_check(req)
303        elif t == b"upload":
304            # like PUT, but get the file data from an HTML form's input field
305            # We could get here from POST /uri/mutablefilecap?t=upload,
306            # or POST /uri/path/file?t=upload, or
307            # POST /uri/path/dir?t=upload&name=foo . All have the same
308            # behavior, we just ignore any name= argument
309            if self.node.is_mutable():
310                d = self.replace_my_contents_with_a_formpost(req)
311            else:
312                if not replace:
313                    raise ExistingChildError()
314                assert self.parentnode and self.name
315                d = self.replace_me_with_a_formpost(req, self.client, replace)
316        else:
317            raise WebError("POST to file: bad t=%s" % str(t, "ascii"))
318
319        return handle_when_done(req, d)
320
321    def _maybe_literal(self, res, Results_Class):
322        if res:
323            return Results_Class(self.client, res)
324        return LiteralCheckResultsRenderer(self.client)
325
326    def _POST_check(self, req):
327        verify = boolean_of_arg(get_arg(req, "verify", "false"))
328        repair = boolean_of_arg(get_arg(req, "repair", "false"))
329        add_lease = boolean_of_arg(get_arg(req, "add-lease", "false"))
330        if repair:
331            d = self.node.check_and_repair(Monitor(), verify, add_lease)
332            d.addCallback(self._maybe_literal, CheckAndRepairResultsRenderer)
333        else:
334            d = self.node.check(Monitor(), verify, add_lease)
335            d.addCallback(self._maybe_literal, CheckResultsRenderer)
336        return d
337
338    @render_exception
339    def render_DELETE(self, req):
340        assert self.parentnode and self.name
341        d = self.parentnode.delete(self.name)
342        d.addCallback(lambda res: self.node.get_uri())
343        return d
344
345    def replace_my_contents(self, req):
346        req.content.seek(0)
347        new_contents = MutableFileHandle(req.content)
348        d = self.node.overwrite(new_contents)
349        d.addCallback(lambda res: self.node.get_uri())
350        return d
351
352
353    def update_my_contents(self, req, offset):
354        req.content.seek(0)
355        added_contents = MutableFileHandle(req.content)
356
357        d = self.node.get_best_mutable_version()
358        d.addCallback(lambda mv:
359            mv.update(added_contents, offset))
360        d.addCallback(lambda ignored:
361            self.node.get_uri())
362        return d
363
364
365    def replace_my_contents_with_a_formpost(self, req):
366        # we have a mutable file. Get the data from the formpost, and replace
367        # the mutable file's contents with it.
368        new_contents = req.fields['file']
369        new_contents = MutableFileHandle(new_contents.file)
370
371        d = self.node.overwrite(new_contents)
372        d.addCallback(lambda res: self.node.get_uri())
373        return d
374
375
376class FileDownloader(Resource, object):
377    def __init__(self, filenode, filename):
378        super(FileDownloader, self).__init__()
379        self.filenode = filenode
380        self.filename = filename
381
382    def parse_range_header(self, range_header):
383        # Parse a byte ranges according to RFC 2616 "14.35.1 Byte
384        # Ranges".  Returns None if the range doesn't make sense so it
385        # can be ignored (per the spec).  When successful, returns a
386        # list of (first,last) inclusive range tuples.
387
388        filesize = self.filenode.get_size()
389        assert isinstance(filesize, int), filesize
390
391        try:
392            # byte-ranges-specifier
393            units, rangeset = range_header.split('=', 1)
394            if units != 'bytes':
395                return None     # nothing else supported
396
397            def parse_range(r):
398                first, last = r.split('-', 1)
399
400                if first == '':
401                    # suffix-byte-range-spec
402                    first = filesize - int(last)
403                    last = filesize - 1
404                else:
405                    # byte-range-spec
406
407                    # first-byte-pos
408                    first = int(first)
409
410                    # last-byte-pos
411                    if last == '':
412                        last = filesize - 1
413                    else:
414                        last = int(last)
415
416                if last < first:
417                    raise ValueError
418
419                return (first, last)
420
421            # byte-range-set
422            #
423            # Note: the spec uses "1#" for the list of ranges, which
424            # implicitly allows whitespace around the ',' separators,
425            # so strip it.
426            return [ parse_range(r.strip()) for r in rangeset.split(',') ]
427        except ValueError:
428            return None
429
430    @render_exception
431    def render(self, req):
432        gte = static.getTypeAndEncoding
433        ctype, encoding = gte(self.filename,
434                              static.File.contentTypes,
435                              static.File.contentEncodings,
436                              defaultType="text/plain")
437        req.setHeader("content-type", ctype)
438        if encoding:
439            req.setHeader("content-encoding", encoding)
440
441        if boolean_of_arg(get_arg(req, "save", "False")):
442            # tell the browser to save the file rather display it we don't
443            # try to encode the filename, instead we echo back the exact same
444            # bytes we were given in the URL. See the comment in
445            # FileNodeHandler.render_GET for the sad details.
446            req.setHeader("content-disposition",
447                          b'attachment; filename="%s"' % self.filename)
448
449        filesize = self.filenode.get_size()
450        assert isinstance(filesize, int), filesize
451        first, size = 0, None
452        contentsize = filesize
453        req.setHeader("accept-ranges", "bytes")
454
455        # TODO: for mutable files, use the roothash. For LIT, hash the data.
456        # or maybe just use the URI for CHK and LIT.
457        rangeheader = req.getHeader('range')
458        if rangeheader:
459            ranges = self.parse_range_header(rangeheader)
460
461            # ranges = None means the header didn't parse, so ignore
462            # the header as if it didn't exist.  If is more than one
463            # range, then just return the first for now, until we can
464            # generate multipart/byteranges.
465            if ranges is not None:
466                first, last = ranges[0]
467
468                if first >= filesize:
469                    raise WebError('First beyond end of file',
470                                   http.REQUESTED_RANGE_NOT_SATISFIABLE)
471                else:
472                    first = max(0, first)
473                    last = min(filesize-1, last)
474
475                    req.setResponseCode(http.PARTIAL_CONTENT)
476                    req.setHeader('content-range',"bytes %s-%s/%s" %
477                                  (str(first), str(last),
478                                   str(filesize)))
479                    contentsize = last - first + 1
480                    size = contentsize
481
482        req.setHeader("content-length", b"%d" % contentsize)
483        if req.method == b"HEAD":
484            return b""
485
486        d = self.filenode.read(req, first, size)
487
488        def _error(f):
489            if f.check(defer.CancelledError):
490                # The HTTP connection was lost and we no longer have anywhere
491                # to send our result.  Let this pass through.
492                return f
493            if req.startedWriting:
494                # The content-type is already set, and the response code has
495                # already been sent, so we can't provide a clean error
496                # indication. We can emit text (which a browser might
497                # interpret as something else), and if we sent a Size header,
498                # they might notice that we've truncated the data. Keep the
499                # error message small to improve the chances of having our
500                # error response be shorter than the intended results.
501                #
502                # We don't have a lot of options, unfortunately.
503                return b"problem during download\n"
504            else:
505                # We haven't written anything yet, so we can provide a
506                # sensible error message.
507                return f
508        d.addCallbacks(
509            lambda ignored: None,
510            _error,
511        )
512        return d
513
514
515def _file_json_metadata(req, filenode, edge_metadata):
516    rw_uri = filenode.get_write_uri()
517    ro_uri = filenode.get_readonly_uri()
518    data = ("filenode", get_filenode_metadata(filenode))
519    if ro_uri:
520        data[1]['ro_uri'] = ro_uri
521    if rw_uri:
522        data[1]['rw_uri'] = rw_uri
523    verifycap = filenode.get_verify_cap()
524    if verifycap:
525        data[1]['verify_uri'] = verifycap.to_string()
526    if edge_metadata is not None:
527        data[1]['metadata'] = edge_metadata
528
529    return text_plain(json.dumps(data, indent=1) + "\n", req)
530
531
532def _file_uri(req, filenode):
533    return text_plain(filenode.get_uri(), req)
534
535
536def _file_read_only_uri(req, filenode):
537    if filenode.is_readonly():
538        return text_plain(filenode.get_uri(), req)
539    return text_plain(filenode.get_readonly_uri(), req)
540
541
542class FileNodeDownloadHandler(FileNodeHandler):
543
544    @exception_to_child
545    def getChild(self, name, req):
546        return FileNodeDownloadHandler(self.client, self.node, name=name)
Note: See TracBrowser for help on using the repository browser.