source: trunk/src/allmydata/node.py

Last change on this file was c5a426b, checked in by Itamar Turner-Trauring <itamar@…>, at 2021-02-12T16:47:11Z

More unicode-of-bytes fixes.

  • Property mode set to 100644
File size: 36.9 KB
Line 
1"""
2This module contains classes and functions to implement and manage
3a node for Tahoe-LAFS.
4
5Ported to Python 3.
6"""
7from __future__ import absolute_import
8from __future__ import division
9from __future__ import print_function
10from __future__ import unicode_literals
11
12from future.utils import PY2
13if PY2:
14    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
15from six import ensure_str, ensure_text
16
17import datetime
18import os.path
19import re
20import types
21import errno
22from base64 import b32decode, b32encode
23from errno import ENOENT, EPERM
24from warnings import warn
25
26try:
27    from typing import Union
28except ImportError:
29    pass
30
31import attr
32
33# On Python 2 this will be the backported package.
34import configparser
35
36from twisted.python.filepath import (
37    FilePath,
38)
39from twisted.python import log as twlog
40from twisted.application import service
41from twisted.python.failure import Failure
42from foolscap.api import Tub
43
44import foolscap.logging.log
45
46from allmydata.util import log
47from allmydata.util import fileutil, iputil
48from allmydata.util.fileutil import abspath_expanduser_unicode
49from allmydata.util.encodingutil import get_filesystem_encoding, quote_output
50from allmydata.util import configutil
51from allmydata.util.yamlutil import (
52    safe_load,
53)
54
55from . import (
56    __full_version__,
57)
58
59def _common_valid_config():
60    return configutil.ValidConfiguration({
61        "connections": (
62            "tcp",
63        ),
64        "node": (
65            "log_gatherer.furl",
66            "nickname",
67            "reveal-ip-address",
68            "tempdir",
69            "timeout.disconnect",
70            "timeout.keepalive",
71            "tub.location",
72            "tub.port",
73            "web.port",
74            "web.static",
75        ),
76        "i2p": (
77            "enabled",
78            "i2p.configdir",
79            "i2p.executable",
80            "launch",
81            "sam.port",
82            "dest",
83            "dest.port",
84            "dest.private_key_file",
85        ),
86        "tor": (
87            "control.port",
88            "enabled",
89            "launch",
90            "socks.port",
91            "tor.executable",
92            "onion",
93            "onion.local_port",
94            "onion.external_port",
95            "onion.private_key_file",
96        ),
97    })
98
99# group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
100ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$")
101
102# this is put into README in new node-directories (for client and introducers)
103PRIV_README = """
104This directory contains files which contain private data for the Tahoe node,
105such as private keys.  On Unix-like systems, the permissions on this directory
106are set to disallow users other than its owner from reading the contents of
107the files.   See the 'configuration.rst' documentation file for details.
108"""
109
110
111def formatTimeTahoeStyle(self, when):
112    """
113    Format the given (UTC) timestamp in the way Tahoe-LAFS expects it,
114    for example: 2007-10-12 00:26:28.566Z
115
116    :param when: UTC POSIX timestamp
117    :type when: float
118    :returns: datetime.datetime
119    """
120    d = datetime.datetime.utcfromtimestamp(when)
121    if d.microsecond:
122        return d.isoformat(ensure_str(" "))[:-3]+"Z"
123    return d.isoformat(ensure_str(" ")) + ".000Z"
124
125PRIV_README = """
126This directory contains files which contain private data for the Tahoe node,
127such as private keys.  On Unix-like systems, the permissions on this directory
128are set to disallow users other than its owner from reading the contents of
129the files.   See the 'configuration.rst' documentation file for details."""
130
131class _None(object):
132    """
133    This class is to be used as a marker in get_config()
134    """
135    pass
136
137class MissingConfigEntry(Exception):
138    """ A required config entry was not found. """
139
140class OldConfigError(Exception):
141    """ An obsolete config file was found. See
142    docs/historical/configuration.rst. """
143    def __str__(self):
144        return ("Found pre-Tahoe-LAFS-v1.3 configuration file(s):\n"
145                "%s\n"
146                "See docs/historical/configuration.rst."
147                % "\n".join([quote_output(fname) for fname in self.args[0]]))
148
149class OldConfigOptionError(Exception):
150    """Indicate that outdated configuration options are being used."""
151    pass
152
153class UnescapedHashError(Exception):
154    """Indicate that a configuration entry contains an unescaped '#' character."""
155    def __str__(self):
156        return ("The configuration entry %s contained an unescaped '#' character."
157                % quote_output("[%s]%s = %s" % self.args))
158
159class PrivacyError(Exception):
160    """reveal-IP-address = false, but the node is configured in such a way
161    that the IP address could be revealed"""
162
163
164def create_node_dir(basedir, readme_text):
165    """
166    Create new new 'node directory' at 'basedir'. This includes a
167    'private' subdirectory. If basedir (and privdir) already exists,
168    nothing is done.
169
170    :param readme_text: text to put in <basedir>/private/README
171    """
172    if not os.path.exists(basedir):
173        fileutil.make_dirs(basedir)
174    privdir = os.path.join(basedir, "private")
175    if not os.path.exists(privdir):
176        fileutil.make_dirs(privdir, 0o700)
177        readme_text = ensure_text(readme_text)
178        with open(os.path.join(privdir, 'README'), 'w') as f:
179            f.write(readme_text)
180
181
182def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
183    """
184    Read and validate configuration.
185
186    :param unicode basedir: directory where configuration data begins
187
188    :param unicode portnumfile: filename fragment for "port number" files
189
190    :param list generated_files: a list of automatically-generated
191        configuration files.
192
193    :param ValidConfiguration _valid_config: (internal use, optional) a
194        structure defining valid configuration sections and keys
195
196    :returns: :class:`allmydata.node._Config` instance
197    """
198    basedir = abspath_expanduser_unicode(ensure_text(basedir))
199    if _valid_config is None:
200        _valid_config = _common_valid_config()
201
202    # complain if there's bad stuff in the config dir
203    _error_about_old_config_files(basedir, generated_files)
204
205    # canonicalize the portnum file
206    portnumfile = os.path.join(basedir, portnumfile)
207
208    config_path = FilePath(basedir).child("tahoe.cfg")
209    try:
210        config_str = config_path.getContent()
211    except EnvironmentError as e:
212        if e.errno != errno.ENOENT:
213            raise
214        # The file is missing, just create empty ConfigParser.
215        config_str = u""
216    else:
217        config_str = config_str.decode("utf-8-sig")
218
219    return config_from_string(
220        basedir,
221        portnumfile,
222        config_str,
223        _valid_config,
224        config_path,
225    )
226
227
228def config_from_string(basedir, portnumfile, config_str, _valid_config=None, fpath=None):
229    """
230    load and validate configuration from in-memory string
231    """
232    if _valid_config is None:
233        _valid_config = _common_valid_config()
234
235    if isinstance(config_str, bytes):
236        config_str = config_str.decode("utf-8")
237
238    # load configuration from in-memory string
239    parser = configutil.get_config_from_string(config_str)
240
241    configutil.validate_config(
242        "<string>" if fpath is None else fpath.path,
243        parser,
244        _valid_config,
245    )
246
247    return _Config(
248        parser,
249        portnumfile,
250        basedir,
251        fpath,
252        _valid_config,
253    )
254
255
256def _error_about_old_config_files(basedir, generated_files):
257    """
258    If any old configuration files are detected, raise
259    OldConfigError.
260    """
261    oldfnames = set()
262    old_names = [
263        'nickname', 'webport', 'keepalive_timeout', 'log_gatherer.furl',
264        'disconnect_timeout', 'advertised_ip_addresses', 'introducer.furl',
265        'helper.furl', 'key_generator.furl', 'stats_gatherer.furl',
266        'no_storage', 'readonly_storage', 'sizelimit',
267        'debug_discard_storage', 'run_helper'
268    ]
269    for fn in generated_files:
270        old_names.remove(fn)
271    for name in old_names:
272        fullfname = os.path.join(basedir, name)
273        if os.path.exists(fullfname):
274            oldfnames.add(fullfname)
275    if oldfnames:
276        e = OldConfigError(oldfnames)
277        twlog.msg(e)
278        raise e
279
280
281def ensure_text_and_abspath_expanduser_unicode(basedir):
282    # type: (Union[bytes, str]) -> str
283    return abspath_expanduser_unicode(ensure_text(basedir))
284
285
286@attr.s
287class _Config(object):
288    """
289    Manages configuration of a Tahoe 'node directory'.
290
291    Note: all this code and functionality was formerly in the Node
292    class; names and funtionality have been kept the same while moving
293    the code. It probably makes sense for several of these APIs to
294    have better names.
295
296    :ivar ConfigParser config: The actual configuration values.
297
298    :ivar str portnum_fname: filename to use for the port-number file (a
299        relative path inside basedir).
300
301    :ivar str _basedir: path to our "node directory", inside which all
302        configuration is managed.
303
304    :ivar (FilePath|NoneType) config_path: The path actually used to create
305        the configparser (might be ``None`` if using in-memory data).
306
307    :ivar ValidConfiguration valid_config_sections: The validator for the
308        values in this configuration.
309    """
310    config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser))
311    portnum_fname = attr.ib()
312    _basedir = attr.ib(
313        converter=ensure_text_and_abspath_expanduser_unicode,
314    )  # type: str
315    config_path = attr.ib(
316        validator=attr.validators.optional(
317            attr.validators.instance_of(FilePath),
318        ),
319    )
320    valid_config_sections = attr.ib(
321        default=configutil.ValidConfiguration.everything(),
322        validator=attr.validators.instance_of(configutil.ValidConfiguration),
323    )
324
325    @property
326    def nickname(self):
327        nickname = self.get_config("node", "nickname", u"<unspecified>")
328        assert isinstance(nickname, str)
329        return nickname
330
331    @property
332    def _config_fname(self):
333        if self.config_path is None:
334            return "<string>"
335        return self.config_path.path
336
337    def write_config_file(self, name, value, mode="w"):
338        """
339        writes the given 'value' into a file called 'name' in the config
340        directory
341        """
342        fn = os.path.join(self._basedir, name)
343        try:
344            fileutil.write(fn, value, mode)
345        except EnvironmentError:
346            log.err(
347                Failure(),
348                "Unable to write config file '{}'".format(fn),
349            )
350
351    def items(self, section, default=_None):
352        try:
353            return self.config.items(section)
354        except configparser.NoSectionError:
355            if default is _None:
356                raise
357            return default
358
359    def get_config(self, section, option, default=_None, boolean=False):
360        try:
361            if boolean:
362                return self.config.getboolean(section, option)
363
364            item = self.config.get(section, option)
365            if option.endswith(".furl") and '#' in item:
366                raise UnescapedHashError(section, option, item)
367
368            return item
369        except (configparser.NoOptionError, configparser.NoSectionError):
370            if default is _None:
371                raise MissingConfigEntry(
372                    "{} is missing the [{}]{} entry".format(
373                        quote_output(self._config_fname),
374                        section,
375                        option,
376                    )
377                )
378            return default
379
380    def set_config(self, section, option, value):
381        """
382        Set a config option in a section and re-write the tahoe.cfg file
383
384        :param str section: The name of the section in which to set the
385            option.
386
387        :param str option: The name of the option to set.
388
389        :param str value: The value of the option.
390
391        :raise UnescapedHashError: If the option holds a fURL and there is a
392            ``#`` in the value.
393        """
394        if option.endswith(".furl") and "#" in value:
395            raise UnescapedHashError(section, option, value)
396
397        copied_config = configutil.copy_config(self.config)
398        configutil.set_config(copied_config, section, option, value)
399        configutil.validate_config(
400            self._config_fname,
401            copied_config,
402            self.valid_config_sections,
403        )
404        if self.config_path is not None:
405            configutil.write_config(self.config_path, copied_config)
406        self.config = copied_config
407
408    def get_config_from_file(self, name, required=False):
409        """Get the (string) contents of a config file, or None if the file
410        did not exist. If required=True, raise an exception rather than
411        returning None. Any leading or trailing whitespace will be stripped
412        from the data."""
413        fn = os.path.join(self._basedir, name)
414        try:
415            return fileutil.read(fn).strip()
416        except EnvironmentError as e:
417            if e.errno != errno.ENOENT:
418                raise  # we only care about "file doesn't exist"
419            if not required:
420                return None
421            raise
422
423    def get_or_create_private_config(self, name, default=_None):
424        """Try to get the (string) contents of a private config file (which
425        is a config file that resides within the subdirectory named
426        'private'), and return it. Any leading or trailing whitespace will be
427        stripped from the data.
428
429        If the file does not exist, and default is not given, report an error.
430        If the file does not exist and a default is specified, try to create
431        it using that default, and then return the value that was written.
432        If 'default' is a string, use it as a default value. If not, treat it
433        as a zero-argument callable that is expected to return a string.
434        """
435        privname = os.path.join(self._basedir, "private", name)
436        try:
437            value = fileutil.read(privname, mode="r")
438        except EnvironmentError as e:
439            if e.errno != errno.ENOENT:
440                raise  # we only care about "file doesn't exist"
441            if default is _None:
442                raise MissingConfigEntry("The required configuration file %s is missing."
443                                         % (quote_output(privname),))
444            if isinstance(default, bytes):
445                default = str(default, "utf-8")
446            if isinstance(default, str):
447                value = default
448            else:
449                value = default()
450            fileutil.write(privname, value)
451        return value.strip()
452
453    def write_private_config(self, name, value):
454        """Write the (string) contents of a private config file (which is a
455        config file that resides within the subdirectory named 'private'), and
456        return it.
457        """
458        if isinstance(value, str):
459            value = value.encode("utf-8")
460        privname = os.path.join(self._basedir, "private", name)
461        with open(privname, "wb") as f:
462            f.write(value)
463
464    def get_private_config(self, name, default=_None):
465        """Read the (native string) contents of a private config file (a
466        config file that resides within the subdirectory named 'private'),
467        and return it. Return a default, or raise an error if one was not
468        given.
469        """
470        privname = os.path.join(self._basedir, "private", name)
471        try:
472            return fileutil.read(privname, mode="r").strip()
473        except EnvironmentError as e:
474            if e.errno != errno.ENOENT:
475                raise  # we only care about "file doesn't exist"
476            if default is _None:
477                raise MissingConfigEntry("The required configuration file %s is missing."
478                                         % (quote_output(privname),))
479            return default
480
481    def get_private_path(self, *args):
482        """
483        returns an absolute path inside the 'private' directory with any
484        extra args join()-ed
485        """
486        return os.path.join(self._basedir, "private", *args)
487
488    def get_config_path(self, *args):
489        """
490        returns an absolute path inside the config directory with any
491        extra args join()-ed
492        """
493        # note: we re-expand here (_basedir already went through this
494        # expanduser function) in case the path we're being asked for
495        # has embedded ".."'s in it
496        return abspath_expanduser_unicode(
497            os.path.join(self._basedir, *args)
498        )
499
500    def get_introducer_configuration(self):
501        """
502        Get configuration for introducers.
503
504        :return {unicode: (unicode, FilePath)}: A mapping from introducer
505            petname to a tuple of the introducer's fURL and local cache path.
506        """
507        introducers_yaml_filename = self.get_private_path("introducers.yaml")
508        introducers_filepath = FilePath(introducers_yaml_filename)
509
510        def get_cache_filepath(petname):
511            return FilePath(
512                self.get_private_path("introducer_{}_cache.yaml".format(petname)),
513            )
514
515        try:
516            with introducers_filepath.open() as f:
517                introducers_yaml = safe_load(f)
518                if introducers_yaml is None:
519                    raise EnvironmentError(
520                        EPERM,
521                        "Can't read '{}'".format(introducers_yaml_filename),
522                        introducers_yaml_filename,
523                    )
524                introducers = {
525                    petname: config["furl"]
526                    for petname, config
527                    in introducers_yaml.get("introducers", {}).items()
528                }
529                non_strs = list(
530                    k
531                    for k
532                    in introducers.keys()
533                    if not isinstance(k, str)
534                )
535                if non_strs:
536                    raise TypeError(
537                        "Introducer petnames {!r} should have been str".format(
538                            non_strs,
539                        ),
540                    )
541                non_strs = list(
542                    v
543                    for v
544                    in introducers.values()
545                    if not isinstance(v, str)
546                )
547                if non_strs:
548                    raise TypeError(
549                        "Introducer fURLs {!r} should have been str".format(
550                            non_strs,
551                        ),
552                    )
553                log.msg(
554                    "found {} introducers in {!r}".format(
555                        len(introducers),
556                        introducers_yaml_filename,
557                    )
558                )
559        except EnvironmentError as e:
560            if e.errno != ENOENT:
561                raise
562            introducers = {}
563
564        # supported the deprecated [client]introducer.furl item in tahoe.cfg
565        tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
566        if tahoe_cfg_introducer_furl == "None":
567            raise ValueError(
568                "tahoe.cfg has invalid 'introducer.furl = None':"
569                " to disable it omit the key entirely"
570            )
571        if tahoe_cfg_introducer_furl:
572            warn(
573                "tahoe.cfg [client]introducer.furl is deprecated; "
574                "use private/introducers.yaml instead.",
575                category=DeprecationWarning,
576                stacklevel=-1,
577            )
578            if "default" in introducers:
579                raise ValueError(
580                    "'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml;"
581                    " please fix impossible configuration."
582                )
583            introducers['default'] = tahoe_cfg_introducer_furl
584
585        return {
586            petname: (furl, get_cache_filepath(petname))
587            for (petname, furl)
588            in introducers.items()
589        }
590
591
592def create_tub_options(config):
593    """
594    :param config: a _Config instance
595
596    :returns: dict containing all Foolscap Tub-related options,
597        overriding defaults with appropriate config from `config`
598        instance.
599    """
600    # We can't unify the camelCase vs. dashed-name divide here,
601    # because these are options for Foolscap
602    tub_options = {
603        "logLocalFailures": True,
604        "logRemoteFailures": True,
605        "expose-remote-exception-types": False,
606        "accept-gifts": False,
607    }
608
609    # see #521 for a discussion of how to pick these timeout values.
610    keepalive_timeout_s = config.get_config("node", "timeout.keepalive", "")
611    if keepalive_timeout_s:
612        tub_options["keepaliveTimeout"] = int(keepalive_timeout_s)
613    disconnect_timeout_s = config.get_config("node", "timeout.disconnect", "")
614    if disconnect_timeout_s:
615        # N.B.: this is in seconds, so use "1800" to get 30min
616        tub_options["disconnectTimeout"] = int(disconnect_timeout_s)
617    return tub_options
618
619
620def _make_tcp_handler():
621    """
622    :returns: a Foolscap default TCP handler
623    """
624    # this is always available
625    from foolscap.connections.tcp import default
626    return default()
627
628
629def create_default_connection_handlers(config, handlers):
630    """
631    :return: A dictionary giving the default connection handlers.  The keys
632        are strings like "tcp" and the values are strings like "tor" or
633        ``None``.
634    """
635    reveal_ip = config.get_config("node", "reveal-IP-address", True, boolean=True)
636
637    # Remember the default mappings from tahoe.cfg
638    default_connection_handlers = {
639        name: name
640        for name
641        in handlers
642    }
643    tcp_handler_name = config.get_config("connections", "tcp", "tcp").lower()
644    if tcp_handler_name == "disabled":
645        default_connection_handlers["tcp"] = None
646    else:
647        if tcp_handler_name not in handlers:
648            raise ValueError(
649                "'tahoe.cfg [connections] tcp=' uses "
650                "unknown handler type '{}'".format(
651                    tcp_handler_name
652                )
653            )
654        if not handlers[tcp_handler_name]:
655            raise ValueError(
656                "'tahoe.cfg [connections] tcp=' uses "
657                "unavailable/unimportable handler type '{}'. "
658                "Please pip install tahoe-lafs[{}] to fix.".format(
659                    tcp_handler_name,
660                    tcp_handler_name,
661                )
662            )
663        default_connection_handlers["tcp"] = tcp_handler_name
664
665    if not reveal_ip:
666        if default_connection_handlers.get("tcp") == "tcp":
667            raise PrivacyError(
668                "Privacy requested with `reveal-IP-address = false` "
669                "but `tcp = tcp` conflicts with this.",
670            )
671    return default_connection_handlers
672
673
674def create_connection_handlers(config, i2p_provider, tor_provider):
675    """
676    :returns: 2-tuple of default_connection_handlers, foolscap_connection_handlers
677    """
678    # We store handlers for everything. None means we were unable to
679    # create that handler, so hints which want it will be ignored.
680    handlers = {
681        "tcp": _make_tcp_handler(),
682        "tor": tor_provider.get_client_endpoint(),
683        "i2p": i2p_provider.get_client_endpoint(),
684    }
685    log.msg(
686        format="built Foolscap connection handlers for: %(known_handlers)s",
687        known_handlers=sorted([k for k,v in handlers.items() if v]),
688        facility="tahoe.node",
689        umid="PuLh8g",
690    )
691    return create_default_connection_handlers(
692        config,
693        handlers,
694    ), handlers
695
696
697def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
698               handler_overrides={}, **kwargs):
699    """
700    Create a Tub with the right options and handlers. It will be
701    ephemeral unless the caller provides certFile= in kwargs
702
703    :param handler_overrides: anything in this will override anything
704        in `default_connection_handlers` for just this call.
705
706    :param dict tub_options: every key-value pair in here will be set in
707        the new Tub via `Tub.setOption`
708    """
709    tub = Tub(**kwargs)
710    for (name, value) in list(tub_options.items()):
711        tub.setOption(name, value)
712    handlers = default_connection_handlers.copy()
713    handlers.update(handler_overrides)
714    tub.removeAllConnectionHintHandlers()
715    for hint_type, handler_name in list(handlers.items()):
716        handler = foolscap_connection_handlers.get(handler_name)
717        if handler:
718            tub.addConnectionHintHandler(hint_type, handler)
719    return tub
720
721
722def _convert_tub_port(s):
723    """
724    :returns: a proper Twisted endpoint string like (`tcp:X`) is `s`
725        is a bare number, or returns `s` as-is
726    """
727    us = s
728    if isinstance(s, bytes):
729        us = s.decode("utf-8")
730    if re.search(r'^\d+$', us):
731        return "tcp:{}".format(int(us))
732    return us
733
734
735class PortAssignmentRequired(Exception):
736    """
737    A Tub port number was configured to be 0 where this is not allowed.
738    """
739
740
741def _tub_portlocation(config, get_local_addresses_sync, allocate_tcp_port):
742    """
743    Figure out the network location of the main tub for some configuration.
744
745    :param get_local_addresses_sync: A function like
746        ``iputil.get_local_addresses_sync``.
747
748    :param allocate_tcp_port: A function like ``iputil.allocate_tcp_port``.
749
750    :returns: None or tuple of (port, location) for the main tub based
751        on the given configuration. May raise ValueError or PrivacyError
752        if there are problems with the config
753    """
754    cfg_tubport = config.get_config("node", "tub.port", None)
755    cfg_location = config.get_config("node", "tub.location", None)
756    reveal_ip = config.get_config("node", "reveal-IP-address", True, boolean=True)
757    tubport_disabled = False
758
759    if cfg_tubport is not None:
760        cfg_tubport = cfg_tubport.strip()
761        if cfg_tubport == "":
762            raise ValueError("tub.port must not be empty")
763        if cfg_tubport == "disabled":
764            tubport_disabled = True
765
766    location_disabled = False
767    if cfg_location is not None:
768        cfg_location = cfg_location.strip()
769        if cfg_location == "":
770            raise ValueError("tub.location must not be empty")
771        if cfg_location == "disabled":
772            location_disabled = True
773
774    if tubport_disabled and location_disabled:
775        return None
776    if tubport_disabled and not location_disabled:
777        raise ValueError("tub.port is disabled, but not tub.location")
778    if location_disabled and not tubport_disabled:
779        raise ValueError("tub.location is disabled, but not tub.port")
780
781    if cfg_tubport is None:
782        # For 'tub.port', tahoe.cfg overrides the individual file on
783        # disk. So only read config.portnum_fname if tahoe.cfg doesn't
784        # provide a value.
785        if os.path.exists(config.portnum_fname):
786            file_tubport = fileutil.read(config.portnum_fname).strip()
787            tubport = _convert_tub_port(file_tubport)
788        else:
789            tubport = "tcp:%d" % (allocate_tcp_port(),)
790            fileutil.write_atomically(config.portnum_fname, tubport + "\n",
791                                      mode="")
792    else:
793        tubport = _convert_tub_port(cfg_tubport)
794
795    for port in tubport.split(","):
796        if port in ("0", "tcp:0"):
797            raise PortAssignmentRequired()
798
799    if cfg_location is None:
800        cfg_location = "AUTO"
801
802    local_portnum = None # needed to hush lgtm.com static analyzer
803    # Replace the location "AUTO", if present, with the detected local
804    # addresses. Don't probe for local addresses unless necessary.
805    split_location = cfg_location.split(",")
806    if "AUTO" in split_location:
807        if not reveal_ip:
808            raise PrivacyError("tub.location uses AUTO")
809        local_addresses = get_local_addresses_sync()
810        # tubport must be like "tcp:12345" or "tcp:12345:morestuff"
811        local_portnum = int(tubport.split(":")[1])
812    new_locations = []
813    for loc in split_location:
814        if loc == "AUTO":
815            new_locations.extend(["tcp:%s:%d" % (ip, local_portnum)
816                                  for ip in local_addresses])
817        else:
818            if not reveal_ip:
819                # Legacy hints are "host:port". We use Foolscap's utility
820                # function to convert all hints into the modern format
821                # ("tcp:host:port") because that's what the receiving
822                # client will probably do. We test the converted hint for
823                # TCP-ness, but publish the original hint because that
824                # was the user's intent.
825                from foolscap.connections.tcp import convert_legacy_hint
826                converted_hint = convert_legacy_hint(loc)
827                hint_type = converted_hint.split(":")[0]
828                if hint_type == "tcp":
829                    raise PrivacyError("tub.location includes tcp: hint")
830            new_locations.append(loc)
831    location = ",".join(new_locations)
832
833    # Lacking this, Python 2 blows up in Foolscap when it is confused by a
834    # Unicode FURL.
835    location = location.encode("utf-8")
836
837    return tubport, location
838
839
840def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location):
841    """
842    Assign a Tub its listener locations.
843
844    :param i2p_provider: See ``allmydata.util.i2p_provider.create``.
845    :param tor_provider: See ``allmydata.util.tor_provider.create``.
846    """
847    for port in tubport.split(","):
848        if port == "listen:i2p":
849            # the I2P provider will read its section of tahoe.cfg and
850            # return either a fully-formed Endpoint, or a descriptor
851            # that will create one, so we don't have to stuff all the
852            # options into the tub.port string (which would need a lot
853            # of escaping)
854            port_or_endpoint = i2p_provider.get_listener()
855        elif port == "listen:tor":
856            port_or_endpoint = tor_provider.get_listener()
857        else:
858            port_or_endpoint = port
859        # Foolscap requires native strings:
860        if isinstance(port_or_endpoint, (bytes, str)):
861            port_or_endpoint = ensure_str(port_or_endpoint)
862        tub.listenOn(port_or_endpoint)
863    # This last step makes the Tub is ready for tub.registerReference()
864    tub.setLocation(location)
865
866
867def create_main_tub(config, tub_options,
868                    default_connection_handlers, foolscap_connection_handlers,
869                    i2p_provider, tor_provider,
870                    handler_overrides={}, cert_filename="node.pem"):
871    """
872    Creates a 'main' Foolscap Tub, typically for use as the top-level
873    access point for a running Node.
874
875    :param Config: a `_Config` instance
876
877    :param dict tub_options: any options to change in the tub
878
879    :param default_connection_handlers: default Foolscap connection
880        handlers
881
882    :param foolscap_connection_handlers: Foolscap connection
883        handlers for this tub
884
885    :param i2p_provider: None, or a _Provider instance if I2P is
886        installed.
887
888    :param tor_provider: None, or a _Provider instance if txtorcon +
889        Tor are installed.
890    """
891    portlocation = _tub_portlocation(
892        config,
893        iputil.get_local_addresses_sync,
894        iputil.allocate_tcp_port,
895    )
896
897    # FIXME? "node.pem" was the CERTFILE option/thing
898    certfile = config.get_private_path("node.pem")
899
900    tub = create_tub(
901        tub_options,
902        default_connection_handlers,
903        foolscap_connection_handlers,
904        handler_overrides=handler_overrides,
905        certFile=certfile,
906    )
907    if portlocation is None:
908        log.msg("Tub is not listening")
909    else:
910        tubport, location = portlocation
911        tub_listen_on(
912            i2p_provider,
913            tor_provider,
914            tub,
915            tubport,
916            location,
917        )
918        log.msg("Tub location set to %r" % (location,))
919    return tub
920
921
922def create_control_tub():
923    """
924    Creates a Foolscap Tub for use by the control port. This is a
925    localhost-only ephemeral Tub, with no control over the listening
926    port or location
927    """
928    control_tub = Tub()
929    portnum = iputil.listenOnUnused(control_tub)
930    log.msg("Control Tub location set to 127.0.0.1:%s" % (portnum,))
931    return control_tub
932
933
934class Node(service.MultiService):
935    """
936    This class implements common functionality of both Client nodes and Introducer nodes.
937    """
938    NODETYPE = "unknown NODETYPE"
939    CERTFILE = "node.pem"
940
941    def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider):
942        """
943        Initialize the node with the given configuration. Its base directory
944        is the current directory by default.
945        """
946        service.MultiService.__init__(self)
947
948        self.config = config
949        self.get_config = config.get_config # XXX stopgap
950        self.nickname = config.nickname # XXX stopgap
951
952        # this can go away once Client.init_client_storage_broker is moved into create_client()
953        # (tests sometimes have None here)
954        self._i2p_provider = i2p_provider
955        self._tor_provider = tor_provider
956
957        self.create_log_tub()
958        self.logSource = "Node"
959        self.setup_logging()
960
961        self.tub = main_tub
962        if self.tub is not None:
963            self.nodeid = b32decode(self.tub.tubID.upper())  # binary format
964            self.short_nodeid = b32encode(self.nodeid).lower()[:8]  # for printing
965            self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + b"\n", mode="wb")
966            self.tub.setServiceParent(self)
967        else:
968            self.nodeid = self.short_nodeid = None
969
970        self.control_tub = control_tub
971        if self.control_tub is not None:
972            self.control_tub.setServiceParent(self)
973
974        self.log("Node constructed. " + __full_version__)
975        iputil.increase_rlimits()
976
977    def _is_tub_listening(self):
978        """
979        :returns: True if the main tub is listening
980        """
981        return len(self.tub.getListeners()) > 0
982
983    # pull this outside of Node's __init__ too, see:
984    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2948
985    def create_log_tub(self):
986        # The logport uses a localhost-only ephemeral Tub, with no control
987        # over the listening port or location. This might change if we
988        # discover a compelling reason for it in the future (e.g. being able
989        # to use "flogtool tail" against a remote server), but for now I
990        # think we can live without it.
991        self.log_tub = Tub()
992        portnum = iputil.listenOnUnused(self.log_tub)
993        self.log("Log Tub location set to 127.0.0.1:%s" % (portnum,))
994        self.log_tub.setServiceParent(self)
995
996    def startService(self):
997        # Note: this class can be started and stopped at most once.
998        self.log("Node.startService")
999        # Record the process id in the twisted log, after startService()
1000        # (__init__ is called before fork(), but startService is called
1001        # after). Note that Foolscap logs handle pid-logging by itself, no
1002        # need to send a pid to the foolscap log here.
1003        twlog.msg("My pid: %s" % os.getpid())
1004        try:
1005            os.chmod("twistd.pid", 0o644)
1006        except EnvironmentError:
1007            pass
1008
1009        service.MultiService.startService(self)
1010        self.log("%s running" % self.NODETYPE)
1011        twlog.msg("%s running" % self.NODETYPE)
1012
1013    def stopService(self):
1014        self.log("Node.stopService")
1015        return service.MultiService.stopService(self)
1016
1017    def shutdown(self):
1018        """Shut down the node. Returns a Deferred that fires (with None) when
1019        it finally stops kicking."""
1020        self.log("Node.shutdown")
1021        return self.stopService()
1022
1023    def setup_logging(self):
1024        # we replace the formatTime() method of the log observer that
1025        # twistd set up for us, with a method that uses our preferred
1026        # timestamp format.
1027        for o in twlog.theLogPublisher.observers:
1028            # o might be a FileLogObserver's .emit method
1029            if type(o) is type(self.setup_logging): # bound method
1030                ob = o.__self__
1031                if isinstance(ob, twlog.FileLogObserver):
1032                    newmeth = types.MethodType(formatTimeTahoeStyle, ob)
1033                    ob.formatTime = newmeth
1034        # TODO: twisted >2.5.0 offers maxRotatedFiles=50
1035
1036        lgfurl_file = self.config.get_private_path("logport.furl").encode(get_filesystem_encoding())
1037        if os.path.exists(lgfurl_file):
1038            os.remove(lgfurl_file)
1039        self.log_tub.setOption("logport-furlfile", lgfurl_file)
1040        lgfurl = self.config.get_config("node", "log_gatherer.furl", "")
1041        if lgfurl:
1042            # this is in addition to the contents of log-gatherer-furlfile
1043            lgfurl = lgfurl.encode("utf-8")
1044            self.log_tub.setOption("log-gatherer-furl", lgfurl)
1045        self.log_tub.setOption("log-gatherer-furlfile",
1046                               self.config.get_config_path("log_gatherer.furl"))
1047
1048        incident_dir = self.config.get_config_path("logs", "incidents")
1049        foolscap.logging.log.setLogDir(incident_dir)
1050        twlog.msg("Foolscap logging initialized")
1051        twlog.msg("Note to developers: twistd.log does not receive very much.")
1052        twlog.msg("Use 'flogtool tail -c NODEDIR/private/logport.furl' instead")
1053        twlog.msg("and read docs/logging.rst")
1054
1055    def log(self, *args, **kwargs):
1056        return log.msg(*args, **kwargs)
Note: See TracBrowser for help on using the repository browser.