Changeset 2f0a5d9 in trunk

Timestamp:

2020-10-27T20:58:46Z (5 years ago)

Author:

GitHub <noreply@…>

Branches:

master

Children:

91c055a

Parents:

874bfa72 (diff), b79504a (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

Itamar Turner-Trauring <itamar@…> (2020-10-27 20:58:46)

git-committer:

GitHub <noreply@…> (2020-10-27 20:58:46)

Message:

Merge pull request #877 from tahoe-lafs/3485.backported-configparser-for-py-2

Backported configparser for Python 2

Fixes ticket:3485

Files:

• newsfragments/3485.minor (added)
• nix/tahoe-lafs.nix (modified) (2 diffs)
• setup.py (modified) (1 diff)
• src/allmydata/client.py (modified) (8 diffs)
• src/allmydata/frontends/ftpd.py (modified) (2 diffs)
• src/allmydata/node.py (modified) (8 diffs)
• src/allmydata/scripts/common.py (modified) (1 diff)
• src/allmydata/storage_client.py (modified) (1 diff)
• src/allmydata/test/test_configutil.py (modified) (1 diff)
• src/allmydata/test/test_connections.py (modified) (1 diff)
• src/allmydata/test/test_node.py (modified) (1 diff)
• src/allmydata/util/configutil.py (modified) (3 diffs)
• src/allmydata/webish.py (modified) (2 diffs)

nix/tahoe-lafs.nix ¶

@@ -5 +5 @@
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
 , beautifulsoup4, eliot, autobahn, cryptography, netifaces
-, html5lib, pyutil, distro
+, html5lib, pyutil, distro, configparser
 }:
 python.pkgs.buildPythonPackage rec {
@@ -47 +47 @@
     service-identity pyyaml magic-wormhole treq
     eliot autobahn cryptography netifaces setuptools
-    future pyutil distro
+    future pyutil distro configparser
   ];
 

setup.py ¶

@@ -135 +135 @@
     # Linux distribution detection:
     "distro >= 1.4.0",
+
+    # Backported configparser for Python 2:
+    "configparser ; python_version < '3.0'",
 ]
 

src/allmydata/client.py ¶

@@ -3 +3 @@
 from functools import partial
 from errno import ENOENT, EPERM
-try:
-    from ConfigParser import NoSectionError
-except ImportError:
-    from configparser import NoSectionError
+
+# On Python 2 this will be the backported package:
+from configparser import NoSectionError
 
 from foolscap.furl import (
@@ -36 +35 @@
     yamlutil, configutil,
 )
-from allmydata.util.encodingutil import (get_filesystem_encoding,
-                                         from_utf8_or_none)
+from allmydata.util.encodingutil import get_filesystem_encoding
 from allmydata.util.abbreviate import parse_abbreviated_size
 from allmydata.util.time_format import parse_duration, parse_date
@@ -720 +718 @@
     def init_stats_provider(self):
         gatherer_furl = self.config.get_config("client", "stats_gatherer.furl", None)
+        if gatherer_furl:
+            # FURLs should be bytes:
+            gatherer_furl = gatherer_furl.encode("utf-8")
         self.stats_provider = StatsProvider(self, gatherer_furl)
         self.stats_provider.setServiceParent(self)
@@ -807 +808 @@
         config_storedir = self.get_config(
             "storage", "storage_dir", self.STOREDIR,
-        ).decode('utf-8')
+        )
         storedir = self.config.get_config_path(config_storedir)
 
@@ -935 +936 @@
         if helper_furl in ("None", ""):
             helper_furl = None
+
+        # FURLs need to be bytes:
+        if helper_furl is not None:
+            helper_furl = helper_furl.encode("utf-8")
 
         DEP = self.encoding_params
@@ -1044 +1049 @@
         from allmydata.webish import WebishServer
         nodeurl_path = self.config.get_config_path("node.url")
-        staticdir_config = self.config.get_config("node", "web.static", "public_html").decode("utf-8")
+        staticdir_config = self.config.get_config("node", "web.static", "public_html")
         staticdir = self.config.get_config_path(staticdir_config)
         ws = WebishServer(self, webport, nodeurl_path, staticdir)
@@ -1051 +1056 @@
     def init_ftp_server(self):
         if self.config.get_config("ftpd", "enabled", False, boolean=True):
-            accountfile = from_utf8_or_none(
-                self.config.get_config("ftpd", "accounts.file", None))
+            accountfile = self.config.get_config("ftpd", "accounts.file", None)
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
@@ -1064 +1068 @@
     def init_sftp_server(self):
         if self.config.get_config("sftpd", "enabled", False, boolean=True):
-            accountfile = from_utf8_or_none(
-                self.config.get_config("sftpd", "accounts.file", None))
+            accountfile = self.config.get_config("sftpd", "accounts.file", None)
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
             accounturl = self.config.get_config("sftpd", "accounts.url", None)
             sftp_portstr = self.config.get_config("sftpd", "port", "8022")
-            pubkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_pubkey_file"))
-            privkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_privkey_file"))
+            pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
+            privkey_file = self.config.get_config("sftpd", "host_privkey_file")
 
             from allmydata.frontends import sftpd

src/allmydata/frontends/ftpd.py ¶

@@ +1 @@
+from six import ensure_str
 
 from types import NoneType
@@ -334 +335 @@
 
         f = ftp.FTPFactory(p)
+        # strports requires a native string.
+        ftp_portstr = ensure_str(ftp_portstr)
         s = strports.service(ftp_portstr, f)
         s.setServiceParent(self)

src/allmydata/node.py ¶

@@ -4 +4 @@
 """
 from past.builtins import unicode
+from six import ensure_str
 
 import datetime
@@ -10 +11 @@
 import types
 import errno
-from io import StringIO
 import tempfile
 from base64 import b32decode, b32encode
 
-# Python 2 compatibility
-from six.moves import configparser
-from future.utils import PY2
-if PY2:
-    from io import BytesIO as StringIO  # noqa: F811
+# On Python 2 this will be the backported package.
+import configparser
 
 from twisted.python import log as twlog
@@ -188 +185 @@
     # (try to) read the main config file
     config_fname = os.path.join(basedir, "tahoe.cfg")
-    parser = configparser.SafeConfigParser()
     try:
         parser = configutil.get_config(config_fname)
@@ -194 +190 @@
         if e.errno != errno.ENOENT:
             raise
+        # The file is missing, just create empty ConfigParser.
+        parser = configutil.get_config_from_string(u"")
 
     configutil.validate_config(config_fname, parser, _valid_config)
@@ -210 +208 @@
         _valid_config = _common_valid_config()
 
+    if isinstance(config_str, bytes):
+        config_str = config_str.decode("utf-8")
+
     # load configuration from in-memory string
-    parser = configparser.SafeConfigParser()
-    parser.readfp(StringIO(config_str))
+    parser = configutil.get_config_from_string(config_str)
 
     fname = "<in-memory>"
@@ -640 +640 @@
     location = ",".join(new_locations)
 
+    # Lacking this, Python 2 blows up in Foolscap when it is confused by a
+    # Unicode FURL.
+    location = location.encode("utf-8")
+
     return tubport, location
 
@@ -687 +691 @@
             else:
                 port_or_endpoint = port
+            # Foolscap requires native strings:
+            if isinstance(port_or_endpoint, (bytes, unicode)):
+                port_or_endpoint = ensure_str(port_or_endpoint)
             tub.listenOn(port_or_endpoint)
         tub.setLocation(location)
@@ -840 +847 @@
         if lgfurl:
             # this is in addition to the contents of log-gatherer-furlfile
+            lgfurl = lgfurl.encode("utf-8")
             self.log_tub.setOption("log-gatherer-furl", lgfurl)
         self.log_tub.setOption("log-gatherer-furlfile",

src/allmydata/scripts/common.py ¶

@@ -9 +9 @@
 if PY2:
     from future.builtins import str  # noqa: F401
-from six.moves.configparser import NoSectionError
+
+# On Python 2 this will be the backported package:
+from configparser import NoSectionError
 
 from twisted.python import usage

src/allmydata/storage_client.py ¶

@@ -32 +32 @@
 
 import re, time, hashlib
-try:
-    from ConfigParser import (
-        NoSectionError,
-    )
-except ImportError:
-    from configparser import NoSectionError
+
+# On Python 2 this will be the backport.
+from configparser import NoSectionError
+
 import attr
 from zope.interface import (

src/allmydata/test/test_configutil.py ¶

@@ -155 +155 @@
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
+
+    def test_duplicate_sections(self):
+        """
+        Duplicate section names are merged.
+        """
+        fname = self.create_tahoe_cfg('[node]\na = foo\n[node]\n b = bar\n')
+        config = configutil.get_config(fname)
+        self.assertEqual(config.get("node", "a"), "foo")
+        self.assertEqual(config.get("node", "b"), "bar")

src/allmydata/test/test_connections.py ¶

@@ -169 +169 @@
             tor_provider.get_tor_handler()
         self.assertIn(
-            "invalid literal for int() with base 10: 'kumquat'",
+            "invalid literal for int()",
+            str(ctx.exception)
+        )
+        self.assertIn(
+            "kumquat",
             str(ctx.exception)
         )

src/allmydata/test/test_node.py ¶

@@ -152 +152 @@
 
         config = read_config(basedir, "")
-        self.failUnlessEqual(config.get_config("node", "nickname").decode('utf-8'),
+        self.failUnlessEqual(config.get_config("node", "nickname"),
                              u"\u2621")
 

src/allmydata/util/configutil.py ¶

@@ -2 +2 @@
 Read/write config files.
 
-Configuration is returned as native strings.
+Configuration is returned as Unicode strings.
 
 Ported to Python 3.
@@ -13 +13 @@
 from future.utils import PY2
 if PY2:
-    # We don't do open(), because we want files to read/write native strs when
-    # we do "r" or "w".
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-if PY2:
-    # In theory on Python 2 configparser also works, but then code gets the
-    # wrong exceptions and they don't get handled. So just use native parser
-    # for now.
-    from ConfigParser import SafeConfigParser
-else:
-    from configparser import SafeConfigParser
+# On Python 2 we use the backport package; that means we always get unicode
+# out.
+from configparser import ConfigParser
 
 import attr
@@ -37 +31 @@
 
 def get_config(tahoe_cfg):
-    """Load the config, returning a SafeConfigParser.
+    """Load the config, returning a ConfigParser.
 
-    Configuration is returned as native strings.
+    Configuration is returned as Unicode strings.
     """
-    config = SafeConfigParser()
-    with open(tahoe_cfg, "r") as f:
-        # On Python 2, where we read in bytes, skip any initial Byte Order
-        # Mark. Since this is an ordinary file, we don't need to handle
-        # incomplete reads, and can assume seekability.
-        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
-            f.seek(0)
-        config.readfp(f)
-    return config
+    # Byte Order Mark is an optional garbage code point you sometimes get at
+    # the start of UTF-8 encoded files. Especially on Windows. Skip it by using
+    # utf-8-sig. https://en.wikipedia.org/wiki/Byte_order_mark
+    with open(tahoe_cfg, "r", encoding="utf-8-sig") as f:
+        cfg_string = f.read()
+    return get_config_from_string(cfg_string)
+
+
+def get_config_from_string(tahoe_cfg_string):
+    """Load the config from a string, return the ConfigParser.
+
+    Configuration is returned as Unicode strings.
+    """
+    parser = ConfigParser(strict=False)
+    parser.read_string(tahoe_cfg_string)
+    return parser
+
 
 def set_config(config, section, option, value):

src/allmydata/webish.py ¶

@@ +1 @@
+from six import ensure_str
+
 import re, time
 
@@ -187 +189 @@
         if re.search(r'^\d', webport):
             webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
+        # strports must be native strings.
+        webport = ensure_str(webport)
         s = strports.service(webport, self.site)
         s.setServiceParent(self)