Changeset dd55acce in trunk

Timestamp:

2019-06-26T20:42:43Z (6 years ago)

Author:

meejah <meejah@…>

Branches:

master

Children:

47633fb

Parents:

016413b (diff), c169246 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into 3031-replace-pycryptopp

Files:

• .circleci/Dockerfile.centos (modified) (1 diff)
• .circleci/Dockerfile.debian (modified) (1 diff)
• .circleci/Dockerfile.fedora (modified) (1 diff)
• .circleci/Dockerfile.pypy (added)
• .circleci/Dockerfile.slackware (modified) (1 diff)
• .circleci/Dockerfile.ubuntu (modified) (1 diff)
• .circleci/config.yml (modified) (7 diffs)
• .circleci/create-virtualenv.sh (modified) (1 diff)
• .circleci/prepare-image.sh (modified) (1 diff)
• .circleci/run-tests.sh (modified) (2 diffs)
• newsfragments/2479.other (added)
• newsfragments/3041.feature (added)
• newsfragments/3051.feature (added)
• newsfragments/3111.minor (added)
• src/allmydata/_auto_deps.py (modified) (1 diff)
• src/allmydata/client.py (modified) (2 diffs)
• src/allmydata/scripts/tahoe_daemonize.py (modified) (4 diffs)
• src/allmydata/storage_client.py (modified) (3 diffs)
• src/allmydata/test/cli/test_daemonize.py (modified) (4 diffs)
• src/allmydata/test/cli/test_start.py (modified) (2 diffs)
• src/allmydata/test/test_client.py (modified) (3 diffs)
• src/allmydata/test/test_websocket_logs.py (added)
• tox.ini (modified) (1 diff)

.circleci/Dockerfile.centos ¶

@@ -24 +24 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.debian ¶

@@ -24 +24 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
 
 # Only the integration tests currently need this but it doesn't hurt to always

.circleci/Dockerfile.fedora ¶

@@ -24 +24 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.slackware ¶

@@ -47 +47 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.ubuntu ¶

@@ -27 +27 @@
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/config.yml ¶

@@ -25 +25 @@
 
       - "slackware-14.2"
+
+      # Test against PyPy 2.7/7.1.1
+      - "pypy2.7-7.1"
 
       # Other assorted tasks and configurations
@@ -60 +63 @@
       - "build-image-centos-7"
       - "build-image-slackware-14.2"
+      - "build-image-pypy-2.7-7.1.1-jessie"
 
 
@@ -86 +90 @@
 
     environment: &UTF_8_ENVIRONMENT
+      # In general, the test suite is not allowed to fail while the job
+      # succeeds.  But you can set this to "yes" if you want it to be
+      # otherwise.
+      ALLOWED_FAILURE: "no"
       # Tell Hypothesis which configuration we want it to use.
       TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
@@ -92 +100 @@
       LANG: "en_US.UTF-8"
       # Select a tox environment to run for this job.
-      TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
@@ -124 +132 @@
                 "/tmp/venv" \
                 "/tmp/project" \
+                "${ALLOWED_FAILURE}" \
                 "${ARTIFACTS_OUTPUT_PATH}" \
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
@@ -156 +165 @@
       - image: "tahoelafsci/debian:8"
         user: "nobody"
+
+
+  pypy2.7-7.1:
+    <<: *DEBIAN
+    docker:
+      - image: "tahoelafsci/pypy:2.7-7.1.1-jessie"
+        user: "nobody"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage"
+      ALLOWED_FAILURE: "yes"
 
 
@@ -407 +428 @@
       DISTRO: "slackware"
       TAG: "14.2"
+
+
+  build-image-pypy-2.7-7.1.1-jessie:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "pypy"
+      TAG: "2.7-7.1.1-jessie"

.circleci/create-virtualenv.sh ¶

@@ -14 +14 @@
 shift
 
+# The basename of the Python executable (found on PATH) that will be used with
+# this image.  This lets us create a virtualenv that uses the correct Python.
+PYTHON="$1"
+shift
+
 # Set up the virtualenv as a non-root user so we can run the test suite as a
 # non-root user.  See below.
-virtualenv --python python2.7 "${BOOTSTRAP_VENV}"
+virtualenv --python "${PYTHON}" "${BOOTSTRAP_VENV}"
 
 # For convenience.

.circleci/prepare-image.sh ¶

@@ -19 +19 @@
 shift
 
+# The basename of the Python executable (found on PATH) that will be used with
+# this image.  This lets us create a virtualenv that uses the correct Python.
+PYTHON="$1"
+shift
+
 "${PROJECT_ROOT}"/.circleci/fix-permissions.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"
-sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}"
+sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PYTHON}"
 sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/populate-wheelhouse.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"

.circleci/run-tests.sh ¶

@@ -12 +12 @@
 # know what wheels to get/build, of course.
 PROJECT_ROOT="$1"
+shift
+
+ALLOWED_FAILURE="$1"
 shift
 
@@ -55 +58 @@
 export PIP_NO_INDEX="1"
 
+if [ "${ALLOWED_FAILURE}" = "yes" ]; then
+    alternative="true"
+else
+    alternative="false"
+fi
+
 ${BOOTSTRAP_VENV}/bin/tox \
     -c ${PROJECT_ROOT}/tox.ini \
     --workdir /tmp/tahoe-lafs.tox \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
-    ${TAHOE_LAFS_TOX_ARGS}
+    ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
 
 if [ -n "${ARTIFACTS}" ]; then
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
+    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

src/allmydata/_auto_deps.py ¶

@@ -108 +108 @@
     # A great way to define types of values.
     "attrs >= 18.2.0",
+
+    # WebSocket library for twisted and asyncio
+    "autobahn >= 19.5.2",
 ]
 

src/allmydata/client.py ¶

@@ -206 +206 @@
         )
     except Exception:
-        return Failure()
+        return defer.fail()
 
 
@@ -260 +260 @@
         return defer.succeed(client)
     except Exception:
-        return Failure()
+        return defer.fail()
 
 

src/allmydata/scripts/tahoe_daemonize.py ¶

@@ -106 +106 @@
     subCommands = [("DaemonizeTahoeNode", None, usage.Options, "node")]
 
+    stderr = sys.stderr
+
 
 class DaemonizeTheRealService(Service, HookMixin):
@@ -123 +125 @@
             "running": None,
         }
+        self.stderr = options.parent.stderr
 
     def startService(self):
@@ -144 +147 @@
             def handle_config_error(fail):
                 fail.trap(UnknownConfigError)
-                sys.stderr.write("\nConfiguration error:\n{}\n\n".format(fail.value))
+                self.stderr.write("\nConfiguration error:\n{}\n\n".format(fail.value))
                 reactor.stop()
                 return
@@ -205 +208 @@
 
     twistd_config = MyTwistdConfig()
+    twistd_config.stdout = out
+    twistd_config.stderr = err
     try:
         twistd_config.parseOptions(twistd_args)

src/allmydata/storage_client.py ¶

@@ -35 +35 @@
 from twisted.internet import defer
 from twisted.application import service
-
+from eliot import (
+    log_call,
+)
 from foolscap.api import eventually
 from allmydata.interfaces import (
@@ -91 +93 @@
         self._connected_high_water_mark = 0
 
+    @log_call(action_type=u"storage-client:broker:set-static-servers")
     def set_static_servers(self, servers):
-        for (server_id, server) in servers.items():
-            assert isinstance(server_id, unicode) # from YAML
-            server_id = server_id.encode("ascii")
-            self._static_server_ids.add(server_id)
-            handler_overrides = server.get("connections", {})
-            s = NativeStorageServer(server_id, server["ann"],
-                                    self._tub_maker, handler_overrides)
-            s.on_status_changed(lambda _: self._got_connection())
-            s.setServiceParent(self)
-            self.servers[server_id] = s
-            s.start_connecting(self._trigger_connections)
+        # Sorting the items gives us a deterministic processing order.  This
+        # doesn't really matter but it makes the logging behavior more
+        # predictable and easier to test (and at least one test does depend on
+        # this sorted order).
+        for (server_id, server) in sorted(servers.items()):
+            try:
+                storage_server = self._make_storage_server(server_id, server)
+            except Exception:
+                pass
+            else:
+                self._static_server_ids.add(server_id)
+                self.servers[server_id] = storage_server
+                storage_server.setServiceParent(self)
+                storage_server.start_connecting(self._trigger_connections)
+
+    @log_call(
+        action_type=u"storage-client:broker:make-storage-server",
+        include_args=["server_id"],
+        include_result=False,
+    )
+    def _make_storage_server(self, server_id, server):
+        assert isinstance(server_id, unicode) # from YAML
+        server_id = server_id.encode("ascii")
+        handler_overrides = server.get("connections", {})
+        s = NativeStorageServer(server_id, server["ann"],
+                                self._tub_maker, handler_overrides)
+        s.on_status_changed(lambda _: self._got_connection())
+        return s
 
     def when_connected_enough(self, threshold):
@@ -255 +275 @@
         return "?"
 
+
 @implementer(IServer)
 class NativeStorageServer(service.MultiService):

src/allmydata/test/cli/test_daemonize.py ¶

@@ -9 +9 @@
 from allmydata.scripts.tahoe_daemonize import DaemonizeTahoeNodePlugin
 from allmydata.scripts.tahoe_daemonize import DaemonizeOptions
+from allmydata.scripts.tahoe_daemonize import MyTwistdConfig
 
 
 class Util(unittest.TestCase):
+    def setUp(self):
+        self.twistd_options = MyTwistdConfig()
+        self.twistd_options.parseOptions(["DaemonizeTahoeNode"])
+        self.options = self.twistd_options.subOptions
 
     def test_node_type_nothing(self):
@@ -40 +45 @@
             r.stop = lambda: None
             r.callWhenRunning = call
-            service = plug.makeService(None)
+            service = plug.makeService(self.options)
             service.parent = Mock()
             service.startService()
@@ -56 +61 @@
             r.callWhenRunning = call
             r.stop = 'foo'
-            service = plug.makeService(None)
+            service = plug.makeService(self.options)
             service.parent = Mock()
             # we'll raise ValueError because there's no key-generator
@@ -81 +86 @@
             r.stop = lambda: None
             r.callWhenRunning = call
-            service = plug.makeService(None)
+            service = plug.makeService(self.options)
             service.parent = Mock()
             with self.assertRaises(ValueError) as ctx:

src/allmydata/test/cli/test_start.py ¶

@@ -1 +1 @@
 import os
-import sys
 import shutil
 import subprocess
@@ -256 +255 @@
 
         i, o, e = StringIO(), StringIO(), StringIO()
-        with patch.object(sys, 'stdout', o), patch.object(sys, 'stderr', e):
-            runner.dispatch(config, i, o, e)
-
-        output = o.getvalue()
+        runner.dispatch(config, i, o, e)
+
+        output = e.getvalue()
         # should print out the collected logs and an error-code
         self.assertIn(

src/allmydata/test/test_client.py ¶

@@ -2 +2 @@
 import mock
 import twisted
+from yaml import (
+    safe_dump,
+)
+from fixtures import (
+    Fixture,
+    TempDir,
+)
+from eliot.testing import (
+    capture_logging,
+    assertHasAction,
+)
 from twisted.trial import unittest
 from twisted.application import service
 from twisted.internet import defer
+from twisted.python.filepath import (
+    FilePath,
+)
+from testtools.matchers import (
+    Equals,
+    AfterPreprocessing,
+)
+from testtools.twistedsupport import (
+    succeeded,
+)
 
 import allmydata
@@ -21 +42 @@
 from foolscap.api import flushEventualQueue
 import allmydata.test.common_util as testutil
+from allmydata.test.common import (
+    SyncTestCase,
+)
 
 
@@ -667 +691 @@
 
 
+def get_known_server_details(a_client):
+    """
+    Get some details about known storage servers from a client.
+
+    :param _Client a_client: The client to inspect.
+
+    :return: A ``list`` of two-tuples.  Each element of the list corresponds
+        to a "known server".  The first element of each tuple is a server id.
+        The second is the server's announcement.
+    """
+    return list(
+        (s.get_serverid(), s.get_announcement())
+        for s
+        in a_client.storage_broker.get_known_servers()
+    )
+
+
+class StaticServers(Fixture):
+    """
+    Create a ``servers.yaml`` file.
+    """
+    def __init__(self, basedir, server_details):
+        super(StaticServers, self).__init__()
+        self._basedir = basedir
+        self._server_details = server_details
+
+    def _setUp(self):
+        private = self._basedir.child(u"private")
+        private.makedirs()
+        servers = private.child(u"servers.yaml")
+        servers.setContent(safe_dump({
+            u"storage": {
+                serverid: {
+                    u"ann": announcement,
+                }
+                for (serverid, announcement)
+                in self._server_details
+            },
+        }))
+
+
+class StorageClients(SyncTestCase):
+    """
+    Tests for storage-related behavior of ``_Client``.
+    """
+    def setUp(self):
+        super(StorageClients, self).setUp()
+        # Some other tests create Nodes and Node mutates tempfile.tempdir and
+        # that screws us up because we're *not* making a Node.  "Fix" it.  See
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3052 for the real fix,
+        # though.
+        import tempfile
+        tempfile.tempdir = None
+
+        tempdir = TempDir()
+        self.useFixture(tempdir)
+        self.basedir = FilePath(tempdir.path)
+
+    @capture_logging(
+        lambda case, logger: assertHasAction(
+            case,
+            logger,
+            actionType=u"storage-client:broker:set-static-servers",
+            succeeded=True,
+        ),
+    )
+    def test_static_servers(self, logger):
+        """
+        Storage servers defined in ``private/servers.yaml`` are loaded into the
+        storage broker.
+        """
+        serverid = u"v0-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+        announcement = {
+            u"nickname": u"some-storage-server",
+            u"anonymous-storage-FURL": u"pb://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@tcp:storage.example:100/swissnum",
+        }
+        self.useFixture(
+            StaticServers(
+                self.basedir,
+                [(serverid, announcement)],
+            ),
+        )
+        self.assertThat(
+            client.create_client(self.basedir.asTextMode().path),
+            succeeded(
+                AfterPreprocessing(
+                    get_known_server_details,
+                    Equals([(serverid, announcement)]),
+                ),
+            ),
+        )
+
+    @capture_logging(
+        lambda case, logger: assertHasAction(
+            case,
+            logger,
+            actionType=u"storage-client:broker:make-storage-server",
+            succeeded=False,
+        ),
+    )
+    def test_invalid_static_server(self, logger):
+        """
+        An invalid announcement for a static server does not prevent other static
+        servers from being loaded.
+        """
+        # Some good details
+        serverid = u"v1-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+        announcement = {
+            u"nickname": u"some-storage-server",
+            u"anonymous-storage-FURL": u"pb://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@tcp:storage.example:100/swissnum",
+        }
+        self.useFixture(
+            StaticServers(
+                self.basedir,
+                [(serverid, announcement),
+                 # Along with a "bad" server announcement.  Order in this list
+                 # doesn't matter, yaml serializer and Python dicts are going
+                 # to shuffle everything around kind of randomly.
+                 (u"v0-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+                  {u"nickname": u"another-storage-server",
+                   u"anonymous-storage-FURL": None,
+                  }),
+                ],
+            ),
+        )
+        self.assertThat(
+            client.create_client(self.basedir.asTextMode().path),
+            succeeded(
+                AfterPreprocessing(
+                    get_known_server_details,
+                    # It should have the good server details.
+                    Equals([(serverid, announcement)]),
+                ),
+            ),
+        )
+
+
 class Run(unittest.TestCase, testutil.StallMixin):
 

tox.ini ¶

@@ -8 +8 @@
 
 [tox]
-envlist = py27
+envlist = {py27,pypy27}{-coverage,}
 minversion = 2.4