Changeset 30b7be6f in trunk

Timestamp:

2020-11-07T01:07:30Z (5 years ago)

Author:

meejah <meejah@…>

Branches:

master

Children:

c7f4a1a1

Parents:

f9231444

Message:

remove integration tests/refactoring

Location:

integration

Files:

• conftest.py (modified) (5 diffs)
• grid.py (deleted)
• test_grid_manager.py (deleted)
• test_servers_of_happiness.py (modified) (1 diff)
• test_tor.py (modified) (1 diff)
• test_web.py (modified) (4 diffs)
• util.py (modified) (9 diffs)

integration/conftest.py ¶

@@ -37 +37 @@
     TahoeProcess,
 )
-from grid import (
-    create_port_allocator,
-    create_flog_gatherer,
-    create_grid,
-)
 
 
@@ -76 +71 @@
     from twisted.internet import reactor as _reactor
     return _reactor
-
-
-@pytest.fixture(scope='session')
-@log_call(action_type=u"integration:port_allocator", include_result=False)
-def port_allocator(reactor):
-    return create_port_allocator(start_port=45000)
 
 
@@ -118 +107 @@
 @log_call(action_type=u"integration:flog_gatherer", include_args=[])
 def flog_gatherer(reactor, temp_dir, flog_binary, request):
-    fg = pytest_twisted.blockon(
-        create_flog_gatherer(reactor, request, temp_dir, flog_binary)
-    )
-    return fg
-
-
-@pytest.fixture(scope='session')
-@log_call(action_type=u"integration:grid", include_args=[])
-def grid(reactor, request, temp_dir, flog_gatherer, port_allocator):
-    g = pytest_twisted.blockon(
-        create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator)
-    )
-    return g
-
-
-@pytest.fixture(scope='session')
-def introducer(grid):
-    return grid.introducer
+    out_protocol = _CollectOutputProtocol()
+    gather_dir = join(temp_dir, 'flog_gather')
+    reactor.spawnProcess(
+        out_protocol,
+        flog_binary,
+        (
+            'flogtool', 'create-gatherer',
+            '--location', 'tcp:localhost:3117',
+            '--port', '3117',
+            gather_dir,
+        )
+    )
+    pytest_twisted.blockon(out_protocol.done)
+
+    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
+    twistd_process = reactor.spawnProcess(
+        twistd_protocol,
+        which('twistd')[0],
+        (
+            'twistd', '--nodaemon', '--python',
+            join(gather_dir, 'gatherer.tac'),
+        ),
+        path=gather_dir,
+    )
+    pytest_twisted.blockon(twistd_protocol.magic_seen)
+
+    def cleanup():
+        _cleanup_tahoe_process(twistd_process, twistd_protocol.exited)
+
+        flog_file = mktemp('.flog_dump')
+        flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
+        flog_dir = join(temp_dir, 'flog_gather')
+        flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
+
+        print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
+        reactor.spawnProcess(
+            flog_protocol,
+            flog_binary,
+            (
+                'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
+            ),
+        )
+        print("Waiting for flogtool to complete")
+        try:
+            pytest_twisted.blockon(flog_protocol.done)
+        except ProcessTerminated as e:
+            print("flogtool exited unexpectedly: {}".format(str(e)))
+        print("Flogtool completed")
+
+    request.addfinalizer(cleanup)
+
+    with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
+        furl = f.read().strip()
+    return furl
+
+
+@pytest.fixture(scope='session')
+@log_call(
+    action_type=u"integration:introducer",
+    include_args=["temp_dir", "flog_gatherer"],
+    include_result=False,
+)
+def introducer(reactor, temp_dir, flog_gatherer, request):
+    config = '''
+[node]
+nickname = introducer0
+web.port = 4560
+log_gatherer.furl = {log_furl}
+'''.format(log_furl=flog_gatherer)
+
+    intro_dir = join(temp_dir, 'introducer')
+    print("making introducer", intro_dir)
+
+    if not exists(intro_dir):
+        mkdir(intro_dir)
+        done_proto = _ProcessExitedProtocol()
+        _tahoe_runner_optional_coverage(
+            done_proto,
+            reactor,
+            request,
+            (
+                'create-introducer',
+                '--listen=tcp',
+                '--hostname=localhost',
+                intro_dir,
+            ),
+        )
+        pytest_twisted.blockon(done_proto.done)
+
+    # over-write the config file with our stuff
+    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
+        f.write(config)
+
+    # on windows, "tahoe start" means: run forever in the foreground,
+    # but on linux it means daemonize. "tahoe run" is consistent
+    # between platforms.
+    protocol = _MagicTextProtocol('introducer running')
+    transport = _tahoe_runner_optional_coverage(
+        protocol,
+        reactor,
+        request,
+        (
+            'run',
+            intro_dir,
+        ),
+    )
+    request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
+
+    pytest_twisted.blockon(protocol.magic_seen)
+    return TahoeProcess(transport, intro_dir)
 
 
@@ -141 +222 @@
 @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"])
 def introducer_furl(introducer, temp_dir):
-    return introducer.furl
+    furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl')
+    while not exists(furl_fname):
+        print("Don't see {} yet".format(furl_fname))
+        sleep(.1)
+    furl = open(furl_fname, 'r').read()
+    return furl
 
 
@@ -220 +306 @@
 @log_call(
     action_type=u"integration:storage_nodes",
-    include_args=["grid"],
+    include_args=["temp_dir", "introducer_furl", "flog_gatherer"],
     include_result=False,
 )
-def storage_nodes(grid):
+def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
     nodes_d = []
     # start all 5 nodes in parallel
     for x in range(5):
-        #nodes_d.append(grid.add_storage_node())
-        pytest_twisted.blockon(grid.add_storage_node())
-
+        name = 'node{}'.format(x)
+        web_port=  9990 + x
+        nodes_d.append(
+            _create_node(
+                reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
+                web_port="tcp:{}:interface=localhost".format(web_port),
+                storage=True,
+            )
+        )
     nodes_status = pytest_twisted.blockon(DeferredList(nodes_d))
-    for ok, value in nodes_status:
-        assert ok, "Storage node creation failed: {}".format(value)
-    return grid.storage_servers
+    nodes = []
+    for ok, process in nodes_status:
+        assert ok, "Storage node creation failed: {}".format(process)
+        nodes.append(process)
+    return nodes
 
 

integration/test_servers_of_happiness.py ¶

@@ -40 +40 @@
         yield proto.done
         assert False, "should raise exception"
-    except util.ProcessFailed as e:
-        assert "UploadUnhappinessError" in e.output
+    except Exception as e:
+        assert isinstance(e, ProcessTerminated)
 
-    assert "shares could be placed on only" in proto.output.getvalue()
+    output = proto.output.getvalue()
+    assert "shares could be placed on only" in output

integration/test_tor.py ¶

@@ -77 +77 @@
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
-    if exists(node_dir):
-        raise RuntimeError(
-            "A node already exists in '{}'".format(node_dir)
+    if True:
+        print("creating", node_dir)
+        mkdir(node_dir)
+        proto = util._DumpOutputProtocol(None)
+        reactor.spawnProcess(
+            proto,
+            sys.executable,
+            (
+                sys.executable, '-m', 'allmydata.scripts.runner',
+                'create-node',
+                '--nickname', name,
+                '--introducer', introducer_furl,
+                '--hide-ip',
+                '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
+                '--listen', 'tor',
+                node_dir,
+            )
         )
-    print("creating", node_dir)
-    mkdir(node_dir)
-    proto = util._DumpOutputProtocol(None)
-    reactor.spawnProcess(
-        proto,
-        sys.executable,
-        (
-            sys.executable, '-m', 'allmydata.scripts.runner',
-            'create-node',
-            '--nickname', name,
-            '--introducer', introducer_furl,
-            '--hide-ip',
-            '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
-            '--listen', 'tor',
-            node_dir,
-        )
-    )
-    yield proto.done
+        yield proto.done
 
     with open(join(node_dir, 'tahoe.cfg'), 'w') as f:

integration/test_web.py ¶

@@ -97 +97 @@
     """
 
-    url = util.node_url(storage_nodes[0].process.node_dir, "helper_status")
+    url = util.node_url(storage_nodes[0].node_dir, "helper_status")
     resp = requests.get(url)
     assert resp.status_code >= 200 and resp.status_code < 300
@@ -417 +417 @@
 
     requests.get(
-        util.node_url(storage0.process.node_dir, u"storage"),
+        util.node_url(storage0.node_dir, u"storage"),
     )
 
@@ -428 +428 @@
 
     resp = requests.get(
-        util.node_url(storage0.process.node_dir, u"storage"),
+        util.node_url(storage0.node_dir, u"storage"),
         params={u"t": u"json"},
     )
@@ -440 +440 @@
     """
     resp = requests.get(
-        util.node_url(introducer.process.node_dir, u""),
+        util.node_url(introducer.node_dir, u""),
     )
     assert "Introducer" in resp.content
 
     resp = requests.get(
-        util.node_url(introducer.process.node_dir, u""),
+        util.node_url(introducer.node_dir, u""),
         params={u"t": u"json"},
     )

integration/util.py ¶

@@ -6 +6 @@
 from six.moves import StringIO
 from functools import partial
-from shutil import rmtree
 
 from twisted.internet.defer import Deferred, succeed
@@ -37 +36 @@
 
 
-class ProcessFailed(Exception):
-    """
-    A subprocess has failed.
-
-    :ivar ProcessTerminated reason: the original reason from .processExited
-
-    :ivar StringIO output: all stdout and stderr collected to this point.
-    """
-
-    def __init__(self, reason, output):
-        self.reason = reason
-        self.output = output
-
-    def __str__(self):
-        return "<ProcessFailed: {}>:\n{}".format(self.reason, self.output)
-
-
 class _CollectOutputProtocol(ProcessProtocol):
     """
@@ -60 +42 @@
     process exits (for any reason).
     """
-    def __init__(self, stdin=None):
+    def __init__(self):
         self.done = Deferred()
         self.output = StringIO()
-        self._stdin = stdin
-
-    def connectionMade(self):
-        if self._stdin is not None:
-            self.transport.write(self._stdin)
-            self.transport.closeStdin()
 
     def processEnded(self, reason):
@@ -76 +52 @@
     def processExited(self, reason):
         if not isinstance(reason.value, ProcessDone):
-            self.done.errback(ProcessFailed(reason, self.output.getvalue()))
+            self.done.errback(reason)
 
     def outReceived(self, data):
@@ -148 +124 @@
         print("signaling {} with TERM".format(tahoe_transport.pid))
         tahoe_transport.signalProcess('TERM')
-        print("signaled, blocking on exit {}".format(exited))
+        print("signaled, blocking on exit")
         pytest_twisted.blockon(exited)
         print("exited, goodbye")
     except ProcessExitedAlready:
         pass
-
-
-def run_tahoe(reactor, request, *args, **kwargs):
-    """
-    Helper to run tahoe with optional coverage.
-
-    :returns: a Deferred that fires when the command is done (or a
-        ProcessFailed exception if it exits non-zero)
-    """
-    stdin = kwargs.get("stdin", None)
-    protocol = _CollectOutputProtocol(stdin=stdin)
-    process = _tahoe_runner_optional_coverage(protocol, reactor, request, args)
-    process.exited = protocol.done
-    return protocol.done
 
 
@@ -270 +232 @@
         created_d = succeed(None)
     else:
-        print("creating: {}".format(node_dir))
+        print("creating", node_dir)
         mkdir(node_dir)
         done_proto = _ProcessExitedProtocol()
@@ -295 +257 @@
             config_path = join(node_dir, 'tahoe.cfg')
             config = get_config(config_path)
-            set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl)
+            set_config(config, 'node', 'log_gatherer.furl', flog_gatherer)
             write_config(config_path, config)
         created_d.addCallback(created)
@@ -482 +444 @@
 
 
-def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1):
+def await_client_ready(tahoe, timeout=10, liveness=60*2):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
@@ -506 +468 @@
             continue
 
-        if len(js['servers']) < servers:
-            print("waiting because fewer than {} server(s)".format(servers))
+        if len(js['servers']) == 0:
+            print("waiting because no servers at all")
             time.sleep(1)
             continue