Changeset 425d16e in trunk

Timestamp:

2019-08-09T03:03:39Z (6 years ago)

Author:

meejah <meejah@…>

Branches:

master

Children:

ff23ad1

Parents:

9b3d37e

git-author:

meejah <meejah@…> (2019-08-09 01:13:02)

git-committer:

meejah <meejah@…> (2019-08-09 03:03:39)

Message:

get rid of time.sleep usage for 'readiness'

Location:

integration

Files:

• conftest.py (modified) (9 diffs)
• util.py (modified) (2 diffs)

integration/conftest.py ¶

@@ -3 +3 @@
 import sys
 import shutil
+import json
 from time import sleep
 from os import mkdir, listdir, environ
@@ -16 +17 @@
 
 from twisted.python.procutils import which
+from twisted.internet.defer import DeferredList
 from twisted.internet.error import (
     ProcessExitedAlready,
@@ -33 +35 @@
     _cleanup_tahoe_process,
     _tahoe_runner_optional_coverage,
+    await_client_ready,
 )
 
@@ -307 +310 @@
 )
 def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
-    nodes = []
+    nodes_d = []
     # start all 5 nodes in parallel
     for x in range(5):
         name = 'node{}'.format(x)
         # tub_port = 9900 + x
-        nodes.append(
-            pytest_twisted.blockon(
-                _create_node(
-                    reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
-                    web_port=None, storage=True,
-                )
+        nodes_d.append(
+            _create_node(
+                reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
+                web_port=None, storage=True,
             )
         )
-    #nodes = pytest_twisted.blockon(DeferredList(nodes))
+    nodes_status = pytest_twisted.blockon(DeferredList(nodes_d))
+    nodes = []
+    for ok, process in nodes_status:
+        assert ok, "Storage node creation failed: {}".format(process)
+        nodes.append(process)
     return nodes
 
@@ -339 +344 @@
         )
     )
+    await_client_ready(process)
     return process
 
@@ -357 +363 @@
         )
     )
+    await_client_ready(process)
     return process
 
@@ -369 +376 @@
         # storage servers aren't "really" ready to roll yet (uploads fairly
         # consistently fail if we don't hack in this pause...)
-        import time ; time.sleep(5)
         proto = _CollectOutputProtocol()
         _tahoe_runner_optional_coverage(
@@ -410 +416 @@
             magic_text = 'Completed initial Magic Folder scan successfully'
             pytest_twisted.blockon(_run_node(reactor, node_dir, request, magic_text))
+            await_client_ready(alice)
     return invite
 
@@ -447 +454 @@
     magic_text = 'Completed initial Magic Folder scan successfully'
     pytest_twisted.blockon(_run_node(reactor, bob_dir, request, magic_text))
+    await_client_ready(bob)
     return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob'))
 

integration/util.py ¶

@@ -1 +1 @@
 import sys
 import time
+import json
 from os import mkdir
 from os.path import exists, join
@@ -430 +431 @@
 
 
+def await_client_ready(process, timeout=10, liveness=60*2):
+    """
+    Uses the status API to wait for a client-type node to be
+    'ready'. A client is deemed ready if:
+      - it answers http://<node_url>/statistics/?t-json/
+      - there is at least one storage-server connected
+      - every storage-server has a "last_received_data" and it is
+        within the last `liveness` seconds
+
+    We will try for up to `timeout` seconds for the above conditions
+    to be true. Otherwise, an exception is raised
+    """
+    start = time.time()
+    while (time.time() - start) < float(timeout):
+        time.sleep(1)
+        try:
+            data = web_get(process.node_dir, u"", params={u"t": u"json"})
+        except ValueError as e:
+            print("waiting because '{}'".format(e))
+        js = json.loads(data)
+        if len(js['servers']) == 0:
+            print("waiting because no servers at all")
+            continue
+        server_times = [
+            server['last_received_data']
+            for server in js['servers']
+        ]
+        # if any times are null/None that server has never been
+        # contacted (so it's down still, probably)
+        if any([t is None for t in server_times]):
+            print("waiting because at least one server not contacted")
+            continue
+
+        # check that all times are 'recent enough'
+        if any([time.time() - t > liveness for t in server_times]):
+            print("waiting because at least one server too old")
+            continue
+
+        # we have a status with at least one server, and all servers
+        # have been contacted recently
+        return True
+    # we only fall out of the loop when we've timed out
+    raise RuntimeError(
+        "Waited {} seconds for {} to be 'ready' but it never was".format(
+            timeout,
+            process.node_dir,
+        )
+    )
+
+
 def magic_folder_cli(request, reactor, node_dir, *argv):
     return cli(request, reactor, node_dir, "magic-folder", *argv)