source: trunk/integration/conftest.py @ 28e442a

Last change on this file since 28e442a was 28e442a, checked in by GitHub <noreply@…>, at 2023-04-03T16:16:58Z

Merge pull request #1280 from exarkun/4001.propagate-parent-process-env

Propagate parent environment to children in the integration tests

Fixes: ticket:4001

  • Property mode set to 100644
File size: 17.3 KB
Line 
1"""
2Ported to Python 3.
3"""
4
5from __future__ import annotations
6
7import sys
8import shutil
9from time import sleep
10from os import mkdir, listdir, environ
11from os.path import join, exists
12from tempfile import mkdtemp, mktemp
13from functools import partial
14from json import loads
15
16from foolscap.furl import (
17    decode_furl,
18)
19
20from eliot import (
21    to_file,
22    log_call,
23)
24
25from twisted.python.filepath import FilePath
26from twisted.python.procutils import which
27from twisted.internet.defer import DeferredList
28from twisted.internet.error import (
29    ProcessExitedAlready,
30    ProcessTerminated,
31)
32
33import pytest
34import pytest_twisted
35
36from .util import (
37    _CollectOutputProtocol,
38    _MagicTextProtocol,
39    _DumpOutputProtocol,
40    _ProcessExitedProtocol,
41    _create_node,
42    _cleanup_tahoe_process,
43    _tahoe_runner_optional_coverage,
44    await_client_ready,
45    TahoeProcess,
46    cli,
47    generate_ssh_key,
48    block_with_timeout,
49)
50
51
52# pytest customization hooks
53
54def pytest_addoption(parser):
55    parser.addoption(
56        "--keep-tempdir", action="store_true", dest="keep",
57        help="Keep the tmpdir with the client directories (introducer, etc)",
58    )
59    parser.addoption(
60        "--coverage", action="store_true", dest="coverage",
61        help="Collect coverage statistics",
62    )
63    parser.addoption(
64        "--force-foolscap", action="store_true", default=False,
65        dest="force_foolscap",
66        help=("If set, force Foolscap only for the storage protocol. " +
67              "Otherwise HTTP will be used.")
68    )
69    parser.addoption(
70        "--runslow", action="store_true", default=False,
71        dest="runslow",
72        help="If set, run tests marked as slow.",
73    )
74
75def pytest_collection_modifyitems(session, config, items):
76    if not config.option.runslow:
77        # The --runslow option was not given; keep only collected items not
78        # marked as slow.
79        items[:] = [
80            item
81            for item
82            in items
83            if item.get_closest_marker("slow") is None
84        ]
85
86
87@pytest.fixture(autouse=True, scope='session')
88def eliot_logging():
89    with open("integration.eliot.json", "w") as f:
90        to_file(f)
91        yield
92
93
94# I've mostly defined these fixtures from "easiest" to "most
95# complicated", and the dependencies basically go "down the
96# page". They're all session-scoped which has the "pro" that we only
97# set up the grid once, but the "con" that each test has to be a
98# little careful they're not stepping on toes etc :/
99
100@pytest.fixture(scope='session')
101@log_call(action_type=u"integration:reactor", include_result=False)
102def reactor():
103    # this is a fixture in case we might want to try different
104    # reactors for some reason.
105    from twisted.internet import reactor as _reactor
106    return _reactor
107
108
109@pytest.fixture(scope='session')
110@log_call(action_type=u"integration:temp_dir", include_args=[])
111def temp_dir(request) -> str:
112    """
113    Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir
114    """
115    tmp = mkdtemp(prefix="tahoe")
116    if request.config.getoption('keep'):
117        print("\nWill retain tempdir '{}'".format(tmp))
118
119    # I'm leaving this in and always calling it so that the tempdir
120    # path is (also) printed out near the end of the run
121    def cleanup():
122        if request.config.getoption('keep'):
123            print("Keeping tempdir '{}'".format(tmp))
124        else:
125            try:
126                shutil.rmtree(tmp, ignore_errors=True)
127            except Exception as e:
128                print("Failed to remove tmpdir: {}".format(e))
129    request.addfinalizer(cleanup)
130
131    return tmp
132
133
134@pytest.fixture(scope='session')
135@log_call(action_type=u"integration:flog_binary", include_args=[])
136def flog_binary():
137    return which('flogtool')[0]
138
139
140@pytest.fixture(scope='session')
141@log_call(action_type=u"integration:flog_gatherer", include_args=[])
142def flog_gatherer(reactor, temp_dir, flog_binary, request):
143    out_protocol = _CollectOutputProtocol()
144    gather_dir = join(temp_dir, 'flog_gather')
145    reactor.spawnProcess(
146        out_protocol,
147        flog_binary,
148        (
149            'flogtool', 'create-gatherer',
150            '--location', 'tcp:localhost:3117',
151            '--port', '3117',
152            gather_dir,
153        ),
154        env=environ,
155    )
156    pytest_twisted.blockon(out_protocol.done)
157
158    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
159    twistd_process = reactor.spawnProcess(
160        twistd_protocol,
161        which('twistd')[0],
162        (
163            'twistd', '--nodaemon', '--python',
164            join(gather_dir, 'gatherer.tac'),
165        ),
166        path=gather_dir,
167        env=environ,
168    )
169    pytest_twisted.blockon(twistd_protocol.magic_seen)
170
171    def cleanup():
172        _cleanup_tahoe_process(twistd_process, twistd_protocol.exited)
173
174        flog_file = mktemp('.flog_dump')
175        flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
176        flog_dir = join(temp_dir, 'flog_gather')
177        flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
178
179        print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
180        reactor.spawnProcess(
181            flog_protocol,
182            flog_binary,
183            (
184                'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
185            ),
186            env=environ,
187        )
188        print("Waiting for flogtool to complete")
189        try:
190            block_with_timeout(flog_protocol.done, reactor)
191        except ProcessTerminated as e:
192            print("flogtool exited unexpectedly: {}".format(str(e)))
193        print("Flogtool completed")
194
195    request.addfinalizer(cleanup)
196
197    with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
198        furl = f.read().strip()
199    return furl
200
201
202@pytest.fixture(scope='session')
203@log_call(
204    action_type=u"integration:introducer",
205    include_args=["temp_dir", "flog_gatherer"],
206    include_result=False,
207)
208def introducer(reactor, temp_dir, flog_gatherer, request):
209    config = '''
210[node]
211nickname = introducer0
212web.port = 4560
213log_gatherer.furl = {log_furl}
214'''.format(log_furl=flog_gatherer)
215
216    intro_dir = join(temp_dir, 'introducer')
217    print("making introducer", intro_dir)
218
219    if not exists(intro_dir):
220        mkdir(intro_dir)
221        done_proto = _ProcessExitedProtocol()
222        _tahoe_runner_optional_coverage(
223            done_proto,
224            reactor,
225            request,
226            (
227                'create-introducer',
228                '--listen=tcp',
229                '--hostname=localhost',
230                intro_dir,
231            ),
232        )
233        pytest_twisted.blockon(done_proto.done)
234
235    # over-write the config file with our stuff
236    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
237        f.write(config)
238
239    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
240    # "start" command.
241    protocol = _MagicTextProtocol('introducer running')
242    transport = _tahoe_runner_optional_coverage(
243        protocol,
244        reactor,
245        request,
246        (
247            'run',
248            intro_dir,
249        ),
250    )
251    request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
252
253    pytest_twisted.blockon(protocol.magic_seen)
254    return TahoeProcess(transport, intro_dir)
255
256
257@pytest.fixture(scope='session')
258@log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"])
259def introducer_furl(introducer, temp_dir):
260    furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl')
261    while not exists(furl_fname):
262        print("Don't see {} yet".format(furl_fname))
263        sleep(.1)
264    furl = open(furl_fname, 'r').read()
265    tubID, location_hints, name = decode_furl(furl)
266    if not location_hints:
267        # If there are no location hints then nothing can ever possibly
268        # connect to it and the only thing that can happen next is something
269        # will hang or time out.  So just give up right now.
270        raise ValueError(
271            "Introducer ({!r}) fURL has no location hints!".format(
272                introducer_furl,
273            ),
274        )
275    return furl
276
277
278@pytest.fixture(scope='session')
279@log_call(
280    action_type=u"integration:tor:introducer",
281    include_args=["temp_dir", "flog_gatherer"],
282    include_result=False,
283)
284def tor_introducer(reactor, temp_dir, flog_gatherer, request):
285    config = '''
286[node]
287nickname = introducer_tor
288web.port = 4561
289log_gatherer.furl = {log_furl}
290'''.format(log_furl=flog_gatherer)
291
292    intro_dir = join(temp_dir, 'introducer_tor')
293    print("making introducer", intro_dir)
294
295    if not exists(intro_dir):
296        mkdir(intro_dir)
297        done_proto = _ProcessExitedProtocol()
298        _tahoe_runner_optional_coverage(
299            done_proto,
300            reactor,
301            request,
302            (
303                'create-introducer',
304                '--tor-control-port', 'tcp:localhost:8010',
305                '--listen=tor',
306                intro_dir,
307            ),
308        )
309        pytest_twisted.blockon(done_proto.done)
310
311    # over-write the config file with our stuff
312    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
313        f.write(config)
314
315    # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
316    # "start" command.
317    protocol = _MagicTextProtocol('introducer running')
318    transport = _tahoe_runner_optional_coverage(
319        protocol,
320        reactor,
321        request,
322        (
323            'run',
324            intro_dir,
325        ),
326    )
327
328    def cleanup():
329        try:
330            transport.signalProcess('TERM')
331            block_with_timeout(protocol.exited, reactor)
332        except ProcessExitedAlready:
333            pass
334    request.addfinalizer(cleanup)
335
336    pytest_twisted.blockon(protocol.magic_seen)
337    return transport
338
339
340@pytest.fixture(scope='session')
341def tor_introducer_furl(tor_introducer, temp_dir):
342    furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl')
343    while not exists(furl_fname):
344        print("Don't see {} yet".format(furl_fname))
345        sleep(.1)
346    furl = open(furl_fname, 'r').read()
347    return furl
348
349
350@pytest.fixture(scope='session')
351@log_call(
352    action_type=u"integration:storage_nodes",
353    include_args=["temp_dir", "introducer_furl", "flog_gatherer"],
354    include_result=False,
355)
356def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request):
357    nodes_d = []
358    # start all 5 nodes in parallel
359    for x in range(5):
360        name = 'node{}'.format(x)
361        web_port=  9990 + x
362        nodes_d.append(
363            _create_node(
364                reactor, request, temp_dir, introducer_furl, flog_gatherer, name,
365                web_port="tcp:{}:interface=localhost".format(web_port),
366                storage=True,
367            )
368        )
369    nodes_status = pytest_twisted.blockon(DeferredList(nodes_d))
370    nodes = []
371    for ok, process in nodes_status:
372        assert ok, "Storage node creation failed: {}".format(process)
373        nodes.append(process)
374    return nodes
375
376@pytest.fixture(scope="session")
377def alice_sftp_client_key_path(temp_dir):
378    # The client SSH key path is typically going to be somewhere else (~/.ssh,
379    # typically), but for convenience sake for testing we'll put it inside node.
380    return join(temp_dir, "alice", "private", "ssh_client_rsa_key")
381
382@pytest.fixture(scope='session')
383@log_call(action_type=u"integration:alice", include_args=[], include_result=False)
384def alice(
385        reactor,
386        temp_dir,
387        introducer_furl,
388        flog_gatherer,
389        storage_nodes,
390        alice_sftp_client_key_path,
391        request,
392):
393    process = pytest_twisted.blockon(
394        _create_node(
395            reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
396            web_port="tcp:9980:interface=localhost",
397            storage=False,
398            # We're going to kill this ourselves, so no need for finalizer to
399            # do it:
400            finalize=False,
401        )
402    )
403    pytest_twisted.blockon(await_client_ready(process))
404
405    # 1. Create a new RW directory cap:
406    cli(process, "create-alias", "test")
407    rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
408
409    # 2. Enable SFTP on the node:
410    host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
411    accounts_path = join(process.node_dir, "private", "accounts")
412    with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
413        f.write("""\
414[sftpd]
415enabled = true
416port = tcp:8022:interface=127.0.0.1
417host_pubkey_file = {ssh_key_path}.pub
418host_privkey_file = {ssh_key_path}
419accounts.file = {accounts_path}
420""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
421    generate_ssh_key(host_ssh_key_path)
422
423    # 3. Add a SFTP access file with an SSH key for auth.
424    generate_ssh_key(alice_sftp_client_key_path)
425    # Pub key format is "ssh-rsa <thekey> <username>". We want the key.
426    ssh_public_key = open(alice_sftp_client_key_path + ".pub").read().strip().split()[1]
427    with open(accounts_path, "w") as f:
428        f.write("""\
429alice-key ssh-rsa {ssh_public_key} {rwcap}
430""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
431
432    # 4. Restart the node with new SFTP config.
433    pytest_twisted.blockon(process.restart_async(reactor, request))
434    pytest_twisted.blockon(await_client_ready(process))
435    print(f"Alice pid: {process.transport.pid}")
436    return process
437
438
439@pytest.fixture(scope='session')
440@log_call(action_type=u"integration:bob", include_args=[], include_result=False)
441def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request):
442    process = pytest_twisted.blockon(
443        _create_node(
444            reactor, request, temp_dir, introducer_furl, flog_gatherer, "bob",
445            web_port="tcp:9981:interface=localhost",
446            storage=False,
447        )
448    )
449    pytest_twisted.blockon(await_client_ready(process))
450    return process
451
452
453@pytest.fixture(scope='session')
454@pytest.mark.skipif(sys.platform.startswith('win'),
455                    'Tor tests are unstable on Windows')
456def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
457    # Try to find Chutney already installed in the environment.
458    try:
459        import chutney
460    except ImportError:
461        # Nope, we'll get our own in a moment.
462        pass
463    else:
464        # We already have one, just use it.
465        return (
466            # from `checkout/lib/chutney/__init__.py` we want to get back to
467            # `checkout` because that's the parent of the directory with all
468            # of the network definitions.  So, great-grand-parent.
469            FilePath(chutney.__file__).parent().parent().parent().path,
470            # There's nothing to add to the environment.
471            {},
472        )
473
474    chutney_dir = join(temp_dir, 'chutney')
475    mkdir(chutney_dir)
476
477    missing = [exe for exe in ["tor", "tor-gencert"] if not which(exe)]
478    if missing:
479        pytest.skip(f"Some command-line tools not found: {missing}")
480
481    # XXX yuck! should add a setup.py to chutney so we can at least
482    # "pip install <path to tarball>" and/or depend on chutney in "pip
483    # install -e .[dev]" (i.e. in the 'dev' extra)
484    #
485    # https://trac.torproject.org/projects/tor/ticket/20343
486    proto = _DumpOutputProtocol(None)
487    reactor.spawnProcess(
488        proto,
489        'git',
490        (
491            'git', 'clone',
492            'https://git.torproject.org/chutney.git',
493            chutney_dir,
494        ),
495        env=environ,
496    )
497    pytest_twisted.blockon(proto.done)
498
499    # XXX: Here we reset Chutney to a specific revision known to work,
500    # since there are no stability guarantees or releases yet.
501    proto = _DumpOutputProtocol(None)
502    reactor.spawnProcess(
503        proto,
504        'git',
505        (
506            'git', '-C', chutney_dir,
507            'reset', '--hard',
508            'c825cba0bcd813c644c6ac069deeb7347d3200ee'
509        ),
510        env=environ,
511    )
512    pytest_twisted.blockon(proto.done)
513
514    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
515
516
517@pytest.fixture(scope='session')
518@pytest.mark.skipif(sys.platform.startswith('win'),
519                    reason='Tor tests are unstable on Windows')
520def tor_network(reactor, temp_dir, chutney, request):
521    """
522    Build a basic Tor network.
523
524    :param chutney: The root directory of a Chutney checkout and a dict of
525        additional environment variables to set so a Python process can use
526        it.
527
528    :return: None
529    """
530    chutney_root, chutney_env = chutney
531    basic_network = join(chutney_root, 'networks', 'basic')
532
533    env = environ.copy()
534    env.update(chutney_env)
535    chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
536    def chutney(argv):
537        proto = _DumpOutputProtocol(None)
538        reactor.spawnProcess(
539            proto,
540            sys.executable,
541            chutney_argv + argv,
542            path=join(chutney_root),
543            env=env,
544        )
545        return proto.done
546
547    # now, as per Chutney's README, we have to create the network
548    # ./chutney configure networks/basic
549    # ./chutney start networks/basic
550    pytest_twisted.blockon(chutney(("configure", basic_network)))
551    pytest_twisted.blockon(chutney(("start", basic_network)))
552
553    # print some useful stuff
554    try:
555        pytest_twisted.blockon(chutney(("status", basic_network)))
556    except ProcessTerminated:
557        print("Chutney.TorNet status failed (continuing)")
558
559    def cleanup():
560        print("Tearing down Chutney Tor network")
561        try:
562            block_with_timeout(chutney(("stop", basic_network)), reactor)
563        except ProcessTerminated:
564            # If this doesn't exit cleanly, that's fine, that shouldn't fail
565            # the test suite.
566            pass
567
568    request.addfinalizer(cleanup)
Note: See TracBrowser for help on using the repository browser.