source: trunk/.circleci/config.yml

Last change on this file was 49df402, checked in by Jean-Paul Calderone <exarkun@…>, at 2021-09-29T19:48:33Z

maybe this is the right url

  • Property mode set to 100644
File size: 16.7 KB
Line 
1# https://circleci.com/docs/2.0/
2
3# We use version 2.1 of CircleCI's configuration format (the docs are still at
4# the 2.0 link) in order to have access to Windows executors. This means we
5# can't use dots in job names anymore. They have a new "parameters" feature
6# that is supposed to remove the need to have version numbers in job names (the
7# source of our dots), but switching to that is going to be a bigger refactor:
8#
9#   https://discuss.circleci.com/t/v2-1-job-name-validation/31123
10#   https://circleci.com/docs/2.0/reusing-config/
11#
12version: 2.1
13
14workflows:
15  ci:
16    jobs:
17      # Start with jobs testing various platforms.
18      - "debian-9":
19          {}
20      - "debian-10":
21          requires:
22            - "debian-9"
23
24      - "ubuntu-20-04":
25          {}
26      - "ubuntu-18-04":
27          requires:
28            - "ubuntu-20-04"
29      - "ubuntu-16-04":
30          requires:
31            - "ubuntu-20-04"
32
33      - "fedora-29":
34          {}
35      - "fedora-28":
36          requires:
37            - "fedora-29"
38
39      - "centos-8":
40          {}
41
42      - "nixos-19-09":
43          {}
44
45      - "nixos-21-05":
46          {}
47
48      # Test against PyPy 2.7
49      - "pypy27-buster":
50          {}
51
52      # Just one Python 3.6 configuration while the port is in-progress.
53      - "python36":
54          {}
55
56      # Other assorted tasks and configurations
57      - "lint":
58          {}
59      - "codechecks3":
60          {}
61      - "pyinstaller":
62          {}
63      - "deprecations":
64          {}
65      - "c-locale":
66          {}
67      # Any locale other than C or UTF-8.
68      - "another-locale":
69          {}
70
71      - "integration":
72          requires:
73            # If the unit test suite doesn't pass, don't bother running the
74            # integration tests.
75            - "debian-9"
76
77      - "typechecks":
78          {}
79      - "docs":
80          {}
81
82  images:
83    # Build the Docker images used by the ci jobs.  This makes the ci jobs
84    # faster and takes various spurious failures out of the critical path.
85    triggers:
86      # Build once a day
87      - schedule:
88          cron: "0 0 * * *"
89          filters:
90            branches:
91              only:
92                - "master"
93
94    jobs:
95      # Every job that pushes a Docker image from Docker Hub needs to provide
96      # credentials.  Use this first job to define a yaml anchor that can be
97      # used to supply a CircleCI job context which makes Docker Hub
98      # credentials available in the environment.
99      #
100      # Contexts are managed in the CircleCI web interface:
101      #
102      #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
103      - "build-image-debian-10": &DOCKERHUB_CONTEXT
104          context: "dockerhub-auth"
105      - "build-image-debian-9":
106          <<: *DOCKERHUB_CONTEXT
107      - "build-image-ubuntu-16-04":
108          <<: *DOCKERHUB_CONTEXT
109      - "build-image-ubuntu-18-04":
110          <<: *DOCKERHUB_CONTEXT
111      - "build-image-ubuntu-20-04":
112          <<: *DOCKERHUB_CONTEXT
113      - "build-image-fedora-28":
114          <<: *DOCKERHUB_CONTEXT
115      - "build-image-fedora-29":
116          <<: *DOCKERHUB_CONTEXT
117      - "build-image-centos-8":
118          <<: *DOCKERHUB_CONTEXT
119      - "build-image-pypy27-buster":
120          <<: *DOCKERHUB_CONTEXT
121      - "build-image-python36-ubuntu":
122          <<: *DOCKERHUB_CONTEXT
123
124
125jobs:
126  dockerhub-auth-template:
127    # This isn't a real job.  It doesn't get scheduled as part of any
128    # workflow.  Instead, it's just a place we can hang a yaml anchor to
129    # finish the Docker Hub authentication configuration.  Workflow jobs using
130    # the DOCKERHUB_CONTEXT anchor will have access to the environment
131    # variables used here.  These variables will allow the Docker Hub image
132    # pull to be authenticated and hopefully avoid hitting and rate limits.
133    docker: &DOCKERHUB_AUTH
134      - image: "null"
135        auth:
136          username: $DOCKERHUB_USERNAME
137          password: $DOCKERHUB_PASSWORD
138
139    steps:
140      - run:
141          name: "CircleCI YAML schema conformity"
142          command: |
143            # This isn't a real command.  We have to have something in this
144            # space, though, or the CircleCI yaml schema validator gets angry.
145            # Since this job is never scheduled this step is never run so the
146            # actual value here is irrelevant.
147
148  lint:
149    docker:
150      - <<: *DOCKERHUB_AUTH
151        image: "circleci/python:2"
152
153    steps:
154      - "checkout"
155
156      - run:
157          name: "Install tox"
158          command: |
159            pip install --user tox
160
161      - run:
162          name: "Static-ish code checks"
163          command: |
164            ~/.local/bin/tox -e codechecks
165
166  codechecks3:
167    docker:
168      - <<: *DOCKERHUB_AUTH
169        image: "circleci/python:3"
170
171    steps:
172      - "checkout"
173
174      - run:
175          name: "Install tox"
176          command: |
177            pip install --user tox
178
179      - run:
180          name: "Static-ish code checks"
181          command: |
182            ~/.local/bin/tox -e codechecks3
183
184  pyinstaller:
185    docker:
186      - <<: *DOCKERHUB_AUTH
187        image: "circleci/python:2"
188
189    steps:
190      - "checkout"
191
192      - run:
193          name: "Install tox"
194          command: |
195            pip install --user tox
196
197      - run:
198          name: "Make PyInstaller executable"
199          command: |
200            ~/.local/bin/tox -e pyinstaller
201
202      - run:
203          # To verify that the resultant PyInstaller-generated binary executes
204          # cleanly (i.e., that it terminates with an exit code of 0 and isn't
205          # failing due to import/packaging-related errors, etc.).
206          name: "Test PyInstaller executable"
207          command: |
208            dist/Tahoe-LAFS/tahoe --version
209
210  debian-9: &DEBIAN
211    docker:
212      - <<: *DOCKERHUB_AUTH
213        image: "tahoelafsci/debian:9-py2.7"
214        user: "nobody"
215
216    environment: &UTF_8_ENVIRONMENT
217      # In general, the test suite is not allowed to fail while the job
218      # succeeds.  But you can set this to "yes" if you want it to be
219      # otherwise.
220      ALLOWED_FAILURE: "no"
221      # Tell Hypothesis which configuration we want it to use.
222      TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
223      # Tell the C runtime things about character encoding (mainly to do with
224      # filenames and argv).
225      LANG: "en_US.UTF-8"
226      # Select a tox environment to run for this job.
227      TAHOE_LAFS_TOX_ENVIRONMENT: "py27"
228      # Additional arguments to pass to tox.
229      TAHOE_LAFS_TOX_ARGS: ""
230      # The path in which test artifacts will be placed.
231      ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
232      # Convince all of our pip invocations to look at the cached wheelhouse
233      # we maintain.
234      WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
235      PIP_FIND_LINKS: "file:///tmp/wheelhouse"
236      # Upload the coverage report.
237      UPLOAD_COVERAGE: ""
238
239    # pip cannot install packages if the working directory is not readable.
240    # We want to run a lot of steps as nobody instead of as root.
241    working_directory: "/tmp/project"
242
243    steps:
244      - "checkout"
245      - run: &SETUP_VIRTUALENV
246          name: "Setup virtualenv"
247          command: |
248            /tmp/project/.circleci/setup-virtualenv.sh \
249                "/tmp/venv" \
250                "/tmp/project" \
251                "${WHEELHOUSE_PATH}" \
252                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
253                "${TAHOE_LAFS_TOX_ARGS}"
254
255      - run: &RUN_TESTS
256          name: "Run test suite"
257          command: |
258            /tmp/project/.circleci/run-tests.sh \
259                "/tmp/venv" \
260                "/tmp/project" \
261                "${ALLOWED_FAILURE}" \
262                "${ARTIFACTS_OUTPUT_PATH}" \
263                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
264                "${TAHOE_LAFS_TOX_ARGS}"
265          # trial output gets directed straight to a log.  avoid the circleci
266          # timeout while the test suite runs.
267          no_output_timeout: "20m"
268
269      - store_test_results: &STORE_TEST_RESULTS
270          path: "/tmp/artifacts/junit"
271
272      - store_artifacts: &STORE_TEST_LOG
273          # Despite passing --workdir /tmp to tox above, it still runs trial
274          # in the project source checkout.
275          path: "/tmp/project/_trial_temp/test.log"
276
277      - store_artifacts: &STORE_ELIOT_LOG
278          # Despite passing --workdir /tmp to tox above, it still runs trial
279          # in the project source checkout.
280          path: "/tmp/project/eliot.log"
281
282      - store_artifacts: &STORE_OTHER_ARTIFACTS
283          # Store any other artifacts, too.  This is handy to allow other jobs
284          # sharing most of the definition of this one to be able to
285          # contribute artifacts easily.
286          path: "/tmp/artifacts"
287
288      - run: &SUBMIT_COVERAGE
289          name: "Submit coverage results"
290          command: |
291            if [ -n "${UPLOAD_COVERAGE}" ]; then
292              /tmp/venv/bin/codecov
293            fi
294
295
296  debian-10:
297    <<: *DEBIAN
298    docker:
299      - <<: *DOCKERHUB_AUTH
300        image: "tahoelafsci/debian:10-py2.7"
301        user: "nobody"
302
303
304  pypy27-buster:
305    <<: *DEBIAN
306    docker:
307      - <<: *DOCKERHUB_AUTH
308        image: "tahoelafsci/pypy:buster-py2"
309        user: "nobody"
310
311    environment:
312      <<: *UTF_8_ENVIRONMENT
313      # We don't do coverage since it makes PyPy far too slow:
314      TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
315      # Since we didn't collect it, don't upload it.
316      UPLOAD_COVERAGE: ""
317
318
319  c-locale:
320    <<: *DEBIAN
321
322    environment:
323      <<: *UTF_8_ENVIRONMENT
324      LANG: "C"
325
326
327  another-locale:
328    <<: *DEBIAN
329
330    environment:
331      <<: *UTF_8_ENVIRONMENT
332      # aka "Latin 1"
333      LANG: "en_US.ISO-8859-1"
334
335
336  deprecations:
337    <<: *DEBIAN
338
339    environment:
340      <<: *UTF_8_ENVIRONMENT
341      # Select the deprecations tox environments.
342      TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
343      # Put the logs somewhere we can report them.
344      TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
345      # The deprecations tox environments don't do coverage measurement.
346      UPLOAD_COVERAGE: ""
347
348
349  integration:
350    <<: *DEBIAN
351
352    environment:
353      <<: *UTF_8_ENVIRONMENT
354      # Select the integration tests tox environments.
355      TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
356      # Disable artifact collection because py.test can't produce any.
357      ARTIFACTS_OUTPUT_PATH: ""
358
359    steps:
360      - "checkout"
361      # DRY, YAML-style.  See the debian-9 steps.
362      - run: *SETUP_VIRTUALENV
363      - run: *RUN_TESTS
364
365
366  ubuntu-16-04:
367    <<: *DEBIAN
368    docker:
369      - <<: *DOCKERHUB_AUTH
370        image: "tahoelafsci/ubuntu:16.04-py2.7"
371        user: "nobody"
372
373
374  ubuntu-18-04: &UBUNTU_18_04
375    <<: *DEBIAN
376    docker:
377      - <<: *DOCKERHUB_AUTH
378        image: "tahoelafsci/ubuntu:18.04-py2.7"
379        user: "nobody"
380
381
382  python36:
383    <<: *UBUNTU_18_04
384    docker:
385      - <<: *DOCKERHUB_AUTH
386        image: "tahoelafsci/ubuntu:18.04-py3"
387        user: "nobody"
388
389    environment:
390      <<: *UTF_8_ENVIRONMENT
391      # The default trial args include --rterrors which is incompatible with
392      # this reporter on Python 3.  So drop that and just specify the
393      # reporter.
394      TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
395      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
396
397
398  ubuntu-20-04:
399    <<: *DEBIAN
400    docker:
401      - <<: *DOCKERHUB_AUTH
402        image: "tahoelafsci/ubuntu:20.04"
403        user: "nobody"
404
405
406  centos-8: &RHEL_DERIV
407    docker:
408      - <<: *DOCKERHUB_AUTH
409        image: "tahoelafsci/centos:8-py2"
410        user: "nobody"
411
412    environment: *UTF_8_ENVIRONMENT
413
414    # pip cannot install packages if the working directory is not readable.
415    # We want to run a lot of steps as nobody instead of as root.
416    working_directory: "/tmp/project"
417
418    steps:
419      - "checkout"
420      - run: *SETUP_VIRTUALENV
421      - run: *RUN_TESTS
422      - store_test_results: *STORE_TEST_RESULTS
423      - store_artifacts: *STORE_TEST_LOG
424      - store_artifacts: *STORE_ELIOT_LOG
425      - store_artifacts: *STORE_OTHER_ARTIFACTS
426      - run: *SUBMIT_COVERAGE
427
428
429  fedora-28:
430    <<: *RHEL_DERIV
431    docker:
432      - <<: *DOCKERHUB_AUTH
433        image: "tahoelafsci/fedora:28-py"
434        user: "nobody"
435
436
437  fedora-29:
438    <<: *RHEL_DERIV
439    docker:
440      - <<: *DOCKERHUB_AUTH
441        image: "tahoelafsci/fedora:29-py"
442        user: "nobody"
443
444  nixos-19-09: &NIXOS
445    docker:
446      # Run in a highly Nix-capable environment.
447      - <<: *DOCKERHUB_AUTH
448        image: "nixorg/nix:circleci"
449
450    environment:
451      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
452      SOURCE: "nix/"
453
454    steps:
455      - "checkout"
456      - "run":
457          name: "Build and Test"
458          command: |
459            # CircleCI build environment looks like it has a zillion and a
460            # half cores.  Don't let Nix autodetect this high core count
461            # because it blows up memory usage and fails the test run.  Pick a
462            # number of cores that suites the build environment we're paying
463            # for (the free one!).
464            #
465            # Also, let it run more than one job at a time because we have to
466            # build a couple simple little dependencies that don't take
467            # advantage of multiple cores and we get a little speedup by doing
468            # them in parallel.
469            nix-build --cores 3 --max-jobs 2 "$SOURCE"
470
471  nixos-21-05:
472    <<: *NIXOS
473
474    environment:
475      # Note this doesn't look more similar to the 19.09 NIX_PATH URL because
476      # there was some internal shuffling by the NixOS project about how they
477      # publish stable revisions.
478      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz"
479      SOURCE: "nix/py3.nix"
480
481  typechecks:
482    docker:
483      - <<: *DOCKERHUB_AUTH
484        image: "tahoelafsci/ubuntu:18.04-py3"
485
486    steps:
487      - "checkout"
488      - run:
489          name: "Validate Types"
490          command: |
491            /tmp/venv/bin/tox -e typechecks
492
493  docs:
494    docker:
495      - <<: *DOCKERHUB_AUTH
496        image: "tahoelafsci/ubuntu:18.04-py3"
497
498    steps:
499      - "checkout"
500      - run:
501          name: "Build documentation"
502          command: |
503            /tmp/venv/bin/tox -e docs
504
505  build-image: &BUILD_IMAGE
506    # This is a template for a job to build a Docker image that has as much of
507    # the setup as we can manage already done and baked in.  This cuts down on
508    # the per-job setup time the actual testing jobs have to perform - by
509    # perhaps 10% - 20%.
510    #
511    # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
512    docker:
513      - <<: *DOCKERHUB_AUTH
514        image: "docker:17.05.0-ce-git"
515
516    environment:
517      DISTRO: "tahoelafsci/<DISTRO>:foo-py2"
518      TAG: "tahoelafsci/distro:<TAG>-py2"
519      PYTHON_VERSION: "tahoelafsci/distro:tag-py<PYTHON_VERSION}"
520
521    steps:
522      - "checkout"
523      - "setup_remote_docker"
524      - run:
525          name: "Log in to Dockerhub"
526          command: |
527            docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
528      - run:
529          name: "Build image"
530          command: |
531            docker \
532                build \
533                --build-arg TAG=${TAG} \
534                --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
535                -t tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} \
536                -f ~/project/.circleci/Dockerfile.${DISTRO} \
537                ~/project/
538      - run:
539          name: "Push image"
540          command: |
541            docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
542
543
544  build-image-debian-10:
545    <<: *BUILD_IMAGE
546
547    environment:
548      DISTRO: "debian"
549      TAG: "10"
550      PYTHON_VERSION: "2.7"
551
552
553  build-image-debian-9:
554    <<: *BUILD_IMAGE
555
556    environment:
557      DISTRO: "debian"
558      TAG: "9"
559      PYTHON_VERSION: "2.7"
560
561
562  build-image-ubuntu-16-04:
563    <<: *BUILD_IMAGE
564
565    environment:
566      DISTRO: "ubuntu"
567      TAG: "16.04"
568      PYTHON_VERSION: "2.7"
569
570
571  build-image-ubuntu-18-04:
572    <<: *BUILD_IMAGE
573
574    environment:
575      DISTRO: "ubuntu"
576      TAG: "18.04"
577      PYTHON_VERSION: "2.7"
578
579
580  build-image-python36-ubuntu:
581    <<: *BUILD_IMAGE
582
583    environment:
584      DISTRO: "ubuntu"
585      TAG: "18.04"
586      PYTHON_VERSION: "3"
587
588
589  build-image-ubuntu-20-04:
590    <<: *BUILD_IMAGE
591
592    environment:
593      DISTRO: "ubuntu"
594      TAG: "20.04"
595      PYTHON_VERSION: "2.7"
596
597
598  build-image-centos-8:
599    <<: *BUILD_IMAGE
600
601    environment:
602      DISTRO: "centos"
603      TAG: "8"
604      PYTHON_VERSION: "2"
605
606
607  build-image-fedora-28:
608    <<: *BUILD_IMAGE
609
610    environment:
611      DISTRO: "fedora"
612      TAG: "28"
613      # The default on Fedora (this version anyway) is still Python 2.
614      PYTHON_VERSION: ""
615
616
617  build-image-fedora-29:
618    <<: *BUILD_IMAGE
619
620    environment:
621      DISTRO: "fedora"
622      TAG: "29"
623
624
625  build-image-pypy27-buster:
626    <<: *BUILD_IMAGE
627
628    environment:
629      DISTRO: "pypy"
630      TAG: "buster"
631      # We only have Python 2 for PyPy right now so there's no support for
632      # setting up PyPy 3 in the image building toolchain.  This value is just
633      # for constructing the right Docker image tag.
634      PYTHON_VERSION: "2"
Note: See TracBrowser for help on using the repository browser.