Changeset e8ddd8a in trunk

Timestamp:

2024-12-12T19:16:38Z (4 months ago)

Author:

GitHub <noreply@…>

Children:

4809397, ef55c29

Parents:

dc6ea19 (diff), 4de3508 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

meejah <meejah@…> (2024-12-12 19:16:38)

git-committer:

GitHub <noreply@…> (2024-12-12 19:16:38)

Message:

Merge pull request #1403 from LeastAuthority?/4134.fix-circleci-nix

4134: Fix Nix jobs on CircleCI

Files:

• .circleci/config.yml (modified) (4 diffs)
• .circleci/lib.sh (deleted)
• flake.lock (modified) (4 diffs)
• flake.nix (modified) (4 diffs)
• newsfragments/4134.minor (added)
• nix/klein.nix (deleted)
• nix/pycddl.nix (deleted)
• nix/pyopenssl.nix (deleted)
• nix/python-overrides.nix (modified) (1 diff)
• nix/service-identity.nix (deleted)
• nix/tahoe-lafs.nix (modified) (3 diffs)
• nix/twisted.patch (deleted)
• nix/txi2p.nix (deleted)
• nix/txtorcon.nix (deleted)

.circleci/config.yml ¶

@@ -88 +88 @@
 
       - "nixos":
-          name: "nixos-<<matrix.pythonVersion>>"
-          nixpkgs: "nixpkgs-unstable"
+          name: "<<matrix.nixpkgs>>-<<matrix.pythonVersion>>"
           matrix:
             parameters:
+              nixpkgs:
+                - "nixpkgs-24_11"
               pythonVersion:
-                - "python39"
                 - "python310"
                 - "python311"
+                - "python312"
 
       # Eventually, test against PyPy 3.8
@@ -556 +557 @@
             - "run":
                 name: "Unit Test"
+                environment:
+                  # Once dependencies are built, we can allow some more concurrency for our own
+                  # test suite.
+                  UNITTEST_CORES: 8
                 command: |
-                  source .circleci/lib.sh
-
-                  # Translate the nixpkgs selection into a flake reference we
-                  # can use to override the default nixpkgs input.
-                  NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
-
-                  cache_if_able nix run \
-                    --override-input nixpkgs "$NIXPKGS" \
+                  nix run \
                     .#<<parameters.pythonVersion>>-unittest -- \
                     --jobs $UNITTEST_CORES \
@@ -694 +692 @@
       # Run in a highly Nix-capable environment.
       - <<: *DOCKERHUB_AUTH
-        image: "nixos/nix:2.16.1"
-    environment:
-      # currently, all NixOS builds are broken; ignore them
-      ALLOWED_FAILURE: "yes"
-
-      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
-      # to push to CACHIX_NAME.  CACHIX_NAME tells cachix which cache to push
-      # to.
-      CACHIX_NAME: "tahoe-lafs-opensource"
+        image: "nixos/nix:2.25.3"
+    environment:
       # Let us use features marked "experimental".  For example, most/all of
       # the `nix <subcommand>` forms.
@@ -725 +716 @@
 
     steps:
-      - "run":
-          # Get cachix for Nix-friendly caching.
-          name: "Install Basic Dependencies"
-          command: |
-            # Get some build environment dependencies and let them float on a
-            # certain release branch.  These aren't involved in the actual
-            # package build (only in CI environment setup) so the fact that
-            # they float shouldn't hurt reproducibility.
-            NIXPKGS="nixpkgs/nixos-23.05"
-            nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp
-
-            # Activate our cachix cache for "binary substitution".  This sets
-            # up configuration tht lets Nix download something from the cache
-            # instead of building it locally, if possible.
-            cachix use "${CACHIX_NAME}"
-
       - "checkout"
 
       - "run":
           name: "Build Package"
-          command: |
-            source .circleci/lib.sh
-            NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
-            cache_if_able nix build \
+          environment:
+            # CircleCI build environment looks like it has a zillion and a half cores.
+            # Don't let Nix autodetect this high core count because it blows up memory
+            # usage and fails the test run.  Pick a number of cores that suits the build
+            # environment we're paying for (the free one!).
+            DEPENDENCY_CORES: 3
+          command: |
+            nix build \
               --verbose \
               --print-build-logs \
               --cores "$DEPENDENCY_CORES" \
-              --override-input nixpkgs "$NIXPKGS" \
               .#<<parameters.pythonVersion>>-tahoe-lafs
 

flake.lock ¶

@@ -4 +4 @@
       "flake": false,
       "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "lastModified": 1733328505,
+        "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
         "type": "github"
       },
@@ -22 +22 @@
       },
       "locked": {
-        "lastModified": 1687709756,
-        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
@@ -35 +35 @@
       }
     },
-    "nixpkgs-22_11": {
+    "nixpkgs-24_11": {
       "locked": {
-        "lastModified": 1688392541,
-        "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
+        "lastModified": 1733261153,
+        "narHash": "sha256-eq51hyiaIwtWo19fPEeE0Zr2s83DYMKJoukNLgGGpek=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
+        "rev": "b681065d0919f7eb5309a93cea2cfa84dec9aa88",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-23_05": {
-      "locked": {
-        "lastModified": 1689885880,
-        "narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-unstable": {
-      "locked": {
-        "lastModified": 1689791806,
-        "narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "pull/244135/head",
+        "ref": "nixos-24.11",
         "repo": "nixpkgs",
         "type": "github"
@@ -88 +56 @@
         "flake-utils": "flake-utils",
         "nixpkgs": [
-          "nixpkgs-unstable"
+          "nixpkgs-24_11"
         ],
-        "nixpkgs-22_11": "nixpkgs-22_11",
-        "nixpkgs-23_05": "nixpkgs-23_05",
-        "nixpkgs-unstable": "nixpkgs-unstable"
+        "nixpkgs-24_11": "nixpkgs-24_11"
       }
     },

flake.nix ¶

@@ -1 +1 @@
 {
   description = "Tahoe-LAFS, free and open decentralized data store";
-
-  nixConfig = {
-    # Supply configuration for the build cache updated by our CI system.  This
-    # should allow most users to avoid having to build a large number of
-    # packages (otherwise necessary due to our Python package overrides).
-    substituters = ["https://tahoe-lafs-opensource.cachix.org"];
-    trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="];
-  };
 
   inputs = {
@@ -21 +13 @@
     # releases of NixOS at a time is worthwhile and then pins like these will
     # help us test each of those releases.
-    "nixpkgs-22_11" = {
-      url = github:NixOS/nixpkgs?ref=nixos-22.11;
+    "nixpkgs-24_11" = {
+      url = github:NixOS/nixpkgs?ref=nixos-24.11;
     };
-    "nixpkgs-23_05" = {
-      url = github:NixOS/nixpkgs?ref=nixos-23.05;
-    };
-
-    # We depend on a very new python-cryptography which is not yet available
-    # from any release branch of nixpkgs.  However, it is contained in a PR
-    # currently up for review.  Point our nixpkgs at that for now.
-    "nixpkgs-unstable" = {
-      url = github:NixOS/nixpkgs?ref=pull/244135/head;
-    };
-
-    # Point the default nixpkgs at one of those.  This avoids having getting a
-    # _third_ package set involved and gives a way to provide what should be a
-    # working experience by default (that is, if nixpkgs doesn't get
-    # overridden).
-    nixpkgs.follows = "nixpkgs-unstable";
+
+    # Point the default nixpkgs at one of those.
+    nixpkgs.follows = "nixpkgs-24_11";
 
     # Also get flake-utils for simplified multi-system definitions.
@@ -154 +133 @@
         tahoe-lafs.passthru.extras.i2p ++
         tahoe-lafs.passthru.extras.tor ++
-        tahoe-lafs.passthru.extras.unittest
+        tahoe-lafs.passthru.extras.unittest ++
+        [ hatchling hatch-vcs ]
       )).overrideAttrs (old: {
         # See the similar override in makeRuntimeEnv'.
@@ -220 +200 @@
                 let
                   python = "${makeTestEnv pyVersion}/bin/python";
+                  hatchling = "${makeTestEnv pyVersion}/bin/hatchling";
                 in
                   writeScript "unit-tests"
                     ''
-                    ${python} setup.py update_version
+                    ${hatchling} build --hooks-only # Write _version.py
                     export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
                     export PYTHONPATH=$PWD/src

nix/python-overrides.nix ¶

@@ -43 +43 @@
   };
 
-  # Some dependencies aren't packaged in nixpkgs so supply our own packages.
-  pycddl = self.callPackage ./pycddl.nix { };
-  txi2p = self.callPackage ./txi2p.nix { };
-
-  # Some packages are of somewhat too-old versions - update them.
-  klein = self.callPackage ./klein.nix {
-    # Avoid infinite recursion.
-    inherit (super) klein;
-  };
-  txtorcon = self.callPackage ./txtorcon.nix {
-    inherit (super) txtorcon;
-  };
-
-  # With our customized package set a Twisted unit test fails.  Patch the
-  # Twisted test suite to skip that test.
-  # Filed upstream at https://github.com/twisted/twisted/issues/11877
-  twisted = super.twisted.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [ ./twisted.patch ];
-  });
-
-  # Update the version of pyopenssl - and since we're doing that anyway, we
-  # don't need the docs.  Unfortunately this triggers a lot of rebuilding of
-  # dependent packages.
-  pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix {
-    inherit (super) pyopenssl;
-  });
-
-  # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade
-  # that we did breaks service-identity ... so get a newer version that works.
-  service-identity = self.callPackage ./service-identity.nix { };
-
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
   # we know where a working version lives.
   collections-extended = self.callPackage ./collections-extended.nix {
+    # Avoid infinite recursion.
     inherit (super) collections-extended;
   };

nix/tahoe-lafs.nix ¶

@@ -10 +10 @@
 buildPythonPackage rec {
   inherit pname version;
+  pyproject = true;
   src = tahoe-lafs-src;
   propagatedBuildInputs = with pythonPackages; [
@@ -23 +24 @@
     foolscap
     future
+    hatchling
+    hatch-vcs
     klein
     magic-wormhole
@@ -51 +54 @@
       ];
       i2p = [
-        txi2p
+        txi2p-tahoe
       ];
       unittest = [