Merge branch 'ci-efficiency' into 'next'

make CI more efficient See merge request famedly/conduit!596
2024-09-28 20:08:55 +02:00 · 2024-03-06 00:17:39 +00:00 · 2024-03-06 00:17:39 +00:00 · 20bb214d7e
commit 20bb214d7e
parent 732d331847 ae69da635b
5 changed files with 75 additions and 114 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -15,6 +15,10 @@ before_script:
  - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi
  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi

+  # Add alternate binary cache
+  - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
+
  # Add crane binary cache
  - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi
@ -34,8 +38,11 @@ before_script:

 ci:
  stage: ci
-  image: nixos/nix:2.19.2
+  image: nixos/nix:2.20.4
  script:
+    # Cache the inputs required for the devShell
+    - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+
    - direnv exec . engage
  cache:
    key: nix
@ -43,90 +50,42 @@ ci:
      - target
      - .gitlab-ci.d

-static:x86_64-unknown-linux-musl:
+artifacts:
  stage: artifacts
-  image: nixos/nix:2.19.2
+  image: nixos/nix:2.20.4
  script:
-    # Push artifacts and build requirements to binary cache
    - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+    - cp result/bin/conduit x86_64-unknown-linux-musl

-    # Make the output less difficult to find
-    - cp result/bin/conduit conduit
-  artifacts:
-    paths:
-      - conduit
+    - mkdir -p target/release
+    - cp result/bin/conduit target/release
+    - direnv exec . cargo deb --no-build
+    - mv target/debian/*.deb x86_64-unknown-linux-musl.deb

-static:aarch64-unknown-linux-musl:
-  stage: artifacts
-  image: nixos/nix:2.19.2
-  script:
-    # Push artifacts and build requirements to binary cache
-    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
-
-    # Make the output less difficult to find
-    - cp result/bin/conduit conduit
-  artifacts:
-    paths:
-      - conduit
-
-# Note that although we have an `oci-image-x86_64-unknown-linux-musl` output,
-# we don't build it because it would be largely redundant to this one since it's
-# all containerized anyway.
-oci-image:x86_64-unknown-linux-gnu:
-  stage: artifacts
-  image: nixos/nix:2.19.2
-  script:
-    # Push artifacts and build requirements to binary cache
-    #
    # Since the OCI image package is based on the binary package, this has the
    # fun side effect of uploading the normal binary too. Conduit users who are
    # deploying with Nix can leverage this fact by adding our binary cache to
    # their systems.
+    #
+    # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
+    # output, we don't build it because it would be largely redundant to this
+    # one since it's all containerized anyway.
    - ./bin/nix-build-and-cache .#oci-image
-
-    # Make the output less difficult to find
    - cp result oci-image-amd64.tar.gz
-  artifacts:
-    paths:
-      - oci-image-amd64.tar.gz

-oci-image:aarch64-unknown-linux-musl:
-  stage: artifacts
-  needs:
-    # Wait for the static binary job to finish before starting so we don't have
-    # to build that twice for no reason
-    - static:aarch64-unknown-linux-musl
-  image: nixos/nix:2.19.2
-  script:
-    # Push artifacts and build requirements to binary cache
+    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
+    - cp result/bin/conduit aarch64-unknown-linux-musl
+
    - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
-
-    # Make the output less difficult to find
    - cp result oci-image-arm64v8.tar.gz
  artifacts:
    paths:
+      - x86_64-unknown-linux-musl
+      - aarch64-unknown-linux-musl
+      - x86_64-unknown-linux-musl.deb
+      - oci-image-amd64.tar.gz
      - oci-image-arm64v8.tar.gz

-debian:x86_64-unknown-linux-gnu:
-  stage: artifacts
-  # See also `rust-toolchain.toml`
-  image: rust:1.75.0
-  script:
-    - apt-get update && apt-get install -y --no-install-recommends libclang-dev
-    - cargo install cargo-deb
-    - cargo deb
-
-    # Make the output less difficult to find
-    - mv target/debian/*.deb conduit.deb
-  artifacts:
-    paths:
-      - conduit.deb
-  cache:
-    key: debian
-    paths:
-      - target
-      - .gitlab-ci.d
-
 .push-oci-image:
  stage: publish
  image: docker:25.0.0
@ -158,8 +117,7 @@ debian:x86_64-unknown-linux-gnu:
        docker manifest push $IMAGE_NAME:latest
      fi
  dependencies:
-    - oci-image:x86_64-unknown-linux-gnu
-    - oci-image:aarch64-unknown-linux-musl
+    - artifacts
  only:
    - next
    - master
@ -177,4 +135,4 @@ oci-image:push-dockerhub:
  variables:
    IMAGE_NAME: matrixconduit/matrix-conduit
  before_script:
-    - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD
+    - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD
--- a/DEPLOY.md
+++ b/DEPLOY.md
@ -34,11 +34,11 @@ If you use a system with an older glibc version (e.g. RHEL8), you might need to

 | Target | Type | Download |
 |-|-|-|
-| `x86_64-unknown-linux-gnu` | Dynamically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit.deb?job=debian:x86_64-unknown-linux-gnu) |
-| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:x86_64-unknown-linux-musl) |
-| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/conduit?job=static:aarch64-unknown-linux-musl) |
-| `x86_64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=oci-image:x86_64-unknown-linux-musl) |
-| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=oci-image:aarch64-unknown-linux-musl) |
+| `x86_64-unknown-linux-musl` | Statically linked Debian package | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl.deb?job=artifacts) |
+| `x86_64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/x86_64-unknown-linux-musl?job=artifacts) |
+| `aarch64-unknown-linux-musl` | Statically linked binary | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/aarch64-unknown-linux-musl?job=artifacts) |
+| `x86_64-unknown-linux-gnu` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-amd64.tar.gz?job=artifacts) |
+| `aarch64-unknown-linux-musl` | OCI image | [link](https://gitlab.com/api/v4/projects/famedly%2Fconduit/jobs/artifacts/next/raw/oci-image-arm64v8.tar.gz?job=artifacts) |

 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>
--- a/bin/nix-build-and-cache
+++ b/bin/nix-build-and-cache
@ -9,23 +9,18 @@ INSTALLABLE="$1"
 nix build "$@"

 if [ ! -z ${ATTIC_TOKEN+x} ]; then
-
-nix run --inputs-from . attic -- login \
-    conduit \
-        https://nix.computer.surgery/conduit \
+    nix run --inputs-from . attic -- \
+        login \
+        conduit \
+        "${ATTIC_ENDPOINT:-https://nix.computer.surgery/conduit}" \
        "$ATTIC_TOKEN"

-    push_args=(
-        # Attic and its build dependencies
-        "$(nix path-info --inputs-from . attic)"
-        "$(nix path-info --inputs-from . attic --derivation)"
-
-        # The target installable and its build dependencies
-        "$(nix path-info "$INSTALLABLE" --derivation)"
+    # Push the target installable and its build dependencies
+    nix run --inputs-from . attic -- \
+        push \
+        conduit \
+        "$(nix path-info "$INSTALLABLE" --derivation)" \
        "$(nix path-info "$INSTALLABLE")"
-    )
-
-    nix run --inputs-from . attic -- push conduit "${push_args[@]}"
 else
    echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
 fi
--- a/flake.lock
+++ b/flake.lock
@ -9,11 +9,11 @@
        "nixpkgs-stable": "nixpkgs-stable"
      },
      "locked": {
-        "lastModified": 1705617092,
-        "narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=",
+        "lastModified": 1707922053,
+        "narHash": "sha256-wSZjK+rOXn+UQiP1NbdNn5/UW6UcBxjvlqr2wh++MbM=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b",
+        "rev": "6eabc3f02fae3683bffab483e614bebfcd476b21",
        "type": "github"
      },
      "original": {
@ -51,17 +51,17 @@
        ]
      },
      "locked": {
-        "lastModified": 1706473964,
-        "narHash": "sha256-Fq6xleee/TsX6NbtoRuI96bBuDHMU57PrcK9z1QEKbk=",
+        "lastModified": 1707685877,
+        "narHash": "sha256-XoXRS+5whotelr1rHiZle5t5hDg9kpguS5yk8c8qzOc=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "c798790eabec3e3da48190ae3698ac227aab770c",
+        "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
        "type": "github"
      },
      "original": {
        "owner": "ipetkov",
-        "ref": "master",
        "repo": "crane",
+        "rev": "2c653e4478476a52c6aa3ac0495e4dea7449ea0e",
        "type": "github"
      }
    },
@ -73,11 +73,11 @@
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
-        "lastModified": 1705559032,
-        "narHash": "sha256-Cb+Jd1+Gz4Wi+8elPnUIHnqQmE1qjDRZ+PsJaPaAffY=",
+        "lastModified": 1709619709,
+        "narHash": "sha256-l6EPVJfwfelWST7qWQeP6t/TDK3HHv5uUB1b2vw4mOQ=",
        "owner": "nix-community",
        "repo": "fenix",
-        "rev": "e132ea0eb0c799a2109a91688e499d7bf4962801",
+        "rev": "c8943ea9e98d41325ff57d4ec14736d330b321b2",
        "type": "github"
      },
      "original": {
@ -138,11 +138,11 @@
        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1705309234,
-        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "lastModified": 1709126324,
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
        "type": "github"
      },
      "original": {
@ -200,11 +200,11 @@
    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1705496572,
-        "narHash": "sha256-rPIe9G5EBLXdBdn9ilGc0nq082lzQd0xGGe092R/5QE=",
+        "lastModified": 1709479366,
+        "narHash": "sha256-n6F0n8UV6lnTZbYPl1A9q1BS0p4hduAv1mGAP17CVd0=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "842d9d80cfd4560648c785f8a4e6f3b096790e19",
+        "rev": "b8697e57f10292a6165a20f03d2f42920dfaf973",
        "type": "github"
      },
      "original": {
@ -228,11 +228,11 @@
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
-        "lastModified": 1705523001,
-        "narHash": "sha256-TWq5vJ6m+9HGSDMsQAmz1TMegMi79R3TTyKjnPWsQp8=",
+        "lastModified": 1709571018,
+        "narHash": "sha256-ISFrxHxE0J5g7lDAscbK88hwaT5uewvWoma9TlFmRzM=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
-        "rev": "9d9b34354d2f13e33568c9c55b226dd014a146a0",
+        "rev": "9f14343f9ee24f53f17492c5f9b653427e2ad15e",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -13,7 +13,12 @@
      inputs.nixpkgs.follows = "nixpkgs";
    };
    crane = {
-      url = "github:ipetkov/crane?ref=master";
+      # Pin latest crane that's not affected by the following bugs:
+      #
+      # * <https://github.com/ipetkov/crane/issues/527#issuecomment-1978079140>
+      # * <https://github.com/toml-rs/toml/issues/691>
+      # * <https://github.com/toml-rs/toml/issues/267>
+      url = "github:ipetkov/crane?rev=2c653e4478476a52c6aa3ac0495e4dea7449ea0e";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    attic.url = "github:zhaofengli/attic?ref=main";
@ -50,7 +55,7 @@
        # bindgen needs the build platform's libclang. Apparently due to
        # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
        # quite do the right thing here.
-        pkgs.buildPackages.rustPlatform.bindgenHook
+        pkgs.pkgsBuildHost.rustPlatform.bindgenHook
      ];

      env = pkgs: {
@ -78,7 +83,7 @@
              # these flags when using a different linker. Don't ask me why,
              # though, because I don't know. All I know is it breaks otherwise.
              #
-              # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39
+              # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
              (
                # Nixpkgs doesn't check for x86_64 here but we do, because I
                # observed a failure building statically for x86_64 without
@ -102,7 +107,7 @@
      # even covers the case of build scripts that need native code compiled and
      # run on the build platform (I think).
      #
-      # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78
+      # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
      // (
        let
          inherit (pkgs.rust.lib) envVars;
@ -140,8 +145,8 @@
          "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
          "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
          "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
-          HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc";
-          HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++";
+          HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc";
+          HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++";
        }
      ));

@ -247,6 +252,9 @@
        ] ++ (with pkgsHost; [
          engage

+          # Needed for producing Debian packages
+          cargo-deb
+
          # Needed for Complement
          go
          olm