mirror of
https://gitlab.com/famedly/conduit.git
synced 2024-08-19 04:24:41 +02:00
324 lines
13 KiB
YAML
324 lines
13 KiB
YAML
stages:
|
|
- build
|
|
- build docker image
|
|
- test
|
|
- upload artifacts
|
|
|
|
variables:
|
|
GIT_SUBMODULE_STRATEGY: recursive
|
|
FF_USE_FASTZIP: 1
|
|
CACHE_COMPRESSION_LEVEL: fastest
|
|
|
|
# --------------------------------------------------------------------- #
|
|
# Cargo: Compiling for different architectures #
|
|
# --------------------------------------------------------------------- #
|
|
|
|
.build-cargo-shared-settings:
|
|
stage: "build"
|
|
needs: []
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "master"'
|
|
- if: '$CI_COMMIT_BRANCH == "next"'
|
|
- if: "$CI_COMMIT_TAG"
|
|
interruptible: true
|
|
image: "rust:1.58"
|
|
tags: ["docker"]
|
|
services: ["docker:dind"]
|
|
variables:
|
|
DOCKER_HOST: tcp://docker:2375/
|
|
DOCKER_TLS_CERTDIR: ""
|
|
DOCKER_DRIVER: overlay2
|
|
SHARED_PATH: $CI_PROJECT_DIR/shared
|
|
CARGO_PROFILE_RELEASE_LTO: "true"
|
|
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
|
|
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
|
|
before_script:
|
|
- 'echo "Building for target $TARGET"'
|
|
- "rustup show && rustc --version && cargo --version" # Print version info for debugging
|
|
# install cross-compiling prerequisites
|
|
- 'apt-get update && apt-get install -y docker.io && docker version' # install docker
|
|
- 'cargo install cross && cross --version' # install cross
|
|
# fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
|
|
- 'mkdir -p $SHARED_PATH/cargo'
|
|
- 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo'
|
|
- 'cp -r $RUSTUP_HOME $SHARED_PATH'
|
|
- 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup'
|
|
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
|
|
- if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi
|
|
script:
|
|
# cross-compile conduit for target
|
|
- 'time ./cross/build.sh --locked --release'
|
|
- 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
|
|
# print information about linking for debugging
|
|
- 'file conduit-$TARGET' # print file information
|
|
- 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
|
|
cache:
|
|
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
|
|
key: 'cargo-cache-$TARGET'
|
|
paths:
|
|
- $SHARED_PATH/cargo/registry/index
|
|
- $SHARED_PATH/cargo/registry/cache
|
|
- $SHARED_PATH/cargo/git/db
|
|
artifacts:
|
|
expire_in: never
|
|
|
|
build:release:cargo:x86_64-unknown-linux-musl-with-debug:
|
|
extends: .build-cargo-shared-settings
|
|
variables:
|
|
CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
|
|
TARGET: "x86_64-unknown-linux-musl"
|
|
after_script:
|
|
- "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug"
|
|
artifacts:
|
|
name: "conduit-x86_64-unknown-linux-musl-with-debug"
|
|
paths:
|
|
- "conduit-x86_64-unknown-linux-musl-with-debug"
|
|
expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug"
|
|
|
|
build:release:cargo:x86_64-unknown-linux-musl:
|
|
extends: .build-cargo-shared-settings
|
|
variables:
|
|
TARGET: "x86_64-unknown-linux-musl"
|
|
artifacts:
|
|
name: "conduit-x86_64-unknown-linux-musl"
|
|
paths:
|
|
- "conduit-x86_64-unknown-linux-musl"
|
|
expose_as: "Conduit for x86_64-unknown-linux-musl"
|
|
|
|
build:release:cargo:arm-unknown-linux-musleabihf:
|
|
extends: .build-cargo-shared-settings
|
|
variables:
|
|
TARGET: "arm-unknown-linux-musleabihf"
|
|
artifacts:
|
|
name: "conduit-arm-unknown-linux-musleabihf"
|
|
paths:
|
|
- "conduit-arm-unknown-linux-musleabihf"
|
|
expose_as: "Conduit for arm-unknown-linux-musleabihf"
|
|
|
|
build:release:cargo:armv7-unknown-linux-musleabihf:
|
|
extends: .build-cargo-shared-settings
|
|
variables:
|
|
TARGET: "armv7-unknown-linux-musleabihf"
|
|
artifacts:
|
|
name: "conduit-armv7-unknown-linux-musleabihf"
|
|
paths:
|
|
- "conduit-armv7-unknown-linux-musleabihf"
|
|
expose_as: "Conduit for armv7-unknown-linux-musleabihf"
|
|
|
|
build:release:cargo:aarch64-unknown-linux-musl:
|
|
extends: .build-cargo-shared-settings
|
|
variables:
|
|
TARGET: "aarch64-unknown-linux-musl"
|
|
artifacts:
|
|
name: "conduit-aarch64-unknown-linux-musl"
|
|
paths:
|
|
- "conduit-aarch64-unknown-linux-musl"
|
|
expose_as: "Conduit for aarch64-unknown-linux-musl"
|
|
|
|
.cargo-debug-shared-settings:
|
|
extends: ".build-cargo-shared-settings"
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH != "master"'
|
|
cache:
|
|
key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
|
|
script:
|
|
# cross-compile conduit for target
|
|
- 'time ./cross/build.sh --locked'
|
|
- 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
|
|
# print information about linking for debugging
|
|
- 'file conduit-debug-$TARGET' # print file information
|
|
- 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
|
|
artifacts:
|
|
expire_in: 4 weeks
|
|
|
|
build:debug:cargo:x86_64-unknown-linux-musl:
|
|
extends: ".cargo-debug-shared-settings"
|
|
variables:
|
|
TARGET: "x86_64-unknown-linux-musl"
|
|
artifacts:
|
|
name: "conduit-debug-x86_64-unknown-linux-musl"
|
|
paths:
|
|
- "conduit-debug-x86_64-unknown-linux-musl"
|
|
expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
|
|
|
|
# --------------------------------------------------------------------- #
|
|
# Create and publish docker image #
|
|
# --------------------------------------------------------------------- #
|
|
|
|
.docker-shared-settings:
|
|
stage: "build docker image"
|
|
image: jdrouet/docker-with-buildx:stable
|
|
tags: ["docker"]
|
|
services:
|
|
- docker:dind
|
|
needs:
|
|
- "build:release:cargo:x86_64-unknown-linux-musl"
|
|
- "build:release:cargo:arm-unknown-linux-musleabihf"
|
|
- "build:release:cargo:armv7-unknown-linux-musleabihf"
|
|
- "build:release:cargo:aarch64-unknown-linux-musl"
|
|
variables:
|
|
DOCKER_HOST: tcp://docker:2375/
|
|
DOCKER_TLS_CERTDIR: ""
|
|
DOCKER_DRIVER: overlay2
|
|
PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
|
|
DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
|
|
cache:
|
|
paths:
|
|
- docker_cache
|
|
key: "$CI_JOB_NAME"
|
|
before_script:
|
|
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
|
|
# Only log in to Dockerhub if the credentials are given:
|
|
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
|
|
script:
|
|
# Prepare buildx to build multiarch stuff:
|
|
- docker context create 'ci-context'
|
|
- docker buildx create --name 'multiarch-builder' --use 'ci-context'
|
|
# Copy binaries to their docker arch path
|
|
- mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
|
|
- mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
|
|
- mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
|
|
- mv ./conduit-aarch64-unknown-linux-musl linux/arm64
|
|
- 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
|
|
# Build and push image:
|
|
- >
|
|
docker buildx build
|
|
--pull
|
|
--push
|
|
--cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
|
|
--cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
|
|
--build-arg CREATED=$CREATED
|
|
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
|
|
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
|
|
--platform "$PLATFORMS"
|
|
--tag "$TAG"
|
|
--tag "$TAG-alpine"
|
|
--tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
|
--file "$DOCKER_FILE" .
|
|
|
|
docker:next:gitlab:
|
|
extends: .docker-shared-settings
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "next"'
|
|
variables:
|
|
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
|
|
|
|
docker:next:dockerhub:
|
|
extends: .docker-shared-settings
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
|
|
variables:
|
|
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
|
|
|
|
docker:master:gitlab:
|
|
extends: .docker-shared-settings
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "master"'
|
|
variables:
|
|
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
|
|
|
|
docker:master:dockerhub:
|
|
extends: .docker-shared-settings
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
|
|
variables:
|
|
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
|
|
|
|
# --------------------------------------------------------------------- #
|
|
# Run tests #
|
|
# --------------------------------------------------------------------- #
|
|
|
|
test:cargo:
|
|
stage: "test"
|
|
needs: []
|
|
image: "rust:latest"
|
|
tags: ["docker"]
|
|
variables:
|
|
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
|
|
interruptible: true
|
|
before_script:
|
|
# - mkdir -p $CARGO_HOME
|
|
- apt-get update -yqq
|
|
- apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev
|
|
- rustup component add clippy rustfmt
|
|
- curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report
|
|
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
|
|
- if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
|
|
script:
|
|
- rustc --version && cargo --version # Print version info for debugging
|
|
- cargo fmt --all -- --check
|
|
- "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
|
|
- "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
|
|
artifacts:
|
|
when: always
|
|
reports:
|
|
junit: report.xml
|
|
codequality: gl-code-quality-report.json
|
|
|
|
test:sytest:
|
|
stage: "test"
|
|
allow_failure: true
|
|
needs:
|
|
- "build:debug:cargo:x86_64-unknown-linux-musl"
|
|
image:
|
|
name: "valkum/sytest-conduit:latest"
|
|
entrypoint: [""]
|
|
tags: ["docker"]
|
|
variables:
|
|
PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
|
|
before_script:
|
|
- "mkdir -p /app"
|
|
- "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
|
|
- "chmod +x /app/conduit"
|
|
- "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
|
|
- "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
|
|
- "cd /"
|
|
script:
|
|
- "SYTEST_EXIT_CODE=0"
|
|
- "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
|
|
- 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap'
|
|
- "exit $SYTEST_EXIT_CODE"
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- "$CI_PROJECT_DIR/sytest.xml"
|
|
- "$CI_PROJECT_DIR/results.tap"
|
|
reports:
|
|
junit: "$CI_PROJECT_DIR/sytest.xml"
|
|
|
|
# --------------------------------------------------------------------- #
|
|
# Store binaries as package so they have download urls #
|
|
# --------------------------------------------------------------------- #
|
|
|
|
publish:package:
|
|
stage: "upload artifacts"
|
|
needs:
|
|
- "build:release:cargo:x86_64-unknown-linux-musl"
|
|
- "build:release:cargo:arm-unknown-linux-musleabihf"
|
|
- "build:release:cargo:armv7-unknown-linux-musleabihf"
|
|
- "build:release:cargo:aarch64-unknown-linux-musl"
|
|
# - "build:cargo-deb:x86_64-unknown-linux-gnu"
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == "master"'
|
|
- if: '$CI_COMMIT_BRANCH == "next"'
|
|
- if: "$CI_COMMIT_TAG"
|
|
image: curlimages/curl:latest
|
|
tags: ["docker"]
|
|
variables:
|
|
GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
|
|
script:
|
|
- 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
|
|
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
|
|
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"'
|
|
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"'
|
|
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"'
|
|
|
|
# Avoid duplicate pipelines
|
|
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
|
|
workflow:
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
|
|
when: never
|
|
- if: "$CI_COMMIT_BRANCH"
|