forked from MirrorHub/synapse
Add --editable
flag to complement.sh
which uses an editable install of Synapse for faster turn-around times whilst developing iteratively. (#14548)
Co-authored-by: Mathieu Velten <mathieuv@matrix.org>
This commit is contained in:
parent
22e91b8019
commit
be3a8a85e3
5 changed files with 160 additions and 18 deletions
1
changelog.d/14548.misc
Normal file
1
changelog.d/14548.misc
Normal file
|
@ -0,0 +1 @@
|
||||||
|
Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively.
|
|
@ -1,6 +1,7 @@
|
||||||
# syntax=docker/dockerfile:1
|
# syntax=docker/dockerfile:1
|
||||||
|
|
||||||
ARG SYNAPSE_VERSION=latest
|
ARG SYNAPSE_VERSION=latest
|
||||||
|
ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
|
||||||
|
|
||||||
# first of all, we create a base image with an nginx which we can copy into the
|
# first of all, we create a base image with an nginx which we can copy into the
|
||||||
# target image. For repeated rebuilds, this is much faster than apt installing
|
# target image. For repeated rebuilds, this is much faster than apt installing
|
||||||
|
@ -23,7 +24,7 @@ FROM debian:bullseye-slim AS deps_base
|
||||||
FROM redis:6-bullseye AS redis_base
|
FROM redis:6-bullseye AS redis_base
|
||||||
|
|
||||||
# now build the final image, based on the the regular Synapse docker image
|
# now build the final image, based on the the regular Synapse docker image
|
||||||
FROM matrixdotorg/synapse:$SYNAPSE_VERSION
|
FROM $FROM
|
||||||
|
|
||||||
# Install supervisord with pip instead of apt, to avoid installing a second
|
# Install supervisord with pip instead of apt, to avoid installing a second
|
||||||
# copy of python.
|
# copy of python.
|
||||||
|
|
|
@ -7,8 +7,9 @@
|
||||||
# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
|
# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
|
||||||
|
|
||||||
ARG SYNAPSE_VERSION=latest
|
ARG SYNAPSE_VERSION=latest
|
||||||
|
ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
|
||||||
|
|
||||||
FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
|
FROM $FROM
|
||||||
# First of all, we copy postgres server from the official postgres image,
|
# First of all, we copy postgres server from the official postgres image,
|
||||||
# since for repeated rebuilds, this is much faster than apt installing
|
# since for repeated rebuilds, this is much faster than apt installing
|
||||||
# postgres each time.
|
# postgres each time.
|
||||||
|
|
75
docker/editable.Dockerfile
Normal file
75
docker/editable.Dockerfile
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
# This dockerfile builds an editable install of Synapse.
|
||||||
|
#
|
||||||
|
# Used by `complement.sh`. Not suitable for production use.
|
||||||
|
|
||||||
|
ARG PYTHON_VERSION=3.9
|
||||||
|
|
||||||
|
###
|
||||||
|
### Stage 0: generate requirements.txt
|
||||||
|
###
|
||||||
|
# We hardcode the use of Debian bullseye here because this could change upstream
|
||||||
|
# and other Dockerfiles used for testing are expecting bullseye.
|
||||||
|
FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
|
||||||
|
|
||||||
|
# Install Rust and other dependencies (stolen from normal Dockerfile)
|
||||||
|
# install the OS build deps
|
||||||
|
RUN \
|
||||||
|
--mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||||
|
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||||
|
apt-get update -qq && apt-get install -yqq \
|
||||||
|
build-essential \
|
||||||
|
libffi-dev \
|
||||||
|
libjpeg-dev \
|
||||||
|
libpq-dev \
|
||||||
|
libssl-dev \
|
||||||
|
libwebp-dev \
|
||||||
|
libxml++2.6-dev \
|
||||||
|
libxslt1-dev \
|
||||||
|
openssl \
|
||||||
|
zlib1g-dev \
|
||||||
|
git \
|
||||||
|
curl \
|
||||||
|
gosu \
|
||||||
|
libjpeg62-turbo \
|
||||||
|
libpq5 \
|
||||||
|
libwebp6 \
|
||||||
|
xmlsec1 \
|
||||||
|
libjemalloc2 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
ENV RUSTUP_HOME=/rust
|
||||||
|
ENV CARGO_HOME=/cargo
|
||||||
|
ENV PATH=/cargo/bin:/rust/bin:$PATH
|
||||||
|
RUN mkdir /rust /cargo
|
||||||
|
RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
|
||||||
|
|
||||||
|
|
||||||
|
# Make a base copy of the editable source tree, so that we have something to
|
||||||
|
# install and build now — even though it's going to be covered up by a mount
|
||||||
|
# at runtime.
|
||||||
|
COPY synapse /editable-src/synapse/
|
||||||
|
COPY rust /editable-src/rust/
|
||||||
|
# ... and what we need to `pip install`.
|
||||||
|
COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/
|
||||||
|
|
||||||
|
RUN pip install poetry
|
||||||
|
RUN poetry config virtualenvs.create false
|
||||||
|
RUN cd /editable-src && poetry install --extras all
|
||||||
|
|
||||||
|
# Make copies of useful things for inspection:
|
||||||
|
# - the Rust module (must be copied to the editable source tree before startup)
|
||||||
|
# - poetry.lock is useful for checking if dependencies have changed.
|
||||||
|
RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak
|
||||||
|
RUN cp /editable-src/poetry.lock /poetry.lock.bak
|
||||||
|
|
||||||
|
|
||||||
|
### Extra setup from original Dockerfile
|
||||||
|
COPY ./docker/start.py /start.py
|
||||||
|
COPY ./docker/conf /conf
|
||||||
|
|
||||||
|
EXPOSE 8008/tcp 8009/tcp 8448/tcp
|
||||||
|
|
||||||
|
ENTRYPOINT ["/start.py"]
|
||||||
|
|
||||||
|
HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
|
||||||
|
CMD curl -fSs http://localhost:8008/health || exit 1
|
|
@ -53,6 +53,12 @@ Run the complement test suite on Synapse.
|
||||||
Only build the Docker images. Don't actually run Complement.
|
Only build the Docker images. Don't actually run Complement.
|
||||||
Conflicts with -f/--fast.
|
Conflicts with -f/--fast.
|
||||||
|
|
||||||
|
-e, --editable
|
||||||
|
Use an editable build of Synapse, rebuilding the image if necessary.
|
||||||
|
This is suitable for use in development where a fast turn-around time
|
||||||
|
is important.
|
||||||
|
Not suitable for use in CI in case the editable environment is impure.
|
||||||
|
|
||||||
For help on arguments to 'go test', run 'go help testflag'.
|
For help on arguments to 'go test', run 'go help testflag'.
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
@ -73,6 +79,9 @@ while [ $# -ge 1 ]; do
|
||||||
"--build-only")
|
"--build-only")
|
||||||
skip_complement_run=1
|
skip_complement_run=1
|
||||||
;;
|
;;
|
||||||
|
"-e"|"--editable")
|
||||||
|
use_editable_synapse=1
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
# unknown arg: presumably an argument to gotest. break the loop.
|
# unknown arg: presumably an argument to gotest. break the loop.
|
||||||
break
|
break
|
||||||
|
@ -96,25 +105,76 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then
|
||||||
echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
|
echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -n "$use_editable_synapse" ]; then
|
||||||
|
if [[ -e synapse/synapse_rust.abi3.so ]]; then
|
||||||
|
# In an editable install, back up the host's compiled Rust module to prevent
|
||||||
|
# inconvenience; the container will overwrite the module with its own copy.
|
||||||
|
mv -n synapse/synapse_rust.abi3.so synapse/synapse_rust.abi3.so~host
|
||||||
|
# And restore it on exit:
|
||||||
|
synapse_pkg=`realpath synapse`
|
||||||
|
trap "mv -f '$synapse_pkg/synapse_rust.abi3.so~host' '$synapse_pkg/synapse_rust.abi3.so'" EXIT
|
||||||
|
fi
|
||||||
|
|
||||||
|
editable_mount="$(realpath .):/editable-src:z"
|
||||||
|
if docker inspect complement-synapse-editable &>/dev/null; then
|
||||||
|
# complement-synapse-editable already exists: see if we can still use it:
|
||||||
|
# - The Rust module must still be importable; it will fail to import if the Rust source has changed.
|
||||||
|
# - The Poetry lock file must be the same (otherwise we assume dependencies have changed)
|
||||||
|
|
||||||
|
# First set up the module in the right place for an editable installation.
|
||||||
|
docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
|
||||||
|
|
||||||
|
if (docker run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \
|
||||||
|
&& docker run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then
|
||||||
|
skip_docker_build=1
|
||||||
|
else
|
||||||
|
echo "Editable Synapse image is stale. Will rebuild."
|
||||||
|
unset skip_docker_build
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -z "$skip_docker_build" ]; then
|
if [ -z "$skip_docker_build" ]; then
|
||||||
# Build the base Synapse image from the local checkout
|
if [ -n "$use_editable_synapse" ]; then
|
||||||
echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
|
|
||||||
docker build -t matrixdotorg/synapse \
|
|
||||||
--build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
|
|
||||||
--build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
|
|
||||||
-f "docker/Dockerfile" .
|
|
||||||
echo_if_github "::endgroup::"
|
|
||||||
|
|
||||||
# Build the workers docker image (from the base Synapse image we just built).
|
# Build a special image designed for use in development with editable
|
||||||
echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
|
# installs.
|
||||||
docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
|
docker build -t synapse-editable \
|
||||||
echo_if_github "::endgroup::"
|
-f "docker/editable.Dockerfile" .
|
||||||
|
|
||||||
# Build the unified Complement image (from the worker Synapse image we just built).
|
docker build -t synapse-workers-editable \
|
||||||
echo_if_github "::group::Build Docker image: complement/Dockerfile"
|
--build-arg FROM=synapse-editable \
|
||||||
docker build -t complement-synapse \
|
-f "docker/Dockerfile-workers" .
|
||||||
-f "docker/complement/Dockerfile" "docker/complement"
|
|
||||||
echo_if_github "::endgroup::"
|
docker build -t complement-synapse-editable \
|
||||||
|
--build-arg FROM=synapse-workers-editable \
|
||||||
|
-f "docker/complement/Dockerfile" "docker/complement"
|
||||||
|
|
||||||
|
# Prepare the Rust module
|
||||||
|
docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so
|
||||||
|
|
||||||
|
else
|
||||||
|
|
||||||
|
# Build the base Synapse image from the local checkout
|
||||||
|
echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
|
||||||
|
docker build -t matrixdotorg/synapse \
|
||||||
|
--build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
|
||||||
|
--build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
|
||||||
|
-f "docker/Dockerfile" .
|
||||||
|
echo_if_github "::endgroup::"
|
||||||
|
|
||||||
|
# Build the workers docker image (from the base Synapse image we just built).
|
||||||
|
echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
|
||||||
|
docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
|
||||||
|
echo_if_github "::endgroup::"
|
||||||
|
|
||||||
|
# Build the unified Complement image (from the worker Synapse image we just built).
|
||||||
|
echo_if_github "::group::Build Docker image: complement/Dockerfile"
|
||||||
|
docker build -t complement-synapse \
|
||||||
|
-f "docker/complement/Dockerfile" "docker/complement"
|
||||||
|
echo_if_github "::endgroup::"
|
||||||
|
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -n "$skip_complement_run" ]; then
|
if [ -n "$skip_complement_run" ]; then
|
||||||
|
@ -123,6 +183,10 @@ if [ -n "$skip_complement_run" ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
export COMPLEMENT_BASE_IMAGE=complement-synapse
|
export COMPLEMENT_BASE_IMAGE=complement-synapse
|
||||||
|
if [ -n "$use_editable_synapse" ]; then
|
||||||
|
export COMPLEMENT_BASE_IMAGE=complement-synapse-editable
|
||||||
|
export COMPLEMENT_HOST_MOUNTS="$editable_mount"
|
||||||
|
fi
|
||||||
|
|
||||||
extra_test_args=()
|
extra_test_args=()
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue