Factor out some of the code shared between the sytest scripts (#974)

* Factor out some of the code shared between the different sytest jenkins scripts

* Exclude jenkins from the MANIFEST

* Fix dendron build

* Missing new line

* Poke jenkins

* Export the PORT_BASE and PORT_COUNT

* Poke jenkins
This commit is contained in:
Mark Haines 2016-08-02 20:42:30 +01:00 committed by GitHub
parent 06f812b95c
commit 9a2f296fa2
6 changed files with 71 additions and 69 deletions

View file

@ -24,5 +24,7 @@ recursive-include synapse/static *.js
exclude jenkins.sh exclude jenkins.sh
exclude jenkins*.sh exclude jenkins*.sh
exclude jenkins*
recursive-exclude jenkins *.sh
prune demo/etc prune demo/etc

View file

@ -22,24 +22,10 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
rm .coverage* || echo "No coverage files to remove" rm .coverage* || echo "No coverage files to remove"
tox --notest -e py27 ./jenkins/prepare_synapse.sh
TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install ./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git
$TOX_BIN/pip install psycopg2
$TOX_BIN/pip install lxml
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
if [[ ! -e .dendron-base ]]; then
git clone https://github.com/matrix-org/dendron.git .dendron-base --mirror
else
(cd .dendron-base; git fetch -p)
fi
rm -rf dendron
git clone .dendron-base dendron --shared
cd dendron
: ${GOPATH:=${WORKSPACE}/.gopath} : ${GOPATH:=${WORKSPACE}/.gopath}
if [[ "${GOPATH}" != *:* ]]; then if [[ "${GOPATH}" != *:* ]]; then
@ -48,35 +34,26 @@ if [[ "${GOPATH}" != *:* ]]; then
fi fi
export GOPATH export GOPATH
git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) cd dendron
go get github.com/constabulary/gb/... go get github.com/constabulary/gb/...
gb generate gb generate
gb build gb build
cd .. cd ../sytest
if [[ ! -e .sytest-base ]]; then
git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
else
(cd .sytest-base; git fetch -p)
fi
rm -rf sytest
git clone .sytest-base sytest --shared
cd sytest
git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
: ${PORT_BASE:=20000} : ${PORT_BASE:=20000}
: ${PORT_COUNT=100} : ${PORT_COUNT=100}
export PORT_BASE
export PORT_COUNT
./jenkins/prep_sytest_for_postgres.sh ./jenkins/prep_sytest_for_postgres.sh
mkdir -p var mkdir -p var
echo >&2 "Running sytest with PostgreSQL"; echo >&2 "Running sytest with PostgreSQL";
TOX_BIN=$WORKSPACE/.tox/py27/bin
./jenkins/install_and_run.sh --python $TOX_BIN/python \ ./jenkins/install_and_run.sh --python $TOX_BIN/python \
--synapse-directory $WORKSPACE \ --synapse-directory $WORKSPACE \
--dendron $WORKSPACE/dendron/bin/dendron \ --dendron $WORKSPACE/dendron/bin/dendron \

View file

@ -22,37 +22,26 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
rm .coverage* || echo "No coverage files to remove" rm .coverage* || echo "No coverage files to remove"
tox --notest -e py27 ./jenkins/prepare_synapse.sh
TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
$TOX_BIN/pip install psycopg2
$TOX_BIN/pip install lxml
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
if [[ ! -e .sytest-base ]]; then
git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
else
(cd .sytest-base; git fetch -p)
fi
rm -rf sytest
git clone .sytest-base sytest --shared
cd sytest
git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
: ${PORT_BASE:=20000} : ${PORT_BASE:=20000}
: ${PORT_COUNT=100} : ${PORT_COUNT=100}
export PORT_BASE
export PORT_COUNT
cd sytest
./jenkins/prep_sytest_for_postgres.sh ./jenkins/prep_sytest_for_postgres.sh
echo >&2 "Running sytest with PostgreSQL"; echo >&2 "Running sytest with PostgreSQL";
TOX_BIN=$WORKSPACE/.tox/py27/bin
./jenkins/install_and_run.sh --coverage \ ./jenkins/install_and_run.sh --coverage \
--python $TOX_BIN/python \ --python $TOX_BIN/python \
--synapse-directory $WORKSPACE \ --synapse-directory $WORKSPACE \
--port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) \ --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1))
cd .. cd ..
cp sytest/.coverage.* . cp sytest/.coverage.* .

View file

@ -4,6 +4,7 @@ set -eux
: ${WORKSPACE:="$(pwd)"} : ${WORKSPACE:="$(pwd)"}
export WORKSPACE
export PYTHONDONTWRITEBYTECODE=yep export PYTHONDONTWRITEBYTECODE=yep
export SYNAPSE_CACHE_FACTOR=1 export SYNAPSE_CACHE_FACTOR=1
@ -22,28 +23,18 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
rm .coverage* || echo "No coverage files to remove" rm .coverage* || echo "No coverage files to remove"
tox --notest -e py27 ./jenkins/prepare_synapse.sh
TOX_BIN=$WORKSPACE/.tox/py27/bin
python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
$TOX_BIN/pip install lxml
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git
if [[ ! -e .sytest-base ]]; then
git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
else
(cd .sytest-base; git fetch -p)
fi
rm -rf sytest
git clone .sytest-base sytest --shared
cd sytest
git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
: ${PORT_BASE:=20000} : ${PORT_BASE:=20000}
: ${PORT_COUNT=100} : ${PORT_COUNT=100}
export PORT_BASE
export PORT_COUNT
cd sytest
TOX_BIN=$WORKSPACE/.tox/py27/bin
./jenkins/install_and_run.sh --coverage \ ./jenkins/install_and_run.sh --coverage \
--python $TOX_BIN/python \ --python $TOX_BIN/python \
--synapse-directory $WORKSPACE \ --synapse-directory $WORKSPACE \

24
jenkins/clone.sh Executable file
View file

@ -0,0 +1,24 @@
#! /bin/bash
NAME=$1
PROJECT=$2
BASE=".$NAME-base"
# update our clone
if [ ! -d .$NAME-base ]; then
git clone $PROJECT $BASE --mirror
else
(cd $BASE; git fetch -p)
fi
rm -rf $NAME
git clone $BASE $NAME --shared
: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
cd $NAME
# check out the relevant branch
git checkout "${GIT_BRANCH}" || (
echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop"
git checkout "origin/develop"
)
git clean -df .

19
jenkins/prepare_synapse.sh Executable file
View file

@ -0,0 +1,19 @@
#! /bin/bash
cd "`dirname $0`/.."
TOX_DIR=$WORKSPACE/.tox
mkdir -p $TOX_DIR
if ! [ $TOX_DIR -ef .tox ]; then
ln -s "$TOX_DIR" .tox
fi
# set up the virtualenv
tox -e py27 --notest -v
TOX_BIN=$TOX_DIR/py27/bin
python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install
$TOX_BIN/pip install lxml
$TOX_BIN/pip install psycopg2