From 9a2f296fa2e6bd42f10b12a81dba2279b8482fcc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Aug 2016 20:42:30 +0100 Subject: Factor out some of the code shared between the sytest scripts (#974) * Factor out some of the code shared between the different sytest jenkins scripts * Exclude jenkins from the MANIFEST * Fix dendron build * Missing new line * Poke jenkins * Export the PORT_BASE and PORT_COUNT * Poke jenkins --- jenkins-dendron-postgres.sh | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) (limited to 'jenkins-dendron-postgres.sh') diff --git a/jenkins-dendron-postgres.sh b/jenkins-dendron-postgres.sh index f715cd559a..e6e94cc8bc 100755 --- a/jenkins-dendron-postgres.sh +++ b/jenkins-dendron-postgres.sh @@ -22,24 +22,10 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w rm .coverage* || echo "No coverage files to remove" -tox --notest -e py27 +./jenkins/prepare_synapse.sh -TOX_BIN=$WORKSPACE/.tox/py27/bin -python synapse/python_dependencies.py | xargs -n1 $TOX_BIN/pip install -$TOX_BIN/pip install psycopg2 -$TOX_BIN/pip install lxml - -: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} - -if [[ ! -e .dendron-base ]]; then - git clone https://github.com/matrix-org/dendron.git .dendron-base --mirror -else - (cd .dendron-base; git fetch -p) -fi - -rm -rf dendron -git clone .dendron-base dendron --shared -cd dendron +./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git +./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git : ${GOPATH:=${WORKSPACE}/.gopath} if [[ "${GOPATH}" != *:* ]]; then @@ -48,35 +34,26 @@ if [[ "${GOPATH}" != *:* ]]; then fi export GOPATH -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +cd dendron go get github.com/constabulary/gb/... gb generate gb build -cd .. - - -if [[ ! -e .sytest-base ]]; then - git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror -else - (cd .sytest-base; git fetch -p) -fi - -rm -rf sytest -git clone .sytest-base sytest --shared -cd sytest - -git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) +cd ../sytest : ${PORT_BASE:=20000} : ${PORT_COUNT=100} +export PORT_BASE +export PORT_COUNT ./jenkins/prep_sytest_for_postgres.sh mkdir -p var echo >&2 "Running sytest with PostgreSQL"; + +TOX_BIN=$WORKSPACE/.tox/py27/bin ./jenkins/install_and_run.sh --python $TOX_BIN/python \ --synapse-directory $WORKSPACE \ --dendron $WORKSPACE/dendron/bin/dendron \ -- cgit 1.4.1 From 1fc50712d6729c85680e8ebe1c8612c304217c47 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 4 Aug 2016 12:31:55 +0100 Subject: Factor out more common code from the jenkins scripts (#980) * Factor out more common code from the jenkins scripts * Fix install_and_run path * Poke jenkins * Poke jenkins --- jenkins-dendron-postgres.sh | 63 +++++++-------------------------------------- jenkins-postgres.sh | 44 +++---------------------------- jenkins-sqlite.sh | 39 ++-------------------------- jenkins/clone.sh | 36 ++++++++++++++++++++------ 4 files changed, 44 insertions(+), 138 deletions(-) (limited to 'jenkins-dendron-postgres.sh') diff --git a/jenkins-dendron-postgres.sh b/jenkins-dendron-postgres.sh index e6e94cc8bc..68912a8967 100755 --- a/jenkins-dendron-postgres.sh +++ b/jenkins-dendron-postgres.sh @@ -4,62 +4,19 @@ set -eux : ${WORKSPACE:="$(pwd)"} +export WORKSPACE export PYTHONDONTWRITEBYTECODE=yep export SYNAPSE_CACHE_FACTOR=1 -# Output test results as junit xml -export TRIAL_FLAGS="--reporter=subunit" -export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" -# Write coverage reports to a separate file for each process -export COVERAGE_OPTS="-p" -export DUMP_COVERAGE_COMMAND="coverage help" - -# Output flake8 violations to violations.flake8.log -# Don't exit with non-0 status code on Jenkins, -# so that the build steps continue and a later step can decided whether to -# UNSTABLE or FAILURE this build. -export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" - -rm .coverage* || echo "No coverage files to remove" - ./jenkins/prepare_synapse.sh - ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git ./jenkins/clone.sh dendron https://github.com/matrix-org/dendron.git - -: ${GOPATH:=${WORKSPACE}/.gopath} -if [[ "${GOPATH}" != *:* ]]; then - mkdir -p "${GOPATH}" - export PATH="${GOPATH}/bin:${PATH}" -fi -export GOPATH - -cd dendron - -go get github.com/constabulary/gb/... -gb generate -gb build - -cd ../sytest - -: ${PORT_BASE:=20000} -: ${PORT_COUNT=100} -export PORT_BASE -export PORT_COUNT - -./jenkins/prep_sytest_for_postgres.sh - -mkdir -p var - -echo >&2 "Running sytest with PostgreSQL"; - -TOX_BIN=$WORKSPACE/.tox/py27/bin -./jenkins/install_and_run.sh --python $TOX_BIN/python \ - --synapse-directory $WORKSPACE \ - --dendron $WORKSPACE/dendron/bin/dendron \ - --pusher \ - --synchrotron \ - --federation-reader \ - --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) - -cd .. +./dendron/jenkins/build_dendron.sh +./sytest/jenkins/prep_sytest_for_postgres.sh + +./sytest/jenkins/install_and_run.sh \ + --synapse-directory $WORKSPACE \ + --dendron $WORKSPACE/dendron/bin/dendron \ + --pusher \ + --synchrotron \ + --federation-reader \ diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh index edf61a45bb..f2ca8ccdff 100755 --- a/jenkins-postgres.sh +++ b/jenkins-postgres.sh @@ -4,50 +4,14 @@ set -eux : ${WORKSPACE:="$(pwd)"} +export WORKSPACE export PYTHONDONTWRITEBYTECODE=yep export SYNAPSE_CACHE_FACTOR=1 -# Output test results as junit xml -export TRIAL_FLAGS="--reporter=subunit" -export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" -# Write coverage reports to a separate file for each process -export COVERAGE_OPTS="-p" -export DUMP_COVERAGE_COMMAND="coverage help" - -# Output flake8 violations to violations.flake8.log -# Don't exit with non-0 status code on Jenkins, -# so that the build steps continue and a later step can decided whether to -# UNSTABLE or FAILURE this build. -export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" - -rm .coverage* || echo "No coverage files to remove" - ./jenkins/prepare_synapse.sh - ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git -: ${PORT_BASE:=20000} -: ${PORT_COUNT=100} -export PORT_BASE -export PORT_COUNT - -cd sytest - -./jenkins/prep_sytest_for_postgres.sh - -echo >&2 "Running sytest with PostgreSQL"; - -TOX_BIN=$WORKSPACE/.tox/py27/bin -./jenkins/install_and_run.sh --coverage \ - --python $TOX_BIN/python \ - --synapse-directory $WORKSPACE \ - --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) - -cd .. -cp sytest/.coverage.* . +./sytest/jenkins/prep_sytest_for_postgres.sh -# Combine the coverage reports -echo "Combining:" .coverage.* -$TOX_BIN/python -m coverage combine -# Output coverage to coverage.xml -$TOX_BIN/coverage xml -o coverage.xml +./sytest/jenkins/install_and_run.sh \ + --synapse-directory $WORKSPACE \ diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh index 1c3530ebb3..84613d979c 100755 --- a/jenkins-sqlite.sh +++ b/jenkins-sqlite.sh @@ -8,43 +8,8 @@ export WORKSPACE export PYTHONDONTWRITEBYTECODE=yep export SYNAPSE_CACHE_FACTOR=1 -# Output test results as junit xml -export TRIAL_FLAGS="--reporter=subunit" -export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" -# Write coverage reports to a separate file for each process -export COVERAGE_OPTS="-p" -export DUMP_COVERAGE_COMMAND="coverage help" - -# Output flake8 violations to violations.flake8.log -# Don't exit with non-0 status code on Jenkins, -# so that the build steps continue and a later step can decided whether to -# UNSTABLE or FAILURE this build. -export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" - -rm .coverage* || echo "No coverage files to remove" - ./jenkins/prepare_synapse.sh - ./jenkins/clone.sh sytest https://github.com/matrix-org/sytest.git -: ${PORT_BASE:=20000} -: ${PORT_COUNT=100} -export PORT_BASE -export PORT_COUNT - -cd sytest - -TOX_BIN=$WORKSPACE/.tox/py27/bin -./jenkins/install_and_run.sh --coverage \ - --python $TOX_BIN/python \ - --synapse-directory $WORKSPACE \ - --port-range ${PORT_BASE}:$((PORT_BASE+PORT_COUNT-1)) \ - -cd .. -cp sytest/.coverage.* . - -# Combine the coverage reports -echo "Combining:" .coverage.* -$TOX_BIN/python -m coverage combine -# Output coverage to coverage.xml -$TOX_BIN/coverage xml -o coverage.xml +./sytest/jenkins/install_and_run.sh \ + --synapse-directory $WORKSPACE \ diff --git a/jenkins/clone.sh b/jenkins/clone.sh index f56d076ea9..ab30ac7782 100755 --- a/jenkins/clone.sh +++ b/jenkins/clone.sh @@ -1,24 +1,44 @@ #! /bin/bash +# This clones a project from github into a named subdirectory +# If the project has a branch with the same name as this branch +# then it will checkout that branch after cloning. +# Otherwise it will checkout "origin/develop." +# The first argument is the name of the directory to checkout +# the branch into. +# The second argument is the URL of the remote repository to checkout. +# Usually something like https://github.com/matrix-org/sytest.git + +set -eux + NAME=$1 PROJECT=$2 BASE=".$NAME-base" -# update our clone -if [ ! -d .$NAME-base ]; then - git clone $PROJECT $BASE --mirror +# Update our mirror. +if [ ! -d ".$NAME-base" ]; then + # Create a local mirror of the source repository. + # This saves us from having to download the entire repository + # when this script is next run. + git clone "$PROJECT" "$BASE" --mirror else - (cd $BASE; git fetch -p) + # Fetch any updates from the source repository. + (cd "$BASE"; git fetch -p) fi -rm -rf $NAME -git clone $BASE $NAME --shared +# Remove the existing repository so that we have a clean copy +rm -rf "$NAME" +# Cloning with --shared means that we will share portions of the +# .git directory with our local mirror. +git clone "$BASE" "$NAME" --shared +# Jenkins may have supplied us with the name of the branch in the +# environment. Otherwise we will have to guess based on the current +# commit. : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} -cd $NAME +cd "$NAME" # check out the relevant branch git checkout "${GIT_BRANCH}" || ( echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" git checkout "origin/develop" ) -git clean -df . -- cgit 1.4.1