From fc9c7b6cbc27b17aa08cc77832249a00cdbe0338 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 3 Mar 2016 15:26:07 +0000
Subject: Add jenkins-postgres.sh

---
 jenkins-postgres.sh | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100755 jenkins-postgres.sh

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
new file mode 100755
index 0000000000..7dca840e39
--- /dev/null
+++ b/jenkins-postgres.sh
@@ -0,0 +1,87 @@
+#!/bin/bash -eu
+
+: ${WORKSPACE:="$(pwd)"}
+
+export PYTHONDONTWRITEBYTECODE=yep
+export SYNAPSE_CACHE_FACTOR=1
+
+# Output test results as junit xml
+export TRIAL_FLAGS="--reporter=subunit"
+export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
+# Write coverage reports to a separate file for each process
+export COVERAGE_OPTS="-p"
+export DUMP_COVERAGE_COMMAND="coverage help"
+
+# Output flake8 violations to violations.flake8.log
+# Don't exit with non-0 status code on Jenkins,
+# so that the build steps continue and a later step can decided whether to
+# UNSTABLE or FAILURE this build.
+export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
+
+rm .coverage* || echo "No coverage files to remove"
+
+tox --notest
+
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
+
+TOX_BIN=$WORKSPACE/.tox/py27/bin
+
+if [[ ! -e .sytest-base ]]; then
+  git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
+else
+  (cd .sytest-base; git fetch -p)
+fi
+
+rm -rf sytest
+git clone .sytest-base sytest --shared
+cd sytest
+
+git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
+
+: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
+: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
+: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
+export PERL5LIB PERL_MB_OPT PERL_MM_OPT
+
+./install-deps.pl
+
+: ${PORT_BASE:=8000}
+
+
+if [[ -z "$POSTGRES_DB_1" ]]; then
+    echo >&2 "Variable POSTGRES_DB_1 not set"
+    exit 1
+fi
+
+if [[ -z "$POSTGRES_DB_2" ]]; then
+    echo >&2 "Variable POSTGRES_DB_2 not set"
+    exit 1
+fi
+
+cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF
+name: psycopg2
+args:
+    database: $POSTGRES_DB_1
+EOF
+
+cat > localhost-$(($PORT_BASE + 2))/database.yaml << EOF
+name: psycopg2
+args:
+    database: $POSTGRES_DB_2
+EOF
+
+
+# Run if both postgresql databases exist
+echo >&2 "Running sytest with PostgreSQL";
+$TOX_BIN/pip install psycopg2
+./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
+    --python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
+
+cd ..
+cp sytest/.coverage.* .
+
+# Combine the coverage reports
+echo "Combining:" .coverage.*
+$TOX_BIN/python -m coverage combine
+# Output coverage to coverage.xml
+$TOX_BIN/coverage xml -o coverage.xml
-- 
cgit 1.5.1


From 7678ec3f9b9a529c7d8c8aa8a38572c5cf2283d6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 3 Mar 2016 15:42:07 +0000
Subject: Mkdir

---
 jenkins-postgres.sh | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index 7dca840e39..1708cbfaaa 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -58,6 +58,9 @@ if [[ -z "$POSTGRES_DB_2" ]]; then
     exit 1
 fi
 
+mkdir -p "localhost-$(($PORT_BASE + 1))"
+mkdir -p "localhost-$(($PORT_BASE + 2))"
+
 cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF
 name: psycopg2
 args:
-- 
cgit 1.5.1


From 62d808beccaf4ef94e743c57b9570bea99071eb7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Thu, 3 Mar 2016 11:38:36 +0000
Subject: jenkins-*.sh: set -x

Also move the options from the shebang line to the body of the script, so that
they take effect even if somebody explicitly runs "bash jenkins.sh"
---
 jenkins-flake8.sh    | 4 +++-
 jenkins-postgres.sh  | 4 +++-
 jenkins-sqlite.sh    | 4 +++-
 jenkins-unittests.sh | 4 +++-
 4 files changed, 12 insertions(+), 4 deletions(-)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-flake8.sh b/jenkins-flake8.sh
index cbcb0ae4c0..11f1cab6c8 100755
--- a/jenkins-flake8.sh
+++ b/jenkins-flake8.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -eu
+#!/bin/bash
+
+set -eux
 
 : ${WORKSPACE:="$(pwd)"}
 
diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index 1708cbfaaa..d1fed590a2 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -eu
+#!/bin/bash
+
+set -eux
 
 : ${WORKSPACE:="$(pwd)"}
 
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
index 2d98a0af93..57fd8de54d 100755
--- a/jenkins-sqlite.sh
+++ b/jenkins-sqlite.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -eu
+#!/bin/bash
+
+set -eux
 
 : ${WORKSPACE:="$(pwd)"}
 
diff --git a/jenkins-unittests.sh b/jenkins-unittests.sh
index 2fa2f1b1de..104d511994 100755
--- a/jenkins-unittests.sh
+++ b/jenkins-unittests.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -eu
+#!/bin/bash
+
+set -eux
 
 : ${WORKSPACE:="$(pwd)"}
 
-- 
cgit 1.5.1


From 1748d4b739824dfee92a8ea65a0cd95839af8f25 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 8 Mar 2016 23:37:12 +0000
Subject: Use sytest build scripts, rather than reinventing the wheel

---
 jenkins-postgres.sh | 44 +++++---------------------------------------
 jenkins-sqlite.sh   | 15 ++-------------
 2 files changed, 7 insertions(+), 52 deletions(-)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index d1fed590a2..e9f22cc04b 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -24,9 +24,10 @@ rm .coverage* || echo "No coverage files to remove"
 
 tox --notest
 
-: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
-
 TOX_BIN=$WORKSPACE/.tox/py27/bin
+$TOX_BIN/pip install psycopg2
+
+: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
 
 if [[ ! -e .sytest-base ]]; then
   git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
@@ -40,47 +41,12 @@ cd sytest
 
 git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
 
-: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
-: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
-: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
-export PERL5LIB PERL_MB_OPT PERL_MM_OPT
-
-./install-deps.pl
-
 : ${PORT_BASE:=8000}
 
+./jenkins/prep_sytest_for_postgres.sh
 
-if [[ -z "$POSTGRES_DB_1" ]]; then
-    echo >&2 "Variable POSTGRES_DB_1 not set"
-    exit 1
-fi
-
-if [[ -z "$POSTGRES_DB_2" ]]; then
-    echo >&2 "Variable POSTGRES_DB_2 not set"
-    exit 1
-fi
-
-mkdir -p "localhost-$(($PORT_BASE + 1))"
-mkdir -p "localhost-$(($PORT_BASE + 2))"
-
-cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF
-name: psycopg2
-args:
-    database: $POSTGRES_DB_1
-EOF
-
-cat > localhost-$(($PORT_BASE + 2))/database.yaml << EOF
-name: psycopg2
-args:
-    database: $POSTGRES_DB_2
-EOF
-
-
-# Run if both postgresql databases exist
 echo >&2 "Running sytest with PostgreSQL";
-$TOX_BIN/pip install psycopg2
-./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
-    --python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
+./jenkins/install_and_run.sh --python $TOX_BIN/python --port-base $PORT_BASE
 
 cd ..
 cp sytest/.coverage.* .
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
index 57fd8de54d..a68f491cab 100755
--- a/jenkins-sqlite.sh
+++ b/jenkins-sqlite.sh
@@ -23,11 +23,10 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
 rm .coverage* || echo "No coverage files to remove"
 
 tox --notest
+TOX_BIN=$WORKSPACE/.tox/py27/bin
 
 : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
 
-TOX_BIN=$WORKSPACE/.tox/py27/bin
-
 if [[ ! -e .sytest-base ]]; then
   git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
 else
@@ -40,18 +39,8 @@ cd sytest
 
 git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
 
-: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5}
-: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5}
-: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5}
-export PERL5LIB PERL_MB_OPT PERL_MM_OPT
-
-./install-deps.pl
-
 : ${PORT_BASE:=8500}
-
-echo >&2 "Running sytest with SQLite3";
-./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
-    --python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap
+./jenkins/install_and_run.sh --python $TOX_BIN/python --port-base $PORT_BASE
 
 cd ..
 cp sytest/.coverage.* .
-- 
cgit 1.5.1


From 866d0e7cb8483789b61040d7cc5ef7155bcfcf62 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Tue, 8 Mar 2016 23:47:50 +0000
Subject: Only build py27 tox env for integ tests

---
 jenkins-postgres.sh | 2 +-
 jenkins-sqlite.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index e9f22cc04b..14e60482cd 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -22,7 +22,7 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
 
 rm .coverage* || echo "No coverage files to remove"
 
-tox --notest
+tox --notest -e py27
 
 TOX_BIN=$WORKSPACE/.tox/py27/bin
 $TOX_BIN/pip install psycopg2
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
index a68f491cab..9e474aa9f6 100755
--- a/jenkins-sqlite.sh
+++ b/jenkins-sqlite.sh
@@ -22,7 +22,7 @@ export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished w
 
 rm .coverage* || echo "No coverage files to remove"
 
-tox --notest
+tox --notest -e py27
 TOX_BIN=$WORKSPACE/.tox/py27/bin
 
 : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
-- 
cgit 1.5.1


From 4814e7c9b2b9cc14d2890483747bcb1655579ec5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 Mar 2016 00:06:57 +0000
Subject: Specify synapse-directory for integ tests

---
 jenkins-postgres.sh | 4 +++-
 jenkins-sqlite.sh   | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index 14e60482cd..876fabdc71 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -46,7 +46,9 @@ git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling b
 ./jenkins/prep_sytest_for_postgres.sh
 
 echo >&2 "Running sytest with PostgreSQL";
-./jenkins/install_and_run.sh --python $TOX_BIN/python --port-base $PORT_BASE
+./jenkins/install_and_run.sh --python $TOX_BIN/python \
+                             --synapse-directory $WORKSPACE \
+                             --port-base $PORT_BASE
 
 cd ..
 cp sytest/.coverage.* .
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
index 9e474aa9f6..ff6664276d 100755
--- a/jenkins-sqlite.sh
+++ b/jenkins-sqlite.sh
@@ -40,7 +40,9 @@ cd sytest
 git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
 
 : ${PORT_BASE:=8500}
-./jenkins/install_and_run.sh --python $TOX_BIN/python --port-base $PORT_BASE
+./jenkins/install_and_run.sh --python $TOX_BIN/python \
+                             --synapse-directory $WORKSPACE \
+                             --port-base $PORT_BASE
 
 cd ..
 cp sytest/.coverage.* .
-- 
cgit 1.5.1


From ce829c2aefc5e40eacbf6b4bce8a57fe54ae4a1c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 9 Mar 2016 00:14:19 +0000
Subject: Reinstate coverage checks for integ tests

---
 jenkins-postgres.sh | 3 ++-
 jenkins-sqlite.sh   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'jenkins-postgres.sh')

diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh
index 876fabdc71..9ac86d2593 100755
--- a/jenkins-postgres.sh
+++ b/jenkins-postgres.sh
@@ -46,7 +46,8 @@ git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling b
 ./jenkins/prep_sytest_for_postgres.sh
 
 echo >&2 "Running sytest with PostgreSQL";
-./jenkins/install_and_run.sh --python $TOX_BIN/python \
+./jenkins/install_and_run.sh --coverage \
+                             --python $TOX_BIN/python \
                              --synapse-directory $WORKSPACE \
                              --port-base $PORT_BASE
 
diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh
index ff6664276d..345d01936c 100755
--- a/jenkins-sqlite.sh
+++ b/jenkins-sqlite.sh
@@ -40,7 +40,8 @@ cd sytest
 git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop)
 
 : ${PORT_BASE:=8500}
-./jenkins/install_and_run.sh --python $TOX_BIN/python \
+./jenkins/install_and_run.sh --coverage \
+                             --python $TOX_BIN/python \
                              --synapse-directory $WORKSPACE \
                              --port-base $PORT_BASE
 
-- 
cgit 1.5.1