summary refs log tree commit diff
path: root/.github/workflows/latest_deps.yml
blob: d5a68ffa1f12be01cf5e9e660028921da4e15306 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
# dependencies which match the broad requirements. Since most CI runs are against
# the locked poetry environment, run specifically against the latest dependencies to
# know if there's an upcoming breaking change.
#
# As an overview this workflow:
# - checks out develop,
# - installs from source, pulling in the dependencies like a fresh `pip install` would, and
# - runs mypy and test suites in that checkout.
#
# Based on the twisted trunk CI job.

name: Latest dependencies

on:
  schedule:
    - cron: 0 7 * * *
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
        uses: dtolnay/rust-toolchain@fc3253060d0c959bea12a59f10f8391454a0b02d
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2

      # The dev dependencies aren't exposed in the wheel metadata (at least with current
      # poetry-core versions), so we install with poetry.
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: "3.x"
          poetry-version: "1.3.2"
          extras: "all"
      # Dump installed versions for debugging.
      - run: poetry run pip list > before.txt
      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
      # `pip install matrix-synapse[all]` as closely as possible.
      - run: poetry update --no-dev
      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
      - name: Remove warn_unused_ignores from mypy config
        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
      - run: poetry run mypy
  trial:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - database: "sqlite"
          - database: "postgres"
            postgres-version: "14"

    steps:
      - uses: actions/checkout@v3

      - name: Install Rust
        uses: dtolnay/rust-toolchain@fc3253060d0c959bea12a59f10f8391454a0b02d
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2

      - run: sudo apt-get -qq install xmlsec1
      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
        if: ${{ matrix.postgres-version }}
        run: |
          docker run -d -p 5432:5432 \
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
            postgres:${{ matrix.postgres-version }}
      - uses: actions/setup-python@v4
        with:
          python-version: "3.x"
      - run: pip install .[all,test]
      - name: Await PostgreSQL
        if: ${{ matrix.postgres-version }}
        timeout-minutes: 2
        run: until pg_isready -h localhost; do sleep 1; done

      # We nuke the local copy, as we've installed synapse into the virtualenv
      # (rather than use an editable install, which we no longer support). If we
      # don't do this then python can't find the native lib.
      - run: rm -rf synapse/

      - run: python -m twisted.trial --jobs=2 tests
        env:
          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: localhost
          SYNAPSE_POSTGRES_USER: postgres
          SYNAPSE_POSTGRES_PASSWORD: postgres
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
        # Note: Dumps to workflow logs instead of using actions/upload-artifact
        #       This keeps logs colocated with failing jobs
        #       It also ignores find's exit code; this is a best effort affair
        run: >-
          find _trial_temp -name '*.log'
          -exec echo "::group::{}" \;
          -exec cat {} \;
          -exec echo "::endgroup::" \;
          || true


  sytest:
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:testing
      volumes:
        - ${{ github.workspace }}:/src
    strategy:
      fail-fast: false
      matrix:
        include:
          - sytest-tag: focal

          - sytest-tag: focal
            postgres: postgres
            workers: workers
            redis: redis
    env:
      POSTGRES: ${{ matrix.postgres && 1}}
      WORKERS: ${{ matrix.workers && 1 }}
      REDIS: ${{ matrix.redis && 1 }}
      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}

    steps:
      - uses: actions/checkout@v3

      - name: Install Rust
        uses: dtolnay/rust-toolchain@fc3253060d0c959bea12a59f10f8391454a0b02d
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@v2

      - name: Ensure sytest runs `pip install`
        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
        run: rm /src/poetry.lock
        working-directory: /src
      - name: Prepare test blacklist
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
      - name: Upload SyTest logs
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
          path: |
            /logs/results.tap
            /logs/**/*.log*


  complement:
    if: "${{ !failure() && !cancelled() }}"
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
          - arrangement: monolith
            database: SQLite

          - arrangement: monolith
            database: Postgres

          - arrangement: workers
            database: Postgres

    steps:
      - name: Run actions/checkout@v3 for synapse
        uses: actions/checkout@v3
        with:
          path: synapse

      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - run: |
          set -o pipefail
          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
        shell: bash
        name: Run Complement Tests

  # Open an issue if the build fails, so we know about it.
  # Only do this if we're not experimenting with this action in a PR.
  open-issue:
    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
    needs:
      # TODO: should mypy be included here? It feels more brittle than the others.
      - mypy
      - trial
      - sytest
      - complement

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          update_existing: true
          filename: .ci/latest_deps_build_failed_issue_template.md