Add a nix flake that sets up a development environment (via devenv) (#15495)

4 files changed, 484 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index 9d037f28e7..8cf504324b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,9 +15,10 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want the poetry and cargo lockfile.
+# We do want poetry, cargo and flake lockfiles.
 !poetry.lock
 !Cargo.lock
+!flake.lock
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -38,6 +39,9 @@ __pycache__/
 /.envrc
 .direnv/
 
+# For nix/devenv users
+.devenv/
+
 # IDEs
 /.idea/
 /.ropeproject/
diff --git a/changelog.d/15495.misc b/changelog.d/15495.misc
new file mode 100644
index 0000000000..ff7b5cbddf
--- /dev/null
+++ b/changelog.d/15495.misc
@@ -0,0 +1 @@
+Add a Nix flake for use as a development environment.
\ No newline at end of file
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000000..85886b730f
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,274 @@
+{
+  "nodes": {
+    "devenv": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nix": "nix",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1682534083,
+        "narHash": "sha256-lBgFaLNHRQtD3InZbBXzIS8HgZUgcPJ6jiqGa4FJPrk=",
+        "owner": "anoadragon453",
+        "repo": "devenv",
+        "rev": "9694bd0a845dd184d4468cc3d3461089aace787a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "anoadragon453",
+        "ref": "anoa/fix_languages_python",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1682490133,
+        "narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "devenv",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
+    "nix": {
+      "inputs": {
+        "lowdown-src": "lowdown-src",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression"
+      },
+      "locked": {
+        "lastModified": 1676545802,
+        "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=",
+        "owner": "domenkozar",
+        "repo": "nix",
+        "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "domenkozar",
+        "ref": "relaxed-flakes",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1678875422,
+        "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1673800717,
+        "narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1682519441,
+        "narHash": "sha256-Vsq/8NOtvW1AoC6shCBxRxZyMQ+LhvPuJT6ltbzuv+Y=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "7a32a141db568abde9bc389845949dc2a454dfd3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "master",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1678376203,
+        "narHash": "sha256-3tyYGyC8h7fBwncLZy5nCUjTJPrHbmNwp47LlNLOHSM=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "1a20b9708962096ec2481eeb2ddca29ed747770a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "fenix": "fenix",
+        "nixpkgs": "nixpkgs_2",
+        "systems": "systems"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1682426789,
+        "narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000000..91916d9abb
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,204 @@
+# A nix flake that sets up a complete Synapse development environment. Dependencies
+# for the SyTest (https://github.com/matrix-org/sytest) and Complement
+# (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
+# installed automatically.
+#
+# You must have already installed nix (https://nixos.org) on your system to use this.
+# nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but nix can be installed inside of WSL2 or even Docker
+# containers. Please refer to https://nixos.org/download for details.
+#
+# You must also enable support for flakes in Nix. See the following for how to
+# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
+#
+# Usage:
+#
+# With nix installed, navigate to the directory containing this flake and run
+# `nix develop --impure`. The `--impure` is necessary in order to store state
+# locally from "services", such as PostgreSQL and Redis.
+#
+# You should now be dropped into a new shell with all programs and dependencies
+# availabile to you!
+#
+# You can start up pre-configured, local PostgreSQL and Redis instances by
+# running: `devenv up`. To stop them, use Ctrl-C.
+#
+# A PostgreSQL database called 'synapse' will be set up for you, along with
+# a PostgreSQL user named 'synapse_user'.
+# The 'host' can be found by running `echo $PGHOST` with the development
+# shell activated. Use these values to configure your Synapse to connect
+# to the local PostgreSQL database. You do not need to specify a password.
+# https://matrix-org.github.io/synapse/latest/postgres
+#
+# All state (the venv, postgres and redis data and config) are stored in
+# .devenv/state. Deleting a file from here and then re-entering the shell
+# will recreate these files from scratch.
+#
+# You can exit the development shell by typing `exit`, or using Ctrl-D.
+#
+# If you would like this development environment to activate automatically
+# upon entering this directory in your terminal, first install `direnv`
+# (https://direnv.net/). Then run `echo 'use flake . --impure' >> .envrc` at
+# the root of the Synapse repo. Finally, run `direnv allow .` to allow the
+# contents of '.envrc' to run every time you enter this directory. Voilà!
+
+{
+  inputs = {
+    # Use the master/unstable branch of nixpkgs. The latest stable, 22.11,
+    # does not contain 'perl536Packages.NetAsyncHTTP', needed by Sytest.
+    nixpkgs.url = "github:NixOS/nixpkgs/master";
+    # Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
+    systems.url = "github:nix-systems/default";
+    # A development environment manager built on Nix. See https://devenv.sh.
+    # This is temporarily overridden to a fork that fixes a quirk between
+    # devenv's service and python language features. This can be removed
+    # when https://github.com/cachix/devenv/pull/559 is merged upstream.
+    devenv.url = "github:anoadragon453/devenv/anoa/fix_languages_python";
+    #devenv.url = "github:cachix/devenv/main";
+    # Rust toolchains and rust-analyzer nightly.
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
+    let
+      forEachSystem = nixpkgs.lib.genAttrs (import systems);
+    in {
+      devShells = forEachSystem (system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+        in {
+          # Everything is configured via devenv - a nix module for creating declarative
+          # developer environments. See https://devenv.sh/reference/options/ for a list
+          # of all possible options.
+          default = devenv.lib.mkShell {
+            inherit inputs pkgs;
+            modules = [
+              {
+                # Make use of the Starship command prompt when this development environment
+                # is manually activated (via `nix develop --impure`).
+                # See https://starship.rs/ for details on the prompt itself.
+                starship.enable = true;
+
+                # Configure packages to install.
+                # Search for package names at https://search.nixos.org/packages?channel=unstable
+                packages = with pkgs; [
+                  # Native dependencies for running Synapse.
+                  icu
+                  libffi
+                  libjpeg
+                  libpqxx
+                  libwebp
+                  libxml2
+                  libxslt
+                  sqlite
+
+                  # Native dependencies for unit tests (SyTest also requires OpenSSL).
+                  openssl
+
+                  # Native dependencies for running Complement.
+                  olm
+                ];
+
+                # Install Python and manage a virtualenv with Poetry.
+                languages.python.enable = true;
+                languages.python.poetry.enable = true;
+                # Automatically activate the poetry virtualenv upon entering the shell.
+                languages.python.poetry.activate.enable = true;
+                # Install all extra Python dependencies; this is needed to run the unit
+                # tests and utilitise all Synapse features.
+                languages.python.poetry.install.arguments = ["--extras all"];
+                # Install the 'matrix-synapse' package from the local checkout.
+                languages.python.poetry.install.installRootPackage = true;
+
+                # This is a work-around for NixOS systems. NixOS is special in
+                # that you can have multiple versions of packages installed at
+                # once, including your libc linker!
+                #
+                # Some binaries built for Linux expect those to be in a certain
+                # filepath, but that is not the case on NixOS. In that case, we
+                # force compiling those binaries locally instead.
+                env.POETRY_INSTALLER_NO_BINARY = "ruff";
+
+                # Install dependencies for the additional programming languages
+                # involved with Synapse development.
+                #
+                # * Rust is used for developing and running Synapse.
+                # * Golang is needed to run the Complement test suite.
+                # * Perl is needed to run the SyTest test suite.
+                languages.go.enable = true;
+                languages.rust.enable = true;
+                languages.rust.version = "stable";
+                languages.perl.enable = true;
+
+                # Postgres is needed to run Synapse with postgres support and
+                # to run certain unit tests that require postgres.
+                services.postgres.enable = true;
+
+                # On the first invocation of `devenv up`, create a database for
+                # Synapse to store data in.
+                services.postgres.initdbArgs = ["--locale=C" "--encoding=UTF8"];
+                services.postgres.initialDatabases = [
+                  { name = "synapse"; }
+                ];
+                # Create a postgres user called 'synapse_user' which has ownership
+                # over the 'synapse' database.
+                services.postgres.initialScript = ''
+                CREATE USER synapse_user;
+                ALTER DATABASE synapse OWNER TO synapse_user;
+                '';
+
+                # Redis is needed in order to run Synapse in worker mode.
+                services.redis.enable = true;
+
+                # Define the perl modules we require to run SyTest.
+                #
+                # This list was compiled by cross-referencing https://metacpan.org/
+                # with the modules defined in './cpanfile' and then finding the
+                # corresponding nix packages on https://search.nixos.org/packages.
+                #
+                # This was done until `./install-deps.pl --dryrun` produced no output.
+                env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
+                  DBI
+                  ClassMethodModifiers
+                  CryptEd25519
+                  DataDump
+                  DBDPg
+                  DigestHMAC
+                  DigestSHA1
+                  EmailAddressXS
+                  EmailMIME
+                  EmailSimple  # required by Email::Mime
+                  EmailMessageID  # required by Email::Mime
+                  EmailMIMEContentType  # required by Email::Mime
+                  TextUnidecode  # required by Email::Mime
+                  ModuleRuntime  # required by Email::Mime
+                  EmailMIMEEncodings  # required by Email::Mime
+                  FilePath
+                  FileSlurper
+                  Future
+                  GetoptLong
+                  HTTPMessage
+                  IOAsync
+                  IOAsyncSSL
+                  IOSocketSSL
+                  NetSSLeay
+                  JSON
+                  ListUtilsBy
+                  ScalarListUtils
+                  ModulePluggable
+                  NetAsyncHTTP
+                  MetricsAny  # required by Net::Async::HTTP
+                  NetAsyncHTTPServer
+                  StructDumb
+                  URI
+                  YAMLLibYAML
+                ]}";
+              }
+            ];
+          };
+        });
+    };
+}