diff options
author | Erik Johnston <erik@matrix.org> | 2022-12-14 11:02:16 +0000 |
---|---|---|
committer | Erik Johnston <erik@matrix.org> | 2022-12-14 11:02:16 +0000 |
commit | c93ef61fa38337691359b65a90f0a6d5bfc299a7 (patch) | |
tree | 5ce79ebd56eac6f417fe9816a050b1a1d84e7f9b | |
parent | Allow selecting "prejoin" events by state keys (#14642) (diff) | |
download | synapse-c93ef61fa38337691359b65a90f0a6d5bfc299a7.tar.xz |
WIP Rust HTTP for federation
-rw-r--r-- | Cargo.lock | 1004 | ||||
-rw-r--r-- | rust/Cargo.toml | 10 | ||||
-rw-r--r-- | rust/src/http/mod.rs | 149 | ||||
-rw-r--r-- | rust/src/http/resolver.rs | 428 | ||||
-rw-r--r-- | rust/src/lib.rs | 2 | ||||
-rw-r--r-- | stubs/synapse/synapse_rust/http.pyi | 16 | ||||
-rw-r--r-- | synapse/__init__.py | 2 | ||||
-rw-r--r-- | synapse/http/matrixfederationclient.py | 115 |
8 files changed, 1679 insertions, 47 deletions
diff --git a/Cargo.lock b/Cargo.lock index 6e97fb8fb1..d211a42928 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] +name = "async-trait" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -54,12 +65,40 @@ dependencies = [ ] [[package]] +name = "bytes" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" + +[[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -70,6 +109,12 @@ dependencies = [ ] [[package]] +name = "data-encoding" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" + +[[package]] name = "digest" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -81,6 +126,146 @@ dependencies = [ ] [[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" + +[[package]] +name = "futures-executor" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] name = "generic-array" version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -91,18 +276,209 @@ dependencies = [ ] [[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] name = "indoc" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3" [[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipconfig" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" +dependencies = [ + "socket2", + "widestring", + "winapi", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" + +[[package]] name = "itoa" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -121,6 +497,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" [[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] name = "lock_api" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -140,6 +522,27 @@ dependencies = [ ] [[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -155,12 +558,97 @@ dependencies = [ ] [[package]] +name = "mio" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.42.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] name = "once_cell" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" [[package]] +name = "openssl" +version = "0.10.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -180,10 +668,40 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] name = "proc-macro2" version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -211,6 +729,19 @@ dependencies = [ ] [[package]] +name = "pyo3-asyncio" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1febe3946b26194628f00526929ee6f8559f9e807f811257e94d4c456103be0e" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + +[[package]] name = "pyo3-build-config" version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -275,6 +806,12 @@ dependencies = [ ] [[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] name = "quote" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -284,6 +821,36 @@ dependencies = [ ] [[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] name = "redox_syscall" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -310,18 +877,70 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] name = "ryu" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys 0.36.1", +] + +[[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] name = "serde" version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -353,12 +972,40 @@ dependencies = [ ] [[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] name = "subtle" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -381,15 +1028,25 @@ version = "0.1.0" dependencies = [ "anyhow", "blake2", + "futures", + "futures-util", "hex", + "http", + "hyper", + "hyper-tls", "lazy_static", "log", + "native-tls", "pyo3", + "pyo3-asyncio", "pyo3-log", "pythonize", "regex", "serde", "serde_json", + "tokio", + "tokio-native-tls", + "trust-dns-resolver", ] [[package]] @@ -399,68 +1056,403 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" [[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.42.0", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "lru-cache", + "parking_lot", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", + "trust-dns-proto", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] name = "typenum" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] name = "unicode-ident" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] name = "unindent" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112" [[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna 0.3.0", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] name = "windows-sys" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", ] [[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cffaa5b51b..f96f8c4041 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,14 +21,24 @@ name = "synapse.synapse_rust" [dependencies] anyhow = "1.0.63" +futures = "0.3.25" +futures-util = "0.3.25" +http = "0.2.8" +hyper = { version = "0.14.23", features = ["client", "http1", "http2", "runtime", "server", "full"] } +hyper-tls = "0.5.0" lazy_static = "1.4.0" log = "0.4.17" +native-tls = "0.2.11" pyo3 = { version = "0.17.1", features = ["extension-module", "macros", "anyhow", "abi3", "abi3-py37"] } +pyo3-asyncio = { version = "0.17.0", features = ["tokio", "tokio-runtime"] } pyo3-log = "0.7.0" pythonize = "0.17.0" regex = "1.6.0" serde = { version = "1.0.144", features = ["derive"] } serde_json = "1.0.85" +tokio = "1.23.0" +tokio-native-tls = "0.3.0" +trust-dns-resolver = "0.22.0" [build-dependencies] blake2 = "0.10.4" diff --git a/rust/src/http/mod.rs b/rust/src/http/mod.rs new file mode 100644 index 0000000000..b533a3d36d --- /dev/null +++ b/rust/src/http/mod.rs @@ -0,0 +1,149 @@ +use std::collections::HashMap; + +use anyhow::Error; +use http::Request; +use hyper::Body; +use log::info; +use pyo3::{ + pyclass, pymethods, + types::{PyBytes, PyModule}, + IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, +}; + +use self::resolver::{MatrixConnector, MatrixResolver}; + +mod resolver; + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { + let child_module = PyModule::new(py, "http")?; + child_module.add_class::<HttpClient>()?; + child_module.add_class::<MatrixResponse>()?; + + m.add_submodule(child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import push` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.http", child_module)?; + + Ok(()) +} + +#[derive(Clone)] +struct Bytes(Vec<u8>); + +impl ToPyObject for Bytes { + fn to_object(&self, py: Python<'_>) -> pyo3::PyObject { + PyBytes::new(py, &self.0).into_py(py) + } +} + +impl IntoPy<PyObject> for Bytes { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +#[pyclass] +pub struct MatrixResponse { + #[pyo3(get)] + code: u16, + #[pyo3(get)] + phrase: &'static str, + #[pyo3(get)] + content: Bytes, + #[pyo3(get)] + headers: HashMap<String, Bytes>, +} + +#[pyclass] +#[derive(Clone)] +pub struct HttpClient { + client: hyper::Client<MatrixConnector>, +} + +impl HttpClient { + pub fn new() -> Result<Self, Error> { + let resolver = MatrixResolver::new()?; + + let client = hyper::Client::builder().build(MatrixConnector::with_resolver(resolver)); + + Ok(HttpClient { client }) + } + + pub async fn async_request( + &self, + url: String, + method: String, + headers: HashMap<Vec<u8>, Vec<Vec<u8>>>, + body: Option<Vec<u8>>, + ) -> Result<MatrixResponse, Error> { + let mut builder = Request::builder().method(&*method).uri(url); + + for (key, values) in headers { + for value in values { + builder = builder.header(key.clone(), value); + } + } + + let request = if let Some(body) = body { + builder.body(Body::from(body))? + } else { + builder.body(Body::empty())? + }; + + let response = self.client.request(request).await?; + + let code = response.status().as_u16(); + let phrase = response.status().canonical_reason().unwrap_or_default(); + + let headers = response + .headers() + .iter() + .map(|(k, v)| (k.to_string(), Bytes(v.as_bytes().to_owned()))) + .collect(); + + let body = response.into_body(); + + let bytes = hyper::body::to_bytes(body).await?; + let content = Bytes(bytes.to_vec()); + + info!("DONE"); + + Ok(MatrixResponse { + code, + phrase, + content, + headers, + }) + } +} + +#[pymethods] +impl HttpClient { + #[new] + fn py_new() -> Result<Self, Error> { + Self::new() + } + + fn request<'a>( + &'a self, + py: Python<'a>, + url: String, + method: String, + headers: HashMap<Vec<u8>, Vec<Vec<u8>>>, + body: Option<Vec<u8>>, + ) -> PyResult<&'a PyAny> { + pyo3::prepare_freethreaded_python(); + info!("REQUEST"); + + let client = self.clone(); + + pyo3_asyncio::tokio::future_into_py(py, async move { + let resp = client.async_request(url, method, headers, body).await?; + Ok(resp) + }) + } +} diff --git a/rust/src/http/resolver.rs b/rust/src/http/resolver.rs new file mode 100644 index 0000000000..77e9bf5c20 --- /dev/null +++ b/rust/src/http/resolver.rs @@ -0,0 +1,428 @@ +use std::collections::BTreeMap; +use std::future::Future; +use std::net::IpAddr; +use std::pin::Pin; +use std::str::FromStr; +use std::{ + io::Cursor, + sync::{Arc, Mutex}, + task::{self, Poll}, +}; + +use anyhow::{bail, Error}; +use futures::{FutureExt, TryFutureExt}; +use futures_util::stream::StreamExt; +use http::Uri; +use hyper::client::connect::Connection; +use hyper::client::connect::{Connected, HttpConnector}; +use hyper::server::conn::Http; +use hyper::service::Service; +use hyper::Client; +use hyper_tls::HttpsConnector; +use hyper_tls::MaybeHttpsStream; +use log::info; +use native_tls::TlsConnector; +use serde::Deserialize; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::TcpStream; +use tokio_native_tls::TlsConnector as AsyncTlsConnector; +use trust_dns_resolver::error::ResolveErrorKind; + +pub struct Endpoint { + pub host: String, + pub port: u16, + + pub host_header: String, + pub tls_name: String, +} + +#[derive(Clone)] +pub struct MatrixResolver { + resolver: trust_dns_resolver::TokioAsyncResolver, + http_client: Client<HttpsConnector<HttpConnector>>, +} + +impl MatrixResolver { + pub fn new() -> Result<MatrixResolver, Error> { + let http_client = hyper::Client::builder().build(HttpsConnector::new()); + + MatrixResolver::with_client(http_client) + } + + pub fn with_client( + http_client: Client<HttpsConnector<HttpConnector>>, + ) -> Result<MatrixResolver, Error> { + let resolver = trust_dns_resolver::TokioAsyncResolver::tokio_from_system_conf()?; + + Ok(MatrixResolver { + resolver, + http_client, + }) + } + + /// Does SRV lookup + pub async fn resolve_server_name_from_uri(&self, uri: &Uri) -> Result<Vec<Endpoint>, Error> { + let host = uri.host().expect("URI has no host").to_string(); + let port = uri.port_u16(); + + self.resolve_server_name_from_host_port(host, port).await + } + + pub async fn resolve_server_name_from_host_port( + &self, + mut host: String, + mut port: Option<u16>, + ) -> Result<Vec<Endpoint>, Error> { + let mut authority = if let Some(p) = port { + format!("{}:{}", host, p) + } else { + host.to_string() + }; + + // If a literal IP or includes port then we shortcircuit. + if host.parse::<IpAddr>().is_ok() || port.is_some() { + return Ok(vec![Endpoint { + host: host.to_string(), + port: port.unwrap_or(8448), + + host_header: authority.to_string(), + tls_name: host.to_string(), + }]); + } + + // Do well-known delegation lookup. + if let Some(server) = get_well_known(&self.http_client, &host).await { + let a = http::uri::Authority::from_str(&server.server)?; + host = a.host().to_string(); + port = a.port_u16(); + authority = a.to_string(); + } + + // If a literal IP or includes port then we shortcircuit. + if host.parse::<IpAddr>().is_ok() || port.is_some() { + return Ok(vec![Endpoint { + host: host.clone(), + port: port.unwrap_or(8448), + + host_header: authority.to_string(), + tls_name: host.clone(), + }]); + } + + let result = self + .resolver + .srv_lookup(format!("_matrix._tcp.{}", host)) + .await; + + let records = match result { + Ok(records) => records, + Err(err) => match err.kind() { + ResolveErrorKind::NoRecordsFound { .. } => { + return Ok(vec![Endpoint { + host: host.clone(), + port: 8448, + host_header: authority.to_string(), + tls_name: host.clone(), + }]) + } + _ => return Err(err.into()), + }, + }; + + let mut priority_map: BTreeMap<u16, Vec<_>> = BTreeMap::new(); + + let mut count = 0; + for record in records { + count += 1; + let priority = record.priority(); + priority_map.entry(priority).or_default().push(record); + } + + let mut results = Vec::with_capacity(count); + + for (_priority, records) in priority_map { + // TODO: Correctly shuffle records + results.extend(records.into_iter().map(|record| Endpoint { + host: record.target().to_utf8(), + port: record.port(), + + host_header: host.to_string(), + tls_name: host.to_string(), + })) + } + + Ok(results) + } +} + +async fn get_well_known<C>(http_client: &Client<C>, host: &str) -> Option<WellKnownServer> +where + C: Service<Uri> + Clone + Sync + Send + 'static, + C::Error: Into<Box<dyn std::error::Error + Send + Sync>>, + C::Future: Unpin + Send, + C::Response: AsyncRead + AsyncWrite + Connection + Unpin + Send + 'static, +{ + // TODO: Add timeout. + + let uri = hyper::Uri::builder() + .scheme("https") + .authority(host) + .path_and_query("/.well-known/matrix/server") + .build() + .ok()?; + + let mut body = http_client.get(uri).await.ok()?.into_body(); + + let mut vec = Vec::new(); + while let Some(next) = body.next().await { + let chunk = next.ok()?; + vec.extend(chunk); + } + + serde_json::from_slice(&vec).ok()? +} + +#[derive(Deserialize)] +struct WellKnownServer { + #[serde(rename = "m.server")] + server: String, +} + +#[derive(Clone)] +pub struct MatrixConnector { + resolver: MatrixResolver, +} + +impl MatrixConnector { + pub fn with_resolver(resolver: MatrixResolver) -> MatrixConnector { + MatrixConnector { resolver } + } +} + +impl Service<Uri> for MatrixConnector { + type Response = MaybeHttpsStream<TcpStream>; + type Error = Error; + type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>; + + fn poll_ready(&mut self, _: &mut task::Context<'_>) -> Poll<Result<(), Self::Error>> { + // This connector is always ready, but others might not be. + Poll::Ready(Ok(())) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + let resolver = self.resolver.clone(); + + if dst.scheme_str() != Some("matrix") { + return HttpsConnector::new() + .call(dst) + .map_err(|e| Error::msg(e)) + .boxed(); + } + + async move { + let endpoints = resolver + .resolve_server_name_from_host_port( + dst.host().expect("hostname").to_string(), + dst.port_u16(), + ) + .await?; + + for endpoint in endpoints { + match try_connecting(&dst, &endpoint).await { + Ok(r) => return Ok(r), + // Errors here are not unexpected, and we just move on + // with our lives. + Err(e) => info!( + "Failed to connect to {} via {}:{} because {}", + dst.host().expect("hostname"), + endpoint.host, + endpoint.port, + e, + ), + } + } + + bail!( + "failed to resolve host: {:?} port {:?}", + dst.host(), + dst.port() + ) + } + .boxed() + } +} + +/// Attempts to connect to a particular endpoint. +async fn try_connecting( + dst: &Uri, + endpoint: &Endpoint, +) -> Result<MaybeHttpsStream<TcpStream>, Error> { + let tcp = TcpStream::connect((&endpoint.host as &str, endpoint.port)).await?; + + let connector: AsyncTlsConnector = if dst.host().expect("hostname").contains("localhost") { + TlsConnector::builder() + .danger_accept_invalid_certs(true) + .build()? + .into() + } else { + TlsConnector::new().unwrap().into() + }; + + let tls = connector.connect(&endpoint.tls_name, tcp).await?; + + Ok(tls.into()) +} + +/// A connector that reutrns a connection which returns 200 OK to all connections. +#[derive(Clone)] +pub struct TestConnector; + +impl Service<Uri> for TestConnector { + type Response = TestConnection; + type Error = Error; + type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>; + + fn poll_ready(&mut self, _: &mut task::Context<'_>) -> Poll<Result<(), Self::Error>> { + // This connector is always ready, but others might not be. + Poll::Ready(Ok(())) + } + + fn call(&mut self, _dst: Uri) -> Self::Future { + let (client, server) = TestConnection::double_ended(); + + { + let service = hyper::service::service_fn(|_| async move { + Ok(hyper::Response::new(hyper::Body::from("Hello World"))) + as Result<_, hyper::http::Error> + }); + let fut = Http::new().serve_connection(server, service); + tokio::spawn(fut); + } + + futures::future::ok(client).boxed() + } +} + +#[derive(Default)] +struct TestConnectionInner { + outbound_buffer: Cursor<Vec<u8>>, + inbound_buffer: Cursor<Vec<u8>>, + wakers: Vec<futures::task::Waker>, +} + +/// A in memory connection for use with tests. +#[derive(Clone, Default)] +pub struct TestConnection { + inner: Arc<Mutex<TestConnectionInner>>, + direction: bool, +} + +impl TestConnection { + pub fn double_ended() -> (TestConnection, TestConnection) { + let inner: Arc<Mutex<TestConnectionInner>> = Arc::default(); + + let a = TestConnection { + inner: inner.clone(), + direction: false, + }; + + let b = TestConnection { + inner, + direction: true, + }; + + (a, b) + } +} + +impl AsyncRead for TestConnection { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll<Result<(), std::io::Error>> { + let mut conn = self.inner.lock().expect("mutex"); + + let buffer = if self.direction { + &mut conn.inbound_buffer + } else { + &mut conn.outbound_buffer + }; + + let bytes_read = std::io::Read::read(buffer, buf.initialize_unfilled())?; + buf.advance(bytes_read); + if bytes_read > 0 { + Poll::Ready(Ok(())) + } else { + conn.wakers.push(cx.waker().clone()); + Poll::Pending + } + } +} + +impl AsyncWrite for TestConnection { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut task::Context<'_>, + buf: &[u8], + ) -> Poll<Result<usize, std::io::Error>> { + let mut conn = self.inner.lock().expect("mutex"); + + if self.direction { + conn.outbound_buffer.get_mut().extend_from_slice(buf); + } else { + conn.inbound_buffer.get_mut().extend_from_slice(buf); + } + + for waker in conn.wakers.drain(..) { + waker.wake() + } + + Poll::Ready(Ok(buf.len())) + } + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll<Result<(), std::io::Error>> { + let mut conn = self.inner.lock().expect("mutex"); + + if self.direction { + Pin::new(&mut conn.outbound_buffer).poll_flush(cx) + } else { + Pin::new(&mut conn.inbound_buffer).poll_flush(cx) + } + } + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll<Result<(), std::io::Error>> { + let mut conn = self.inner.lock().expect("mutex"); + + if self.direction { + Pin::new(&mut conn.outbound_buffer).poll_shutdown(cx) + } else { + Pin::new(&mut conn.inbound_buffer).poll_shutdown(cx) + } + } +} + +impl Connection for TestConnection { + fn connected(&self) -> Connected { + Connected::new() + } +} + +#[tokio::test] +async fn test_memory_connection() { + let client: hyper::Client<_, hyper::Body> = hyper::Client::builder().build(TestConnector); + + let response = client + .get("http://localhost".parse().unwrap()) + .await + .unwrap(); + + assert!(response.status().is_success()); + + let bytes = hyper::body::to_bytes(response.into_body()).await.unwrap(); + assert_eq!(&bytes[..], b"Hello World"); +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index c7b60e58a7..1a19705b4f 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,5 +1,6 @@ use pyo3::prelude::*; +pub mod http; pub mod push; /// Returns the hash of all the rust source files at the time it was compiled. @@ -26,6 +27,7 @@ fn synapse_rust(py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?; push::register_module(py, m)?; + http::register_module(py, m)?; Ok(()) } diff --git a/stubs/synapse/synapse_rust/http.pyi b/stubs/synapse/synapse_rust/http.pyi new file mode 100644 index 0000000000..fa630c720f --- /dev/null +++ b/stubs/synapse/synapse_rust/http.pyi @@ -0,0 +1,16 @@ +from typing import Dict, List, Optional + +class MatrixResponse: + code: int + phrase: str + content: bytes + headers: Dict[str, str] + +class HttpClient: + async def request( + self, + url: str, + method: str, + headers: Dict[bytes, List[bytes]], + body: Optional[bytes], + ) -> MatrixResponse: ... diff --git a/synapse/__init__.py b/synapse/__init__.py index fbfd506a43..08806dbb98 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -29,7 +29,7 @@ if sys.version_info < (3, 7): sys.exit(1) # Allow using the asyncio reactor via env var. -if strtobool(os.environ.get("SYNAPSE_ASYNC_IO_REACTOR", "0")): +if strtobool(os.environ.get("SYNAPSE_ASYNC_IO_REACTOR", "0")) or True: from incremental import Version import twisted diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b92f1d3d1a..670164923a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import abc +import asyncio import cgi import codecs import logging @@ -42,14 +43,18 @@ from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json from typing_extensions import Literal +from zope.interface import implementer from twisted.internet import defer from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IReactorTime +from twisted.internet.protocol import Protocol from twisted.internet.task import Cooperator -from twisted.web.client import ResponseFailed +from twisted.internet.testing import StringTransport +from twisted.python.failure import Failure +from twisted.web.client import Response, ResponseDone, ResponseFailed from twisted.web.http_headers import Headers -from twisted.web.iweb import IBodyProducer, IResponse +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse import synapse.metrics import synapse.util.retryutils @@ -75,6 +80,7 @@ from synapse.http.types import QueryParams from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag, start_active_span, tags +from synapse.synapse_rust.http import HttpClient from synapse.types import JsonDict from synapse.util import json_decoder from synapse.util.async_helpers import AwakenableSleeper, timeout_deferred @@ -199,6 +205,33 @@ class JsonParser(ByteParser[Union[JsonDict, list]]): return json_decoder.decode(self._buffer.getvalue()) +@attr.s(auto_attribs=True) +@implementer(IResponse) +class RustResponse: + version: tuple + + code: int + + phrase: bytes + + headers: Headers + + length: Union[int, UNKNOWN_LENGTH] + + # request: Optional[IClientRequest] + + # previousResponse: Optional[IResponse] + + _data: bytes + + def deliverBody(self, protocol: Protocol): + protocol.dataReceived(self._data) + protocol.connectionLost(Failure(ResponseDone("Response body fully received"))) + + def setPreviousResponse(self, response: IResponse): + pass + + async def _handle_response( reactor: IReactorTime, timeout_sec: float, @@ -372,6 +405,8 @@ class MatrixFederationHttpClient: self._sleeper = AwakenableSleeper(self.reactor) + self._rust_client = HttpClient() + def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" @@ -556,11 +591,8 @@ class MatrixFederationHttpClient: destination_bytes, method_bytes, url_to_sign_bytes, json ) data = encode_canonical_json(json) - producer: Optional[IBodyProducer] = QuieterFileBodyProducer( - BytesIO(data), cooperator=self._cooperator - ) else: - producer = None + data = None auth_headers = self.build_auth_headers( destination_bytes, method_bytes, url_to_sign_bytes ) @@ -591,23 +623,32 @@ class MatrixFederationHttpClient: # * The `Deferred` that joins the forks back together is # wrapped in `make_deferred_yieldable` to restore the # logging context regardless of the path taken. - request_deferred = run_in_background( - self.agent.request, - method_bytes, - url_bytes, - headers=Headers(headers_dict), - bodyProducer=producer, - ) - request_deferred = timeout_deferred( - request_deferred, - timeout=_sec_timeout, - reactor=self.reactor, + # request_deferred = run_in_background( + # self._rust_client.request, + # url_str, + # request.method, + # headers_dict, + # data, + # ) + # request_deferred = timeout_deferred( + # request_deferred, + # timeout=_sec_timeout, + # reactor=self.reactor, + # ) + + # response = await make_deferred_yieldable(request_deferred) + + response_d = self._rust_client.request( + url_str, + request.method, + headers_dict, + data, ) - - response = await make_deferred_yieldable(request_deferred) + response = await defer.Deferred.fromFuture(response_d) except DNSLookupError as e: raise RequestSendFailed(e, can_retry=retry_on_dns_fail) from e except Exception as e: + logger.exception("ERROR") raise RequestSendFailed(e, can_retry=True) from e incoming_responses_counter.labels( @@ -615,7 +656,7 @@ class MatrixFederationHttpClient: ).inc() set_tag(tags.HTTP_STATUS_CODE, response.code) - response_phrase = response.phrase.decode("ascii", errors="replace") + response_phrase = response.phrase if 200 <= response.code < 300: logger.debug( @@ -635,25 +676,7 @@ class MatrixFederationHttpClient: ) # :'( # Update transactions table? - d = treq.content(response) - d = timeout_deferred( - d, timeout=_sec_timeout, reactor=self.reactor - ) - - try: - body = await make_deferred_yieldable(d) - except Exception as e: - # Eh, we're already going to raise an exception so lets - # ignore if this fails. - logger.warning( - "{%s} [%s] Failed to get error response: %s %s: %s", - request.txn_id, - request.destination, - request.method, - url_str, - _flatten_response_never_received(e), - ) - body = None + body = response.content exc = HttpResponseException( response.code, response_phrase, body @@ -715,7 +738,19 @@ class MatrixFederationHttpClient: _flatten_response_never_received(e), ) raise - return response + + headers = Headers() + for key, value in response.headers.items(): + headers.addRawHeader(key, value) + + return RustResponse( + ("HTTP", 1, 1), + response.code, + response.phrase.encode("ascii"), + headers, + UNKNOWN_LENGTH, + response.content, + ) def build_auth_headers( self, |