Update PyO3 to 0.21 (#17162)

This version change requires a migration to a new API. See
https://pyo3.rs/v0.21.2/migration#from-020-to-021

This will fix the annoying warnings added when using the recent rust
nightly:

> warning: non-local `impl` definition, they should be avoided as they
go against expectation

8 files changed, 90 insertions, 62 deletions

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index d41a216d1c..026487275c 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -30,14 +30,14 @@ http = "1.1.0"
 lazy_static = "1.4.0"
 log = "0.4.17"
 mime = "0.3.17"
-pyo3 = { version = "0.20.0", features = [
+pyo3 = { version = "0.21.0", features = [
     "macros",
     "anyhow",
     "abi3",
     "abi3-py38",
 ] }
-pyo3-log = "0.9.0"
-pythonize = "0.20.0"
+pyo3-log = "0.10.0"
+pythonize = "0.21.0"
 regex = "1.6.0"
 sha2 = "0.10.8"
 serde = { version = "1.0.144", features = ["derive"] }
diff --git a/rust/src/acl/mod.rs b/rust/src/acl/mod.rs
index 286574fb49..982720ba90 100644
--- a/rust/src/acl/mod.rs
+++ b/rust/src/acl/mod.rs
@@ -25,21 +25,21 @@ use std::net::Ipv4Addr;
 use std::str::FromStr;
 
 use anyhow::Error;
-use pyo3::prelude::*;
+use pyo3::{prelude::*, pybacked::PyBackedStr};
 use regex::Regex;
 
 use crate::push::utils::{glob_to_regex, GlobMatchType};
 
 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "acl")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "acl")?;
     child_module.add_class::<ServerAclEvaluator>()?;
 
-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;
 
     // We need to manually add the module to sys.modules to make `from
     // synapse.synapse_rust import acl` work.
-    py.import("sys")?
+    py.import_bound("sys")?
         .getattr("modules")?
         .set_item("synapse.synapse_rust.acl", child_module)?;
 
@@ -59,8 +59,8 @@ impl ServerAclEvaluator {
     #[new]
     pub fn py_new(
         allow_ip_literals: bool,
-        allow: Vec<&str>,
-        deny: Vec<&str>,
+        allow: Vec<PyBackedStr>,
+        deny: Vec<PyBackedStr>,
     ) -> Result<Self, Error> {
         let allow = allow
             .iter()
diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs
index 53c7b1ba61..63774fbd54 100644
--- a/rust/src/events/internal_metadata.rs
+++ b/rust/src/events/internal_metadata.rs
@@ -38,9 +38,10 @@ use anyhow::Context;
 use log::warn;
 use pyo3::{
     exceptions::PyAttributeError,
+    pybacked::PyBackedStr,
     pyclass, pymethods,
-    types::{PyDict, PyString},
-    IntoPy, PyAny, PyObject, PyResult, Python,
+    types::{PyAnyMethods, PyDict, PyDictMethods, PyString},
+    Bound, IntoPy, PyAny, PyObject, PyResult, Python,
 };
 
 /// Definitions of the various fields of the internal metadata.
@@ -59,7 +60,7 @@ enum EventInternalMetadataData {
 
 impl EventInternalMetadataData {
     /// Convert the field to its name and python object.
-    fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a PyString, PyObject) {
+    fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, PyObject) {
         match self {
             EventInternalMetadataData::OutOfBandMembership(o) => {
                 (pyo3::intern!(py, "out_of_band_membership"), o.into_py(py))
@@ -90,10 +91,13 @@ impl EventInternalMetadataData {
     /// Converts from python key/values to the field.
     ///
     /// Returns `None` if the key is a valid but unrecognized string.
-    fn from_python_pair(key: &PyAny, value: &PyAny) -> PyResult<Option<Self>> {
-        let key_str: &str = key.extract()?;
+    fn from_python_pair(
+        key: &Bound<'_, PyAny>,
+        value: &Bound<'_, PyAny>,
+    ) -> PyResult<Option<Self>> {
+        let key_str: PyBackedStr = key.extract()?;
 
-        let e = match key_str {
+        let e = match &*key_str {
             "out_of_band_membership" => EventInternalMetadataData::OutOfBandMembership(
                 value
                     .extract()
@@ -210,11 +214,11 @@ pub struct EventInternalMetadata {
 #[pymethods]
 impl EventInternalMetadata {
     #[new]
-    fn new(dict: &PyDict) -> PyResult<Self> {
+    fn new(dict: &Bound<'_, PyDict>) -> PyResult<Self> {
         let mut data = Vec::with_capacity(dict.len());
 
         for (key, value) in dict.iter() {
-            match EventInternalMetadataData::from_python_pair(key, value) {
+            match EventInternalMetadataData::from_python_pair(&key, &value) {
                 Ok(Some(entry)) => data.push(entry),
                 Ok(None) => {}
                 Err(err) => {
@@ -240,7 +244,7 @@ impl EventInternalMetadata {
     ///
     /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here.
     fn get_dict(&self, py: Python<'_>) -> PyResult<PyObject> {
-        let dict = PyDict::new(py);
+        let dict = PyDict::new_bound(py);
 
         for entry in &self.data {
             let (key, value) = entry.to_python_pair(py);
diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs
index ee857b3d72..a4ade1a178 100644
--- a/rust/src/events/mod.rs
+++ b/rust/src/events/mod.rs
@@ -20,20 +20,23 @@
 
 //! Classes for representing Events.
 
-use pyo3::{types::PyModule, PyResult, Python};
+use pyo3::{
+    types::{PyAnyMethods, PyModule, PyModuleMethods},
+    Bound, PyResult, Python,
+};
 
 mod internal_metadata;
 
 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "events")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "events")?;
     child_module.add_class::<internal_metadata::EventInternalMetadata>()?;
 
-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;
 
     // We need to manually add the module to sys.modules to make `from
     // synapse.synapse_rust import events` work.
-    py.import("sys")?
+    py.import_bound("sys")?
         .getattr("modules")?
         .set_item("synapse.synapse_rust.events", child_module)?;
 
diff --git a/rust/src/http.rs b/rust/src/http.rs
index 74098f4c8b..af052ab721 100644
--- a/rust/src/http.rs
+++ b/rust/src/http.rs
@@ -17,8 +17,8 @@ use headers::{Header, HeaderMapExt};
 use http::{HeaderName, HeaderValue, Method, Request, Response, StatusCode, Uri};
 use pyo3::{
     exceptions::PyValueError,
-    types::{PyBytes, PySequence, PyTuple},
-    PyAny, PyResult,
+    types::{PyAnyMethods, PyBytes, PyBytesMethods, PySequence, PyTuple},
+    Bound, PyAny, PyResult,
 };
 
 use crate::errors::SynapseError;
@@ -28,10 +28,11 @@ use crate::errors::SynapseError;
 /// # Errors
 ///
 /// Returns an error if calling the `read` on the Python object failed
-fn read_io_body(body: &PyAny, chunk_size: usize) -> PyResult<Bytes> {
+fn read_io_body(body: &Bound<'_, PyAny>, chunk_size: usize) -> PyResult<Bytes> {
     let mut buf = BytesMut::new();
     loop {
-        let bytes: &PyBytes = body.call_method1("read", (chunk_size,))?.downcast()?;
+        let bound = &body.call_method1("read", (chunk_size,))?;
+        let bytes: &Bound<'_, PyBytes> = bound.downcast()?;
         if bytes.as_bytes().is_empty() {
             return Ok(buf.into());
         }
@@ -50,17 +51,19 @@ fn read_io_body(body: &PyAny, chunk_size: usize) -> PyResult<Bytes> {
 /// # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
-pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
+pub fn http_request_from_twisted(request: &Bound<'_, PyAny>) -> PyResult<Request<Bytes>> {
     let content = request.getattr("content")?;
-    let body = read_io_body(content, 4096)?;
+    let body = read_io_body(&content, 4096)?;
 
     let mut req = Request::new(body);
 
-    let uri: &PyBytes = request.getattr("uri")?.downcast()?;
+    let bound = &request.getattr("uri")?;
+    let uri: &Bound<'_, PyBytes> = bound.downcast()?;
     *req.uri_mut() =
         Uri::try_from(uri.as_bytes()).map_err(|_| PyValueError::new_err("invalid uri"))?;
 
-    let method: &PyBytes = request.getattr("method")?.downcast()?;
+    let bound = &request.getattr("method")?;
+    let method: &Bound<'_, PyBytes> = bound.downcast()?;
     *req.method_mut() = Method::from_bytes(method.as_bytes())
         .map_err(|_| PyValueError::new_err("invalid method"))?;
 
@@ -71,14 +74,17 @@ pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
 
     for header in headers_iter {
         let header = header?;
-        let header: &PyTuple = header.downcast()?;
-        let name: &PyBytes = header.get_item(0)?.downcast()?;
+        let header: &Bound<'_, PyTuple> = header.downcast()?;
+        let bound = &header.get_item(0)?;
+        let name: &Bound<'_, PyBytes> = bound.downcast()?;
         let name = HeaderName::from_bytes(name.as_bytes())
             .map_err(|_| PyValueError::new_err("invalid header name"))?;
 
-        let values: &PySequence = header.get_item(1)?.downcast()?;
+        let bound = &header.get_item(1)?;
+        let values: &Bound<'_, PySequence> = bound.downcast()?;
         for index in 0..values.len()? {
-            let value: &PyBytes = values.get_item(index)?.downcast()?;
+            let bound = &values.get_item(index)?;
+            let value: &Bound<'_, PyBytes> = bound.downcast()?;
             let value = HeaderValue::from_bytes(value.as_bytes())
                 .map_err(|_| PyValueError::new_err("invalid header value"))?;
             req.headers_mut().append(name.clone(), value);
@@ -100,7 +106,10 @@ pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
 ///  # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
-pub fn http_response_to_twisted<B>(request: &PyAny, response: Response<B>) -> PyResult<()>
+pub fn http_response_to_twisted<B>(
+    request: &Bound<'_, PyAny>,
+    response: Response<B>,
+) -> PyResult<()>
 where
     B: Buf,
 {
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
index 9bd1f17ad9..06477880b9 100644
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -38,7 +38,7 @@ fn reset_logging_config() {
 
 /// The entry point for defining the Python module.
 #[pymodule]
-fn synapse_rust(py: Python<'_>, m: &PyModule) -> PyResult<()> {
+fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
     m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?;
     m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?;
diff --git a/rust/src/push/mod.rs b/rust/src/push/mod.rs
index 7dedbf10b6..2a452b69a3 100644
--- a/rust/src/push/mod.rs
+++ b/rust/src/push/mod.rs
@@ -66,7 +66,7 @@ use log::warn;
 use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
 use pyo3::types::{PyBool, PyList, PyLong, PyString};
-use pythonize::{depythonize, pythonize};
+use pythonize::{depythonize_bound, pythonize};
 use serde::de::Error as _;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -78,19 +78,19 @@ pub mod evaluator;
 pub mod utils;
 
 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "push")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "push")?;
     child_module.add_class::<PushRule>()?;
     child_module.add_class::<PushRules>()?;
     child_module.add_class::<FilteredPushRules>()?;
     child_module.add_class::<PushRuleEvaluator>()?;
     child_module.add_function(wrap_pyfunction!(get_base_rule_ids, m)?)?;
 
-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;
 
     // We need to manually add the module to sys.modules to make `from
     // synapse.synapse_rust import push` work.
-    py.import("sys")?
+    py.import_bound("sys")?
         .getattr("modules")?
         .set_item("synapse.synapse_rust.push", child_module)?;
 
@@ -271,12 +271,12 @@ pub enum SimpleJsonValue {
 
 impl<'source> FromPyObject<'source> for SimpleJsonValue {
     fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(s) = <PyString as pyo3::PyTryFrom>::try_from(ob) {
+        if let Ok(s) = ob.downcast::<PyString>() {
             Ok(SimpleJsonValue::Str(Cow::Owned(s.to_string())))
         // A bool *is* an int, ensure we try bool first.
-        } else if let Ok(b) = <PyBool as pyo3::PyTryFrom>::try_from(ob) {
+        } else if let Ok(b) = ob.downcast::<PyBool>() {
             Ok(SimpleJsonValue::Bool(b.extract()?))
-        } else if let Ok(i) = <PyLong as pyo3::PyTryFrom>::try_from(ob) {
+        } else if let Ok(i) = ob.downcast::<PyLong>() {
             Ok(SimpleJsonValue::Int(i.extract()?))
         } else if ob.is_none() {
             Ok(SimpleJsonValue::Null)
@@ -299,7 +299,7 @@ pub enum JsonValue {
 
 impl<'source> FromPyObject<'source> for JsonValue {
     fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(l) = <PyList as pyo3::PyTryFrom>::try_from(ob) {
+        if let Ok(l) = ob.downcast::<PyList>() {
             match l.iter().map(SimpleJsonValue::extract).collect() {
                 Ok(a) => Ok(JsonValue::Array(a)),
                 Err(e) => Err(PyTypeError::new_err(format!(
@@ -370,8 +370,8 @@ impl IntoPy<PyObject> for Condition {
 }
 
 impl<'source> FromPyObject<'source> for Condition {
-    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        Ok(depythonize(ob)?)
+    fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult<Self> {
+        Ok(depythonize_bound(ob.clone())?)
     }
 }
 
diff --git a/rust/src/rendezvous/mod.rs b/rust/src/rendezvous/mod.rs
index c0f5d8b600..f69f45490f 100644
--- a/rust/src/rendezvous/mod.rs
+++ b/rust/src/rendezvous/mod.rs
@@ -26,8 +26,10 @@ use headers::{
 use http::{header::ETAG, HeaderMap, Response, StatusCode, Uri};
 use mime::Mime;
 use pyo3::{
-    exceptions::PyValueError, pyclass, pymethods, types::PyModule, Py, PyAny, PyObject, PyResult,
-    Python, ToPyObject,
+    exceptions::PyValueError,
+    pyclass, pymethods,
+    types::{PyAnyMethods, PyModule, PyModuleMethods},
+    Bound, Py, PyAny, PyObject, PyResult, Python, ToPyObject,
 };
 use ulid::Ulid;
 
@@ -109,7 +111,7 @@ impl RendezvousHandler {
     #[pyo3(signature = (homeserver, /, capacity=100, max_content_length=4*1024, eviction_interval=60*1000, ttl=60*1000))]
     fn new(
         py: Python<'_>,
-        homeserver: &PyAny,
+        homeserver: &Bound<'_, PyAny>,
         capacity: usize,
         max_content_length: u64,
         eviction_interval: u64,
@@ -150,7 +152,7 @@ impl RendezvousHandler {
     }
 
     fn _evict(&mut self, py: Python<'_>) -> PyResult<()> {
-        let clock = self.clock.as_ref(py);
+        let clock = self.clock.bind(py);
         let now: u64 = clock.call_method0("time_msec")?.extract()?;
         let now = SystemTime::UNIX_EPOCH + Duration::from_millis(now);
         self.evict(now);
@@ -158,12 +160,12 @@ impl RendezvousHandler {
         Ok(())
     }
 
-    fn handle_post(&mut self, py: Python<'_>, twisted_request: &PyAny) -> PyResult<()> {
+    fn handle_post(&mut self, py: Python<'_>, twisted_request: &Bound<'_, PyAny>) -> PyResult<()> {
         let request = http_request_from_twisted(twisted_request)?;
 
         let content_type = self.check_input_headers(request.headers())?;
 
-        let clock = self.clock.as_ref(py);
+        let clock = self.clock.bind(py);
         let now: u64 = clock.call_method0("time_msec")?.extract()?;
         let now = SystemTime::UNIX_EPOCH + Duration::from_millis(now);
 
@@ -197,7 +199,12 @@ impl RendezvousHandler {
         Ok(())
     }
 
-    fn handle_get(&mut self, py: Python<'_>, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_get(
+        &mut self,
+        py: Python<'_>,
+        twisted_request: &Bound<'_, PyAny>,
+        id: &str,
+    ) -> PyResult<()> {
         let request = http_request_from_twisted(twisted_request)?;
 
         let if_none_match: Option<IfNoneMatch> = request.headers().typed_get_optional()?;
@@ -233,7 +240,12 @@ impl RendezvousHandler {
         Ok(())
     }
 
-    fn handle_put(&mut self, py: Python<'_>, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_put(
+        &mut self,
+        py: Python<'_>,
+        twisted_request: &Bound<'_, PyAny>,
+        id: &str,
+    ) -> PyResult<()> {
         let request = http_request_from_twisted(twisted_request)?;
 
         let content_type = self.check_input_headers(request.headers())?;
@@ -281,7 +293,7 @@ impl RendezvousHandler {
         Ok(())
     }
 
-    fn handle_delete(&mut self, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_delete(&mut self, twisted_request: &Bound<'_, PyAny>, id: &str) -> PyResult<()> {
         let _request = http_request_from_twisted(twisted_request)?;
 
         let id: Ulid = id.parse().map_err(|_| NotFoundError::new())?;
@@ -298,16 +310,16 @@ impl RendezvousHandler {
     }
 }
 
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "rendezvous")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "rendezvous")?;
 
     child_module.add_class::<RendezvousHandler>()?;
 
-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;
 
     // We need to manually add the module to sys.modules to make `from
     // synapse.synapse_rust import rendezvous` work.
-    py.import("sys")?
+    py.import_bound("sys")?
         .getattr("modules")?
         .set_item("synapse.synapse_rust.rendezvous", child_module)?;