diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index d2c0aebe48..6576a30098 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 37
+SCHEMA_VERSION = 38
dir_path = os.path.abspath(os.path.dirname(__file__))
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 8f5f8f24a9..8cc9f0353b 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -137,17 +137,8 @@ class PusherStore(SQLBaseStore):
@cachedInlineCallbacks(num_args=1, max_entries=15000)
def get_if_user_has_pusher(self, user_id):
- result = yield self._simple_select_many_batch(
- table='pushers',
- keyvalues={
- 'user_name': 'user_id',
- },
- retcol='user_name',
- desc='get_if_user_has_pusher',
- allow_none=True,
- )
-
- defer.returnValue(bool(result))
+ # This only exists for the cachedList decorator
+ raise NotImplementedError()
@cachedList(cached_method_name="get_if_user_has_pusher",
list_name="user_ids", num_args=1, inlineCallbacks=True)
diff --git a/synapse/storage/schema/delta/38/postgres_fts_gist.sql b/synapse/storage/schema/delta/38/postgres_fts_gist.sql
new file mode 100644
index 0000000000..f090a7b75a
--- /dev/null
+++ b/synapse/storage/schema/delta/38/postgres_fts_gist.sql
@@ -0,0 +1,17 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ INSERT into background_updates (update_name, progress_json)
+ VALUES ('event_search_postgres_gist', '{}');
diff --git a/synapse/storage/search.py b/synapse/storage/search.py
index 12941d1775..8f2b3c4435 100644
--- a/synapse/storage/search.py
+++ b/synapse/storage/search.py
@@ -31,6 +31,7 @@ class SearchStore(BackgroundUpdateStore):
EVENT_SEARCH_UPDATE_NAME = "event_search"
EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order"
+ EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist"
def __init__(self, hs):
super(SearchStore, self).__init__(hs)
@@ -41,6 +42,10 @@ class SearchStore(BackgroundUpdateStore):
self.EVENT_SEARCH_ORDER_UPDATE_NAME,
self._background_reindex_search_order
)
+ self.register_background_update_handler(
+ self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME,
+ self._background_reindex_gist_search
+ )
@defer.inlineCallbacks
def _background_reindex_search(self, progress, batch_size):
@@ -140,6 +145,28 @@ class SearchStore(BackgroundUpdateStore):
defer.returnValue(result)
@defer.inlineCallbacks
+ def _background_reindex_gist_search(self, progress, batch_size):
+ def create_index(conn):
+ conn.rollback()
+ conn.set_session(autocommit=True)
+ c = conn.cursor()
+
+ c.execute(
+ "CREATE INDEX CONCURRENTLY event_search_fts_idx_gist"
+ " ON event_search USING GIST (vector)"
+ )
+
+ c.execute("DROP INDEX event_search_fts_idx")
+
+ conn.set_session(autocommit=False)
+
+ if isinstance(self.database_engine, PostgresEngine):
+ yield self.runWithConnection(create_index)
+
+ yield self._end_background_update(self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME)
+ defer.returnValue(1)
+
+ @defer.inlineCallbacks
def _background_reindex_search_order(self, progress, batch_size):
target_min_stream_id = progress["target_min_stream_id_inclusive"]
max_stream_id = progress["max_stream_id_exclusive"]
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 5055c04b24..adab520c78 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -16,13 +16,12 @@
from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached
-from twisted.internet import defer, reactor
+from twisted.internet import defer
from canonicaljson import encode_canonical_json
from collections import namedtuple
-import itertools
import logging
import ujson as json
@@ -50,20 +49,6 @@ class TransactionStore(SQLBaseStore):
def __init__(self, hs):
super(TransactionStore, self).__init__(hs)
- # New transactions that are currently in flights
- self.inflight_transactions = {}
-
- # Newly delievered transactions that *weren't* persisted while in flight
- self.new_delivered_transactions = {}
-
- # Newly delivered transactions that *were* persisted while in flight
- self.update_delivered_transactions = {}
-
- self.last_transaction = {}
-
- reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns)
- self._clock.looping_call(self._persist_in_mem_txns, 1000)
-
self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)
def get_received_txn_response(self, transaction_id, origin):
@@ -148,46 +133,7 @@ class TransactionStore(SQLBaseStore):
Returns:
list: A list of previous transaction ids.
"""
-
- auto_id = self._transaction_id_gen.get_next()
-
- txn_row = _TransactionRow(
- id=auto_id,
- transaction_id=transaction_id,
- destination=destination,
- ts=origin_server_ts,
- response_code=0,
- response_json=None,
- )
-
- self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row
-
- prev_txn = self.last_transaction.get(destination)
- if prev_txn:
- return defer.succeed(prev_txn)
- else:
- return self.runInteraction(
- "_get_prevs_txn",
- self._get_prevs_txn,
- destination,
- )
-
- def _get_prevs_txn(self, txn, destination):
- # First we find out what the prev_txns should be.
- # Since we know that we are only sending one transaction at a time,
- # we can simply take the last one.
- query = (
- "SELECT * FROM sent_transactions"
- " WHERE destination = ?"
- " ORDER BY id DESC LIMIT 1"
- )
-
- txn.execute(query, (destination,))
- results = self.cursor_to_dict(txn)
-
- prev_txns = [r["transaction_id"] for r in results]
-
- return prev_txns
+ return defer.succeed([])
def delivered_txn(self, transaction_id, destination, code, response_dict):
"""Persists the response for an outgoing transaction.
@@ -198,52 +144,7 @@ class TransactionStore(SQLBaseStore):
code (int)
response_json (str)
"""
-
- txn_row = self.inflight_transactions.get(
- destination, {}
- ).pop(transaction_id, None)
-
- self.last_transaction[destination] = transaction_id
-
- if txn_row:
- d = self.new_delivered_transactions.setdefault(destination, {})
- d[transaction_id] = txn_row._replace(
- response_code=code,
- response_json=None, # For now, don't persist response
- )
- else:
- d = self.update_delivered_transactions.setdefault(destination, {})
- # For now, don't persist response
- d[transaction_id] = _UpdateTransactionRow(code, None)
-
- def get_transactions_after(self, transaction_id, destination):
- """Get all transactions after a given local transaction_id.
-
- Args:
- transaction_id (str)
- destination (str)
-
- Returns:
- list: A list of dicts
- """
- return self.runInteraction(
- "get_transactions_after",
- self._get_transactions_after, transaction_id, destination
- )
-
- def _get_transactions_after(self, txn, transaction_id, destination):
- query = (
- "SELECT * FROM sent_transactions"
- " WHERE destination = ? AND id >"
- " ("
- " SELECT id FROM sent_transactions"
- " WHERE transaction_id = ? AND destination = ?"
- " )"
- )
-
- txn.execute(query, (destination, transaction_id, destination))
-
- return self.cursor_to_dict(txn)
+ pass
@cached(max_entries=10000)
def get_destination_retry_timings(self, destination):
@@ -339,58 +240,11 @@ class TransactionStore(SQLBaseStore):
txn.execute(query, (self._clock.time_msec(),))
return self.cursor_to_dict(txn)
- @defer.inlineCallbacks
- def _persist_in_mem_txns(self):
- try:
- inflight = self.inflight_transactions
- new_delivered = self.new_delivered_transactions
- update_delivered = self.update_delivered_transactions
-
- self.inflight_transactions = {}
- self.new_delivered_transactions = {}
- self.update_delivered_transactions = {}
-
- full_rows = [
- row._asdict()
- for txn_map in itertools.chain(inflight.values(), new_delivered.values())
- for row in txn_map.values()
- ]
-
- def f(txn):
- if full_rows:
- self._simple_insert_many_txn(
- txn=txn,
- table="sent_transactions",
- values=full_rows
- )
-
- for dest, txn_map in update_delivered.items():
- for txn_id, update_row in txn_map.items():
- self._simple_update_one_txn(
- txn,
- table="sent_transactions",
- keyvalues={
- "transaction_id": txn_id,
- "destination": dest,
- },
- updatevalues={
- "response_code": update_row.response_code,
- "response_json": None, # For now, don't persist response
- }
- )
-
- if full_rows or update_delivered:
- yield self.runInteraction("_persist_in_mem_txns", f)
- except:
- logger.exception("Failed to persist transactions!")
-
def _cleanup_transactions(self):
now = self._clock.time_msec()
month_ago = now - 30 * 24 * 60 * 60 * 1000
- six_hours_ago = now - 6 * 60 * 60 * 1000
def _cleanup_transactions_txn(txn):
txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,))
- txn.execute("DELETE FROM sent_transactions WHERE ts < ?", (six_hours_ago,))
- return self.runInteraction("_persist_in_mem_txns", _cleanup_transactions_txn)
+ return self.runInteraction("_cleanup_transactions", _cleanup_transactions_txn)
|
