From 60a0f81c7a2da86bf959227a440e3f7a2b727bb5 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 1 Mar 2016 14:49:41 +0000
Subject: Add a /replication API for extracting the updates that happened on
 synapse

This is necessary for replicating the data in synapse to be visible to a
separate service because presence and typing notifications aren't stored
in a database so won't be visible to another process.

This API can be used to either get the raw data by requesting the tables
themselves or to just receive notifications for updates by following the
streams meta-stream.

Returns updates for each table requested a JSON array of arrays with a
row for each row in the table.

Each table is prefixed by a header row with the: name of the table,
current stream_id position for the table, number of rows, number of
columns and the names of the columns.
This is followed by the rows that have been added to the server since
the requester last asked.

The API has a timeout and is hooked up to the notifier so that a slave
can long poll for updates.
---
 scripts-dev/tail-synapse.py | 67 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 scripts-dev/tail-synapse.py

(limited to 'scripts-dev')

diff --git a/scripts-dev/tail-synapse.py b/scripts-dev/tail-synapse.py
new file mode 100644
index 0000000000..18be711e92
--- /dev/null
+++ b/scripts-dev/tail-synapse.py
@@ -0,0 +1,67 @@
+import requests
+import collections
+import sys
+import time
+import json
+
+Entry = collections.namedtuple("Entry", "name position rows")
+
+ROW_TYPES = {}
+
+
+def row_type_for_columns(name, column_names):
+    column_names = tuple(column_names)
+    row_type = ROW_TYPES.get((name, column_names))
+    if row_type is None:
+        row_type = collections.namedtuple(name, column_names)
+        ROW_TYPES[(name, column_names)] = row_type
+    return row_type
+
+
+def parse_response(content):
+    streams = json.loads(content)
+    result = {}
+    for name, value in streams.items():
+        row_type = row_type_for_columns(name, value["field_names"])
+        position = value["position"]
+        rows = [row_type(*row) for row in value["rows"]]
+        result[name] = Entry(name, position, rows)
+    return result
+
+
+def replicate(server, streams):
+    return parse_response(requests.get(
+        server + "/_synapse/replication",
+        verify=False,
+        params=streams
+    ).content)
+
+
+def main():
+    server = sys.argv[1]
+
+    streams = None
+    while not streams:
+        try:
+            streams = {
+                row.name: row.position
+                for row in replicate(server, {"streams":"-1"})["streams"].rows
+            }
+        except requests.exceptions.ConnectionError as e:
+            time.sleep(0.1)
+
+    while True:
+        try:
+            results = replicate(server, streams)
+        except:
+            sys.stdout.write("connection_lost("+ repr(streams) + ")\n")
+            break
+        for update in results.values():
+            for row in update.rows:
+                sys.stdout.write(repr(row) + "\n")
+            streams[update.name] = update.position
+
+
+
+if __name__=='__main__':
+    main()
-- 
cgit 1.4.1


From 4ebb688f4ffcef2784df0aa7f5b088e72b28c07b Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 17 Mar 2016 10:59:12 +0000
Subject: Add option to definitions.py to search for functions a function
 refers to

---
 scripts-dev/definitions.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

(limited to 'scripts-dev')

diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index 8340c72618..47dac7772d 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -86,9 +86,12 @@ def used_names(prefix, item, defs, names):
     for name, funcs in defs.get('class', {}).items():
         used_names(prefix + name + ".", name, funcs, names)
 
+    path = prefix.rstrip('.')
     for used in defs.get('uses', ()):
         if used in names:
-            names[used].setdefault('used', {}).setdefault(item, []).append(prefix.rstrip('.'))
+            if item:
+                names[item].setdefault('uses', []).append(used)
+            names[used].setdefault('used', {}).setdefault(item, []).append(path)
 
 
 if __name__ == '__main__':
@@ -113,6 +116,10 @@ if __name__ == '__main__':
         "--referrers", default=0, type=int,
         help="Include referrers up to the given depth"
     )
+    parser.add_argument(
+        "--referred", default=0, type=int,
+        help="Include referred down to the given depth"
+    )
     parser.add_argument(
         "--format", default="yaml",
         help="Output format, one of 'yaml' or 'dot'"
@@ -161,6 +168,20 @@ if __name__ == '__main__':
                 continue
             result[name] = definition
 
+    referred_depth = args.referred
+    referred = set()
+    while referred_depth:
+        referred_depth -= 1
+        for entry in result.values():
+            for uses in entry.get("uses", ()):
+                referred.add(uses)
+        for name, definition in names.items():
+            if not name in referred:
+                continue
+            if ignore and any(pattern.match(name) for pattern in ignore):
+                continue
+            result[name] = definition
+
     if args.format == 'yaml':
         yaml.dump(result, sys.stdout, default_flow_style=False)
     elif args.format == 'dot':
-- 
cgit 1.4.1