From f093873d697f99a59cade944bb68850b017afd09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 24 Aug 2015 16:30:35 +0100 Subject: Replace syutil references in scripts --- scripts-dev/check_event_hash.py | 2 +- scripts-dev/check_signature.py | 8 +++----- scripts-dev/convert_server_keys.py | 8 ++++---- scripts-dev/hash_history.py | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/check_event_hash.py b/scripts-dev/check_event_hash.py index 679afbd268..7ccae34d48 100644 --- a/scripts-dev/check_event_hash.py +++ b/scripts-dev/check_event_hash.py @@ -1,5 +1,5 @@ from synapse.crypto.event_signing import * -from syutil.base64util import encode_base64 +from unpaddedbase64 import encode_base64 import argparse import hashlib diff --git a/scripts-dev/check_signature.py b/scripts-dev/check_signature.py index 59e3d603ac..079577908a 100644 --- a/scripts-dev/check_signature.py +++ b/scripts-dev/check_signature.py @@ -1,9 +1,7 @@ -from syutil.crypto.jsonsign import verify_signed_json -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys -) -from syutil.base64util import decode_base64 +from signedjson.sign import verify_signed_json +from signedjson.key import decode_verify_key_bytes, write_signing_keys +from unpaddedbase64 import decode_base64 import urllib2 import json diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index a1ee39059c..151551f22c 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -4,10 +4,10 @@ import sys import json import time import hashlib -from syutil.base64util import encode_base64 -from syutil.crypto.signing_key import read_signing_keys -from syutil.crypto.jsonsign import sign_json -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64 +from signedjson.key import read_signing_keys +from signedjson.sign import sign_json +from canonicaljson import encode_canonical_json def select_v1_keys(connection): diff --git a/scripts-dev/hash_history.py b/scripts-dev/hash_history.py index bdad530af8..616d6a10e7 100644 --- a/scripts-dev/hash_history.py +++ b/scripts-dev/hash_history.py @@ -6,8 +6,8 @@ from synapse.crypto.event_signing import ( add_event_pdu_content_hash, compute_pdu_event_reference_hash ) from synapse.api.events.utils import prune_pdu -from syutil.base64util import encode_base64, decode_base64 -from syutil.jsonutil import encode_canonical_json +from unpaddedbase64 import encode_base64, decode_base64 +from canonicaljson import encode_canonical_json import sqlite3 import sys -- cgit 1.5.1 From c0d1f37baf33aeeab22e635b5fd7905ab07e39e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 13:47:14 +0100 Subject: Don't require pdus in check_auth script --- scripts-dev/check_auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index b889ac7fa7..4fa8792a5f 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -56,10 +56,9 @@ if __name__ == '__main__': js = json.load(args.json) - auth = Auth(Mock()) check_auth( auth, [FrozenEvent(d) for d in js["auth_chain"]], - [FrozenEvent(d) for d in js["pdus"]], + [FrozenEvent(d) for d in js.get("pdus", [])], ) -- cgit 1.5.1 From cc3ab0c214156cdad9731beb27790e8510b4b023 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:13:06 +0100 Subject: Add dev script for finding where functions are called from, and finding functions that aren't called at all --- scripts-dev/definitions.py | 130 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 scripts-dev/definitions.py (limited to 'scripts-dev') diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py new file mode 100755 index 0000000000..09cec9d8ec --- /dev/null +++ b/scripts-dev/definitions.py @@ -0,0 +1,130 @@ +#! /usr/bin/python + +import ast +import yaml + +class DefinitionVisitor(ast.NodeVisitor): + def __init__(self): + super(DefinitionVisitor, self).__init__() + self.functions = {} + self.classes = {} + self.names = {} + self.attrs = set() + self.definitions = { + 'def': self.functions, + 'class': self.classes, + 'names': self.names, + 'attrs': self.attrs, + } + + def visit_Name(self, node): + self.names.setdefault(type(node.ctx).__name__, set()).add(node.id) + + def visit_Attribute(self, node): + self.attrs.add(node.attr) + for child in ast.iter_child_nodes(node): + self.visit(child) + + def visit_ClassDef(self, node): + visitor = DefinitionVisitor() + self.classes[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + def visit_FunctionDef(self, node): + visitor = DefinitionVisitor() + self.functions[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + +def non_empty(defs): + functions = {name: non_empty(f) for name, f in defs['def'].items()} + classes = {name: non_empty(f) for name, f in defs['class'].items()} + result = {} + if functions: result['def'] = functions + if classes: result['class'] = classes + names = defs['names'] + uses = [] + for name in names.get('Load', ()): + if name not in names.get('Param', ()) and name not in names.get('Store', ()): + uses.append(name) + uses.extend(defs['attrs']) + if uses: result['uses'] = uses + result['names'] = names + result['attrs'] = defs['attrs'] + return result + + +def definitions_in_code(input_code): + input_ast = ast.parse(input_code) + visitor = DefinitionVisitor() + visitor.visit(input_ast) + definitions = non_empty(visitor.definitions) + return definitions + + +def definitions_in_file(filepath): + with open(filepath) as f: + return definitions_in_code(f.read()) + + +def defined_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + +def used_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for used in defs.get('uses', ()): + if used in names: + names[used].setdefault('used', []).append(prefix.rstrip('.')) + + +if __name__ == '__main__': + import sys, os + if not sys.argv[1:]: + sys.stderr.write( + "Usage: definitions.py \n" + " definitions.py \n" + "Either list the definitions matching the regexp or list\n" + " 'unused' definitions\n" + ) + + definitions = {} + for root, dirs, files in os.walk(sys.argv[1]): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) + + names = {} + for filepath, defs in definitions.items(): + defined_names(filepath + ":", defs, names) + + for filepath, defs in definitions.items(): + used_names(filepath + ":", defs, names) + + if sys.argv[2:]: + import re + pattern = re.compile(sys.argv[2]) + for name in list(names): + if not pattern.match(name): + del names[name] + else: + for name in list(names): + if 'used' in names[name]: + del names[name] + + yaml.dump(names, sys.stdout, default_flow_style=False) + #yaml.dump(definitions, sys.stdout, default_flow_style=False) -- cgit 1.5.1 From 04abf53a5633d8220b01772aedc573caa0e71f6e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:17:50 +0100 Subject: Use argparse for definition finder --- scripts-dev/definitions.py | 64 +++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 26 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 09cec9d8ec..934bc4a3d8 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -92,21 +92,32 @@ def used_names(prefix, defs, names): if __name__ == '__main__': - import sys, os - if not sys.argv[1:]: - sys.stderr.write( - "Usage: definitions.py \n" - " definitions.py \n" - "Either list the definitions matching the regexp or list\n" - " 'unused' definitions\n" - ) + import sys, os, argparse, re + + parser = argparse.ArgumentParser(description='Find definitions.') + parser.add_argument( + "--unused", action="store_true", help="Only list unused definitions" + ) + parser.add_argument( + "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" + ) + parser.add_argument( + "--pattern", nargs='+', action="append", metavar="REGEXP", + help="Search for a pattern" + ) + parser.add_argument( + "directories", nargs='+', metavar="DIR", + help="Directories to search for definitions" + ) + args = parser.parse_args() definitions = {} - for root, dirs, files in os.walk(sys.argv[1]): - for filename in files: - if filename.endswith(".py"): - filepath = os.path.join(root, filename) - definitions[filepath] = definitions_in_file(filepath) + for directory in args.directories: + for root, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) names = {} for filepath, defs in definitions.items(): @@ -115,16 +126,17 @@ if __name__ == '__main__': for filepath, defs in definitions.items(): used_names(filepath + ":", defs, names) - if sys.argv[2:]: - import re - pattern = re.compile(sys.argv[2]) - for name in list(names): - if not pattern.match(name): - del names[name] - else: - for name in list(names): - if 'used' in names[name]: - del names[name] - - yaml.dump(names, sys.stdout, default_flow_style=False) - #yaml.dump(definitions, sys.stdout, default_flow_style=False) + patterns = [re.compile(pattern) for pattern in args.pattern or ()] + ignore = [re.compile(pattern) for pattern in args.ignore or ()] + + result = {} + for name, definition in names.items(): + if patterns and not any(pattern.match(name) for pattern in patterns): + continue + if ignore and any(pattern.match(name) for pattern in ignore): + continue + if args.unused and definition.get('used'): + continue + result[name] = definition + + yaml.dump(result, sys.stdout, default_flow_style=False) -- cgit 1.5.1 From 314aabba82d382d36f9ac91050e77deb6018fbcd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:45:33 +0100 Subject: Fix scripts-dev/definitions.py argparse options --- scripts-dev/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts-dev') diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 934bc4a3d8..f0d0cd8a3f 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -102,7 +102,7 @@ if __name__ == '__main__': "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" ) parser.add_argument( - "--pattern", nargs='+', action="append", metavar="REGEXP", + "--pattern", action="append", metavar="REGEXP", help="Search for a pattern" ) parser.add_argument( -- cgit 1.5.1