From 74bc42cfddb9bbaf364794857ab902b046c893dc Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Tue, 24 Feb 2015 18:03:39 +0000
Subject: An initial implementation of a 'metrics' instance, similar to a
 'logger' for keeping counter stats on method calls

---
 synapse/metrics/__init__.py | 69 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 synapse/metrics/__init__.py

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
new file mode 100644
index 0000000000..125845eb30
--- /dev/null
+++ b/synapse/metrics/__init__.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .metric import CounterMetric
+
+
+# We'll keep all the available metrics in a single toplevel dict, one shared
+# for the entire process. We don't currently support per-HomeServer instances
+# of metrics, because in practice any one python VM will host only one
+# HomeServer anyway. This makes a lot of implementation neater
+all_metrics = {}
+
+
+class Metrics(object):
+    """ A single Metrics object gives a (mutable) slice view of the all_metrics
+    dict, allowing callers to easily register new metrics that are namespaced
+    nicely."""
+
+    def __init__(self, name):
+        self.name_prefix = name
+
+    def _register(self, metric):
+        all_metrics[metric.name] = metric
+
+    def register_counter(self, name, *args, **kwargs):
+        full_name = "%s.%s" % (self.name_prefix, name)
+
+        metric = CounterMetric(full_name, *args, **kwargs)
+
+        self._register(metric)
+
+        return metric
+
+    def counted(self, func):
+        """ A method decorator that registers a counter, to count invocations
+        of this method. """
+        counter = self.register_counter(func.__name__)
+
+        def wrapped(*args, **kwargs):
+            counter.inc()
+            return func(*args, **kwargs)
+        return wrapped
+
+
+def get_metrics_for(name):
+    """ Returns a Metrics instance for conveniently creating metrics
+    namespaced with the given name prefix. """
+    return Metrics(name)
+
+
+def render_all():
+    strs = []
+
+    for name in sorted(all_metrics.keys()):
+        strs += all_metrics[name].render()
+
+    return "\n".join(strs)
-- 
cgit 1.4.1


From ce8b5769f7e08515edf8988281d17df7b0ddfdaa Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Wed, 4 Mar 2015 15:47:23 +0000
Subject: Create the concept of a cachecounter metric; generating two counters
 specific to caches

---
 synapse/metrics/__init__.py  | 11 ++++++++++-
 synapse/metrics/metric.py    | 43 +++++++++++++++++++++++++++++++++++++------
 tests/metrics/test_metric.py | 27 ++++++++++++++++++++++++++-
 3 files changed, 73 insertions(+), 8 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 125845eb30..d5c30bbe41 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .metric import CounterMetric
+from .metric import CounterMetric, CacheCounterMetric
 
 
 # We'll keep all the available metrics in a single toplevel dict, one shared
@@ -43,6 +43,15 @@ class Metrics(object):
 
         return metric
 
+    def register_cachecounter(self, name, *args, **kwargs):
+        full_name = "%s.%s" % (self.name_prefix, name)
+
+        metric = CacheCounterMetric(full_name, *args, **kwargs)
+
+        self._register(metric)
+
+        return metric
+
     def counted(self, func):
         """ A method decorator that registers a counter, to count invocations
         of this method. """
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index f5a98763cc..00b149f6f6 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -14,16 +14,28 @@
 # limitations under the License.
 
 
-class CounterMetric(object):
+class BaseMetric(object):
 
     def __init__(self, name, keys=[]):
         self.name = name
         self.keys = keys # OK not to clone as we never write it
 
+    def _render_key(self, values):
+        # TODO: some kind of value escape
+        return ",".join(["%s=%s" % kv for kv in zip(self.keys, values)])
+
+
+class CounterMetric(BaseMetric):
+    """The simplest kind of metric; one that stores a monotonically-increasing
+    integer that counts events."""
+
+    def __init__(self, *args, **kwargs):
+        super(CounterMetric, self).__init__(*args, **kwargs)
+
         self.counts = {}
 
         # Scalar metrics are never empty
-        if not len(keys):
+        if not len(self.keys):
             self.counts[()] = 0
 
     def inc(self, *values):
@@ -42,13 +54,32 @@ class CounterMetric(object):
     def fetch(self):
         return dict(self.counts)
 
-    def _render_key(self, values):
-        # TODO: some kind of value escape
-        return ",".join(["%s=%s" % kv for kv in zip(self.keys, values)])
-
     def render(self):
         if not len(self.keys):
             return ["%s %d" % (self.name, self.counts[()])]
 
         return ["%s{%s} %d" % (self.name, self._render_key(k), self.counts[k])
                 for k in sorted(self.counts.keys())]
+
+
+class CacheCounterMetric(object):
+    """A combination of two CounterMetrics, one to count cache hits and one to
+    count misses.
+
+    This metric generates standard metric name pairs, so that monitoring rules
+    can easily be applied to measure hit ratio."""
+
+    def __init__(self, name, keys=[]):
+        self.name = name
+
+        self.hits   = CounterMetric(name + ":hits",   keys=keys)
+        self.misses = CounterMetric(name + ":misses", keys=keys)
+
+    def inc_hits(self, *values):
+        self.hits.inc(*values)
+
+    def inc_misses(self, *values):
+        self.misses.inc(*values)
+
+    def render(self):
+        return self.hits.render() + self.misses.render()
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
index a4fd52a9d5..93e8e27e4f 100644
--- a/tests/metrics/test_metric.py
+++ b/tests/metrics/test_metric.py
@@ -15,7 +15,7 @@
 
 from tests import unittest
 
-from synapse.metrics.metric import CounterMetric
+from synapse.metrics.metric import CounterMetric, CacheCounterMetric
 
 
 class CounterMetricTestCase(unittest.TestCase):
@@ -59,3 +59,28 @@ class CounterMetricTestCase(unittest.TestCase):
             "vector{method=GET} 2",
             "vector{method=PUT} 1",
         ])
+
+
+class CacheCounterMetricTestCase(unittest.TestCase):
+
+    def test_cachecounter(self):
+        counter = CacheCounterMetric("cache")
+
+        self.assertEquals(counter.render(), [
+            "cache:hits 0",
+            "cache:misses 0",
+        ])
+
+        counter.inc_misses()
+
+        self.assertEquals(counter.render(), [
+            "cache:hits 0",
+            "cache:misses 1",
+        ])
+
+        counter.inc_hits()
+
+        self.assertEquals(counter.render(), [
+            "cache:hits 1",
+            "cache:misses 1",
+        ])
-- 
cgit 1.4.1


From d8caa5454d781a76a65fa4ce75336541b973f624 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Wed, 4 Mar 2015 16:46:44 +0000
Subject: Initial attempt at a scalar callback-based metric to give
 instantaneous snapshot gauges

---
 synapse/metrics/__init__.py  | 11 ++++++++++-
 synapse/metrics/metric.py    | 14 ++++++++++++++
 tests/metrics/test_metric.py | 22 +++++++++++++++++++++-
 3 files changed, 45 insertions(+), 2 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index d5c30bbe41..d7584fc0bc 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .metric import CounterMetric, CacheCounterMetric
+from .metric import CounterMetric, CallbackMetric, CacheCounterMetric
 
 
 # We'll keep all the available metrics in a single toplevel dict, one shared
@@ -43,6 +43,15 @@ class Metrics(object):
 
         return metric
 
+    def register_callback(self, name, callback, *args, **kwargs):
+        full_name = "%s.%s" % (self.name_prefix, name)
+
+        metric = CallbackMetric(full_name, *args, callback=callback, **kwargs)
+
+        self._register(metric)
+
+        return metric
+
     def register_cachecounter(self, name, *args, **kwargs):
         full_name = "%s.%s" % (self.name_prefix, name)
 
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 00b149f6f6..8a497fc154 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -62,6 +62,20 @@ class CounterMetric(BaseMetric):
                 for k in sorted(self.counts.keys())]
 
 
+class CallbackMetric(BaseMetric):
+    """A metric that returns the numeric value returned by a callback whenever
+    it is rendered. Typically this is used to implement gauges that yield the
+    size or other state of some in-memory object by actively querying it."""
+
+    def __init__(self, name, callback, keys=[]):
+        super(CallbackMetric, self).__init__(name, keys=keys)
+
+        self.callback = callback
+
+    def render(self):
+        # TODO(paul): work out something we can do with keys and vectors
+        return ["%s %d" % (self.name, self.callback())]
+
 class CacheCounterMetric(object):
     """A combination of two CounterMetrics, one to count cache hits and one to
     count misses.
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
index 93e8e27e4f..b7bb375ce0 100644
--- a/tests/metrics/test_metric.py
+++ b/tests/metrics/test_metric.py
@@ -15,7 +15,9 @@
 
 from tests import unittest
 
-from synapse.metrics.metric import CounterMetric, CacheCounterMetric
+from synapse.metrics.metric import (
+    CounterMetric, CallbackMetric, CacheCounterMetric
+)
 
 
 class CounterMetricTestCase(unittest.TestCase):
@@ -61,6 +63,24 @@ class CounterMetricTestCase(unittest.TestCase):
         ])
 
 
+class CallbackMetricTestCase(unittest.TestCase):
+
+    def test_callback(self):
+        d = dict()
+
+        metric = CallbackMetric("size", lambda: len(d))
+
+        self.assertEquals(metric.render(), [
+            "size 0",
+        ])
+
+        d["key"] = "value"
+
+        self.assertEquals(metric.render(), [
+            "size 1",
+        ])
+
+
 class CacheCounterMetricTestCase(unittest.TestCase):
 
     def test_cachecounter(self):
-- 
cgit 1.4.1


From e02cc249da6ff71dcc1e5560232b302246a11c9d Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Wed, 4 Mar 2015 17:13:09 +0000
Subject: Ensure that exceptions while rendering individual metrics don't stop
 others from being rendered anyway - especially useful for CallbackMetric

---
 synapse/metrics/__init__.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index d7584fc0bc..d967b04eee 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -13,9 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from .metric import CounterMetric, CallbackMetric, CacheCounterMetric
 
 
+logger = logging.getLogger(__name__)
+
+
 # We'll keep all the available metrics in a single toplevel dict, one shared
 # for the entire process. We don't currently support per-HomeServer instances
 # of metrics, because in practice any one python VM will host only one
@@ -82,6 +87,10 @@ def render_all():
     strs = []
 
     for name in sorted(all_metrics.keys()):
-        strs += all_metrics[name].render()
+        try:
+            strs += all_metrics[name].render()
+        except Exception as e:
+            strs += ["# FAILED to render %s" % name]
+            logger.exception("Failed to render %s metric", name)
 
     return "\n".join(strs)
-- 
cgit 1.4.1


From 8664599af77ba0ed6268b3112174dc8e0c91101b Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Wed, 4 Mar 2015 17:34:23 +0000
Subject: Rename CacheCounterMetric to just CacheMetric; add a CallbackMetric
 component to give the size of the cache

---
 synapse/metrics/__init__.py  |  6 +++---
 synapse/metrics/metric.py    | 13 +++++++++----
 synapse/storage/_base.py     |  6 +++---
 tests/metrics/test_metric.py | 24 +++++++++++++++---------
 4 files changed, 30 insertions(+), 19 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index d967b04eee..442fd70cdf 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -15,7 +15,7 @@
 
 import logging
 
-from .metric import CounterMetric, CallbackMetric, CacheCounterMetric
+from .metric import CounterMetric, CallbackMetric, CacheMetric
 
 
 logger = logging.getLogger(__name__)
@@ -57,10 +57,10 @@ class Metrics(object):
 
         return metric
 
-    def register_cachecounter(self, name, *args, **kwargs):
+    def register_cache(self, name, *args, **kwargs):
         full_name = "%s.%s" % (self.name_prefix, name)
 
-        metric = CacheCounterMetric(full_name, *args, **kwargs)
+        metric = CacheMetric(full_name, *args, **kwargs)
 
         self._register(metric)
 
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 8a497fc154..7e47f86155 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -76,19 +76,24 @@ class CallbackMetric(BaseMetric):
         # TODO(paul): work out something we can do with keys and vectors
         return ["%s %d" % (self.name, self.callback())]
 
-class CacheCounterMetric(object):
+class CacheMetric(object):
     """A combination of two CounterMetrics, one to count cache hits and one to
-    count misses.
+    count misses, and a callback metric to yield the current size.
 
     This metric generates standard metric name pairs, so that monitoring rules
     can easily be applied to measure hit ratio."""
 
-    def __init__(self, name, keys=[]):
+    def __init__(self, name, size_callback, keys=[]):
         self.name = name
 
         self.hits   = CounterMetric(name + ":hits",   keys=keys)
         self.misses = CounterMetric(name + ":misses", keys=keys)
 
+        self.size = CallbackMetric(name + ":size",
+            callback=size_callback,
+            keys=keys,
+        )
+
     def inc_hits(self, *values):
         self.hits.inc(*values)
 
@@ -96,4 +101,4 @@ class CacheCounterMetric(object):
         self.misses.inc(*values)
 
     def render(self):
-        return self.hits.render() + self.misses.render()
+        return self.hits.render() + self.misses.render() + self.size.render()
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 804655e34d..d3c2bc7bfb 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -59,7 +59,7 @@ def cached(max_entries=1000):
     def wrap(orig):
         cache = OrderedDict()
 
-        counter = metrics.register_cachecounter(orig.__name__)
+        counter = metrics.register_cache(orig.__name__, lambda: len(cache))
 
         def prefill(key, value):
             while len(cache) > max_entries:
@@ -183,8 +183,8 @@ class SQLBaseStore(object):
         self._get_event_counters = PerformanceCounters()
 
         self._get_event_cache = LruCache(hs.config.event_cache_size)
-        self._get_event_cache_counter = metrics.register_cachecounter(
-            "get_event"
+        self._get_event_cache_counter = metrics.register_cache("get_event",
+            size_callback=lambda: len(self._get_event_cache),
         )
 
     def start_profiling(self):
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
index b7bb375ce0..32fd178ed4 100644
--- a/tests/metrics/test_metric.py
+++ b/tests/metrics/test_metric.py
@@ -16,7 +16,7 @@
 from tests import unittest
 
 from synapse.metrics.metric import (
-    CounterMetric, CallbackMetric, CacheCounterMetric
+    CounterMetric, CallbackMetric, CacheMetric
 )
 
 
@@ -81,26 +81,32 @@ class CallbackMetricTestCase(unittest.TestCase):
         ])
 
 
-class CacheCounterMetricTestCase(unittest.TestCase):
+class CacheMetricTestCase(unittest.TestCase):
 
-    def test_cachecounter(self):
-        counter = CacheCounterMetric("cache")
+    def test_cache(self):
+        d = dict()
 
-        self.assertEquals(counter.render(), [
+        metric = CacheMetric("cache", lambda: len(d))
+
+        self.assertEquals(metric.render(), [
             "cache:hits 0",
             "cache:misses 0",
+            "cache:size 0",
         ])
 
-        counter.inc_misses()
+        metric.inc_misses()
+        d["key"] = "value"
 
-        self.assertEquals(counter.render(), [
+        self.assertEquals(metric.render(), [
             "cache:hits 0",
             "cache:misses 1",
+            "cache:size 1",
         ])
 
-        counter.inc_hits()
+        metric.inc_hits()
 
-        self.assertEquals(counter.render(), [
+        self.assertEquals(metric.render(), [
             "cache:hits 1",
             "cache:misses 1",
+            "cache:size 1",
         ])
-- 
cgit 1.4.1


From f52acf3b129864f400876f9316e1324e62e75eb0 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Wed, 4 Mar 2015 19:43:46 +0000
Subject: Neater register_* methods on overall Metrics container

---
 synapse/metrics/__init__.py | 34 ++++++++++++----------------------
 1 file changed, 12 insertions(+), 22 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 442fd70cdf..e1818ce395 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -15,7 +15,7 @@
 
 import logging
 
-from .metric import CounterMetric, CallbackMetric, CacheMetric
+from .metric import CounterMetric, CallbackMetric, TimerMetric, CacheMetric
 
 
 logger = logging.getLogger(__name__)
@@ -36,35 +36,25 @@ class Metrics(object):
     def __init__(self, name):
         self.name_prefix = name
 
-    def _register(self, metric):
-        all_metrics[metric.name] = metric
-
-    def register_counter(self, name, *args, **kwargs):
-        full_name = "%s.%s" % (self.name_prefix, name)
-
-        metric = CounterMetric(full_name, *args, **kwargs)
-
-        self._register(metric)
-
-        return metric
-
-    def register_callback(self, name, callback, *args, **kwargs):
+    def _register(self, metric_class, name, *args, **kwargs):
         full_name = "%s.%s" % (self.name_prefix, name)
 
-        metric = CallbackMetric(full_name, *args, callback=callback, **kwargs)
-
-        self._register(metric)
+        metric = metric_class(full_name, *args, **kwargs)
 
+        all_metrics[full_name] = metric
         return metric
 
-    def register_cache(self, name, *args, **kwargs):
-        full_name = "%s.%s" % (self.name_prefix, name)
+    def register_counter(self, *args, **kwargs):
+        return self._register(CounterMetric, *args, **kwargs)
 
-        metric = CacheMetric(full_name, *args, **kwargs)
+    def register_callback(self, *args, **kwargs):
+        return self._register(CallbackMetric, *args, **kwargs)
 
-        self._register(metric)
+    def register_timer(self, *args, **kwargs):
+        return self._register(TimerMetric, *args, **kwargs)
 
-        return metric
+    def register_cache(self, *args, **kwargs):
+        return self._register(CacheMetric, *args, **kwargs)
 
     def counted(self, func):
         """ A method decorator that registers a counter, to count invocations
-- 
cgit 1.4.1


From 399689dcc79e73de1ec6bae8aa18bd83f5618f38 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 5 Mar 2015 16:15:21 +0000
Subject: Provide some process resource usage metrics

---
 synapse/metrics/__init__.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e1818ce395..c00f088fff 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -13,7 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Because otherwise 'resource' collides with synapse.metrics.resource
+from __future__ import absolute_import
+
 import logging
+from resource import getrusage, getpagesize, RUSAGE_SELF
 
 from .metric import CounterMetric, CallbackMetric, TimerMetric, CacheMetric
 
@@ -76,6 +80,9 @@ def get_metrics_for(name):
 def render_all():
     strs = []
 
+    # TODO(paul): Internal hack
+    update_resource_metrics()
+
     for name in sorted(all_metrics.keys()):
         try:
             strs += all_metrics[name].render()
@@ -84,3 +91,23 @@ def render_all():
             logger.exception("Failed to render %s metric", name)
 
     return "\n".join(strs)
+
+
+# Now register some standard process-wide state metrics, to give indications of
+# process resource usage
+
+rusage = None
+PAGE_SIZE = getpagesize()
+
+def update_resource_metrics():
+    global rusage
+    rusage = getrusage(RUSAGE_SELF)
+
+resource_metrics = get_metrics_for("process.resource")
+
+# msecs
+resource_metrics.register_callback("utime", lambda: rusage.ru_utime * 1000)
+resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000)
+
+# pages
+resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * PAGE_SIZE)
-- 
cgit 1.4.1


From 0b96bb793e7e5d3935804b8f0ccaf415006388a9 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 6 Mar 2015 15:39:14 +0000
Subject: Have all @metrics.counted use a single metric name vectored on the
 method name, rather than a brand new scalar counter per counted method

---
 synapse/metrics/__init__.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index c00f088fff..443d67f41c 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -63,10 +63,17 @@ class Metrics(object):
     def counted(self, func):
         """ A method decorator that registers a counter, to count invocations
         of this method. """
-        counter = self.register_counter(func.__name__)
+        if not hasattr(self, "method_counter"):
+            self.method_counter = self.register_counter(
+                "calls",
+                labels=["method"]
+            )
+
+        counter = self.method_counter
+        name = func.__name__
 
         def wrapped(*args, **kwargs):
-            counter.inc()
+            counter.inc(name)
             return func(*args, **kwargs)
         return wrapped
 
-- 
cgit 1.4.1


From b0cf86731957876ca877c35bf30c6f695f1a544c Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 6 Mar 2015 16:18:21 +0000
Subject: Use _ instead of . as a metric namespacing separator, for Prometheus

---
 synapse/federation/transaction_queue.py |  4 ++--
 synapse/handlers/presence.py            |  2 +-
 synapse/http/client.py                  |  4 ++--
 synapse/http/matrixfederationclient.py  |  4 ++--
 synapse/http/server.py                  |  4 ++--
 synapse/metrics/__init__.py             | 14 +++++++++++---
 synapse/notifier.py                     |  2 +-
 synapse/storage/_base.py                | 18 +++++++++++++-----
 8 files changed, 34 insertions(+), 18 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index ae62c69fc3..ca5bcf21cf 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -69,11 +69,11 @@ class TransactionQueue(object):
         # HACK to get unique tx id
         self._next_txn_id = int(self._clock.time_msec())
 
-        metrics.register_callback("pending_pdus",
+        metrics.register_callback("pendingPdus",
             lambda: {(dest,): len(pdus[dest]) for dest in pdus.keys()},
             labels=["dest"],
         )
-        metrics.register_callback("pending_edus",
+        metrics.register_callback("pendingEdus",
             lambda: {(dest,): len(edus[dest]) for dest in edus.keys()},
             labels=["dest"],
         )
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 698946a48b..c6d6aef53b 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -136,7 +136,7 @@ class PresenceHandler(BaseHandler):
         self._user_cachemap = {}
         self._user_cachemap_latest_serial = 0
 
-        metrics.register_callback("user_cachemap:size",
+        metrics.register_callback("userCachemap:size",
             lambda: len(self._user_cachemap)
         )
 
diff --git a/synapse/http/client.py b/synapse/http/client.py
index ad2c9c05ec..01737a7188 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -34,10 +34,10 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-outgoing_requests_counter = metrics.register_counter("outgoing_requests",
+outgoing_requests_counter = metrics.register_counter("requests",
     labels=["method"],
 )
-incoming_responses_counter = metrics.register_counter("incoming_responses",
+incoming_responses_counter = metrics.register_counter("responses",
     labels=["method","code"],
 )
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 6b6d79a044..11883d3852 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -43,10 +43,10 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-outgoing_requests_counter = metrics.register_counter("outgoing_requests",
+outgoing_requests_counter = metrics.register_counter("requests",
     labels=["method"],
 )
-incoming_responses_counter = metrics.register_counter("incoming_responses",
+incoming_responses_counter = metrics.register_counter("responses",
     labels=["method","code"],
 )
 
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 35bd3a00ba..23708c08c9 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -37,10 +37,10 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-incoming_requests_counter = metrics.register_counter("incoming_requests",
+incoming_requests_counter = metrics.register_counter("requests",
     labels=["method"],
 )
-outgoing_responses_counter = metrics.register_counter("outgoing_responses",
+outgoing_responses_counter = metrics.register_counter("responses",
     labels=["method","code"],
 )
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 443d67f41c..47e475acd2 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -41,7 +41,12 @@ class Metrics(object):
         self.name_prefix = name
 
     def _register(self, metric_class, name, *args, **kwargs):
-        full_name = "%s.%s" % (self.name_prefix, name)
+        if "_" in name:
+            raise ValueError("Metric names %s is invalid as it cannot contain an underscore"
+                % (name)
+            )
+
+        full_name = "%s_%s" % (self.name_prefix, name)
 
         metric = metric_class(full_name, *args, **kwargs)
 
@@ -78,10 +83,13 @@ class Metrics(object):
         return wrapped
 
 
-def get_metrics_for(name):
+def get_metrics_for(pkg_name):
     """ Returns a Metrics instance for conveniently creating metrics
     namespaced with the given name prefix. """
-    return Metrics(name)
+
+    # Convert a "package.name" to "package_name" because Prometheus doesn't
+    # let us use . in metric names
+    return Metrics(pkg_name.replace(".", "_"))
 
 
 def render_all():
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1f7cad624e..75e8152d03 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -122,7 +122,7 @@ class Notifier(object):
                 all_listeners |= x
 
             return len(all_listeners)
-        metrics.register_callback("all_listeners", count_listeners)
+        metrics.register_callback("listeners", count_listeners)
 
         metrics.register_callback("rooms",
             lambda: count(bool, self.room_to_listeners.values())
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index a38b603584..35d118c586 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -40,7 +40,14 @@ metrics = synapse.metrics.get_metrics_for("synapse.storage")
 
 sql_query_timer = metrics.register_timer("queries", labels=["verb"])
 sql_txn_timer = metrics.register_timer("transactions", labels=["desc"])
-sql_getevents_timer = metrics.register_timer("get_events", labels=["desc"])
+sql_getevents_timer = metrics.register_timer("getEvents", labels=["desc"])
+
+caches_by_name = {}
+cache_counter = metrics.register_cache(
+    "cache",
+    lambda: {(name,): len(caches_by_name[name]) for name in caches_by_name.keys()},
+    labels=["name"],
+)
 
 
 # TODO(paul):
@@ -62,8 +69,9 @@ def cached(max_entries=1000):
     """
     def wrap(orig):
         cache = OrderedDict()
+        name = orig.__name__
 
-        counter = metrics.register_cache(orig.__name__, lambda: len(cache))
+        caches_by_name[name] = cache
 
         def prefill(key, value):
             while len(cache) > max_entries:
@@ -74,10 +82,10 @@ def cached(max_entries=1000):
         @defer.inlineCallbacks
         def wrapped(self, key):
             if key in cache:
-                counter.inc_hits()
+                cache_counter.inc_hits(name)
                 defer.returnValue(cache[key])
 
-            counter.inc_misses()
+            cache_counter.inc_misses(name)
             ret = yield orig(self, key)
             prefill(key, ret)
             defer.returnValue(ret)
@@ -195,7 +203,7 @@ class SQLBaseStore(object):
 
         self._get_event_cache = LruCache(hs.config.event_cache_size)
         self._get_event_cache_counter = metrics.register_cache(
-            "get_event_cache",
+            "getEventCache",
             size_callback=lambda: len(self._get_event_cache),
         )
 
-- 
cgit 1.4.1


From 4d661ec0f3ccced9cb6a0b1441bfb845f70f1270 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 6 Mar 2015 19:08:47 +0000
Subject: Remember to emit final linefeed from /metrics page, or Prometheus
 gets upset

---
 synapse/metrics/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 47e475acd2..1acaa3fd09 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -105,6 +105,8 @@ def render_all():
             strs += ["# FAILED to render %s" % name]
             logger.exception("Failed to render %s metric", name)
 
+    strs.append("") # to generate a final CRLF
+
     return "\n".join(strs)
 
 
-- 
cgit 1.4.1


From f1fbe3e09f5573ac7ea9159635b02cc579e19720 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Tue, 10 Mar 2015 15:21:03 +0000
Subject: Rename TimerMetric to DistributionMetric; as it could count more than
 just time

---
 synapse/metrics/__init__.py  |  8 +++++---
 synapse/metrics/metric.py    | 24 +++++++++++++-----------
 synapse/storage/_base.py     | 14 +++++++-------
 tests/metrics/test_metric.py | 24 ++++++++++++------------
 4 files changed, 37 insertions(+), 33 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 1acaa3fd09..c161c17e9f 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -19,7 +19,9 @@ from __future__ import absolute_import
 import logging
 from resource import getrusage, getpagesize, RUSAGE_SELF
 
-from .metric import CounterMetric, CallbackMetric, TimerMetric, CacheMetric
+from .metric import (
+    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric
+)
 
 
 logger = logging.getLogger(__name__)
@@ -59,8 +61,8 @@ class Metrics(object):
     def register_callback(self, *args, **kwargs):
         return self._register(CallbackMetric, *args, **kwargs)
 
-    def register_timer(self, *args, **kwargs):
-        return self._register(TimerMetric, *args, **kwargs)
+    def register_distribution(self, *args, **kwargs):
+        return self._register(DistributionMetric, *args, **kwargs)
 
     def register_cache(self, *args, **kwargs):
         return self._register(CacheMetric, *args, **kwargs)
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 6b7d3358bc..45d2752a20 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -101,35 +101,37 @@ class CallbackMetric(BaseMetric):
                 for k in sorted(value.keys())]
 
 
-class TimerMetric(CounterMetric):
-    """A combination of an event counter and a time accumulator, which counts
-    both the number of events and how long each one takes.
+class DistributionMetric(CounterMetric):
+    """A combination of an event counter and an accumulator, which counts
+    both the number of events and accumulates the total value. Typically this
+    could be used to keep track of method-running times, or other distributions
+    of values that occur in discrete occurances.
 
     TODO(paul): Try to export some heatmap-style stats?
     """
 
     def __init__(self, *args, **kwargs):
-        super(TimerMetric, self).__init__(*args, **kwargs)
+        super(DistributionMetric, self).__init__(*args, **kwargs)
 
-        self.times = {}
+        self.totals = {}
 
         # Scalar metrics are never empty
         if self.is_scalar():
-            self.times[()] = 0
+            self.totals[()] = 0
 
-    def inc_time(self, msec, *values):
+    def inc_by(self, inc, *values):
         self.inc(*values)
 
-        if values not in self.times:
-            self.times[values] = msec
+        if values not in self.totals:
+            self.totals[values] = inc
         else:
-            self.times[values] += msec
+            self.totals[values] += inc
 
     def render_item(self, k):
         keystr = self._render_key(k)
 
         return ["%s:count%s %d" % (self.name, keystr, self.counts[k]),
-                "%s:msec%s %d" % (self.name, keystr, self.times[k])]
+                "%s:total%s %d" % (self.name, keystr, self.totals[k])]
 
 
 class CacheMetric(object):
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 2708d3c5b6..104e8e3cf6 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -38,9 +38,9 @@ transaction_logger = logging.getLogger("synapse.storage.txn")
 
 metrics = synapse.metrics.get_metrics_for("synapse.storage")
 
-sql_query_timer = metrics.register_timer("queries", labels=["verb"])
-sql_txn_timer = metrics.register_timer("transactions", labels=["desc"])
-sql_getevents_timer = metrics.register_timer("getEvents", labels=["desc"])
+sql_query_timer = metrics.register_distribution("queries", labels=["verb"])
+sql_txn_timer = metrics.register_distribution("transactions", labels=["desc"])
+sql_getevents_timer = metrics.register_distribution("getEvents", labels=["desc"])
 
 caches_by_name = {}
 cache_counter = metrics.register_cache(
@@ -143,7 +143,7 @@ class LoggingTransaction(object):
         finally:
             msecs = (time.time() * 1000) - start
             sql_logger.debug("[SQL time] {%s} %f", self.name, msecs)
-            sql_query_timer.inc_time(msecs, sql.split()[0])
+            sql_query_timer.inc_by(msecs, sql.split()[0])
 
 
 class PerformanceCounters(object):
@@ -268,7 +268,7 @@ class SQLBaseStore(object):
                     self._current_txn_total_time += end - start
                     self._txn_perf_counters.update(desc, start, end)
 
-                    sql_txn_timer.inc_time(self._current_txn_total_time, desc)
+                    sql_txn_timer.inc_by(self._current_txn_total_time, desc)
 
         with PreserveLoggingContext():
             result = yield self._db_pool.runInteraction(
@@ -672,7 +672,7 @@ class SQLBaseStore(object):
 
         def update_counter(desc, last_time):
             curr_time = self._get_event_counters.update(desc, last_time)
-            sql_getevents_timer.inc_time(curr_time - last_time, desc)
+            sql_getevents_timer.inc_by(curr_time - last_time, desc)
             return curr_time
 
         cache = self._get_event_cache.setdefault(event_id, {})
@@ -727,7 +727,7 @@ class SQLBaseStore(object):
 
         def update_counter(desc, last_time):
             curr_time = self._get_event_counters.update(desc, last_time)
-            sql_getevents_timer.inc_time(curr_time - last_time, desc)
+            sql_getevents_timer.inc_by(curr_time - last_time, desc)
             return curr_time
 
         d = json.loads(js)
diff --git a/tests/metrics/test_metric.py b/tests/metrics/test_metric.py
index 193908b44e..1ca3e45a26 100644
--- a/tests/metrics/test_metric.py
+++ b/tests/metrics/test_metric.py
@@ -16,7 +16,7 @@
 from tests import unittest
 
 from synapse.metrics.metric import (
-    CounterMetric, CallbackMetric, TimerMetric, CacheMetric
+    CounterMetric, CallbackMetric, DistributionMetric, CacheMetric
 )
 
 
@@ -97,37 +97,37 @@ class CallbackMetricTestCase(unittest.TestCase):
         ])
 
 
-class TimerMetricTestCase(unittest.TestCase):
+class DistributionMetricTestCase(unittest.TestCase):
 
     def test_scalar(self):
-        metric = TimerMetric("thing")
+        metric = DistributionMetric("thing")
 
         self.assertEquals(metric.render(), [
             'thing:count 0',
-            'thing:msec 0',
+            'thing:total 0',
         ])
 
-        metric.inc_time(500)
+        metric.inc_by(500)
 
         self.assertEquals(metric.render(), [
             'thing:count 1',
-            'thing:msec 500',
+            'thing:total 500',
         ])
 
     def test_vector(self):
-        metric = TimerMetric("queries", labels=["verb"])
+        metric = DistributionMetric("queries", labels=["verb"])
 
         self.assertEquals(metric.render(), [])
 
-        metric.inc_time(300, "SELECT")
-        metric.inc_time(200, "SELECT")
-        metric.inc_time(800, "INSERT")
+        metric.inc_by(300, "SELECT")
+        metric.inc_by(200, "SELECT")
+        metric.inc_by(800, "INSERT")
 
         self.assertEquals(metric.render(), [
             'queries:count{verb="INSERT"} 1',
-            'queries:msec{verb="INSERT"} 800',
+            'queries:total{verb="INSERT"} 800',
             'queries:count{verb="SELECT"} 2',
-            'queries:msec{verb="SELECT"} 500',
+            'queries:total{verb="SELECT"} 500',
         ])
 
 
-- 
cgit 1.4.1


From 493e3fa0ca81b6070648e0a2c00c6c229cec92fe Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Tue, 10 Mar 2015 15:23:33 +0000
Subject: Don't forbid '_' in metric basenames any more, to allow things like
 foo_time

---
 synapse/metrics/__init__.py | 5 -----
 1 file changed, 5 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index c161c17e9f..f85c6418e5 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -43,11 +43,6 @@ class Metrics(object):
         self.name_prefix = name
 
     def _register(self, metric_class, name, *args, **kwargs):
-        if "_" in name:
-            raise ValueError("Metric names %s is invalid as it cannot contain an underscore"
-                % (name)
-            )
-
         full_name = "%s_%s" % (self.name_prefix, name)
 
         metric = metric_class(full_name, *args, **kwargs)
-- 
cgit 1.4.1


From 2e4f0b2bd736fd70040d936145948b65b4e00b12 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Tue, 10 Mar 2015 15:29:22 +0000
Subject: Replace the @metrics.counted annotations in federation with
 specifically-written counters and distributions

---
 synapse/federation/federation_client.py | 27 +++++++++++++++------------
 synapse/federation/federation_server.py | 26 ++++++++++++++------------
 synapse/metrics/__init__.py             | 17 -----------------
 3 files changed, 29 insertions(+), 41 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index ef177b79cc..6811a0e3d1 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -36,7 +36,15 @@ import random
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
+
+# synapse.federation.federation_client is a silly name
+metrics = synapse.metrics.get_metrics_for("synapse.federation.client")
+
+sent_pdus_destination_dist = metrics.register_distribution("sent_pdu_destinations")
+
+sent_edus_counter = metrics.register_counter("sent_edus")
+
+sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"])
 
 
 class FederationClient(FederationBase):
@@ -53,7 +61,6 @@ class FederationClient(FederationBase):
         self._get_pdu_cache.start()
 
     @log_function
-    @metrics.counted
     def send_pdu(self, pdu, destinations):
         """Informs the replication layer about a new PDU generated within the
         home server that should be transmitted to others.
@@ -70,6 +77,8 @@ class FederationClient(FederationBase):
         order = self._order
         self._order += 1
 
+        sent_pdus_destination_dist.inc_by(len(destinations))
+
         logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id)
 
         # TODO, add errback, etc.
@@ -81,7 +90,6 @@ class FederationClient(FederationBase):
         )
 
     @log_function
-    @metrics.counted
     def send_edu(self, destination, edu_type, content):
         edu = Edu(
             origin=self.server_name,
@@ -90,18 +98,18 @@ class FederationClient(FederationBase):
             content=content,
         )
 
+        sent_edus_counter.inc()
+
         # TODO, add errback, etc.
         self._transaction_queue.enqueue_edu(edu)
         return defer.succeed(None)
 
     @log_function
-    @metrics.counted
     def send_failure(self, failure, destination):
         self._transaction_queue.enqueue_failure(failure, destination)
         return defer.succeed(None)
 
     @log_function
-    @metrics.counted
     def make_query(self, destination, query_type, args,
                    retry_on_dns_fail=True):
         """Sends a federation Query to a remote homeserver of the given type
@@ -118,6 +126,8 @@ class FederationClient(FederationBase):
             a Deferred which will eventually yield a JSON object from the
             response
         """
+        sent_queries_counter.inc(query_type)
+
         return self.transport_layer.make_query(
             destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail
         )
@@ -163,7 +173,6 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def get_pdu(self, destinations, event_id, outlier=False):
         """Requests the PDU with given origin and ID from the remote home
         servers.
@@ -253,7 +262,6 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def get_state_for_room(self, destination, room_id, event_id):
         """Requests all of the `current` state PDUs for a given room from
         a remote home server.
@@ -294,7 +302,6 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def get_event_auth(self, destination, room_id, event_id):
         res = yield self.transport_layer.get_event_auth(
             destination, room_id, event_id,
@@ -314,7 +321,6 @@ class FederationClient(FederationBase):
         defer.returnValue(signed_auth)
 
     @defer.inlineCallbacks
-    @metrics.counted
     def make_join(self, destinations, room_id, user_id):
         for destination in destinations:
             try:
@@ -341,7 +347,6 @@ class FederationClient(FederationBase):
         raise RuntimeError("Failed to send to any server.")
 
     @defer.inlineCallbacks
-    @metrics.counted
     def send_join(self, destinations, pdu):
         for destination in destinations:
             try:
@@ -391,7 +396,6 @@ class FederationClient(FederationBase):
         raise RuntimeError("Failed to send to any server.")
 
     @defer.inlineCallbacks
-    @metrics.counted
     def send_invite(self, destination, room_id, event_id, pdu):
         time_now = self._clock.time_msec()
         code, content = yield self.transport_layer.send_invite(
@@ -415,7 +419,6 @@ class FederationClient(FederationBase):
         defer.returnValue(pdu)
 
     @defer.inlineCallbacks
-    @metrics.counted
     def query_auth(self, destination, room_id, event_id, local_auth):
         """
         Params:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 3216fca95f..25c0014f97 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -33,7 +33,14 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-metrics = synapse.metrics.get_metrics_for(__name__)
+# synapse.federation.federation_server is a silly name
+metrics = synapse.metrics.get_metrics_for("synapse.federation.server")
+
+received_pdus_counter = metrics.register_counter("received_pdus")
+
+received_edus_counter = metrics.register_counter("received_edus")
+
+received_queries_counter = metrics.register_counter("received_queries", labels=["type"])
 
 
 class FederationServer(FederationBase):
@@ -75,7 +82,6 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def on_backfill_request(self, origin, room_id, versions, limit):
         pdus = yield self.handler.on_backfill_request(
             origin, room_id, versions, limit
@@ -85,10 +91,11 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def on_incoming_transaction(self, transaction_data):
         transaction = Transaction(**transaction_data)
 
+        received_pdus_counter.inc_by(len(transaction.pdus))
+
         for p in transaction.pdus:
             if "unsigned" in p:
                 unsigned = p["unsigned"]
@@ -158,6 +165,8 @@ class FederationServer(FederationBase):
         defer.returnValue((200, response))
 
     def received_edu(self, origin, edu_type, content):
+        received_edus_counter.inc()
+
         if edu_type in self.edu_handlers:
             self.edu_handlers[edu_type](origin, content)
         else:
@@ -165,7 +174,6 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def on_context_state_request(self, origin, room_id, event_id):
         if event_id:
             pdus = yield self.handler.get_state_for_pdu(
@@ -193,7 +201,6 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def on_pdu_request(self, origin, event_id):
         pdu = yield self._get_persisted_pdu(origin, event_id)
 
@@ -206,13 +213,13 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    @metrics.counted
     def on_pull_request(self, origin, versions):
         raise NotImplementedError("Pull transactions not implemented")
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_query_request(self, query_type, args):
+        received_queries_counter.inc(query_type)
+
         if query_type in self.query_handlers:
             response = yield self.query_handlers[query_type](args)
             defer.returnValue((200, response))
@@ -222,14 +229,12 @@ class FederationServer(FederationBase):
             )
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_make_join_request(self, room_id, user_id):
         pdu = yield self.handler.on_make_join_request(room_id, user_id)
         time_now = self._clock.time_msec()
         defer.returnValue({"event": pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_invite_request(self, origin, content):
         pdu = self.event_from_pdu_json(content)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
@@ -237,7 +242,6 @@ class FederationServer(FederationBase):
         defer.returnValue((200, {"event": ret_pdu.get_pdu_json(time_now)}))
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_send_join_request(self, origin, content):
         logger.debug("on_send_join_request: content: %s", content)
         pdu = self.event_from_pdu_json(content)
@@ -252,7 +256,6 @@ class FederationServer(FederationBase):
         }))
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_event_auth(self, origin, room_id, event_id):
         time_now = self._clock.time_msec()
         auth_pdus = yield self.handler.on_event_auth(event_id)
@@ -261,7 +264,6 @@ class FederationServer(FederationBase):
         }))
 
     @defer.inlineCallbacks
-    @metrics.counted
     def on_query_auth_request(self, origin, content, event_id):
         """
         Content is a dict with keys::
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index f85c6418e5..94164974fc 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -62,23 +62,6 @@ class Metrics(object):
     def register_cache(self, *args, **kwargs):
         return self._register(CacheMetric, *args, **kwargs)
 
-    def counted(self, func):
-        """ A method decorator that registers a counter, to count invocations
-        of this method. """
-        if not hasattr(self, "method_counter"):
-            self.method_counter = self.register_counter(
-                "calls",
-                labels=["method"]
-            )
-
-        counter = self.method_counter
-        name = func.__name__
-
-        def wrapped(*args, **kwargs):
-            counter.inc(name)
-            return func(*args, **kwargs)
-        return wrapped
-
 
 def get_metrics_for(pkg_name):
     """ Returns a Metrics instance for conveniently creating metrics
-- 
cgit 1.4.1


From 128cf2daf76e5b05a4e577b60ea406fdbb6986bf Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 12 Mar 2015 16:24:38 +0000
Subject: Appease pep8

---
 synapse/federation/transaction_queue.py |  9 ++++++---
 synapse/handlers/presence.py            |  5 +++--
 synapse/http/client.py                  | 10 +++++++---
 synapse/http/matrixfederationclient.py  | 10 +++++++---
 synapse/http/server.py                  |  8 +++++---
 synapse/metrics/__init__.py             |  3 ++-
 synapse/metrics/metric.py               | 11 ++++++-----
 synapse/metrics/resource.py             |  2 +-
 synapse/notifier.py                     | 15 +++++++++------
 9 files changed, 46 insertions(+), 27 deletions(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py
index 99e386fa52..4dccd93d0e 100644
--- a/synapse/federation/transaction_queue.py
+++ b/synapse/federation/transaction_queue.py
@@ -57,7 +57,8 @@ class TransactionQueue(object):
         # done
         self.pending_transactions = {}
 
-        metrics.register_callback("pending_destinations",
+        metrics.register_callback(
+            "pending_destinations",
             lambda: len(self.pending_transactions),
         )
 
@@ -67,10 +68,12 @@ class TransactionQueue(object):
         # destination -> list of tuple(edu, deferred)
         self.pending_edus_by_dest = edus = {}
 
-        metrics.register_callback("pending_pdus",
+        metrics.register_callback(
+            "pending_pdus",
             lambda: sum(map(len, pdus.values())),
         )
-        metrics.register_callback("pending_edus",
+        metrics.register_callback(
+            "pending_edus",
             lambda: sum(map(len, edus.values())),
         )
 
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index c6d6aef53b..731df00648 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -136,8 +136,9 @@ class PresenceHandler(BaseHandler):
         self._user_cachemap = {}
         self._user_cachemap_latest_serial = 0
 
-        metrics.register_callback("userCachemap:size",
-            lambda: len(self._user_cachemap)
+        metrics.register_callback(
+            "userCachemap:size",
+            lambda: len(self._user_cachemap),
         )
 
     def _get_or_make_usercache(self, user):
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 01737a7188..2ae1c4d3a4 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -34,11 +34,13 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-outgoing_requests_counter = metrics.register_counter("requests",
+outgoing_requests_counter = metrics.register_counter(
+    "requests",
     labels=["method"],
 )
-incoming_responses_counter = metrics.register_counter("responses",
-    labels=["method","code"],
+incoming_responses_counter = metrics.register_counter(
+    "responses",
+    labels=["method", "code"],
 )
 
 
@@ -64,9 +66,11 @@ class SimpleHttpClient(object):
         def _cb(response):
             incoming_responses_counter.inc(method, response.code)
             return response
+
         def _eb(failure):
             incoming_responses_counter.inc(method, "ERR")
             return failure
+
         d.addCallbacks(_cb, _eb)
 
         return d
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 11883d3852..7fa295cad5 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -43,11 +43,13 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-outgoing_requests_counter = metrics.register_counter("requests",
+outgoing_requests_counter = metrics.register_counter(
+    "requests",
     labels=["method"],
 )
-incoming_responses_counter = metrics.register_counter("responses",
-    labels=["method","code"],
+incoming_responses_counter = metrics.register_counter(
+    "responses",
+    labels=["method", "code"],
 )
 
 
@@ -78,9 +80,11 @@ class MatrixFederationHttpAgent(_AgentBase):
         def _cb(response):
             incoming_responses_counter.inc(method, response.code)
             return response
+
         def _eb(failure):
             incoming_responses_counter.inc(method, "ERR")
             return failure
+
         d.addCallbacks(_cb, _eb)
 
         return d
diff --git a/synapse/http/server.py b/synapse/http/server.py
index a0d190ff78..d77cb77799 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -37,11 +37,13 @@ logger = logging.getLogger(__name__)
 
 metrics = synapse.metrics.get_metrics_for(__name__)
 
-incoming_requests_counter = metrics.register_counter("requests",
+incoming_requests_counter = metrics.register_counter(
+    "requests",
     labels=["method", "servlet"],
 )
-outgoing_responses_counter = metrics.register_counter("responses",
-    labels=["method","code"],
+outgoing_responses_counter = metrics.register_counter(
+    "responses",
+    labels=["method", "code"],
 )
 
 
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 94164974fc..7b9c9c8bab 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -85,7 +85,7 @@ def render_all():
             strs += ["# FAILED to render %s" % name]
             logger.exception("Failed to render %s metric", name)
 
-    strs.append("") # to generate a final CRLF
+    strs.append("")  # to generate a final CRLF
 
     return "\n".join(strs)
 
@@ -96,6 +96,7 @@ def render_all():
 rusage = None
 PAGE_SIZE = getpagesize()
 
+
 def update_resource_metrics():
     global rusage
     rusage = getrusage(RUSAGE_SELF)
diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 12460c99c3..21b37748f6 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -27,7 +27,7 @@ class BaseMetric(object):
 
     def __init__(self, name, labels=[]):
         self.name = name
-        self.labels = labels # OK not to clone as we never write it
+        self.labels = labels  # OK not to clone as we never write it
 
     def dimension(self):
         return len(self.labels)
@@ -66,8 +66,8 @@ class CounterMetric(BaseMetric):
 
     def inc_by(self, incr, *values):
         if len(values) != self.dimension():
-            raise ValueError("Expected as many values to inc() as labels (%d)" %
-                (self.dimension())
+            raise ValueError(
+                "Expected as many values to inc() as labels (%d)" % (self.dimension())
             )
 
         # TODO: should assert that the tag values are all strings
@@ -135,10 +135,11 @@ class CacheMetric(object):
     def __init__(self, name, size_callback, labels=[]):
         self.name = name
 
-        self.hits  = CounterMetric(name + ":hits",  labels=labels)
+        self.hits = CounterMetric(name + ":hits", labels=labels)
         self.total = CounterMetric(name + ":total", labels=labels)
 
-        self.size = CallbackMetric(name + ":size",
+        self.size = CallbackMetric(
+            name + ":size",
             callback=size_callback,
             labels=labels,
         )
diff --git a/synapse/metrics/resource.py b/synapse/metrics/resource.py
index ff7baab018..97ea797bf5 100644
--- a/synapse/metrics/resource.py
+++ b/synapse/metrics/resource.py
@@ -26,7 +26,7 @@ class MetricsResource(Resource):
     isLeaf = True
 
     def __init__(self, hs):
-        Resource.__init__(self) # Resource is old-style, so no super()
+        Resource.__init__(self)  # Resource is old-style, so no super()
 
         self.hs = hs
 
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 88873d4534..7121d659d0 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -127,14 +127,17 @@ class Notifier(object):
             return len(all_listeners)
         metrics.register_callback("listeners", count_listeners)
 
-        metrics.register_callback("rooms",
-            lambda: count(bool, self.room_to_listeners.values())
+        metrics.register_callback(
+            "rooms",
+            lambda: count(bool, self.room_to_listeners.values()),
         )
-        metrics.register_callback("users",
-            lambda: count(bool, self.user_to_listeners.values())
+        metrics.register_callback(
+            "users",
+            lambda: count(bool, self.user_to_listeners.values()),
         )
-        metrics.register_callback("appservices",
-            lambda: count(bool, self.appservice_to_listeners.values())
+        metrics.register_callback(
+            "appservices",
+            lambda: count(bool, self.appservice_to_listeners.values()),
         )
 
     @log_function
-- 
cgit 1.4.1


From 05a056a409042d233972c608a8e526e8fd0af262 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Thu, 12 Mar 2015 16:45:05 +0000
Subject: Appease pyflakes

---
 synapse/metrics/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'synapse/metrics/__init__.py')

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 7b9c9c8bab..dffb8a4861 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -81,7 +81,7 @@ def render_all():
     for name in sorted(all_metrics.keys()):
         try:
             strs += all_metrics[name].render()
-        except Exception as e:
+        except Exception:
             strs += ["# FAILED to render %s" % name]
             logger.exception("Failed to render %s metric", name)
 
-- 
cgit 1.4.1