diff --git a/synapse/metrics/metric.py b/synapse/metrics/metric.py
index 6f82b360b..368fc2498 100644
--- a/synapse/metrics/metric.py
+++ b/synapse/metrics/metric.py
@@ -15,7 +15,6 @@
 
 
 from itertools import chain
-from collections import Counter
 
 
 # TODO(paul): I can't believe Python doesn't have one of these
@@ -56,29 +55,30 @@ class CounterMetric(BaseMetric):
     """The simplest kind of metric; one that stores a monotonically-increasing
     integer that counts events."""
 
-    __slots__ = ("counts")
-
     def __init__(self, *args, **kwargs):
         super(CounterMetric, self).__init__(*args, **kwargs)
 
-        self.counts = Counter()
+        self.counts = {}
 
         # Scalar metrics are never empty
         if self.is_scalar():
             self.counts[()] = 0
 
     def inc_by(self, incr, *values):
-        # if len(values) != self.dimension():
-        #     raise ValueError(
-        #         "Expected as many values to inc() as labels (%d)" % (self.dimension())
-        #     )
+        if len(values) != self.dimension():
+            raise ValueError(
+                "Expected as many values to inc() as labels (%d)" % (self.dimension())
+            )
 
         # TODO: should assert that the tag values are all strings
 
-        self.counts[values] += incr
+        if values not in self.counts:
+            self.counts[values] = incr
+        else:
+            self.counts[values] += incr
 
     def inc(self, *values):
-        self.counts[values] += 1
+        self.inc_by(1, *values)
 
     def render_item(self, k):
         return ["%s%s %d" % (self.name, self._render_key(k), self.counts[k])]
@@ -132,8 +132,6 @@ class CacheMetric(object):
     This metric generates standard metric name pairs, so that monitoring rules
     can easily be applied to measure hit ratio."""
 
-    __slots__ = ("name", "hits", "total", "size")
-
     def __init__(self, name, size_callback, labels=[]):
         self.name = name
 
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 0d6f48e2d..40be7fe7e 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -102,6 +102,15 @@ class ObservableDeferred(object):
     def observers(self):
         return self._observers
 
+    def has_called(self):
+        return self._result is not None
+
+    def has_succeeded(self):
+        return self._result is not None and self._result[0] is True
+
+    def get_result(self):
+        return self._result[1]
+
     def __getattr__(self, name):
         return getattr(self._deferred, name)
 
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 4bbb16ed3..5be409727 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -311,12 +311,12 @@ class CacheListDescriptor(object):
 
                 try:
                     res = cache.get(tuple(key))
-                    if not res.called:
+                    if not res.has_succeeded():
                         res = res.observe()
                         res.addCallback(lambda r, arg: (arg, r), arg)
                         cached_defers[arg] = res
                     else:
-                        results[arg] = res.result
+                        results[arg] = res.get_result()
                 except KeyError:
                     missing.append(arg)