http txns: Do not cache error responses

Previously we did. This meant that, amongst other errors, rate-limiting errors
would be cached and prevent messages with that txn ID being sent.

synapse/rest/client/transactions.py

@@ -81,7 +81,16 @@ class HttpTransactionCache(object):
             Deferred which resolves to a tuple of (response_code, response_dict).
         """
         try:
-            return self.transactions[txn_key][0].observe()
+            observable = self.transactions[txn_key][0]
+            if not observable.has_called() or observable.has_succeeded():
+                return observable.observe()
+            # if the request has already been called with a non-2xx status
+            # (a Twisted failure), remove it from the transaction map.
+            # This is done to ensure that we don't cache rate-limiting errors, etc.
+            res = observable.get_result()
+            if res.value.code >= 300:
+                del self.transactions[txn_key]
+                # fall through
         except (KeyError, IndexError):
             pass  # execute the function instead.