From c94de0ab606e5bbe7331a58456204a5775df779b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 12 Sep 2016 16:55:01 +0100
Subject: [PATCH 1/3] Add WHERE clause support to index creation

---
 synapse/storage/background_updates.py | 22 ++++++++++++++--------
 synapse/storage/event_push_actions.py |  3 ++-
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index 003f5ba20..94b2bcc54 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -219,7 +219,7 @@ class BackgroundUpdateStore(SQLBaseStore):
         self._background_update_handlers[update_name] = update_handler
 
     def register_background_index_update(self, update_name, index_name,
-                                         table, columns):
+                                         table, columns, where_clause=None):
         """Helper for store classes to do a background index addition
 
         To use:
@@ -243,14 +243,20 @@ class BackgroundUpdateStore(SQLBaseStore):
             conc = True
         else:
             conc = False
+            # We don't use partial indices on SQLite as it wasn't introduced
+            # until 3.8, and wheezy has 3.7
+            where_clause = None
 
-        sql = "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" \
-              % {
-                  "conc": "CONCURRENTLY" if conc else "",
-                  "name": index_name,
-                  "table": table,
-                  "columns": ", ".join(columns),
-              }
+        sql = (
+            "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)"
+            " %(where_clause)s"
+        ) % {
+            "conc": "CONCURRENTLY" if conc else "",
+            "name": index_name,
+            "table": table,
+            "columns": ", ".join(columns),
+            "where_clause": "WHERE " + where_clause if where_clause else ""
+        }
 
         def create_index_concurrently(conn):
             conn.rollback()
diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index 51b13e949..efa1db3b9 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -36,7 +36,8 @@ class EventPushActionsStore(SQLBaseStore):
             self.EPA_HIGHLIGHT_INDEX,
             index_name="event_push_actions_u_highlight",
             table="event_push_actions",
-            columns=["user_id", "stream_ordering", "highlight"],
+            columns=["user_id", "stream_ordering"],
+            where_clause="highlight = 1",
         )
 
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):

From 0b32bb20bbf025b6e69bae23a6d4a96903f41885 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 12 Sep 2016 16:57:05 +0100
Subject: [PATCH 2/3] Index contains_url for file search queries

---
 synapse/storage/events.py                       |  8 ++++++++
 .../storage/schema/delta/35/contains_url.sql    | 17 +++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 synapse/storage/schema/delta/35/contains_url.sql

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ed182c8d1..6dc46fa50 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -189,6 +189,14 @@ class EventsStore(SQLBaseStore):
             self._background_reindex_fields_sender,
         )
 
+        self.register_background_index_update(
+            "event_contains_url_index",
+            index_name="event_contains_url_index",
+            table="events",
+            columns=["room_id", "topological_ordering", "stream_ordering"],
+            where_clause="contains_url = true AND outlier = false",
+        )
+
         self._event_persist_queue = _EventPeristenceQueue()
 
     def persist_events(self, events_and_contexts, backfilled=False):
diff --git a/synapse/storage/schema/delta/35/contains_url.sql b/synapse/storage/schema/delta/35/contains_url.sql
new file mode 100644
index 000000000..6cd123027
--- /dev/null
+++ b/synapse/storage/schema/delta/35/contains_url.sql
@@ -0,0 +1,17 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ INSERT into background_updates (update_name, progress_json)
+     VALUES ('event_contains_url_index', '{}');

From b17af156c71b9859edbd27516eacd2c55f488e31 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 12 Sep 2016 17:05:54 +0100
Subject: [PATCH 3/3] Remove where clause

---
 synapse/storage/event_push_actions.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py
index efa1db3b9..9cd923eb9 100644
--- a/synapse/storage/event_push_actions.py
+++ b/synapse/storage/event_push_actions.py
@@ -37,7 +37,6 @@ class EventPushActionsStore(SQLBaseStore):
             index_name="event_push_actions_u_highlight",
             table="event_push_actions",
             columns=["user_id", "stream_ordering"],
-            where_clause="highlight = 1",
         )
 
     def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples):