Flesh out email templating

Mostly WIP porting the room name calculation logic from the web client so our room names in the email mirror the clients.

synapse/push/emailpusher.py

@@ -82,6 +82,13 @@ class EmailPusher(object):
         self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering)
         yield self._process()
 
+    @defer.inlineCallbacks
+    def on_new_receipts(self, min_stream_id, max_stream_id):
+        # We could wake up and cancel the timer but there tend to be quite a
+        # lot of read receipts so it's probably less work to just let the
+        # timer fire
+        return defer.succeed(None)
+
     @defer.inlineCallbacks
     def on_timer(self):
         self.timed_call = None

synapse/push/mailer.py

@@ -14,18 +14,23 @@
 # limitations under the License.
 
 from twisted.internet import defer
-
 from twisted.mail.smtp import sendmail
+
 import email.utils
 import email.mime.multipart
 from email.mime.text import MIMEText
 
+from synapse.util.async import concurrently_execute
+from synapse.util.room_name import calculate_room_name
+
 import jinja2
 
 
 class Mailer(object):
     def __init__(self, hs):
         self.hs = hs
+        self.store = self.hs.get_datastore()
+        self.state_handler = self.hs.get_state_handler()
         loader = jinja2.FileSystemLoader(self.hs.config.email_template_dir)
         env = jinja2.Environment(loader=loader)
         self.notif_template = env.get_template(self.hs.config.email_notif_template_html)
@@ -38,9 +43,41 @@ class Mailer(object):
         if raw_to == '':
             raise RuntimeError("Invalid 'to' address")
 
-        plainText = self.notif_template.render()
+        rooms_in_order = deduped_ordered_list(
+            [pa['room_id'] for pa in push_actions]
+        )
 
-        text_part = MIMEText(plainText, "plain")
+        notifs_by_room = {}
+        for pa in push_actions:
+            notifs_by_room.setdefault(pa["room_id"], []).append(pa)
+
+        # collect the current state for all the rooms in which we have
+        # notifications
+        state_by_room = {}
+
+        @defer.inlineCallbacks
+        def _fetch_room_state(room_id):
+            room_state = yield self.state_handler.get_current_state(room_id)
+            state_by_room[room_id] = room_state
+
+        # Run at most 3 of these at once: sync does 10 at a time but email
+        # notifs are much realtime than sync so we can afford to wait a bit.
+        yield concurrently_execute(_fetch_room_state, rooms_in_order, 3)
+
+        rooms = [
+            self.get_room_vars(
+                r, user_id, notifs_by_room[r], state_by_room[r]
+            ) for r in rooms_in_order
+        ]
+
+        template_vars = {
+            "unsubscribe_link": self.make_unsubscribe_link(),
+            "rooms": rooms,
+        }
+
+        plainText = self.notif_template.render(**template_vars)
+
+        text_part = MIMEText(plainText, "html")
         text_part['Subject'] = "New Matrix Notifications"
         text_part['From'] = self.hs.config.email_notif_from
         text_part['To'] = email_address
@@ -50,3 +87,21 @@ class Mailer(object):
             raw_from, raw_to, text_part.as_string(),
             port=self.hs.config.email_smtp_port
         )
+
+    def get_room_vars(self, room_id, user_id, notifs, room_state):
+        room_vars = {}
+        room_vars['title'] = calculate_room_name(room_state, user_id)
+        return room_vars
+
+    def make_unsubscribe_link(self):
+        return "https://vector.im/#/settings"  # XXX: matrix.to
+
+
+def deduped_ordered_list(l):
+    seen = set()
+    ret = []
+    for item in l:
+        if item not in seen:
+            seen.add(item)
+            ret.append(item)
+    return ret

synapse/python_dependencies.py

@@ -46,7 +46,7 @@ CONDITIONAL_REQUIREMENTS = {
         "netaddr>=0.7.18": ["netaddr"],
     },
     "email.enable_notifs": {
-        "Jinja2": ["Jinja2"],
+        "Jinja2>=2.8": ["Jinja2>=2.8"],
     },
 }
 

synapse/util/room_name.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+# intentionally looser than what aliases we allow to be registered since
+# other HSes may allow aliases that we would not
+ALIAS_RE = re.compile(r"^#.*:.+$")
+
+ALL_ALONE = "Empty Room"
+
+
+def calculate_room_name(room_state, user_id):
+    # does it have a name?
+    if ("m.room.name", "") in room_state:
+        m_room_name = room_state[("m.room.name", "")]
+        if m_room_name.content and m_room_name.content["name"]:
+            return m_room_name.content["name"]
+
+    # does it have a caononical alias?
+    if ("m.room.canonical_alias", "") in room_state:
+        canon_alias = room_state[("m.room.canonical_alias", "")]
+        if (
+            canon_alias.content and canon_alias.content["alias"] and
+            looks_like_an_alias(canon_alias.content["alias"])
+        ):
+            return canon_alias.content["alias"]
+
+    # at this point we're going to need to search the state by all state keys
+    # for an event type, so rearrange the data structure
+    room_state_bytype = state_as_two_level_dict(room_state)
+
+    # right then, any aliases at all?
+    if "m.room.aliases" in room_state_bytype:
+        m_room_aliases = room_state_bytype["m.room.aliases"]
+        if len(m_room_aliases.values()) > 0:
+            first_alias_event = m_room_aliases.values()[0]
+            if first_alias_event.content and first_alias_event.content["aliases"]:
+                the_aliases = first_alias_event.content["aliases"]
+                if len(the_aliases) > 0 and looks_like_an_alias(the_aliases[0]):
+                    return the_aliases[0]
+
+    my_member_event = None
+    if ("m.room.member", user_id) in room_state:
+        my_member_event = room_state[("m.room.member", user_id)]
+
+    if (
+        my_member_event is not None and
+        my_member_event.content['membership'] == "invite"
+    ):
+        if ("m.room.member", my_member_event.sender) in room_state:
+            inviter_member_event = room_state[("m.room.member", my_member_event.sender)]
+            return "Invite from %s" % (name_from_member_event(inviter_member_event),)
+        else:
+            return "Room Invite"
+
+    # we're going to have to generate a name based on who's in the room,
+    # so find out who is in the room that isn't the user.
+    if "m.room.member" in room_state_bytype:
+        all_members = [
+            ev for ev in room_state_bytype["m.room.member"].values()
+            if ev.membership == "join" or ev.membership == "invite"
+        ]
+        other_members = [m for m in all_members if m.sender != user_id]
+    else:
+        other_members = []
+        all_members = []
+
+    if len(other_members) == 0:
+        if len(all_members) == 1:
+            # self-chat, peeked room with 1 participant,
+            # or inbound invite, or outbound 3PID invite.
+            if all_members[0].sender == user_id:
+                if "m.room.third_party_invite" in room_state_bytype:
+                    third_party_invites = room_state_bytype["m.room.third_party_invite"]
+                    if len(third_party_invites) > 0:
+                        # technically third party invite events are not member
+                        # events, but they are close enough
+                        return "Inviting %s" (
+                            descriptor_from_member_events(third_party_invites)
+                        )
+                    else:
+                        return ALL_ALONE
+            else:
+                return name_from_member_event(all_members[0])
+        else:
+            return ALL_ALONE
+    else:
+        return descriptor_from_member_events(other_members)
+
+
+def state_as_two_level_dict(state):
+    ret = {}
+    for k, v in state.items():
+        ret.setdefault(k[0], {})[k[1]] = v
+    return ret
+
+
+def looks_like_an_alias(string):
+    return ALIAS_RE.match(string) is not None
+
+
+def descriptor_from_member_events(member_events):
+    #     else if (otherMembers.length === 1) {
+    #     return otherMembers[0].name;
+    # }
+    # else if (otherMembers.length === 2) {
+    #     return (
+    #         otherMembers[0].name + " and " + otherMembers[1].name
+    #     );
+    # }
+    # else {
+    #     return (
+    #         otherMembers[0].name + " and " + (otherMembers.length - 1) + " others"
+    #     );
+    # }
+    if len(member_events) == 0:
+        return "nobody"
+    elif len(member_events) == 1:
+        return name_from_member_event(member_events[0])
+    return "all the people, so many people. They all go hand in hand, hand in hand in their park life."
+
+
+def name_from_member_event(member_event):
+    if (
+        member_event.content and "displayname" in member_event.content and
+        member_event.content["displayname"]
+    ):
+        return member_event.content["displayname"]
+    return member_event.sender