diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 2f2378263..0639c228c 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -17,7 +17,7 @@ import logging
 
 import idna
 from service_identity import VerificationError
-from service_identity.pyopenssl import verify_hostname
+from service_identity.pyopenssl import verify_hostname, verify_ip_address
 from zope.interface import implementer
 
 from OpenSSL import SSL, crypto
@@ -156,7 +156,7 @@ class ConnectionVerifier(object):
 
         if isIPAddress(hostname) or isIPv6Address(hostname):
             self._hostnameBytes = hostname.encode('ascii')
-            self._sendSNI = False
+            self._is_ip_address = True
         else:
             # twisted's ClientTLSOptions falls back to the stdlib impl here if
             # idna is not installed, but points out that lacks support for
@@ -164,17 +164,20 @@ class ConnectionVerifier(object):
             #
             # We can rely on having idna.
             self._hostnameBytes = idna.encode(hostname)
-            self._sendSNI = True
+            self._is_ip_address = False
 
         self._hostnameASCII = self._hostnameBytes.decode("ascii")
 
     def verify_context_info_cb(self, ssl_connection, where):
-        if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI:
+        if where & SSL.SSL_CB_HANDSHAKE_START and not self._is_ip_address:
             ssl_connection.set_tlsext_host_name(self._hostnameBytes)
 
         if where & SSL.SSL_CB_HANDSHAKE_DONE and self._verify_certs:
             try:
-                verify_hostname(ssl_connection, self._hostnameASCII)
+                if self._is_ip_address:
+                    verify_ip_address(ssl_connection, self._hostnameASCII)
+                else:
+                    verify_hostname(ssl_connection, self._hostnameASCII)
             except VerificationError:
                 f = Failure()
                 tls_protocol = ssl_connection.get_app_data()
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index db09ff285..6efd81f20 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -45,7 +45,9 @@ REQUIREMENTS = [
     "signedjson>=1.0.0",
     "pynacl>=1.2.1",
     "idna>=2",
-    "service_identity>=16.0.0",
+
+    # validating SSL certs for IP addresses requires service_identity 18.1.
+    "service_identity>=18.1.0",
 
     # our logcontext handling relies on the ability to cancel inlineCallbacks
     # (https://twistedmatrix.com/trac/ticket/4632) which landed in Twisted 18.7.