maxmustermann/synapse
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Relax ignore-missing-imports for modules that have stubs now and update mypy (#11006)

Browse source 
Updating mypy past version 0.9 means that third-party stubs are no-longer distributed with typeshed. See http://mypy-lang.blogspot.com/2021/06/mypy-0900-released.html for details.
We therefore pull in stub packages in setup.py

Additionally, some modules that we were previously ignoring import failures for now have stubs. So let's use them.

The rest of this change consists of fixups to make the newer mypy + stubs pass CI.

Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
This commit is contained in:
David Robertson 2021-10-08 14:49:41 +01:00 committed by GitHub
parent 670a8d9a1e
commit 797ee7812d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 132 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/11006.misc Normal file
View file

@ -0,0 +1 @@
Bump mypy version for CI to 0.910, and pull in new type stubs for dependencies.

133
mypy.ini
View file

@ -198,98 +198,97 @@ disallow_untyped_defs = True
[mypy-tests.storage.test_user_directory]
disallow_untyped_defs = True
[mypy-pymacaroons.*]
ignore_missing_imports = True
;; Dependencies without annotations
;; Before ignoring a module, check to see if type stubs are available.
;; The `typeshed` project maintains stubs here:
;; https://github.com/python/typeshed/tree/master/stubs
;; and for each package `foo` there's a corresponding `types-foo` package on PyPI,
;; which we can pull in as a dev dependency by adding to `setup.py`'s
;; `CONDITIONAL_REQUIREMENTS["mypy"]` list.
[mypy-zope]
[mypy-authlib.*]
ignore_missing_imports = True
[mypy-bcrypt]
ignore_missing_imports = True
[mypy-constantly]
ignore_missing_imports = True
[mypy-twisted.*]
ignore_missing_imports = True
[mypy-treq.*]
ignore_missing_imports = True
[mypy-hyperlink]
ignore_missing_imports = True
[mypy-h11]
ignore_missing_imports = True
[mypy-msgpack]
ignore_missing_imports = True
[mypy-opentracing]
ignore_missing_imports = True
[mypy-OpenSSL.*]
ignore_missing_imports = True
[mypy-netaddr]
ignore_missing_imports = True
[mypy-saml2.*]
ignore_missing_imports = True
[mypy-canonicaljson]
ignore_missing_imports = True
[mypy-jaeger_client.*]
ignore_missing_imports = True
[mypy-jsonschema]
ignore_missing_imports = True
[mypy-signedjson.*]
ignore_missing_imports = True
[mypy-prometheus_client.*]
ignore_missing_imports = True
[mypy-service_identity.*]
[mypy-constantly]
ignore_missing_imports = True
[mypy-daemonize]
ignore_missing_imports = True
[mypy-sentry_sdk]
ignore_missing_imports = True
[mypy-PIL.*]
ignore_missing_imports = True
[mypy-lxml]
ignore_missing_imports = True
[mypy-jwt.*]
ignore_missing_imports = True
[mypy-authlib.*]
ignore_missing_imports = True
[mypy-rust_python_jaeger_reporter.*]
ignore_missing_imports = True
[mypy-nacl.*]
[mypy-h11]
ignore_missing_imports = True
[mypy-hiredis]
ignore_missing_imports = True
[mypy-hyperlink]
ignore_missing_imports = True
[mypy-ijson.*]
ignore_missing_imports = True
[mypy-jaeger_client.*]
ignore_missing_imports = True
[mypy-josepy.*]
ignore_missing_imports = True
[mypy-pympler.*]
[mypy-jwt.*]
ignore_missing_imports = True
[mypy-lxml]
ignore_missing_imports = True
[mypy-msgpack]
ignore_missing_imports = True
[mypy-nacl.*]
ignore_missing_imports = True
[mypy-netaddr]
ignore_missing_imports = True
[mypy-opentracing]
ignore_missing_imports = True
[mypy-phonenumbers.*]
ignore_missing_imports = True
[mypy-ijson.*]
[mypy-prometheus_client.*]
ignore_missing_imports = True
[mypy-pymacaroons.*]
ignore_missing_imports = True
[mypy-pympler.*]
ignore_missing_imports = True
[mypy-rust_python_jaeger_reporter.*]
ignore_missing_imports = True
[mypy-saml2.*]
ignore_missing_imports = True
[mypy-sentry_sdk]
ignore_missing_imports = True
[mypy-service_identity.*]
ignore_missing_imports = True
[mypy-signedjson.*]
ignore_missing_imports = True
[mypy-treq.*]
ignore_missing_imports = True
[mypy-twisted.*]
ignore_missing_imports = True
[mypy-zope]
ignore_missing_imports = True

11
setup.py
View file

@ -112,7 +112,16 @@ CONDITIONAL_REQUIREMENTS["dev"] = CONDITIONAL_REQUIREMENTS["lint"] + [
"pygithub==1.55",
]
CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
CONDITIONAL_REQUIREMENTS["mypy"] = [
"mypy==0.910",
"mypy-zope==0.3.2",
"types-bleach>=4.1.0",
"types-jsonschema>=3.2.0",
"types-Pillow>=8.3.4",
"types-pyOpenSSL>=20.0.7",
"types-PyYAML>=5.4.10",
"types-setuptools>=57.4.0",
]
# Dependencies which are exclusively required by unit test code. This is
# NOT a list of all modules that are necessary to run the unit tests.

9
synapse/config/tls.py
View file

@ -172,9 +172,12 @@ class TlsConfig(Config):
)
# YYYYMMDDhhmmssZ -- in UTC
expires_on = datetime.strptime(
tls_certificate.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ"
)
expiry_data = tls_certificate.get_notAfter()
if expiry_data is None:
raise ValueError(
"TLS Certificate has no expiry date, and this is not permitted"
)
expires_on = datetime.strptime(expiry_data.decode("ascii"), "%Y%m%d%H%M%SZ")
now = datetime.utcnow()
days_remaining = (expires_on - now).days
return days_remaining

2
synapse/http/client.py
View file

@ -912,7 +912,7 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory):
def __init__(self):
self._context = SSL.Context(SSL.SSLv23_METHOD)
self._context.set_verify(VERIFY_NONE, lambda *_: None)
self._context.set_verify(VERIFY_NONE, lambda *_: False)
def getContext(self, hostname=None, port=None):
return self._context

16
synapse/logging/context.py
View file

@ -52,7 +52,7 @@ try:
is_thread_resource_usage_supported = True
def get_thread_resource_usage() -> "Optional[resource._RUsage]":
def get_thread_resource_usage() -> "Optional[resource.struct_rusage]":
return resource.getrusage(RUSAGE_THREAD)
@ -61,7 +61,7 @@ except Exception:
# won't track resource usage.
is_thread_resource_usage_supported = False
def get_thread_resource_usage() -> "Optional[resource._RUsage]":
def get_thread_resource_usage() -> "Optional[resource.struct_rusage]":
return None
@ -226,10 +226,10 @@ class _Sentinel:
def copy_to(self, record):
pass
def start(self, rusage: "Optional[resource._RUsage]"):
def start(self, rusage: "Optional[resource.struct_rusage]"):
pass
def stop(self, rusage: "Optional[resource._RUsage]"):
def stop(self, rusage: "Optional[resource.struct_rusage]"):
pass
def add_database_transaction(self, duration_sec):
@ -289,7 +289,7 @@ class LoggingContext:
# The thread resource usage when the logcontext became active. None
# if the context is not currently active.
self.usage_start: Optional[resource._RUsage] = None
self.usage_start: Optional[resource.struct_rusage] = None
self.main_thread = get_thread_id()
self.request = None
@ -410,7 +410,7 @@ class LoggingContext:
# we also track the current scope:
record.scope = self.scope
def start(self, rusage: "Optional[resource._RUsage]") -> None:
def start(self, rusage: "Optional[resource.struct_rusage]") -> None:
"""
Record that this logcontext is currently running.
@ -435,7 +435,7 @@ class LoggingContext:
else:
self.usage_start = rusage
def stop(self, rusage: "Optional[resource._RUsage]") -> None:
def stop(self, rusage: "Optional[resource.struct_rusage]") -> None:
"""
Record that this logcontext is no longer running.
@ -490,7 +490,7 @@ class LoggingContext:
return res
def _get_cputime(self, current: "resource._RUsage") -> Tuple[float, float]:
def _get_cputime(self, current: "resource.struct_rusage") -> Tuple[float, float]:
"""Get the cpu usage time between start() and the given rusage
Args:

2
synapse/metrics/background_process_metrics.py
View file

@ -265,7 +265,7 @@ class BackgroundProcessLoggingContext(LoggingContext):
super().__init__("%s-%s" % (name, instance_id))
self._proc = _BackgroundProcess(name, self)
def start(self, rusage: "Optional[resource._RUsage]"):
def start(self, rusage: "Optional[resource.struct_rusage]"):
"""Log context has started running (again)."""
super().start(rusage)

2
synapse/push/mailer.py
View file

@ -892,7 +892,7 @@ def safe_text(raw_text: str) -> jinja2.Markup:
A Markup object ready to safely use in a Jinja template.
"""
return jinja2.Markup(
bleach.linkify(bleach.clean(raw_text, tags=[], attributes={}, strip=False))
bleach.linkify(bleach.clean(raw_text, tags=[], attributes=[], strip=False))
)

38
synapse/rest/media/v1/__init__.py
View file

@ -12,33 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import PIL.Image
from PIL.features import check_codec
# check for JPEG support.
try:
PIL.Image._getdecoder("rgb", "jpeg", None)
except OSError as e:
if str(e).startswith("decoder jpeg not available"):
raise Exception(
"FATAL: jpeg codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)
except Exception:
# any other exception is fine
pass
if not check_codec("jpg"):
raise Exception(
"FATAL: jpeg codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)
# check for PNG support.
try:
PIL.Image._getdecoder("rgb", "zip", None)
except OSError as e:
if str(e).startswith("decoder zip not available"):
raise Exception(
"FATAL: zip codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)
except Exception:
# any other exception is fine
pass
if not check_codec("zlib"):
raise Exception(
"FATAL: zip codec not supported. Install pillow correctly! "
" 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
" pip install pillow --user'"
)

21
synapse/rest/media/v1/thumbnailer.py
View file

@ -61,9 +61,19 @@ class Thumbnailer:
self.transpose_method = None
try:
# We don't use ImageOps.exif_transpose since it crashes with big EXIF
image_exif = self.image._getexif()
#
# Ignore safety: Pillow seems to acknowledge that this method is
# "private, experimental, but generally widely used". Pillow 6
# includes a public getexif() method (no underscore) that we might
# consider using instead when we can bump that dependency.
#
# At the time of writing, Debian buster (currently oldstable)
# provides version 5.4.1. It's expected to EOL in mid-2022, see
# https://wiki.debian.org/DebianReleases#Production_Releases
image_exif = self.image._getexif() # type: ignore
if image_exif is not None:
image_orientation = image_exif.get(EXIF_ORIENTATION_TAG)
assert isinstance(image_orientation, int)
self.transpose_method = EXIF_TRANSPOSE_MAPPINGS.get(image_orientation)
except Exception as e:
# A lot of parsing errors can happen when parsing EXIF
@ -76,7 +86,10 @@ class Thumbnailer:
A tuple containing the new image size in pixels as (width, height).
"""
if self.transpose_method is not None:
self.image = self.image.transpose(self.transpose_method)
# Safety: `transpose` takes an int rather than e.g. an IntEnum.
# self.transpose_method is set above to be a value in
# EXIF_TRANSPOSE_MAPPINGS, and that only contains correct values.
self.image = self.image.transpose(self.transpose_method) # type: ignore[arg-type]
self.width, self.height = self.image.size
self.transpose_method = None
# We don't need EXIF any more
@ -101,7 +114,7 @@ class Thumbnailer:
else:
return (max_height * self.width) // self.height, max_height
def _resize(self, width: int, height: int) -> Image:
def _resize(self, width: int, height: int) -> Image.Image:
# 1-bit or 8-bit color palette images need converting to RGB
# otherwise they will be scaled using nearest neighbour which
# looks awful.
@ -151,7 +164,7 @@ class Thumbnailer:
cropped = scaled_image.crop((crop_left, 0, crop_right, height))
return self._encode_image(cropped, output_type)
def _encode_image(self, output_image: Image, output_type: str) -> BytesIO:
def _encode_image(self, output_image: Image.Image, output_type: str) -> BytesIO:
output_bytes_io = BytesIO()
fmt = self.FORMATS[output_type]
if fmt == "JPEG":

4
synapse/storage/prepare_database.py
View file

@ -487,6 +487,10 @@ def _upgrade_existing_database(
spec = importlib.util.spec_from_file_location(
module_name, absolute_path
)
if spec is None:
raise RuntimeError(
f"Could not build a module spec for {module_name} at {absolute_path}"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore

5
synapse/util/__init__.py
View file

@ -51,7 +51,10 @@ def _handle_frozendict(obj: Any) -> Dict[Any, Any]:
# fishing the protected dict out of the object is a bit nasty,
# but we don't really want the overhead of copying the dict.
try:
return obj._dict
# Safety: we catch the AttributeError immediately below.
# See https://github.com/matrix-org/python-canonicaljson/issues/36#issuecomment-927816293
# for discussion on how frozendict's internals have changed over time.
return obj._dict # type: ignore[attr-defined]
except AttributeError:
# When the C implementation of frozendict is used,
# there isn't a `_dict` attribute with a dict