This commit is contained in:
Erik Johnston 2018-01-12 11:15:31 +00:00
parent 8f03aa9f61
commit 227c491510
3 changed files with 65 additions and 15 deletions

View file

@ -134,9 +134,9 @@ def respond_with_responder(request, responder, media_type, file_size, upload_nam
Args: Args:
request (twisted.web.http.Request) request (twisted.web.http.Request)
responder (Responder) responder (Responder|None)
media_type (str): The media/content type. media_type (str): The media/content type.
file_size (int): Size in bytes of the media, if known. file_size (int): Size in bytes of the media. If not known it should be None
upload_name (str): The name of the requested file, if any. upload_name (str): The name of the requested file, if any.
""" """
if not responder: if not responder:
@ -179,13 +179,12 @@ class FileInfo(object):
or None if local. or None if local.
file_id (str): The local ID of the file. For local files this is the file_id (str): The local ID of the file. For local files this is the
same as the media_id same as the media_id
media_type (str): Type of the file
url_cache (bool): If the file is for the url preview cache url_cache (bool): If the file is for the url preview cache
thumbnail (bool): Whether the file is a thumbnail or not. thumbnail (bool): Whether the file is a thumbnail or not.
thumbnail_width (int) thumbnail_width (int)
thumbnail_height (int) thumbnail_height (int)
thumbnail_method (int) thumbnail_method (str)
thumbnail_type (str) thumbnail_type (str): Content type of thumbnail, e.g. image/png
""" """
def __init__(self, server_name, file_id, url_cache=False, def __init__(self, server_name, file_id, url_cache=False,
thumbnail=False, thumbnail_width=None, thumbnail_height=None, thumbnail=False, thumbnail_width=None, thumbnail_height=None,

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd # Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vecotr Ltd # Copyright 2018 New Vector Ltd
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -76,7 +76,7 @@ class MediaRepository(object):
self.recently_accessed_remotes = set() self.recently_accessed_remotes = set()
# List of StorageProvider's where we should search for media and # List of StorageProviders where we should search for media and
# potentially upload to. # potentially upload to.
self.storage_providers = [] self.storage_providers = []
@ -158,6 +158,16 @@ class MediaRepository(object):
@defer.inlineCallbacks @defer.inlineCallbacks
def get_local_media(self, request, media_id, name): def get_local_media(self, request, media_id, name):
"""Responds to reqests for local media, if exists, or returns 404. """Responds to reqests for local media, if exists, or returns 404.
Args:
request(twisted.web.http.Request)
media_id (str)
name (str|None): Optional name that, if specified, will be used as
the filename in the Content-Disposition header of the response.
Retruns:
Deferred: Resolves once a response has successfully been written
to request
""" """
media_info = yield self.store.get_local_media(media_id) media_info = yield self.store.get_local_media(media_id)
if not media_info or media_info["quarantined_by"]: if not media_info or media_info["quarantined_by"]:
@ -182,18 +192,29 @@ class MediaRepository(object):
@defer.inlineCallbacks @defer.inlineCallbacks
def get_remote_media(self, request, server_name, media_id, name): def get_remote_media(self, request, server_name, media_id, name):
"""Respond to requests for remote media. """Respond to requests for remote media.
Args:
request(twisted.web.http.Request)
server_name (str): Remote server_name where the media originated.
media_id (str)
name (str|None): Optional name that, if specified, will be used as
the filename in the Content-Disposition header of the response.
Retruns:
Deferred: Resolves once a response has successfully been written
to request
""" """
self.recently_accessed_remotes.add((server_name, media_id)) self.recently_accessed_remotes.add((server_name, media_id))
# We linearize here to ensure that we don't try and download remote # We linearize here to ensure that we don't try and download remote
# media mutliple times concurrently # media multiple times concurrently
key = (server_name, media_id) key = (server_name, media_id)
with (yield self.remote_media_linearizer.queue(key)): with (yield self.remote_media_linearizer.queue(key)):
responder, media_info = yield self._get_remote_media_impl( responder, media_info = yield self._get_remote_media_impl(
server_name, media_id, server_name, media_id,
) )
# We purposefully stream the file outside the lock # We deliberately stream the file outside the lock
if responder: if responder:
media_type = media_info["media_type"] media_type = media_info["media_type"]
media_length = media_info["media_length"] media_length = media_info["media_length"]
@ -210,7 +231,7 @@ class MediaRepository(object):
download from remote server. download from remote server.
Returns: Returns:
Deferred((Respodner, media_info)) Deferred[(Responder, media_info)]
""" """
media_info = yield self.store.get_cached_remote_media( media_info = yield self.store.get_cached_remote_media(
server_name, media_id server_name, media_id
@ -251,6 +272,14 @@ class MediaRepository(object):
def _download_remote_file(self, server_name, media_id, file_id): def _download_remote_file(self, server_name, media_id, file_id):
"""Attempt to download the remote file from the given server name, """Attempt to download the remote file from the given server name,
using the given file_id as the local id. using the given file_id as the local id.
Args:
server_name (str): Originating server
media_id (str)
file_id (str): Local file ID
Returns:
Deferred[MediaInfo]
""" """
file_info = FileInfo( file_info = FileInfo(

View file

@ -30,6 +30,12 @@ logger = logging.getLogger(__name__)
class MediaStorage(object): class MediaStorage(object):
"""Responsible for storing/fetching files from local sources. """Responsible for storing/fetching files from local sources.
Args:
local_media_directory (str): Base path where we store media on disk
filepaths (MediaFilePaths)
storage_providers ([StorageProvider]): List of StorageProvider that are
used to fetch and store files.
""" """
def __init__(self, local_media_directory, filepaths, storage_providers): def __init__(self, local_media_directory, filepaths, storage_providers):
@ -68,9 +74,16 @@ class MediaStorage(object):
"""Context manager used to get a file like object to write into, as """Context manager used to get a file like object to write into, as
described by file_info. described by file_info.
Actually yields a 3-tuple (file, fname, finish_cb), where finish_cb is a Actually yields a 3-tuple (file, fname, finish_cb), where file is a file
function that returns a Deferred that must be waited on after the file like object that can be written to, fname is the absolute path of file
has been successfully written to. on disk, and finish_cb is a function that returns a Deferred.
fname can be used to read the contents from after upload, e.g. to
generate thumbnails.
finish_cb must be called and waited on after the file has been
successfully been written to. Should not be called if there was an
error.
Args: Args:
file_info (FileInfo): Info about the file to store file_info (FileInfo): Info about the file to store
@ -109,7 +122,7 @@ class MediaStorage(object):
raise e raise e
if not finished_called: if not finished_called:
raise Exception("Fnished callback not called") raise Exception("Finished callback not called")
@defer.inlineCallbacks @defer.inlineCallbacks
def fetch_media(self, file_info): def fetch_media(self, file_info):
@ -120,7 +133,7 @@ class MediaStorage(object):
file_info (FileInfo) file_info (FileInfo)
Returns: Returns:
Deferred(Responder): Returns a Responder if the file was found, Deferred[Responder|None]: Returns a Responder if the file was found,
otherwise None. otherwise None.
""" """
@ -138,6 +151,15 @@ class MediaStorage(object):
def _file_info_to_path(self, file_info): def _file_info_to_path(self, file_info):
"""Converts file_info into a relative path. """Converts file_info into a relative path.
The path is suitable for storing files under a directory, e.g. used to
store files on local FS under the base media repository directory.
Args:
file_info (FileInfo)
Returns:
str
""" """
if file_info.url_cache: if file_info.url_cache:
return self.filepaths.url_cache_filepath_rel(file_info.file_id) return self.filepaths.url_cache_filepath_rel(file_info.file_id)