#!/usr/bin/env python # Copyright 2017 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Moves a list of remote media from one media store to another. The input should be a list of media files to be moved, one per line. Each line should be formatted:: | This can be extracted from postgres with:: psql --tuples-only -A -c "select media_origin, filesystem_id from matrix.remote_media_cache where ..." To use, pipe the above into:: PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py """ import argparse import logging import os import shutil import sys from synapse.rest.media.v1.filepath import MediaFilePaths logger = logging.getLogger() def main(src_repo, dest_repo): src_paths = MediaFilePaths(src_repo) dest_paths = MediaFilePaths(dest_repo) for line in sys.stdin: line = line.strip() parts = line.split("|") if len(parts) != 2: print("Unable to parse input line %s" % line, file=sys.stderr) sys.exit(1) move_media(parts[0], parts[1], src_paths, dest_paths) def move_media(origin_server, file_id, src_paths, dest_paths): """Move the given file, and any thumbnails, to the dest repo Args: origin_server (str): file_id (str): src_paths (MediaFilePaths): dest_paths (MediaFilePaths): """ logger.info("%s/%s", origin_server, file_id) # check that the original exists original_file = src_paths.remote_media_filepath(origin_server, file_id) if not os.path.exists(original_file): logger.warning( "Original for %s/%s (%s) does not exist", origin_server, file_id, original_file, ) else: mkdir_and_move( original_file, dest_paths.remote_media_filepath(origin_server, file_id) ) # now look for thumbnails original_thumb_dir = src_paths.remote_media_thumbnail_dir(origin_server, file_id) if not os.path.exists(original_thumb_dir): return mkdir_and_move( original_thumb_dir, dest_paths.remote_media_thumbnail_dir(origin_server, file_id), ) def mkdir_and_move(original_file, dest_file): dirname = os.path.dirname(dest_file) if not os.path.exists(dirname): logger.debug("mkdir %s", dirname) os.makedirs(dirname) logger.debug("mv %s %s", original_file, dest_file) shutil.move(original_file, dest_file) if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("-v", action="store_true", help="enable debug logging") parser.add_argument("src_repo", help="Path to source content repo") parser.add_argument("dest_repo", help="Path to source content repo") args = parser.parse_args() logging_config = { "level": logging.DEBUG if args.v else logging.INFO, "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s", } logging.basicConfig(**logging_config) main(args.src_repo, args.dest_repo)