Merge remote-tracking branch 'origin/master' into user_page

synapse/api/urls.py

@@ -17,4 +17,5 @@
 
 CLIENT_PREFIX = "/matrix/client/api/v1"
 FEDERATION_PREFIX = "/matrix/federation/v1"
-WEB_CLIENT_PREFIX = "/matrix/client"
+WEB_CLIENT_PREFIX = "/matrix/client"
+CONTENT_REPO_PREFIX = "/matrix/content"

synapse/app/homeserver.py

@@ -24,9 +24,11 @@ from twisted.python.log import PythonLoggingObserver
 from twisted.web.resource import Resource
 from twisted.web.static import File
 from twisted.web.server import Site
-from synapse.http.server import JsonResource, RootRedirect
+from synapse.http.server import JsonResource, RootRedirect, ContentRepoResource
 from synapse.http.client import TwistedHttpClient
-from synapse.api.urls import CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX
+from synapse.api.urls import (
+    CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX
+)
 
 from daemonize import Daemonize
 
@@ -53,6 +55,9 @@ class SynapseHomeServer(HomeServer):
     def build_resource_for_web_client(self):
         return File("webclient")  # TODO configurable?
 
+    def build_resource_for_content_repo(self):
+        return ContentRepoResource("uploads", self.auth)
+
     def build_db_pool(self):
         """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we
         don't have to worry about overwriting existing content.
@@ -101,7 +106,8 @@ class SynapseHomeServer(HomeServer):
         # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ]
         desired_tree = [
             (CLIENT_PREFIX, self.get_resource_for_client()),
-            (FEDERATION_PREFIX, self.get_resource_for_federation())
+            (FEDERATION_PREFIX, self.get_resource_for_federation()),
+            (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo())
         ]
         if web_client:
             logger.info("Adding the web client.")

synapse/http/server.py

@@ -17,16 +17,23 @@
 from syutil.jsonutil import (
     encode_canonical_json, encode_pretty_printed_json
 )
-from synapse.api.errors import cs_exception, CodeMessageException
+from synapse.api.errors import (
+    cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+)
+from synapse.util.stringutils import random_string
 
 from twisted.internet import defer, reactor
+from twisted.protocols.basic import FileSender
 from twisted.web import server, resource
 from twisted.web.server import NOT_DONE_YET
 from twisted.web.util import redirectTo
 
+import base64
 import collections
+import json
 import logging
-
+import os
+import re
 
 logger = logging.getLogger(__name__)
 
@@ -176,6 +183,141 @@ class RootRedirect(resource.Resource):
         return resource.Resource.getChild(self, name, request)
 
 
+class ContentRepoResource(resource.Resource):
+    """Provides file uploading and downloading.
+
+    Uploads are POSTed to wherever this Resource is linked to. This resource
+    returns a "content token" which can be used to GET this content again. The
+    token is typically a path, but it may not be. Tokens can expire, be one-time
+    uses, etc.
+
+    In this case, the token is a path to the file and contains 3 interesting
+    sections:
+        - User ID base64d (for namespacing content to each user)
+        - random 24 char string
+        - Content type base64d (so we can return it when clients GET it)
+
+    """
+    isLeaf = True
+
+    def __init__(self, directory, auth):
+        resource.Resource.__init__(self)
+        self.directory = directory
+        self.auth = auth
+
+        if not os.path.isdir(self.directory):
+            os.mkdir(self.directory)
+            logger.info("ContentRepoResource : Created %s directory.",
+                        self.directory)
+
+    @defer.inlineCallbacks
+    def map_request_to_name(self, request):
+        # auth the user
+        auth_user = yield self.auth.get_user_by_req(request)
+
+        # namespace all file uploads on the user
+        prefix = base64.urlsafe_b64encode(
+            auth_user.to_string()
+        ).replace('=', '')
+
+        # use a random string for the main portion
+        main_part = random_string(24)
+
+        # suffix with a file extension if we can make one. This is nice to
+        # provide a hint to clients on the file information. We will also reuse
+        # this info to spit back the content type to the client.
+        suffix = ""
+        if request.requestHeaders.hasHeader("Content-Type"):
+            content_type = request.requestHeaders.getRawHeaders(
+                "Content-Type")[0]
+            suffix = "." + base64.urlsafe_b64encode(content_type)
+            if (content_type.split("/")[0].lower() in
+                    ["image", "video", "audio"]):
+                file_ext = content_type.split("/")[-1]
+                # be a little paranoid and only allow a-z
+                file_ext = re.sub("[^a-z]", "", file_ext)
+                suffix += "." + file_ext
+
+        file_path = os.path.join(self.directory, prefix + main_part + suffix)
+        logger.info("User %s is uploading a file to path %s",
+                    auth_user.to_string(),
+                    file_path)
+
+        # keep trying to make a non-clashing file, with a sensible max attempts
+        attempts = 0
+        while os.path.exists(file_path):
+            main_part = random_string(24)
+            file_path = os.path.join(self.directory,
+                                     prefix + main_part + suffix)
+            attempts += 1
+            if attempts > 25:  # really? Really?
+                raise SynapseError(500, "Unable to create file.")
+
+        defer.returnValue(file_path)
+
+    def render_GET(self, request):
+        # no auth here on purpose, to allow anyone to view, even across home
+        # servers.
+
+        # TODO: A little crude here, we could do this better.
+        filename = request.path.split(self.directory + "/")[1]
+        # be paranoid
+        filename = re.sub("[^0-9A-z.-_]", "", filename)
+
+        file_path = self.directory + "/" + filename
+        if os.path.isfile(file_path):
+            # filename has the content type
+            base64_contentype = filename.split(".")[1]
+            content_type = base64.urlsafe_b64decode(base64_contentype)
+            logger.info("Sending file %s", file_path)
+            f = open(file_path, 'rb')
+            request.setHeader('Content-Type', content_type)
+            d = FileSender().beginFileTransfer(f, request)
+
+            # after the file has been sent, clean up and finish the request
+            def cbFinished(ignored):
+                f.close()
+                request.finish()
+            d.addCallback(cbFinished)
+        else:
+            respond_with_json_bytes(
+                request,
+                404,
+                json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
+                send_cors=True)
+
+        return server.NOT_DONE_YET
+
+    def render_POST(self, request):
+        self._async_render(request)
+        return server.NOT_DONE_YET
+
+    @defer.inlineCallbacks
+    def _async_render(self, request):
+        try:
+            fname = yield self.map_request_to_name(request)
+
+            # TODO I have a suspcious feeling this is just going to block
+            with open(fname, "wb") as f:
+                f.write(request.content.read())
+
+            respond_with_json_bytes(request, 200,
+                                    json.dumps({"content_token": fname}),
+                                    send_cors=True)
+
+        except CodeMessageException as e:
+            logger.exception(e)
+            respond_with_json_bytes(request, e.code,
+                                    json.dumps(cs_exception(e)))
+        except Exception as e:
+            logger.error("Failed to store file: %s" % e)
+            respond_with_json_bytes(
+                request,
+                500,
+                json.dumps({"error": "Internal server error"}),
+                send_cors=True)
+
+
 def respond_with_json_bytes(request, code, json_bytes, send_cors=False):
     """Sends encoded JSON in response to the given request.
 

synapse/server.py

@@ -72,6 +72,7 @@ class BaseHomeServer(object):
         'resource_for_client',
         'resource_for_federation',
         'resource_for_web_client',
+        'resource_for_content_repo',
     ]
 
     def __init__(self, hostname, **kwargs):
@@ -140,6 +141,7 @@ class HomeServer(BaseHomeServer):
         resource_for_client
         resource_for_web_client
         resource_for_federation
+        resource_for_content_repo
         http_client
         db_pool
     """

webclient/components/fileUpload/file-upload-service.js

@@ -16,11 +16,12 @@
 
 'use strict';
 
+// TODO determine if this is really required as a separate service to matrixService.
 /*
  * Upload an HTML5 file to a server
  */
 angular.module('mFileUpload', [])
-.service('mFileUpload', ['$http', '$q', function ($http, $q) {
+.service('mFileUpload', ['matrixService', '$q', function (matrixService, $q) {
         
     /*
      * Upload an HTML5 file to a server and returned a promise
@@ -28,20 +29,19 @@ angular.module('mFileUpload', [])
      */
     this.uploadFile = function(file) {
         var deferred = $q.defer();
-        
+        console.log("Uploading " + file.name + "... to /matrix/content");
-        // @TODO: This service runs with the do_POST hacky implementation of /synapse/demos/webserver.py.
+        matrixService.uploadContent(file).then(
-        // This is temporary until we have a true file upload service
+            function(response) {
-        console.log("Uploading " + file.name + "...");
+                var content_url = location.origin + "/matrix/content/" + response.data.content_token;
-        $http.post(file.name, file)
+                console.log("   -> Successfully uploaded! Available at " + content_url);
-        .success(function(data, status, headers, config) {
+                deferred.resolve(content_url);
-            deferred.resolve(location.origin + data.url);
+            },
-            console.log("   -> Successfully uploaded! Available at " + location.origin + data.url);
+            function(error) {
-        }).
+                console.log("   -> Failed to upload "  + file.name);
-        error(function(data, status, headers, config) {
+                deferred.reject(error);
-            console.log("   -> Failed to upload"  + file.name);
+            }
-            deferred.reject();
+        );
-        });
         
         return deferred.promise;
     };
-}]);
+}]);

webclient/components/matrix/matrix-service.js

@@ -54,13 +54,14 @@ angular.module('matrixService', [])
         
         params.access_token = config.access_token;
         
+        if (path.indexOf(prefixPath) !== 0) {
+            path = prefixPath + path;
+        }
+        
         return doBaseRequest(config.homeserver, method, path, params, data, undefined);
     };
 
     var doBaseRequest = function(baseUrl, method, path, params, data, headers) {
-        if (path.indexOf(prefixPath) !== 0) {
-            path = prefixPath + path;
-        }
         return $http({
             method: method,
             url: baseUrl + path,
@@ -319,6 +320,17 @@ angular.module('matrixService', [])
             return doBaseRequest(config.identityServer, "POST", path, {}, data, headers); 
         },
         
+        uploadContent: function(file) {
+            var path = "/matrix/content";
+            var headers = {
+                "Content-Type": undefined // undefined means angular will figure it out
+            };
+            var params = {
+                access_token: config.access_token
+            };
+            return doBaseRequest(config.homeserver, "POST", path, params, file, headers);
+        },
+        
         // start listening on /events
         getEventStream: function(from, timeout) {
             var path = "/events";

webclient/room/room-controller.js

@@ -147,7 +147,7 @@ angular.module('RoomController', ['ngSanitize'])
                 if (document.hidden) {
                     var notification = new window.Notification(
                         ($scope.members[event.user_id].displayname || event.user_id) +
-                        " (" + $scope.room_alias + ")",
+                        " (" + ($scope.room_alias || $scope.room_id) + ")", // FIXME: don't leak room_ids here
                     {
                         "body": event.content.body,
                         "icon": $scope.members[event.user_id].avatar_url,