diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py
index 2216c0f1ca..9bff9ec169 100644
--- a/synapse/http/__init__.py
+++ b/synapse/http/__init__.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 5a5e40a26e..ebf1aa47c4 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
from twisted.internet import defer, reactor
+from twisted.internet.error import DNSLookupError
from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer
from twisted.web.http_headers import Headers
@@ -23,7 +24,7 @@ from synapse.util.async import sleep
from syutil.jsonutil import encode_canonical_json
-from synapse.api.errors import CodeMessageException
+from synapse.api.errors import CodeMessageException, SynapseError
from StringIO import StringIO
@@ -45,6 +46,7 @@ _destination_mappings = {
class HttpClient(object):
""" Interface for talking json over http
"""
+ RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
def put_json(self, destination, path, data):
""" Sends the specifed json data using PUT
@@ -144,13 +146,23 @@ class TwistedHttpClient(HttpClient):
destination = _destination_mappings[destination]
logger.debug("get_json args: %s", args)
+
+ retry_on_dns_fail = True
+ if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args:
+ # FIXME: This isn't ideal, but the interface exposed in get_json
+ # isn't comprehensive enough to give caller's any control over
+ # their connection mechanics.
+ retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES)
+
query_bytes = urllib.urlencode(args, True)
+ logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
response = yield self._create_request(
destination.encode("ascii"),
"GET",
path.encode("ascii"),
- query_bytes=query_bytes
+ query_bytes=query_bytes,
+ retry_on_dns_fail=retry_on_dns_fail
)
body = yield readBody(response)
@@ -179,7 +191,8 @@ class TwistedHttpClient(HttpClient):
@defer.inlineCallbacks
def _create_request(self, destination, method, path_bytes, param_bytes=b"",
- query_bytes=b"", producer=None, headers_dict={}):
+ query_bytes=b"", producer=None, headers_dict={},
+ retry_on_dns_fail=True):
""" Creates and sends a request to the given url
"""
headers_dict[b"User-Agent"] = [b"Synapse"]
@@ -218,6 +231,11 @@ class TwistedHttpClient(HttpClient):
logger.debug("Got response to %s", method)
break
except Exception as e:
+ if not retry_on_dns_fail and isinstance(e, DNSLookupError):
+ logger.warn("DNS Lookup failed to %s with %s", destination,
+ e)
+ raise SynapseError(400, "Domain specified not found.")
+
logger.exception("Got error in _create_request")
_print_ex(e)
diff --git a/synapse/http/content_repository.py b/synapse/http/content_repository.py
new file mode 100644
index 0000000000..7dd4a859f8
--- /dev/null
+++ b/synapse/http/content_repository.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .server import respond_with_json_bytes
+
+from synapse.util.stringutils import random_string
+from synapse.api.errors import (
+ cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+)
+
+from twisted.protocols.basic import FileSender
+from twisted.web import server, resource
+from twisted.internet import defer
+
+import base64
+import json
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+class ContentRepoResource(resource.Resource):
+ """Provides file uploading and downloading.
+
+ Uploads are POSTed to wherever this Resource is linked to. This resource
+ returns a "content token" which can be used to GET this content again. The
+ token is typically a path, but it may not be. Tokens can expire, be one-time
+ uses, etc.
+
+ In this case, the token is a path to the file and contains 3 interesting
+ sections:
+ - User ID base64d (for namespacing content to each user)
+ - random 24 char string
+ - Content type base64d (so we can return it when clients GET it)
+
+ """
+ isLeaf = True
+
+ def __init__(self, hs, directory, auth, external_addr):
+ resource.Resource.__init__(self)
+ self.hs = hs
+ self.directory = directory
+ self.auth = auth
+ self.external_addr = external_addr.rstrip('/')
+ self.max_upload_size = hs.config.max_upload_size
+
+ if not os.path.isdir(self.directory):
+ os.mkdir(self.directory)
+ logger.info("ContentRepoResource : Created %s directory.",
+ self.directory)
+
+ @defer.inlineCallbacks
+ def map_request_to_name(self, request):
+ # auth the user
+ auth_user = yield self.auth.get_user_by_req(request)
+
+ # namespace all file uploads on the user
+ prefix = base64.urlsafe_b64encode(
+ auth_user.to_string()
+ ).replace('=', '')
+
+ # use a random string for the main portion
+ main_part = random_string(24)
+
+ # suffix with a file extension if we can make one. This is nice to
+ # provide a hint to clients on the file information. We will also reuse
+ # this info to spit back the content type to the client.
+ suffix = ""
+ if request.requestHeaders.hasHeader("Content-Type"):
+ content_type = request.requestHeaders.getRawHeaders(
+ "Content-Type")[0]
+ suffix = "." + base64.urlsafe_b64encode(content_type)
+ if (content_type.split("/")[0].lower() in
+ ["image", "video", "audio"]):
+ file_ext = content_type.split("/")[-1]
+ # be a little paranoid and only allow a-z
+ file_ext = re.sub("[^a-z]", "", file_ext)
+ suffix += "." + file_ext
+
+ file_name = prefix + main_part + suffix
+ file_path = os.path.join(self.directory, file_name)
+ logger.info("User %s is uploading a file to path %s",
+ auth_user.to_string(),
+ file_path)
+
+ # keep trying to make a non-clashing file, with a sensible max attempts
+ attempts = 0
+ while os.path.exists(file_path):
+ main_part = random_string(24)
+ file_name = prefix + main_part + suffix
+ file_path = os.path.join(self.directory, file_name)
+ attempts += 1
+ if attempts > 25: # really? Really?
+ raise SynapseError(500, "Unable to create file.")
+
+ defer.returnValue(file_path)
+
+ def render_GET(self, request):
+ # no auth here on purpose, to allow anyone to view, even across home
+ # servers.
+
+ # TODO: A little crude here, we could do this better.
+ filename = request.path.split('/')[-1]
+ # be paranoid
+ filename = re.sub("[^0-9A-z.-_]", "", filename)
+
+ file_path = self.directory + "/" + filename
+
+ logger.debug("Searching for %s", file_path)
+
+ if os.path.isfile(file_path):
+ # filename has the content type
+ base64_contentype = filename.split(".")[1]
+ content_type = base64.urlsafe_b64decode(base64_contentype)
+ logger.info("Sending file %s", file_path)
+ f = open(file_path, 'rb')
+ request.setHeader('Content-Type', content_type)
+ d = FileSender().beginFileTransfer(f, request)
+
+ # after the file has been sent, clean up and finish the request
+ def cbFinished(ignored):
+ f.close()
+ request.finish()
+ d.addCallback(cbFinished)
+ else:
+ respond_with_json_bytes(
+ request,
+ 404,
+ json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
+ send_cors=True)
+
+ return server.NOT_DONE_YET
+
+ def render_POST(self, request):
+ self._async_render(request)
+ return server.NOT_DONE_YET
+
+ def render_OPTIONS(self, request):
+ respond_with_json_bytes(request, 200, {}, send_cors=True)
+ return server.NOT_DONE_YET
+
+ @defer.inlineCallbacks
+ def _async_render(self, request):
+ try:
+ # TODO: The checks here are a bit late. The content will have
+ # already been uploaded to a tmp file at this point
+ content_length = request.getHeader("Content-Length")
+ if content_length is None:
+ raise SynapseError(
+ msg="Request must specify a Content-Length", code=400
+ )
+ if int(content_length) > self.max_upload_size:
+ raise SynapseError(
+ msg="Upload request body is too large",
+ code=413,
+ )
+
+ fname = yield self.map_request_to_name(request)
+
+ # TODO I have a suspcious feeling this is just going to block
+ with open(fname, "wb") as f:
+ f.write(request.content.read())
+
+
+ # FIXME (erikj): These should use constants.
+ file_name = os.path.basename(fname)
+ # FIXME: we can't assume what the public mounted path of the repo is
+ # ...plus self-signed SSL won't work to remote clients anyway
+ # ...and we can't assume that it's SSL anyway, as we might want to
+ # server it via the non-SSL listener...
+ url = "%s/_matrix/content/%s" % (
+ self.external_addr, file_name
+ )
+
+ respond_with_json_bytes(request, 200,
+ json.dumps({"content_token": url}),
+ send_cors=True)
+
+ except CodeMessageException as e:
+ logger.exception(e)
+ respond_with_json_bytes(request, e.code,
+ json.dumps(cs_exception(e)))
+ except Exception as e:
+ logger.error("Failed to store file: %s" % e)
+ respond_with_json_bytes(
+ request,
+ 500,
+ json.dumps({"error": "Internal server error"}),
+ send_cors=True)
+
+
+
diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py
index 6c1fdcb853..7018ee3458 100644
--- a/synapse/http/endpoint.py
+++ b/synapse/http/endpoint.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 74c220e869..8d419c02dd 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2014 matrix.org
+# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,22 +18,16 @@ from syutil.jsonutil import (
encode_canonical_json, encode_pretty_printed_json
)
from synapse.api.errors import (
- cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+ cs_exception, SynapseError, CodeMessageException
)
-from synapse.util.stringutils import random_string
from twisted.internet import defer, reactor
-from twisted.protocols.basic import FileSender
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo
-import base64
import collections
-import json
import logging
-import os
-import re
logger = logging.getLogger(__name__)
@@ -198,161 +192,6 @@ class RootRedirect(resource.Resource):
return resource.Resource.getChild(self, name, request)
-class ContentRepoResource(resource.Resource):
- """Provides file uploading and downloading.
-
- Uploads are POSTed to wherever this Resource is linked to. This resource
- returns a "content token" which can be used to GET this content again. The
- token is typically a path, but it may not be. Tokens can expire, be one-time
- uses, etc.
-
- In this case, the token is a path to the file and contains 3 interesting
- sections:
- - User ID base64d (for namespacing content to each user)
- - random 24 char string
- - Content type base64d (so we can return it when clients GET it)
-
- """
- isLeaf = True
-
- def __init__(self, hs, directory, auth):
- resource.Resource.__init__(self)
- self.hs = hs
- self.directory = directory
- self.auth = auth
-
- if not os.path.isdir(self.directory):
- os.mkdir(self.directory)
- logger.info("ContentRepoResource : Created %s directory.",
- self.directory)
-
- @defer.inlineCallbacks
- def map_request_to_name(self, request):
- # auth the user
- auth_user = yield self.auth.get_user_by_req(request)
-
- # namespace all file uploads on the user
- prefix = base64.urlsafe_b64encode(
- auth_user.to_string()
- ).replace('=', '')
-
- # use a random string for the main portion
- main_part = random_string(24)
-
- # suffix with a file extension if we can make one. This is nice to
- # provide a hint to clients on the file information. We will also reuse
- # this info to spit back the content type to the client.
- suffix = ""
- if request.requestHeaders.hasHeader("Content-Type"):
- content_type = request.requestHeaders.getRawHeaders(
- "Content-Type")[0]
- suffix = "." + base64.urlsafe_b64encode(content_type)
- if (content_type.split("/")[0].lower() in
- ["image", "video", "audio"]):
- file_ext = content_type.split("/")[-1]
- # be a little paranoid and only allow a-z
- file_ext = re.sub("[^a-z]", "", file_ext)
- suffix += "." + file_ext
-
- file_name = prefix + main_part + suffix
- file_path = os.path.join(self.directory, file_name)
- logger.info("User %s is uploading a file to path %s",
- auth_user.to_string(),
- file_path)
-
- # keep trying to make a non-clashing file, with a sensible max attempts
- attempts = 0
- while os.path.exists(file_path):
- main_part = random_string(24)
- file_name = prefix + main_part + suffix
- file_path = os.path.join(self.directory, file_name)
- attempts += 1
- if attempts > 25: # really? Really?
- raise SynapseError(500, "Unable to create file.")
-
- defer.returnValue(file_path)
-
- def render_GET(self, request):
- # no auth here on purpose, to allow anyone to view, even across home
- # servers.
-
- # TODO: A little crude here, we could do this better.
- filename = request.path.split('/')[-1]
- # be paranoid
- filename = re.sub("[^0-9A-z.-_]", "", filename)
-
- file_path = self.directory + "/" + filename
-
- logger.debug("Searching for %s", file_path)
-
- if os.path.isfile(file_path):
- # filename has the content type
- base64_contentype = filename.split(".")[1]
- content_type = base64.urlsafe_b64decode(base64_contentype)
- logger.info("Sending file %s", file_path)
- f = open(file_path, 'rb')
- request.setHeader('Content-Type', content_type)
- d = FileSender().beginFileTransfer(f, request)
-
- # after the file has been sent, clean up and finish the request
- def cbFinished(ignored):
- f.close()
- request.finish()
- d.addCallback(cbFinished)
- else:
- respond_with_json_bytes(
- request,
- 404,
- json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
- send_cors=True)
-
- return server.NOT_DONE_YET
-
- def render_POST(self, request):
- self._async_render(request)
- return server.NOT_DONE_YET
-
- def render_OPTIONS(self, request):
- respond_with_json_bytes(request, 200, {}, send_cors=True)
- return server.NOT_DONE_YET
-
- @defer.inlineCallbacks
- def _async_render(self, request):
- try:
- fname = yield self.map_request_to_name(request)
-
- # TODO I have a suspcious feeling this is just going to block
- with open(fname, "wb") as f:
- f.write(request.content.read())
-
-
- # FIXME (erikj): These should use constants.
- file_name = os.path.basename(fname)
- # FIXME: we can't assume what the public mounted path of the repo is
- # ...plus self-signed SSL won't work to remote clients anyway
- # ...and we can't assume that it's SSL anyway, as we might want to
- # server it via the non-SSL listener...
- url = "https://%s/_matrix/content/%s" % (
- self.hs.domain_with_port, file_name
- )
-
- respond_with_json_bytes(request, 200,
- json.dumps({"content_token": url}),
- send_cors=True)
-
- except CodeMessageException as e:
- logger.exception(e)
- respond_with_json_bytes(request, e.code,
- json.dumps(cs_exception(e)))
- except Exception as e:
- logger.error("Failed to store file: %s" % e)
- respond_with_json_bytes(
- request,
- 500,
- json.dumps({"error": "Internal server error"}),
- send_cors=True)
-
-
def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
response_code_message=None):
"""Sends encoded JSON in response to the given request.
|
