diff options
Diffstat (limited to 'synapse/http')
| -rw-r--r-- | synapse/http/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/http/client.py | 66 | ||||
| -rw-r--r-- | synapse/http/content_repository.py | 206 | ||||
| -rw-r--r-- | synapse/http/endpoint.py | 2 | ||||
| -rw-r--r-- | synapse/http/server.py | 179 |
5 files changed, 276 insertions, 179 deletions
diff --git a/synapse/http/__init__.py b/synapse/http/__init__.py index 2216c0f1ca..9bff9ec169 100644 --- a/synapse/http/__init__.py +++ b/synapse/http/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/client.py b/synapse/http/client.py index 093bdf0e3f..ebf1aa47c4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,8 @@ from twisted.internet import defer, reactor -from twisted.web.client import _AgentBase, _URI, readBody +from twisted.internet.error import DNSLookupError +from twisted.web.client import _AgentBase, _URI, readBody, FileBodyProducer from twisted.web.http_headers import Headers from synapse.http.endpoint import matrix_endpoint @@ -23,7 +24,9 @@ from synapse.util.async import sleep from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException +from synapse.api.errors import CodeMessageException, SynapseError + +from StringIO import StringIO import json import logging @@ -43,6 +46,7 @@ _destination_mappings = { class HttpClient(object): """ Interface for talking json over http """ + RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" def put_json(self, destination, path, data): """ Sends the specifed json data using PUT @@ -142,13 +146,43 @@ class TwistedHttpClient(HttpClient): destination = _destination_mappings[destination] logger.debug("get_json args: %s", args) + + retry_on_dns_fail = True + if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args: + # FIXME: This isn't ideal, but the interface exposed in get_json + # isn't comprehensive enough to give caller's any control over + # their connection mechanics. + retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES) + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) response = yield self._create_request( destination.encode("ascii"), "GET", path.encode("ascii"), - query_bytes=query_bytes + query_bytes=query_bytes, + retry_on_dns_fail=retry_on_dns_fail + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def post_urlencoded_get_json(self, destination, path, args={}): + if destination in _destination_mappings: + destination = _destination_mappings[destination] + + logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + producer=FileBodyProducer(StringIO(urllib.urlencode(args))), + headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} ) body = yield readBody(response) @@ -157,7 +191,8 @@ class TwistedHttpClient(HttpClient): @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}): + query_bytes=b"", producer=None, headers_dict={}, + retry_on_dns_fail=True): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [b"Synapse"] @@ -178,10 +213,7 @@ class TwistedHttpClient(HttpClient): retries_left = 5 # TODO: setup and pass in an ssl_context to enable TLS - endpoint = matrix_endpoint( - reactor, destination, timeout=10, - ssl_context_factory=self.hs.tls_context_factory - ) + endpoint = self._getEndpoint(reactor, destination); while True: try: @@ -199,6 +231,11 @@ class TwistedHttpClient(HttpClient): logger.debug("Got response to %s", method) break except Exception as e: + if not retry_on_dns_fail and isinstance(e, DNSLookupError): + logger.warn("DNS Lookup failed to %s with %s", destination, + e) + raise SynapseError(400, "Domain specified not found.") + logger.exception("Got error in _create_request") _print_ex(e) @@ -223,6 +260,17 @@ class TwistedHttpClient(HttpClient): defer.returnValue(response) + def _getEndpoint(self, reactor, destination): + return matrix_endpoint( + reactor, destination, timeout=10, + ssl_context_factory=self.hs.tls_context_factory + ) + + +class PlainHttpClient(TwistedHttpClient): + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + def _print_ex(e): if hasattr(e, "reasons") and e.reasons: diff --git a/synapse/http/content_repository.py b/synapse/http/content_repository.py new file mode 100644 index 0000000000..7dd4a859f8 --- /dev/null +++ b/synapse/http/content_repository.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .server import respond_with_json_bytes + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException, Codes, cs_error +) + +from twisted.protocols.basic import FileSender +from twisted.web import server, resource +from twisted.internet import defer + +import base64 +import json +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +class ContentRepoResource(resource.Resource): + """Provides file uploading and downloading. + + Uploads are POSTed to wherever this Resource is linked to. This resource + returns a "content token" which can be used to GET this content again. The + token is typically a path, but it may not be. Tokens can expire, be one-time + uses, etc. + + In this case, the token is a path to the file and contains 3 interesting + sections: + - User ID base64d (for namespacing content to each user) + - random 24 char string + - Content type base64d (so we can return it when clients GET it) + + """ + isLeaf = True + + def __init__(self, hs, directory, auth, external_addr): + resource.Resource.__init__(self) + self.hs = hs + self.directory = directory + self.auth = auth + self.external_addr = external_addr.rstrip('/') + self.max_upload_size = hs.config.max_upload_size + + if not os.path.isdir(self.directory): + os.mkdir(self.directory) + logger.info("ContentRepoResource : Created %s directory.", + self.directory) + + @defer.inlineCallbacks + def map_request_to_name(self, request): + # auth the user + auth_user = yield self.auth.get_user_by_req(request) + + # namespace all file uploads on the user + prefix = base64.urlsafe_b64encode( + auth_user.to_string() + ).replace('=', '') + + # use a random string for the main portion + main_part = random_string(24) + + # suffix with a file extension if we can make one. This is nice to + # provide a hint to clients on the file information. We will also reuse + # this info to spit back the content type to the client. + suffix = "" + if request.requestHeaders.hasHeader("Content-Type"): + content_type = request.requestHeaders.getRawHeaders( + "Content-Type")[0] + suffix = "." + base64.urlsafe_b64encode(content_type) + if (content_type.split("/")[0].lower() in + ["image", "video", "audio"]): + file_ext = content_type.split("/")[-1] + # be a little paranoid and only allow a-z + file_ext = re.sub("[^a-z]", "", file_ext) + suffix += "." + file_ext + + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + logger.info("User %s is uploading a file to path %s", + auth_user.to_string(), + file_path) + + # keep trying to make a non-clashing file, with a sensible max attempts + attempts = 0 + while os.path.exists(file_path): + main_part = random_string(24) + file_name = prefix + main_part + suffix + file_path = os.path.join(self.directory, file_name) + attempts += 1 + if attempts > 25: # really? Really? + raise SynapseError(500, "Unable to create file.") + + defer.returnValue(file_path) + + def render_GET(self, request): + # no auth here on purpose, to allow anyone to view, even across home + # servers. + + # TODO: A little crude here, we could do this better. + filename = request.path.split('/')[-1] + # be paranoid + filename = re.sub("[^0-9A-z.-_]", "", filename) + + file_path = self.directory + "/" + filename + + logger.debug("Searching for %s", file_path) + + if os.path.isfile(file_path): + # filename has the content type + base64_contentype = filename.split(".")[1] + content_type = base64.urlsafe_b64decode(base64_contentype) + logger.info("Sending file %s", file_path) + f = open(file_path, 'rb') + request.setHeader('Content-Type', content_type) + d = FileSender().beginFileTransfer(f, request) + + # after the file has been sent, clean up and finish the request + def cbFinished(ignored): + f.close() + request.finish() + d.addCallback(cbFinished) + else: + respond_with_json_bytes( + request, + 404, + json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), + send_cors=True) + + return server.NOT_DONE_YET + + def render_POST(self, request): + self._async_render(request) + return server.NOT_DONE_YET + + def render_OPTIONS(self, request): + respond_with_json_bytes(request, 200, {}, send_cors=True) + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _async_render(self, request): + try: + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, + ) + + fname = yield self.map_request_to_name(request) + + # TODO I have a suspcious feeling this is just going to block + with open(fname, "wb") as f: + f.write(request.content.read()) + + + # FIXME (erikj): These should use constants. + file_name = os.path.basename(fname) + # FIXME: we can't assume what the public mounted path of the repo is + # ...plus self-signed SSL won't work to remote clients anyway + # ...and we can't assume that it's SSL anyway, as we might want to + # server it via the non-SSL listener... + url = "%s/_matrix/content/%s" % ( + self.external_addr, file_name + ) + + respond_with_json_bytes(request, 200, + json.dumps({"content_token": url}), + send_cors=True) + + except CodeMessageException as e: + logger.exception(e) + respond_with_json_bytes(request, e.code, + json.dumps(cs_exception(e))) + except Exception as e: + logger.error("Failed to store file: %s" % e) + respond_with_json_bytes( + request, + 500, + json.dumps({"error": "Internal server error"}), + send_cors=True) + + + diff --git a/synapse/http/endpoint.py b/synapse/http/endpoint.py index 6c1fdcb853..7018ee3458 100644 --- a/synapse/http/endpoint.py +++ b/synapse/http/endpoint.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/http/server.py b/synapse/http/server.py index 0b87718bfa..8d419c02dd 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2014 matrix.org +# Copyright 2014 OpenMarket Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,22 +18,16 @@ from syutil.jsonutil import ( encode_canonical_json, encode_pretty_printed_json ) from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error + cs_exception, SynapseError, CodeMessageException ) -from synapse.util.stringutils import random_string from twisted.internet import defer, reactor -from twisted.protocols.basic import FileSender from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET from twisted.web.util import redirectTo -import base64 import collections -import json import logging -import os -import re logger = logging.getLogger(__name__) @@ -140,7 +134,8 @@ class JsonResource(HttpServer, resource.Resource): self._send_response( request, e.code, - cs_exception(e) + cs_exception(e), + response_code_message=e.response_code_message ) except Exception as e: logger.exception(e) @@ -150,7 +145,8 @@ class JsonResource(HttpServer, resource.Resource): {"error": "Internal server error"} ) - def _send_response(self, request, code, response_json_object): + def _send_response(self, request, code, response_json_object, + response_code_message=None): # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -166,7 +162,8 @@ class JsonResource(HttpServer, resource.Resource): json_bytes = encode_pretty_printed_json(response_json_object) # TODO: Only enable CORS for the requests that need it. - respond_with_json_bytes(request, code, json_bytes, send_cors=True) + respond_with_json_bytes(request, code, json_bytes, send_cors=True, + response_code_message=response_code_message) @staticmethod def _request_user_agent_is_curl(request): @@ -195,162 +192,8 @@ class RootRedirect(resource.Resource): return resource.Resource.getChild(self, name, request) -class ContentRepoResource(resource.Resource): - """Provides file uploading and downloading. - - Uploads are POSTed to wherever this Resource is linked to. This resource - returns a "content token" which can be used to GET this content again. The - token is typically a path, but it may not be. Tokens can expire, be one-time - uses, etc. - - In this case, the token is a path to the file and contains 3 interesting - sections: - - User ID base64d (for namespacing content to each user) - - random 24 char string - - Content type base64d (so we can return it when clients GET it) - - """ - isLeaf = True - - def __init__(self, hs, directory, auth): - resource.Resource.__init__(self) - self.hs = hs - self.directory = directory - self.auth = auth - - if not os.path.isdir(self.directory): - os.mkdir(self.directory) - logger.info("ContentRepoResource : Created %s directory.", - self.directory) - - @defer.inlineCallbacks - def map_request_to_name(self, request): - # auth the user - auth_user = yield self.auth.get_user_by_req(request) - - # namespace all file uploads on the user - prefix = base64.urlsafe_b64encode( - auth_user.to_string() - ).replace('=', '') - - # use a random string for the main portion - main_part = random_string(24) - - # suffix with a file extension if we can make one. This is nice to - # provide a hint to clients on the file information. We will also reuse - # this info to spit back the content type to the client. - suffix = "" - if request.requestHeaders.hasHeader("Content-Type"): - content_type = request.requestHeaders.getRawHeaders( - "Content-Type")[0] - suffix = "." + base64.urlsafe_b64encode(content_type) - if (content_type.split("/")[0].lower() in - ["image", "video", "audio"]): - file_ext = content_type.split("/")[-1] - # be a little paranoid and only allow a-z - file_ext = re.sub("[^a-z]", "", file_ext) - suffix += "." + file_ext - - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - logger.info("User %s is uploading a file to path %s", - auth_user.to_string(), - file_path) - - # keep trying to make a non-clashing file, with a sensible max attempts - attempts = 0 - while os.path.exists(file_path): - main_part = random_string(24) - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - attempts += 1 - if attempts > 25: # really? Really? - raise SynapseError(500, "Unable to create file.") - - defer.returnValue(file_path) - - def render_GET(self, request): - # no auth here on purpose, to allow anyone to view, even across home - # servers. - - # TODO: A little crude here, we could do this better. - filename = request.path.split('/')[-1] - # be paranoid - filename = re.sub("[^0-9A-z.-_]", "", filename) - - file_path = self.directory + "/" + filename - - logger.debug("Searching for %s", file_path) - - if os.path.isfile(file_path): - # filename has the content type - base64_contentype = filename.split(".")[1] - content_type = base64.urlsafe_b64decode(base64_contentype) - logger.info("Sending file %s", file_path) - f = open(file_path, 'rb') - request.setHeader('Content-Type', content_type) - d = FileSender().beginFileTransfer(f, request) - - # after the file has been sent, clean up and finish the request - def cbFinished(ignored): - f.close() - request.finish() - d.addCallback(cbFinished) - else: - respond_with_json_bytes( - request, - 404, - json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), - send_cors=True) - - return server.NOT_DONE_YET - - def render_POST(self, request): - self._async_render(request) - return server.NOT_DONE_YET - - def render_OPTIONS(self, request): - respond_with_json_bytes(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render(self, request): - try: - fname = yield self.map_request_to_name(request) - - # TODO I have a suspcious feeling this is just going to block - with open(fname, "wb") as f: - f.write(request.content.read()) - - - # FIXME (erikj): These should use constants. - file_name = os.path.basename(fname) - # FIXME: we can't assume what the public mounted path of the repo is - # ...plus self-signed SSL won't work to remote clients anyway - # ...and we can't assume that it's SSL anyway, as we might want to - # server it via the non-SSL listener... - url = "https://%s/_matrix/content/%s" % ( - self.hs.domain_with_port, file_name - ) - - respond_with_json_bytes(request, 200, - json.dumps({"content_token": url}), - send_cors=True) - - except CodeMessageException as e: - logger.exception(e) - respond_with_json_bytes(request, e.code, - json.dumps(cs_exception(e))) - except Exception as e: - logger.error("Failed to store file: %s" % e) - respond_with_json_bytes( - request, - 500, - json.dumps({"error": "Internal server error"}), - send_cors=True) - - -def respond_with_json_bytes(request, code, json_bytes, send_cors=False): +def respond_with_json_bytes(request, code, json_bytes, send_cors=False, + response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -362,7 +205,7 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False): Returns: twisted.web.server.NOT_DONE_YET""" - request.setResponseCode(code) + request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") if send_cors: |