From b61401082982eec6d8c5d78adeb59db18260fa76 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 19 Aug 2016 16:45:30 +0100 Subject: [PATCH 01/74] Docs --- coreapi/client.py | 7 +- coreapi/exceptions.py | 10 ++ coreapi/transports/base.py | 2 +- docs/api-guide/client.md | 81 ++++++++++++++ docs/api-guide/codecs.md | 201 +++++++++++++++++++++++++++++++++++ docs/api-guide/document.md | 27 +++++ docs/api-guide/exceptions.md | 16 +++ docs/api-guide/transports.md | 0 docs/api-guide/utils.md | 29 +++++ docs/index.md | 17 +++ mkdocs.yml | 1 + 11 files changed, 384 insertions(+), 7 deletions(-) create mode 100644 docs/api-guide/client.md create mode 100644 docs/api-guide/codecs.md create mode 100644 docs/api-guide/document.md create mode 100644 docs/api-guide/exceptions.md create mode 100644 docs/api-guide/transports.md create mode 100644 docs/api-guide/utils.md create mode 100644 docs/index.md create mode 100644 mkdocs.yml diff --git a/coreapi/client.py b/coreapi/client.py index 90fa4c0..1e432af 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -83,12 +83,7 @@ def get(self, url, force_codec=False): return transport.transition(link, self.decoders, force_codec=force_codec) def reload(self, document, force_codec=False): - url = document.url - link = Link(url, action='get') - - # Perform the action, and return a new document. - transport = determine_transport(self.transports, link.url) - return transport.transition(link, self.decoders, force_codec=force_codec) + return self.get(document.url, force_codec=force_codec) def action(self, document, keys, params=None, action=None, encoding=None, transform=None): if isinstance(keys, string_types): diff --git a/coreapi/exceptions.py b/coreapi/exceptions.py index 0b17dfa..a88b902 100644 --- a/coreapi/exceptions.py +++ b/coreapi/exceptions.py @@ -38,6 +38,16 @@ class LinkLookupError(Exception): pass +class ValidationError(Exception): + """ + Raised when the parameters passed do not match the link fields. + + * One or more parameters were passed that do not have a corresponding field. + * One or more required fields did not have a corresponding parameter passed. + """ + pass + + class ErrorMessage(Exception): """ Raised when the transition returns an error message. diff --git a/coreapi/transports/base.py b/coreapi/transports/base.py index 08c2c60..b19f0cc 100644 --- a/coreapi/transports/base.py +++ b/coreapi/transports/base.py @@ -5,5 +5,5 @@ class BaseTransport(itypes.Object): schemes = None - def transition(self, link, params=None, decoders=None, link_ancestors=None, force_codec=False): + def transition(self, link, decoders, params=None, link_ancestors=None, force_codec=False): raise NotImplementedError() # pragma: nocover diff --git a/docs/api-guide/client.md b/docs/api-guide/client.md new file mode 100644 index 0000000..ee1d654 --- /dev/null +++ b/docs/api-guide/client.md @@ -0,0 +1,81 @@ +# Clients + +In order to interact with an API using Core API, a client instance is required. + +The client is used to fetch the initial API description, and to then perform +interactions against the API. + +An example client session might look something like this: + + client = Client() + document = client.get('https://api.example.org/') + data = client.action(document, ['flights', 'search'], params={ + 'from': 'LHR', + 'to': 'PA', + 'date': '2016-10-12' + }) + +--- + +## Instantiating a client + +The default client may be obtained by instantiating an object, without +passing any parameters. + + client = Client() + +A client instance holds the configuration about which transports are available +for making network requests, and which codecs are available for decoding the +content of network responses. + +This configuration is set by passing either or both of the `decoders` and +`transports` arguments. The signature of the `Client` class is: + + Client(decoders=None, transports=None) + +For example the following would instantiate a client that is capable of +decoding either Core JSON schema responses, or decoding plain JSON +data responses: + + decoders = [ + codecs.CoreJSONCodec(), + codecs.JSONCodec() + ] + client = Client(decoders=decoders) + +When no arguments are passed, the following defaults are used: + + decoders = [ + codecs.CoreJSONCodec(), # application/vnd.coreapi+json + codecs.JSONCodec(), # application/json + codecs.TextCodec() # text/* + ] + + transports = [ + transports.HTTPTransport() # http, https + ] + +The configured decoders and transports are made available as read-only +properties on a client instance: + +* `.decoders` +* `.transports` + +--- + +## Making an initial request + +* get(url) + +Make a network request to the given URL, and return a decoded `Document`. + +--- + +## Interacting with an API + +* action(self, document, keys, params=None, overrides=None, validate=True) + +Effect an interaction against the given document. + +* `keys` - A list of strings that index a link within the document. +* `params` - A dictionary of parameters to use for the API interaction. diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md new file mode 100644 index 0000000..3466ba8 --- /dev/null +++ b/docs/api-guide/codecs.md @@ -0,0 +1,201 @@ +# Codecs + +Codecs are responsible for decoding a bytestring into a `Document` instance, +or for encoding a `Document` instance into a bytestring. + +A codec is associated with a media type. For example in HTTP responses, +the `Content-Type` header is used to indicate the media type of +the bytestring returned in the body of the response. + +When using a Core API client, HTTP responses are decoded with an appropriate +codec, based on the `Content-Type` of the response. + +## Using codecs + +All the codecs provided by the `coreapi` library are instantiated without +arguments, for example: + + codec = codecs.CoreJSONCodec() + +A codec will provide either one or both of the `decode` or `encode` methods. + +### Decoding + +* decode(content, url=None) + +### Encoding + +* encode(document, **options) + +### Attributes + +The following attributes are available on codec instances: + +* `media_type` - A string indicating the media type that the codec represents. +* `supports` - A list of strings, indicating the operations that the codec supports. + +The `supports` option should be one of the four following options: + +* `['decode', 'encode']` # Supports both decoding and encoding documents. +* `['decode']` # Supports decoding documents only. +* `['encode']` # Supports encoding documents only. +* `['data']` # Indicates that the codec supports decoding, + # but that it is expected to return plain data, + # rather than a `Document` object. + +--- + +## Available Codecs + +### CoreJSONCodec + +Supports decoding or encoding the Core JSON format. + +**.media_type**: `application/vnd.coreapi+json` + +**.supports**: `['decode', 'encode']` + +Example of decoding a Core JSON bytestring into a `Document` instance: + + >>> codec = codecs.TextCodec() + >>> content = b'{"_type": "document", ...' + >>> document = codec.decode(content) + >>> print(document) + + 'search': link(from, to, date) + +Example of encoding a `Document` instance into a Core JSON bytestring: + + >>> content = codec.encode(document, indent=True) + >>> print(content) + { + "_type": "document" + } + +--- + +### JSONCodec + +Supports decoding JSON data. + +**.media_type**: `application/json` + +**.supports**: `['data']` + +Example: + + >>> codec = codecs.TextCodec() + >>> content = b'{"string": "abc", "boolean": true, "null": null}' + >>> data = codec.decode(content) + >>> print(data) + {"string": "abc", "boolean": True, "null": None} + +--- + +### TextCodec + +Supports decoding plain-text responses. + +**.media_type**: `text/*` + +**.supports**: `['data']` + +Example: + + >>> codec = codecs.TextCodec() + >>> data = codec.decode(b'hello, world!') + >>> print(data) + hello, world! + +--- + +### DisplayCodec + +Supports encoding a `Document` to a display representation. + +**.media_type**: `text/plain` + +**.supports**: `['encoding']` + +Example: + + >>> codec = codecs.DisplayCodec() + >>> content = codec.encode(document, indent=True) + >>> print(content) + + 'search': link(from, to, date) + +--- + +### PythonCodec + +Supports encoding a `Document` to an its Python representation. + +**.media_type**: `text/python` + +**.supports**: `['encoding']` + +Example: + + >>> codec = codecs.PythonCodec() + >>> content = codec.encode(document, indent=True) + >>> print(content) + Document( + title='Flight Search API', + url='http://api.example.com/', + content={ + 'search': Link( + url='/search/', + action='get', + fields=[ + Field(name='from'), + Field(name='to'), + Field(name='date') + ] + ) + } + ) + +--- + +## Custom Codecs + +Custom codec classes may be created by inheriting from `BaseCodec`, setting +the `media_type` and `supports` properties, and implementing one or both +of the `decode` or `encode` methods. + +For example: + + class YAMLCodec(codecs.BaseCodec): + media_type = 'application/yaml' + supports = ['data'] + + def decode(content, url=None): + return yaml... + +### The codec registry + +Tools such as the Core API command line client require a method of discovering +which codecs are installed on the system. This is enabled by using a registry +system. + +In order to register a custom codec, the PyPI package must contain a correctly +configured `entry_points` option. Typically this needs to be added in a +`setup.py` module, which is run whenever publishing a new package version. + +The `entry_points` option must be a dictionary, containing a `coreapi.codecs` +item listing the available codec classes. As an example, the listing for the +codecs which are registered by the `coreapi` package itself is as follows: + + setup( + name='coreapi', + license='BSD', + ... + entry_points={ + 'coreapi.codecs': [ + 'corejson=coreapi.codecs:CoreJSONCodec', + 'json=coreapi.codecs:JSONCodec', + 'text=coreapi.codecs:TextCodec', + ] + } + ) diff --git a/docs/api-guide/document.md b/docs/api-guide/document.md new file mode 100644 index 0000000..37e116b --- /dev/null +++ b/docs/api-guide/document.md @@ -0,0 +1,27 @@ +# Documents + +## Document + +* url +* title +* content + +## Link + +* url +* action +* encoding +* transform +* description + +## Field + +* name +* required +* location +* description + +## Error + +* title +* content diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md new file mode 100644 index 0000000..5a95c3a --- /dev/null +++ b/docs/api-guide/exceptions.md @@ -0,0 +1,16 @@ +# Exceptions + +ResponseFailed + +* ParseError +* UnsupportedContentType +* NotAcceptable +* TransportError + +InvalidAction + +* LinkLookupError +* ValidationError + + +* ErrorMessage diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/api-guide/utils.md b/docs/api-guide/utils.md new file mode 100644 index 0000000..56b7317 --- /dev/null +++ b/docs/api-guide/utils.md @@ -0,0 +1,29 @@ +# Utils + +## lookup_elements(document, keys, strict_types=True) + +Given a document and a list of keys [...] + +## determine_transport(transports, url) + +Given a list of transports and a URL, return the appropriate transport for +making network requests to that URL. + +May raise `TransportError` + + +## negotiate_decoder(decoders, content_type=None) + +Given a list of codecs, and the value of an HTTP response 'Content-Type' header, +return the appropriate codec for decoding the response content. + + +May raise `UnsupportedContentType` + + +## negotiate_encoder(encoders, accept=None) + +Given a list of codecs, and the value of an incoming HTTP request 'Accept' +header, return the appropriate codec for encoding the outgoing response content. + +May raise `NotAcceptable` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..da37213 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](http://mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs help` - Print this help message. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..c97182f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1 @@ +site_name: My Docs From ec171c7c859877da998cde4e5c5e258b885d6198 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 23 Aug 2016 15:54:36 +0100 Subject: [PATCH 02/74] Add DownloadCodec --- coreapi/client.py | 22 ++++++++----- coreapi/codecs/__init__.py | 3 +- coreapi/codecs/download.py | 63 ++++++++++++++++++++++++++++++++++++++ coreapi/utils.py | 4 ++- setup.py | 1 + 5 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 coreapi/codecs/download.py diff --git a/coreapi/client.py b/coreapi/client.py index 1e432af..c9282ec 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -51,19 +51,27 @@ def _lookup_link(document, keys): return (node, link_ancestors) -class Client(itypes.Object): - DEFAULT_TRANSPORTS = [ - transports.HTTPTransport() +def get_default_decoders(): + return [ + codecs.CoreJSONCodec(), + codecs.JSONCodec(), + codecs.TextCodec(), + codecs.DownloadCodec() ] - DEFAULT_DECODERS = [ - codecs.CoreJSONCodec(), codecs.JSONCodec(), codecs.TextCodec() + + +def get_default_transports(): + return [ + transports.HTTPTransport() ] + +class Client(itypes.Object): def __init__(self, decoders=None, transports=None): if decoders is None: - decoders = self.DEFAULT_DECODERS + decoders = get_default_decoders() if transports is None: - transports = self.DEFAULT_TRANSPORTS + transports = get_default_transports() self._decoders = itypes.List(decoders) self._transports = itypes.List(transports) diff --git a/coreapi/codecs/__init__.py b/coreapi/codecs/__init__.py index 2649513..4fa6a1a 100644 --- a/coreapi/codecs/__init__.py +++ b/coreapi/codecs/__init__.py @@ -2,6 +2,7 @@ from coreapi.codecs.base import BaseCodec from coreapi.codecs.corejson import CoreJSONCodec from coreapi.codecs.display import DisplayCodec +from coreapi.codecs.download import DownloadCodec from coreapi.codecs.jsondata import JSONCodec from coreapi.codecs.python import PythonCodec from coreapi.codecs.text import TextCodec @@ -9,5 +10,5 @@ __all__ = [ 'BaseCodec', 'CoreJSONCodec', 'DisplayCodec', - 'JSONCodec', 'PythonCodec', 'TextCodec' + 'JSONCodec', 'PythonCodec', 'TextCodec', 'DownloadCodec' ] diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py new file mode 100644 index 0000000..66fb86c --- /dev/null +++ b/coreapi/codecs/download.py @@ -0,0 +1,63 @@ +# coding: utf-8 +from coreapi.codecs.base import BaseCodec +from coreapi.compat import urlparse +import os +import posixpath +import shutil +import tempfile + + +def _safe_filename(filename): + keepcharacters = (' ', '.', '_') + filename = "".join( + char for char in filename + if char.isalnum() or char in keepcharacters + ).strip() + + if filename == '..': + return '' + return filename + + +class DownloadCodec(BaseCodec): + media_type = '*/*' + supports = ['data'] + + def __init__(self, download_dir=None): + self._temporary = False + self._download_dir = download_dir + + def __del__(self): + if self._temporary and self._download_dir: + shutil.rmtree(self._download_dir) + + @property + def download_dir(self): + if self._download_dir is None: + self._temporary = True + self._download_dir = tempfile.mkdtemp(prefix='temp-coreapi-download-') + return self._download_dir + + def load(self, bytes, base_url=None): + fd, pathname = tempfile.mkstemp(dir=self.download_dir, suffix='.download') + file_handle = os.fdopen(fd, 'wb') + file_handle.write(bytes) + file_handle.close() + + filename = None + if base_url is not None: + url = urlparse.urlparse(base_url) + filename = _safe_filename(posixpath.basename(url.path)) + if not filename: + # Fallback for no filename, or empty filename generated. + filename = os.path.basename(pathname) + + filename = os.path.join(self.download_dir, filename) + basename, ext = os.path.splitext(filename) + idx = 0 + while os.path.exists(filename): + idx += 1 + filename = "%s (%d)%s" % (basename, idx, ext) + + os.rename(pathname, filename) + return open(filename, 'rb') diff --git a/coreapi/utils.py b/coreapi/utils.py index 758dc6d..8e7de86 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -33,8 +33,10 @@ def negotiate_decoder(decoders, content_type=None): content_type = content_type.split(';')[0].strip().lower() main_type = content_type.split('/')[0] + '/*' + wildcard_type = '*/*' + for codec in decoders: - if (codec.media_type == content_type) or (codec.media_type == main_type): + if codec.media_type in (content_type, main_type, wildcard_type): return codec msg = "Unsupported media in Content-Type header '%s'" % content_type diff --git a/setup.py b/setup.py index 0c4cc24..5571b8e 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ def get_package_data(package): 'corejson=coreapi.codecs:CoreJSONCodec', 'json=coreapi.codecs:JSONCodec', 'text=coreapi.codecs:TextCodec', + 'download=coreapi.codecs:DownloadCodec', ] }, classifiers=[ From 3f6c066eac378226e8fc56735960b4048101c0d8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Aug 2016 08:52:27 +0100 Subject: [PATCH 03/74] Tidy up function signatures for codecs --- coreapi/codecs/base.py | 4 +-- coreapi/codecs/corejson.py | 18 ++++++----- coreapi/codecs/display.py | 4 +-- coreapi/codecs/download.py | 60 ++++++++++++++++++++---------------- coreapi/codecs/jsondata.py | 7 +++-- coreapi/codecs/python.py | 14 ++++----- coreapi/codecs/text.py | 4 +-- coreapi/document.py | 4 +-- coreapi/transports/http.py | 6 +++- coreapi/utils.py | 15 +++++++++ docs/api-guide/utils.md | 2 +- docs/topics/release-notes.md | 0 tests/test_codecs.py | 24 +++++++-------- tests/test_integration.py | 6 ++-- tests/test_transport.py | 2 +- 15 files changed, 102 insertions(+), 68 deletions(-) create mode 100644 docs/topics/release-notes.md diff --git a/coreapi/codecs/base.py b/coreapi/codecs/base.py index 02694c0..c21e083 100644 --- a/coreapi/codecs/base.py +++ b/coreapi/codecs/base.py @@ -5,8 +5,8 @@ class BaseCodec(itypes.Object): media_type = None supports = [] # 'encoding', 'decoding', 'data' - def load(self, bytes, base_url=None): + def decode(self, bytestring, **options): raise NotImplementedError() # pragma: nocover - def dump(self, document, **kwargs): + def encode(self, document, **options): raise NotImplementedError() # pragma: nocover diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 3eb1fa5..edc19e7 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -228,14 +228,16 @@ def _primative_to_document(data, base_url=None): class CoreJSONCodec(BaseCodec): media_type = 'application/vnd.coreapi+json' - supports = ['encoding', 'decoding'] + supports = ['encode', 'decode'] - def load(self, bytes, base_url=None): + def decode(self, bytestring, **options): """ Takes a bytestring and returns a document. """ + base_url = options.get('base_url') + try: - data = json.loads(bytes.decode('utf-8')) + data = json.loads(bytestring.decode('utf-8')) except ValueError as exc: raise ParseError('Malformed JSON. %s' % exc) @@ -248,22 +250,24 @@ def load(self, bytes, base_url=None): return doc - def dump(self, document, indent=False, **kwargs): + def encode(self, document, **options): """ Takes a document and returns a bytestring. """ + indent = options.get('indent') + if indent: - options = { + kwargs = { 'ensure_ascii': False, 'indent': 4, 'separators': VERBOSE_SEPARATORS } else: - options = { + kwargs = { 'ensure_ascii': False, 'indent': None, 'separators': COMPACT_SEPARATORS } data = _document_to_primative(document) - return force_bytes(json.dumps(data, **options)) + return force_bytes(json.dumps(data, **kwargs)) diff --git a/coreapi/codecs/display.py b/coreapi/codecs/display.py index 62e4c32..663ba24 100644 --- a/coreapi/codecs/display.py +++ b/coreapi/codecs/display.py @@ -115,7 +115,7 @@ class DisplayCodec(BaseCodec): A plaintext representation of a Document, intended for readability. """ media_type = 'text/plain' - supports = ['encoding'] + supports = ['encode'] - def dump(self, node, colorize=False, **kwargs): + def encode(self, node, colorize=False, **kwargs): return _to_plaintext(node, colorize=colorize) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 66fb86c..249c8f9 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -1,22 +1,26 @@ # coding: utf-8 from coreapi.codecs.base import BaseCodec from coreapi.compat import urlparse +from coreapi.utils import safe_filename import os import posixpath import shutil import tempfile -def _safe_filename(filename): - keepcharacters = (' ', '.', '_') - filename = "".join( - char for char in filename - if char.isalnum() or char in keepcharacters - ).strip() +def _get_available_path(path): + basename, ext = os.path.splitext(path) + idx = 0 + while os.path.exists(path): + idx += 1 + path = "%s (%d)%s" % (basename, idx, ext) + return path - if filename == '..': - return '' - return filename + +def _get_filename_from_url(url): + parsed = urlparse.urlparse(url) + final_path_component = posixpath.basename(parsed.path.rstrip('/')) + return safe_filename(final_path_component) class DownloadCodec(BaseCodec): @@ -38,26 +42,30 @@ def download_dir(self): self._download_dir = tempfile.mkdtemp(prefix='temp-coreapi-download-') return self._download_dir - def load(self, bytes, base_url=None): - fd, pathname = tempfile.mkstemp(dir=self.download_dir, suffix='.download') + def decode(self, bytestring, **options): + filename = options.get('filename') + base_url = options.get('base_url') + + # Write the download to a temporary .download file. + fd, temp_path = tempfile.mkstemp(dir=self.download_dir, suffix='.download') file_handle = os.fdopen(fd, 'wb') - file_handle.write(bytes) + file_handle.write(bytestring) file_handle.close() - filename = None - if base_url is not None: - url = urlparse.urlparse(base_url) - filename = _safe_filename(posixpath.basename(url.path)) - if not filename: + # Determine the output filename. + output_filename = None + if filename: + output_filename = filename + elif base_url is not None: + output_filename = _get_filename_from_url(base_url) + if not output_filename: # Fallback for no filename, or empty filename generated. - filename = os.path.basename(pathname) + filename = os.path.basename(temp_path) - filename = os.path.join(self.download_dir, filename) - basename, ext = os.path.splitext(filename) - idx = 0 - while os.path.exists(filename): - idx += 1 - filename = "%s (%d)%s" % (basename, idx, ext) + # Determine the full output path. + output_path = os.path.join(self.download_dir, output_filename) + output_path = _get_available_path(output_path) - os.rename(pathname, filename) - return open(filename, 'rb') + # Move the temporary download file to the final location. + os.rename(temp_path, output_path) + return open(output_path, 'rb') diff --git a/coreapi/codecs/jsondata.py b/coreapi/codecs/jsondata.py index bb35a42..10ab488 100644 --- a/coreapi/codecs/jsondata.py +++ b/coreapi/codecs/jsondata.py @@ -9,11 +9,14 @@ class JSONCodec(BaseCodec): media_type = 'application/json' supports = ['data'] - def load(self, bytes, base_url=None): + def decode(self, bytestring, **options): """ Return raw JSON data. """ try: - return json.loads(bytes.decode('utf-8'), object_pairs_hook=collections.OrderedDict) + return json.loads( + bytestring.decode('utf-8'), + object_pairs_hook=collections.OrderedDict + ) except ValueError as exc: raise ParseError('Malformed JSON. %s' % exc) diff --git a/coreapi/codecs/python.py b/coreapi/codecs/python.py index 8117134..2487a89 100644 --- a/coreapi/codecs/python.py +++ b/coreapi/codecs/python.py @@ -64,13 +64,13 @@ class PythonCodec(BaseCodec): A Python representation of a Document, for use with '__repr__'. """ media_type = 'text/python' - supports = ['encoding'] + supports = ['encode'] - def dump(self, node, **kwargs): + def encode(self, document, **options): # Object and Array only have the class name wrapper if they # are the outermost element. - if isinstance(node, Object): - return 'Object(%s)' % _to_repr(node) - elif isinstance(node, Array): - return 'Array(%s)' % _to_repr(node) - return _to_repr(node) + if isinstance(document, Object): + return 'Object(%s)' % _to_repr(document) + elif isinstance(document, Array): + return 'Array(%s)' % _to_repr(document) + return _to_repr(document) diff --git a/coreapi/codecs/text.py b/coreapi/codecs/text.py index 72ce2e3..bc44104 100644 --- a/coreapi/codecs/text.py +++ b/coreapi/codecs/text.py @@ -6,5 +6,5 @@ class TextCodec(BaseCodec): media_type = 'text/*' supports = ['data'] - def load(self, bytes, base_url=None): - return bytes.decode('utf-8') + def decode(self, bytestring, **options): + return bytestring.decode('utf-8') diff --git a/coreapi/document.py b/coreapi/document.py index 57b452e..c87daa5 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -15,12 +15,12 @@ def _to_immutable(value): def _repr(node): from coreapi.codecs.python import PythonCodec - return PythonCodec().dump(node) + return PythonCodec().encode(node) def _str(node): from coreapi.codecs.display import DisplayCodec - return DisplayCodec().dump(node) + return DisplayCodec().encode(node) def _key_sorting(item): diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 36952ee..5398502 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -213,7 +213,11 @@ def _decode_result(response, decoders, force_codec=False): else: content_type = response.headers.get('content-type') codec = negotiate_decoder(decoders, content_type) - result = codec.load(response.content, base_url=response.url) + + options = { + 'base_url': response.url + } + result = codec.decode(response.content, **options) else: # No content returned in response. result = None diff --git a/coreapi/utils.py b/coreapi/utils.py index 8e7de86..cf955f2 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,5 +1,20 @@ from coreapi import exceptions from coreapi.compat import urlparse +import os + + +def safe_filename(filename): + filename = os.path.basename(filename) + + keepcharacters = (' ', '.', '_') + filename = "".join( + char for char in filename + if char.isalnum() or char in keepcharacters + ).strip() + + if filename == '..': + return '' + return filename def determine_transport(transports, url): diff --git a/docs/api-guide/utils.md b/docs/api-guide/utils.md index 56b7317..8a847a8 100644 --- a/docs/api-guide/utils.md +++ b/docs/api-guide/utils.md @@ -1,4 +1,4 @@ -# Utils +# Utilities ## lookup_elements(document, keys, strict_types=True) diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 84af4e3..c761420 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -89,7 +89,7 @@ def test_minimal_document(json_codec): """ Ensure we can load the smallest possible valid JSON encoding. """ - doc = json_codec.load(b'{"_type":"document"}') + doc = json_codec.decode(b'{"_type":"document"}') assert isinstance(doc, Document) assert doc.url == '' assert doc.title == '' @@ -100,7 +100,7 @@ def test_minimal_error(json_codec): """ Ensure we can load a minimal error message encoding. """ - error = json_codec.load(b'{"_type":"error","_meta":{"title":"Failure"},"messages":["failed"]}') + error = json_codec.decode(b'{"_type":"error","_meta":{"title":"Failure"},"messages":["failed"]}') assert error == Error(title="Failure", content={'messages': ['failed']}) @@ -111,27 +111,27 @@ def test_malformed_json(json_codec): Invalid JSON should raise a ParseError. """ with pytest.raises(ParseError): - json_codec.load(b'_') + json_codec.decode(b'_') def test_not_a_document(json_codec): """ Valid JSON that does not return a document should be coerced into one. """ - assert json_codec.load(b'{}') == Document() + assert json_codec.decode(b'{}') == Document() # Encodings may have a verbose and a compact style. def test_compact_style(json_codec): doc = Document(content={'a': 123, 'b': 456}) - bytes = json_codec.dump(doc) + bytes = json_codec.encode(doc) assert bytes == b'{"_type":"document","a":123,"b":456}' def test_verbose_style(json_codec): doc = Document(content={'a': 123, 'b': 456}) - bytes = json_codec.dump(doc, indent=True) + bytes = json_codec.encode(doc, indent=True) assert bytes == b"""{ "_type": "document", "a": 123, @@ -149,7 +149,7 @@ def test_link_encodings(json_codec): fields=['optional', Field('required', required=True, location='path')] ) }) - bytes = json_codec.dump(doc, indent=True) + bytes = json_codec.encode(doc, indent=True) assert bytes == b"""{ "_type": "document", "link": { @@ -173,27 +173,27 @@ def test_link_encodings(json_codec): # Tests for graceful omissions. def test_invalid_document_meta_ignored(json_codec): - doc = json_codec.load(b'{"_type": "document", "_meta": 1, "a": 1}') + doc = json_codec.decode(b'{"_type": "document", "_meta": 1, "a": 1}') assert doc == Document(content={"a": 1}) def test_invalid_document_url_ignored(json_codec): - doc = json_codec.load(b'{"_type": "document", "_meta": {"url": 1}, "a": 1}') + doc = json_codec.decode(b'{"_type": "document", "_meta": {"url": 1}, "a": 1}') assert doc == Document(content={"a": 1}) def test_invalid_document_title_ignored(json_codec): - doc = json_codec.load(b'{"_type": "document", "_meta": {"title": 1}, "a": 1}') + doc = json_codec.decode(b'{"_type": "document", "_meta": {"title": 1}, "a": 1}') assert doc == Document(content={"a": 1}) def test_invalid_link_url_ignored(json_codec): - doc = json_codec.load(b'{"_type": "document", "link": {"_type": "link", "url": 1}}') + doc = json_codec.decode(b'{"_type": "document", "link": {"_type": "link", "url": 1}}') assert doc == Document(content={"link": Link()}) def test_invalid_link_fields_ignored(json_codec): - doc = json_codec.load(b'{"_type": "document", "link": {"_type": "link", "fields": 1}}') + doc = json_codec.decode(b'{"_type": "document", "link": {"_type": "link", "fields": 1}}') assert doc == Document(content={"link": Link()}) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6a761af..b38de9a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -13,7 +13,7 @@ @pytest.fixture def document(): codec = coreapi.codecs.CoreJSONCodec() - return codec.load(encoded) + return codec.decode(encoded) class MockResponse(object): @@ -28,7 +28,7 @@ def __init__(self, content): def test_load(): codec = coreapi.codecs.CoreJSONCodec() - assert codec.load(encoded) == { + assert codec.decode(encoded) == { "a": 123, "next": coreapi.Link(url='http://example.org') } @@ -36,7 +36,7 @@ def test_load(): def test_dump(document): codec = coreapi.codecs.CoreJSONCodec() - content = codec.dump(document) + content = codec.encode(document) assert content == encoded diff --git a/tests/test_transport.py b/tests/test_transport.py index fd0470c..a9aea1d 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -93,7 +93,7 @@ def test_post(monkeypatch, http): def mockreturn(self, request): codec = CoreJSONCodec() body = force_text(request.body) - content = codec.dump(Document(content={'data': json.loads(body)})) + content = codec.encode(Document(content={'data': json.loads(body)})) return MockResponse(content) monkeypatch.setattr(requests.Session, 'send', mockreturn) From 6252b521a01ce2b97ffc23bbb3bc25e8eff885d3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 24 Aug 2016 10:39:59 +0100 Subject: [PATCH 04/74] Smart filename determination in downloads --- coreapi/codecs/download.py | 78 +++++++++++++++++++++++++++++++------- coreapi/transports/http.py | 5 +++ coreapi/utils.py | 15 -------- 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 249c8f9..5c75c40 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -1,14 +1,23 @@ # coding: utf-8 from coreapi.codecs.base import BaseCodec from coreapi.compat import urlparse -from coreapi.utils import safe_filename +import cgi +import mimetypes import os import posixpath import shutil import tempfile -def _get_available_path(path): +def _unique_output_path(path): + """ + Given a path like '/a/b/c.txt' + + Return the first available filename that doesn't already exist, + using an incrementing suffix if needed. + + For example: '/a/b/c.txt' or '/a/b/c (1).txt' or '/a/b/c (2).txt'... + """ basename, ext = os.path.splitext(path) idx = 0 while os.path.exists(path): @@ -17,10 +26,54 @@ def _get_available_path(path): return path -def _get_filename_from_url(url): +def _safe_filename(filename): + filename = os.path.basename(filename) + + keepcharacters = (' ', '.', '_', '-') + filename = ''.join( + char for char in filename + if char.isalnum() or char in keepcharacters + ).strip().strip('.') + + return filename + + +def _get_filename_from_content_disposition(content_disposition): + params = value, params = cgi.parse_header(content_disposition) + + if 'filename*' in params: + try: + charset, lang, filename = params['filename*'].split('\'', 2) + filename = urlparse.unquote(filename) + filename = filename.encode('iso-8859-1').decode(charset) + return _safe_filename(filename) + except (ValueError, LookupError): + pass + + if 'filename' in params: + filename = params['filename'] + return _safe_filename(filename) + + return None + + +def _get_filename_from_url(url, content_type=None): parsed = urlparse.urlparse(url) - final_path_component = posixpath.basename(parsed.path.rstrip('/')) - return safe_filename(final_path_component) + filename = _safe_filename(posixpath.basename(parsed.path.rstrip('/'))) + if filename and ('.' not in filename) and (content_type is not None): + ext = mimetypes.guess_extension(content_type) + if ext: + filename = filename + ext + return filename + + +def _get_filename(base_url=None, content_type=None, content_disposition=None): + filename = None + if content_disposition: + filename = _get_filename_from_content_disposition(content_disposition) + if base_url and not filename: + filename = _get_filename_from_url(base_url, content_type) + return filename class DownloadCodec(BaseCodec): @@ -43,8 +96,9 @@ def download_dir(self): return self._download_dir def decode(self, bytestring, **options): - filename = options.get('filename') base_url = options.get('base_url') + content_type = options.get('content_type') + content_disposition = options.get('content_disposition') # Write the download to a temporary .download file. fd, temp_path = tempfile.mkstemp(dir=self.download_dir, suffix='.download') @@ -53,18 +107,14 @@ def decode(self, bytestring, **options): file_handle.close() # Determine the output filename. - output_filename = None - if filename: - output_filename = filename - elif base_url is not None: - output_filename = _get_filename_from_url(base_url) + output_filename = _get_filename(base_url, content_type, content_disposition) if not output_filename: - # Fallback for no filename, or empty filename generated. - filename = os.path.basename(temp_path) + # Fallback if no output filename could be determined. + output_filename = os.path.basename(temp_path) # Determine the full output path. output_path = os.path.join(self.download_dir, output_filename) - output_path = _get_available_path(output_path) + output_path = _unique_output_path(output_path) # Move the temporary download file to the final location. os.rename(temp_path, output_path) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 5398502..363d656 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -217,6 +217,11 @@ def _decode_result(response, decoders, force_codec=False): options = { 'base_url': response.url } + if 'content-type' in response.headers: + options['content_type'] = response.headers['content-type'] + if 'content-disposition' in response.headers: + options['content_disposition'] = response.headers['content-disposition'] + result = codec.decode(response.content, **options) else: # No content returned in response. diff --git a/coreapi/utils.py b/coreapi/utils.py index cf955f2..8e7de86 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,20 +1,5 @@ from coreapi import exceptions from coreapi.compat import urlparse -import os - - -def safe_filename(filename): - filename = os.path.basename(filename) - - keepcharacters = (' ', '.', '_') - filename = "".join( - char for char in filename - if char.isalnum() or char in keepcharacters - ).strip() - - if filename == '..': - return '' - return filename def determine_transport(transports, url): From dcb378762d553bf65ca19f7aeb1ce8fbec317dbf Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 11:48:17 +0100 Subject: [PATCH 05/74] Default encoding is application/json. --- coreapi/transports/http.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 363d656..34d1be9 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -64,20 +64,14 @@ def _get_params(method, fields, params=None): return Params(path, query, headers, body, data, files) -def _get_encoding(encoding, params): +def _get_encoding(encoding, method): if encoding: return encoding - if params.body is not None: - if is_file(params.body): - return 'application/octet-stream' - return 'application/json' - elif params.files: - return 'multipart/form-data' - elif params.data: - return 'application/json' + if method in ('GET', 'DELETE'): + return '' - return '' + return 'application/json' def _get_url(url, path_params): @@ -287,8 +281,8 @@ def headers(self): def transition(self, link, decoders, params=None, link_ancestors=None, force_codec=False): session = self._session method = _get_method(link.action) + encoding = _get_encoding(link.encoding, method) params = _get_params(method, link.fields, params) - encoding = _get_encoding(link.encoding, params) url = _get_url(link.url, params.path) headers = _get_headers(url, decoders, self.credentials) headers.update(self.headers) From 8ebbf3ce4ecf0c6f6baaf535f9aaa46aec4c8287 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 16:21:46 +0100 Subject: [PATCH 06/74] Force encoding for multipart links, even if no file parameters included. --- coreapi/transports/http.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 34d1be9..ede6fa2 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -18,6 +18,17 @@ empty_params = Params({}, {}, {}, None, {}, {}) +class ForceMultiPartDict(dict): + # A dictionary that always evaluates as True. + # Allows us to force requests to use multipart encoding, even when no + # file parameters are passed. + def __bool__(self): + return True + + def __nonzero__(self): + return True + + def _get_method(action): if not action: return 'GET' @@ -136,7 +147,7 @@ def _build_http_request(session, url, method, headers=None, encoding=None, param opts['json'] = params.data elif encoding == 'multipart/form-data': opts['data'] = params.data - opts['files'] = params.files + opts['files'] = ForceMultiPartDict(params.files) elif encoding == 'application/x-www-form-urlencoded': opts['data'] = params.data elif encoding == 'application/octet-stream': From d04b9942b349d820bd87d91b84a464a58e057930 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 17:45:36 +0100 Subject: [PATCH 07/74] Return DownloadedFile instances from DownloadCodec --- coreapi/codecs/download.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 5c75c40..3ae1a16 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -9,6 +9,15 @@ import tempfile +class DownloadedFile(_TemporaryFileWrapper): + """ + A wrapper around the returned file object, in order to provide + a clearer interface than simply returning a file handle. + """ + def __init__(self, file, name): + super(DownloadedFile, self).__init__(file, name, delete=False) + + def _unique_output_path(path): """ Given a path like '/a/b/c.txt' @@ -27,6 +36,9 @@ def _unique_output_path(path): def _safe_filename(filename): + """ + Sanitize output filenames, to remove any potentially unsafe characters. + """ filename = os.path.basename(filename) keepcharacters = (' ', '.', '_', '-') @@ -39,6 +51,9 @@ def _safe_filename(filename): def _get_filename_from_content_disposition(content_disposition): + """ + Determine an output filename based on the `Content-Disposition` header. + """ params = value, params = cgi.parse_header(content_disposition) if 'filename*' in params: @@ -58,9 +73,15 @@ def _get_filename_from_content_disposition(content_disposition): def _get_filename_from_url(url, content_type=None): + """ + Determine an output filename based on the download URL. + """ parsed = urlparse.urlparse(url) - filename = _safe_filename(posixpath.basename(parsed.path.rstrip('/'))) + final_path_component = posixpath.basename(parsed.path.rstrip('/')) + filename = _safe_filename(final_path_component) if filename and ('.' not in filename) and (content_type is not None): + # If no extension exists then attempt to add one, + # based on the content type. ext = mimetypes.guess_extension(content_type) if ext: filename = filename + ext @@ -68,6 +89,9 @@ def _get_filename_from_url(url, content_type=None): def _get_filename(base_url=None, content_type=None, content_disposition=None): + """ + Determine an output filename to use for the download. + """ filename = None if content_disposition: filename = _get_filename_from_content_disposition(content_disposition) @@ -77,10 +101,16 @@ def _get_filename(base_url=None, content_type=None, content_disposition=None): class DownloadCodec(BaseCodec): + """ + A codec to handle raw file downloads, such as images and other media. + """ media_type = '*/*' supports = ['data'] def __init__(self, download_dir=None): + """ + `download_dir` - The path to use for file downloads. + """ self._temporary = False self._download_dir = download_dir @@ -118,4 +148,5 @@ def decode(self, bytestring, **options): # Move the temporary download file to the final location. os.rename(temp_path, output_path) - return open(output_path, 'rb') + output_file = open(output_path, 'rb') + return DownloadedFile(output_file, output_path) From a671930a666c75138121aaf7b74758847894ce9b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 17:55:12 +0100 Subject: [PATCH 08/74] Add parameter validation --- coreapi/client.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/coreapi/client.py b/coreapi/client.py index c9282ec..8b31078 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -51,6 +51,34 @@ def _lookup_link(document, keys): return (node, link_ancestors) +def _validate_parameters(link, parameters): + """ + Ensure that parameters passed to the link are correct. + Raises a `ValidationError` if any parameters do not validate. + """ + provided = set(parameters.keys()) + required = set([ + field.name for field in link.fields if field.required + ]) + optional = set([ + field.name for field in link.fields if not field.required + ]) + + # Determine any parameter names supplied that are not valid. + unexpected = provided - (optional | required) + unexpected = ['"' + item + '"' for item in sorted(unexpected)] + if unexpected: + prefix = len(unexpected) > 1 and 'parameters: ' or 'parameter: ' + raise exceptions.ValidationError('Unknown ' + prefix + ', '.join(unexpected)) + + # Determine if any required field names not supplied. + missing = required - provided + missing = ['"' + item + '"' for item in sorted(missing)] + if missing: + prefix = len(missing) > 1 and 'parameters: ' or 'parameter: ' + raise exceptions.ValidationError('Missing required ' + prefix + ', '.join(missing)) + + def get_default_decoders(): return [ codecs.CoreJSONCodec(), @@ -93,12 +121,17 @@ def get(self, url, force_codec=False): def reload(self, document, force_codec=False): return self.get(document.url, force_codec=force_codec) - def action(self, document, keys, params=None, action=None, encoding=None, transform=None): + def action(self, document, keys, params=None, validate=True, action=None, encoding=None, transform=None): if isinstance(keys, string_types): keys = [keys] + if params is None: + params = {} + # Validate the keys and link parameters. link, link_ancestors = _lookup_link(document, keys) + if validate: + _validate_parameters(link, params) if (action is not None) or (encoding is not None) or (transform is not None): # Handle any explicit overrides. From 6bb36e2b083af8dd02ffe4c54e7af0cec93a8eae Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 17:55:27 +0100 Subject: [PATCH 09/74] Fix imports --- coreapi/codecs/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 3ae1a16..7e8be26 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -9,7 +9,7 @@ import tempfile -class DownloadedFile(_TemporaryFileWrapper): +class DownloadedFile(tempfile._TemporaryFileWrapper): """ A wrapper around the returned file object, in order to provide a clearer interface than simply returning a file handle. From 7c62c9ec14cc825e2243ebe356e37bba6a7dd801 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 18:02:00 +0100 Subject: [PATCH 10/74] Add overrides argument to client.action() --- coreapi/client.py | 14 ++++++++------ tests/test_transitions.py | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index 8b31078..886478c 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -121,7 +121,7 @@ def get(self, url, force_codec=False): def reload(self, document, force_codec=False): return self.get(document.url, force_codec=force_codec) - def action(self, document, keys, params=None, validate=True, action=None, encoding=None, transform=None): + def action(self, document, keys, params=None, validate=True, overrides=None): if isinstance(keys, string_types): keys = [keys] @@ -133,12 +133,14 @@ def action(self, document, keys, params=None, validate=True, action=None, encodi if validate: _validate_parameters(link, params) - if (action is not None) or (encoding is not None) or (transform is not None): + if overrides: # Handle any explicit overrides. - action = link.action if (action is None) else action - encoding = link.encoding if (encoding is None) else encoding - transform = link.transform if (transform is None) else transform - link = Link(link.url, action=action, encoding=encoding, transform=transform, fields=link.fields) + url = overrides.get('url', link.url) + action = overrides.get('action', link.action) + encoding = overrides.get('encoding', link.encoding) + transform = overrides.get('transform', link.transform) + fields = overrides.get('fields', link.fields) + link = Link(url, action=action, encoding=encoding, transform=transform, fields=fields) # Perform the action, and return a new document. transport = determine_transport(self.transports, link.url) diff --git a/tests/test_transitions.py b/tests/test_transitions.py index 02ae5ff..a31ba13 100644 --- a/tests/test_transitions.py +++ b/tests/test_transitions.py @@ -72,12 +72,12 @@ def test_delete(doc): # Test overrides def test_override_action(doc): - new = client.action(doc, ['nested', 'follow'], action='put') + new = client.action(doc, ['nested', 'follow'], overrides={'action': 'put'}) assert new == {'nested': {'new': 123, 'foo': None}} assert new.title == 'original' def test_override_transform(doc): - new = client.action(doc, ['nested', 'update'], params={'foo': 456}, transform='new') + new = client.action(doc, ['nested', 'update'], params={'foo': 456}, overrides={'transform': 'new'}) assert new == {'new': 123, 'foo': 456} assert new.title == 'new' From 5098ea2fa45858649e5096b234df402a47c8b8e2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 22:13:35 +0100 Subject: [PATCH 11/74] Drop unused location=header --- coreapi/transports/http.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index ede6fa2..51f0bcd 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -14,8 +14,8 @@ import uritemplate -Params = collections.namedtuple('Params', ['path', 'query', 'headers', 'body', 'data', 'files']) -empty_params = Params({}, {}, {}, None, {}, {}) +Params = collections.namedtuple('Params', ['path', 'query', 'body', 'data', 'files']) +empty_params = Params({}, {}, None, {}, {}) class ForceMultiPartDict(dict): @@ -46,7 +46,6 @@ def _get_params(method, fields, params=None): path = {} query = {} - headers = {} body = None data = {} files = {} @@ -62,8 +61,6 @@ def _get_params(method, fields, params=None): path[key] = value elif location == 'query': query[key] = value - elif location == 'header': - headers[key] = value elif location == 'body': body = value elif location == 'form': @@ -72,7 +69,7 @@ def _get_params(method, fields, params=None): else: data[key] = value - return Params(path, query, headers, body, data, files) + return Params(path, query, body, data, files) def _get_encoding(encoding, method): From e866a698e981b487a2ea7209165de5106154e323 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 30 Aug 2016 22:14:11 +0100 Subject: [PATCH 12/74] Start documenting exceptions --- docs/api-guide/exceptions.md | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 5a95c3a..37db17d 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -1,16 +1,27 @@ # Exceptions -ResponseFailed +## ParseError -* ParseError -* UnsupportedContentType -* NotAcceptable -* TransportError +A response was returned with malformed content. -InvalidAction +## UnsupportedContentType -* LinkLookupError -* ValidationError +A response was returned with a content type that is not handled by any of the available codecs. +## NotAcceptable -* ErrorMessage + + +## TransportError + +An issue occurred with the network request. + +## LinkLookupError + +The keys passed in a `client.action()` call did not reference a link in the document. + +## ValidationError + +The parameters passed in a `client.action()` call were invalid. + +## ErrorMessage From 99f73e6b6afb5d028adff7cd2cff3ddb6253a6da Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 1 Sep 2016 00:36:33 +0100 Subject: [PATCH 13/74] Validate parameter types are supported by encodings. --- coreapi/exceptions.py | 4 ++ coreapi/transports/http.py | 68 +++++++++++++++-------------- coreapi/utils.py | 89 +++++++++++++++++++++++++++++++++++++- tests/test_utils.py | 51 ++++++++++++++++++++++ 4 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 tests/test_utils.py diff --git a/coreapi/exceptions.py b/coreapi/exceptions.py index a88b902..93db841 100644 --- a/coreapi/exceptions.py +++ b/coreapi/exceptions.py @@ -48,6 +48,10 @@ class ValidationError(Exception): pass +class InvalidLinkError(Exception): + pass + + class ErrorMessage(Exception): """ Raised when the transition returns an error message. diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 51f0bcd..ac46ea2 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -1,11 +1,10 @@ # coding: utf-8 from __future__ import unicode_literals from collections import OrderedDict +from coreapi import exceptions, utils from coreapi.compat import is_file, urlparse from coreapi.document import Document, Object, Link, Array, Error -from coreapi.exceptions import ErrorMessage from coreapi.transports.base import BaseTransport -from coreapi.utils import negotiate_decoder import collections import requests import itypes @@ -14,8 +13,8 @@ import uritemplate -Params = collections.namedtuple('Params', ['path', 'query', 'body', 'data', 'files']) -empty_params = Params({}, {}, None, {}, {}) +Params = collections.namedtuple('Params', ['path', 'query', 'data', 'files']) +empty_params = Params({}, {}, {}, {}) class ForceMultiPartDict(dict): @@ -35,9 +34,9 @@ def _get_method(action): return action.upper() -def _get_params(method, fields, params=None): +def _get_params(method, encoding, fields, params=None): """ - Separate the params into their location types. + Separate the params into the various types. """ if params is None: return empty_params @@ -46,7 +45,6 @@ def _get_params(method, fields, params=None): path = {} query = {} - body = None data = {} files = {} @@ -58,18 +56,18 @@ def _get_params(method, fields, params=None): location = field_map[key].location if location == 'path': - path[key] = value + path[key] = utils.validate_path_param(value, name=key) elif location == 'query': - query[key] = value + query[key] = utils.validate_query_param(value, name=key) elif location == 'body': - body = value + data = utils.validate_body_param(value, encoding=encoding, name=key) elif location == 'form': if is_file(value): - files[key] = value + files[key] = utils.validate_form_files(value, encoding=encoding, name=key) else: - data[key] = value + data[key] = utils.validate_form_data(value, encoding=encoding, name=key) - return Params(path, query, body, data, files) + return Params(path, query, data, files) def _get_encoding(encoding, method): @@ -113,7 +111,7 @@ def _get_headers(url, decoders, credentials=None): return headers -def _get_content_type(file_obj): +def _get_upload_content_type(file_obj): """ When a raw file upload is made, determine a content-type where possible. """ @@ -121,10 +119,23 @@ def _get_content_type(file_obj): if name is not None: content_type, encoding = mimetypes.guess_type(name) else: - content_type = None + content_type = 'application/octet-stream' return content_type +def _get_upload_content_disposition(file_obj): + """ + When a raw file upload is made, determine a content-type where possible. + """ + name = getattr(file_obj, 'name', None) + if name is not None: + filename = os.path.basename(file_obj.name) + content_disposition = 'attachment; filename="%s"' % filename + else: + content_disposition = 'attachment' + return content_disposition + + def _build_http_request(session, url, method, headers=None, encoding=None, params=empty_params): """ Make an HTTP request and return an HTTP response. @@ -136,28 +147,19 @@ def _build_http_request(session, url, method, headers=None, encoding=None, param if params.query: opts['params'] = params.query - if (params.body is not None) or params.data or params.files: + if params.data or params.files: if encoding == 'application/json': - if params.body is not None: - opts['json'] = params.body - else: - opts['json'] = params.data + opts['json'] = params.data elif encoding == 'multipart/form-data': opts['data'] = params.data opts['files'] = ForceMultiPartDict(params.files) elif encoding == 'application/x-www-form-urlencoded': opts['data'] = params.data elif encoding == 'application/octet-stream': - opts['data'] = params.body - content_type = _get_content_type(params.body) - if content_type: - opts['headers']['content-type'] = content_type - - if hasattr(params.body, 'name'): - filename = os.path.basename(params.body.name) - content_disposition = 'attachment; filename="%s"' % filename - else: - content_disposition = 'attachment' + opts['data'] = params.data + content_type = _get_upload_content_type(params.data) + content_disposition = _get_upload_content_disposition(params.data) + opts['headers']['content-type'] = content_type opts['headers']['content-disposition'] = content_disposition request = requests.Request(method, url, **opts) @@ -214,7 +216,7 @@ def _decode_result(response, decoders, force_codec=False): codec = decoders[0] else: content_type = response.headers.get('content-type') - codec = negotiate_decoder(decoders, content_type) + codec = utils.negotiate_decoder(decoders, content_type) options = { 'base_url': response.url @@ -290,7 +292,7 @@ def transition(self, link, decoders, params=None, link_ancestors=None, force_cod session = self._session method = _get_method(link.action) encoding = _get_encoding(link.encoding, method) - params = _get_params(method, link.fields, params) + params = _get_params(method, encoding, link.fields, params) url = _get_url(link.url, params.path) headers = _get_headers(url, decoders, self.credentials) headers.update(self.headers) @@ -309,6 +311,6 @@ def transition(self, link, decoders, params=None, link_ancestors=None, force_cod result = _handle_inplace_replacements(result, link, link_ancestors) if isinstance(result, Error): - raise ErrorMessage(result) + raise exceptions.ErrorMessage(result) return result diff --git a/coreapi/utils.py b/coreapi/utils.py index 8e7de86..671ea26 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,5 +1,5 @@ from coreapi import exceptions -from coreapi.compat import urlparse +from coreapi.compat import string_types, urlparse def determine_transport(transports, url): @@ -69,3 +69,90 @@ def negotiate_encoder(encoders, accept=None): msg = "Unsupported media in Accept header '%s'" % accept raise exceptions.NotAcceptable(msg) + + +def validate_path_param(value, name): + value = _validate_form_primative(value, name, _allow_list=False) + if not value: + msg = 'Parameter %s: May not be empty.' + raise exceptions.ValidationError(msg % name) + return value + + +def validate_query_param(value, name): + return _validate_form_primative(value, name) + + +def validate_body_param(value, encoding, name): + if encoding == 'application/json': + return _validate_json_data(value, name) + elif encoding in ('multipart/form', 'application/x-www-form-urlencoded'): + if not isinstance(value, dict): + msg = 'Parameter %s: Must be an object.' + raise exceptions.ValidationError(msg % name) + return { + item_key: _validate_form_primative(item_val, name) + for item_key, item_val in value.items() + } + _unsupported_encoding(encoding) + + +def validate_form_data(value, encoding, name): + if encoding == 'application/json': + return _validate_json_data(value, name) + elif encoding in ('multipart/form', 'application/x-www-form-urlencoded'): + return _validate_form_primative(value, name) + _unsupported_encoding(encoding) + + +def validate_form_files(value, encoding, name): + if encoding == 'multipart/form': + return value + elif encoding in ('application/json', 'application/x-www-form-urlencoded'): + msg = 'Parameter %s: File uploads not supported.' + raise exceptions.ValidationError(msg % name) + _unsupported_encoding(encoding) + + +def _validate_form_primative(value, name, _allow_list=True): + """ + Parameters in query parameters or form data should be basic types, that + have a simple string representation. A list of basic types is also valid. + """ + if isinstance(value, string_types): + return value + elif isinstance(value, bool) or (value is None): + return {True: 'true', False: 'false', None: ''}[value] + elif isinstance(value, (int, float)): + return "%s" % value + elif _allow_list and isinstance(value, (list, tuple)): + return [ + _validate_form_primative(item, name, _allow_list=False) + for item in value + ] + + msg = 'Parameter %s: Must be a primative type.' + raise exceptions.ValidationError(msg % name) + + +def _validate_json_data(value, name): + if (value is None) or isinstance(value, string_types + (bool, int, float)): + return value + elif isinstance(value, (list, tuple)): + return [_validate_json_data(item, name) for item in value] + elif isinstance(value, dict): + return { + item_key: _validate_json_data(item_val, name) + for item_key, item_val in value.items() + } + + msg = 'Parameter %s: Must be a JSON primative.' + raise exceptions.ValidationError(msg % name) + + +def _unsupported_encoding(encoding): + if not encoding: + msg = 'Link has no encoding, but includes "body" or "form" parameters.' + raise exceptions.InvalidLinkError(msg) + msg = 'Link has unsupported encoding "%s".' + raise exceptions.InvalidLinkError(msg % encoding) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..077f1bc --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,51 @@ +from coreapi import exceptions, utils +import datetime +import pytest + + +def test_validate_path_param(): + assert utils.validate_path_param(1, name='example') == '1' + assert utils.validate_path_param(True, name='example') == 'true' + with pytest.raises(exceptions.ValidationError): + utils.validate_path_param(None, name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_path_param('', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_path_param({}, name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_path_param([], name='example') + + +def test_validate_query_param(): + assert utils.validate_query_param(1, name='example') == '1' + assert utils.validate_query_param(True, name='example') == 'true' + assert utils.validate_query_param(None, name='example') == '' + assert utils.validate_query_param('', name='example') == '' + assert utils.validate_query_param([1, 2, 3], name='example') == ['1', '2', '3'] + with pytest.raises(exceptions.ValidationError): + utils.validate_query_param({}, name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_query_param([1, 2, {}], name='example') + + +def test_validate_form_data(): + data = { + 'string': 'abc', + 'integer': 123, + 'number': 123.456, + 'boolean': True, + 'null': None, + 'array': [1, 2, 3], + 'object': {'a': 1, 'b': 2, 'c': 3} + } + assert utils.validate_form_data(data, 'application/json', name='example') == data + with pytest.raises(exceptions.ValidationError): + data = datetime.datetime.now() + utils.validate_form_data(data, 'application/json', name='example') + + assert utils.validate_form_data(123, 'application/x-www-form-urlencoded', name='example') == '123' + + with pytest.raises(exceptions.InvalidLinkError): + assert utils.validate_form_data(123, 'invalid/media-type', name='example') + with pytest.raises(exceptions.InvalidLinkError): + assert utils.validate_form_data(123, '', name='example') From 37af2f7883e7b75fa9b02333ab5bee6c47960f58 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 1 Sep 2016 11:13:09 +0100 Subject: [PATCH 14/74] Validate encodings --- coreapi/exceptions.py | 4 --- coreapi/transports/http.py | 38 ++++++++++++++--------- coreapi/utils.py | 62 +++++++++++++++++++------------------- tests/test_utils.py | 43 +++++++++++++++++++++----- 4 files changed, 89 insertions(+), 58 deletions(-) diff --git a/coreapi/exceptions.py b/coreapi/exceptions.py index 93db841..a88b902 100644 --- a/coreapi/exceptions.py +++ b/coreapi/exceptions.py @@ -48,10 +48,6 @@ class ValidationError(Exception): pass -class InvalidLinkError(Exception): - pass - - class ErrorMessage(Exception): """ Raised when the transition returns an error message. diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index ac46ea2..5af7edc 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -34,6 +34,12 @@ def _get_method(action): return action.upper() +def _get_encoding(encoding): + if not encoding: + return 'application/json' + return encoding + + def _get_params(method, encoding, fields, params=None): """ Separate the params into the various types. @@ -48,6 +54,10 @@ def _get_params(method, encoding, fields, params=None): data = {} files = {} + # Ensure graceful behavior in edge-case where both location='body' and + # location='form' fields are present. + seen_body = False + for key, value in params.items(): if key not in field_map or not field_map[key].location: # Default is 'query' for 'GET' and 'DELETE', and 'form' for others. @@ -55,29 +65,27 @@ def _get_params(method, encoding, fields, params=None): else: location = field_map[key].location + if location == 'form' and encoding == 'application/octet-stream': + # Raw uploads should always use 'body', not 'form'. + location = 'body' + if location == 'path': path[key] = utils.validate_path_param(value, name=key) elif location == 'query': query[key] = utils.validate_query_param(value, name=key) elif location == 'body': data = utils.validate_body_param(value, encoding=encoding, name=key) + seen_body = True elif location == 'form': - if is_file(value): - files[key] = utils.validate_form_files(value, encoding=encoding, name=key) - else: - data[key] = utils.validate_form_data(value, encoding=encoding, name=key) + if not seen_body: + data[key] = utils.validate_form_param(value, encoding=encoding, name=key) - return Params(path, query, data, files) + if isinstance(data, dict): + for key, value in list(data.items()): + if is_file(data[key]): + files[key] = data.pop(key) - -def _get_encoding(encoding, method): - if encoding: - return encoding - - if method in ('GET', 'DELETE'): - return '' - - return 'application/json' + return Params(path, query, data, files) def _get_url(url, path_params): @@ -291,7 +299,7 @@ def headers(self): def transition(self, link, decoders, params=None, link_ancestors=None, force_codec=False): session = self._session method = _get_method(link.action) - encoding = _get_encoding(link.encoding, method) + encoding = _get_encoding(link.encoding) params = _get_params(method, encoding, link.fields, params) url = _get_url(link.url, params.path) headers = _get_headers(url, decoders, self.credentials) diff --git a/coreapi/utils.py b/coreapi/utils.py index 671ea26..6afd55c 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,5 +1,5 @@ from coreapi import exceptions -from coreapi.compat import string_types, urlparse +from coreapi.compat import is_file, string_types, text_type, urlparse def determine_transport(transports, url): @@ -72,7 +72,7 @@ def negotiate_encoder(encoders, accept=None): def validate_path_param(value, name): - value = _validate_form_primative(value, name, _allow_list=False) + value = _validate_form_primative(value, name, allow_list=False) if not value: msg = 'Parameter %s: May not be empty.' raise exceptions.ValidationError(msg % name) @@ -86,35 +86,40 @@ def validate_query_param(value, name): def validate_body_param(value, encoding, name): if encoding == 'application/json': return _validate_json_data(value, name) - elif encoding in ('multipart/form', 'application/x-www-form-urlencoded'): - if not isinstance(value, dict): - msg = 'Parameter %s: Must be an object.' + elif encoding == 'multipart/form': + return _validate_form_object(value, name, allow_files=True) + elif encoding == 'application/x-www-form-urlencoded': + return _validate_form_object(value, name) + elif encoding == 'application/octet-stream': + if not is_file(value): + msg = 'Parameter %s: Must be an file upload.' raise exceptions.ValidationError(msg % name) - return { - item_key: _validate_form_primative(item_val, name) - for item_key, item_val in value.items() - } - _unsupported_encoding(encoding) + msg = 'Unsupported encoding "%s" for outgoing request.' + raise exceptions.TransportError(msg % encoding) -def validate_form_data(value, encoding, name): +def validate_form_param(value, encoding, name): if encoding == 'application/json': return _validate_json_data(value, name) - elif encoding in ('multipart/form', 'application/x-www-form-urlencoded'): + elif encoding == 'multipart/form': + return _validate_form_primative(value, name, allow_files=True) + elif encoding == 'application/x-www-form-urlencoded': return _validate_form_primative(value, name) - _unsupported_encoding(encoding) + msg = 'Unsupported encoding "%s" for outgoing request.' + raise exceptions.TransportError(msg % encoding) -def validate_form_files(value, encoding, name): - if encoding == 'multipart/form': - return value - elif encoding in ('application/json', 'application/x-www-form-urlencoded'): - msg = 'Parameter %s: File uploads not supported.' +def _validate_form_object(value, name, allow_files=False): + if not isinstance(value, dict): + msg = 'Parameter %s: Must be an object.' raise exceptions.ValidationError(msg % name) - _unsupported_encoding(encoding) + return { + text_type(item_key): _validate_form_primative(item_val, name, allow_files=allow_files) + for item_key, item_val in value.items() + } -def _validate_form_primative(value, name, _allow_list=True): +def _validate_form_primative(value, name, allow_files=False, allow_list=True): """ Parameters in query parameters or form data should be basic types, that have a simple string representation. A list of basic types is also valid. @@ -125,11 +130,14 @@ def _validate_form_primative(value, name, _allow_list=True): return {True: 'true', False: 'false', None: ''}[value] elif isinstance(value, (int, float)): return "%s" % value - elif _allow_list and isinstance(value, (list, tuple)): + elif allow_list and isinstance(value, (list, tuple)): + # Only the top-level element may be a list. return [ - _validate_form_primative(item, name, _allow_list=False) + _validate_form_primative(item, name, allow_files=False, allow_list=False) for item in value ] + elif allow_files and is_file(value): + return value msg = 'Parameter %s: Must be a primative type.' raise exceptions.ValidationError(msg % name) @@ -142,17 +150,9 @@ def _validate_json_data(value, name): return [_validate_json_data(item, name) for item in value] elif isinstance(value, dict): return { - item_key: _validate_json_data(item_val, name) + text_type(item_key): _validate_json_data(item_val, name) for item_key, item_val in value.items() } msg = 'Parameter %s: Must be a JSON primative.' raise exceptions.ValidationError(msg % name) - - -def _unsupported_encoding(encoding): - if not encoding: - msg = 'Link has no encoding, but includes "body" or "form" parameters.' - raise exceptions.InvalidLinkError(msg) - msg = 'Link has unsupported encoding "%s".' - raise exceptions.InvalidLinkError(msg % encoding) diff --git a/tests/test_utils.py b/tests/test_utils.py index 077f1bc..576991b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -29,6 +29,7 @@ def test_validate_query_param(): def test_validate_form_data(): + # Valid JSON data = { 'string': 'abc', 'integer': 123, @@ -38,14 +39,40 @@ def test_validate_form_data(): 'array': [1, 2, 3], 'object': {'a': 1, 'b': 2, 'c': 3} } - assert utils.validate_form_data(data, 'application/json', name='example') == data + assert utils.validate_form_param(data, 'application/json', name='example') == data + assert utils.validate_body_param(data, 'application/json', name='example') == data + + # Invalid JSON + data = datetime.datetime.now() + with pytest.raises(exceptions.ValidationError): + utils.validate_form_param(data, 'application/json', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_body_param(data, 'application/json', name='example') + + # URL Encoded + assert utils.validate_form_param(123, 'application/x-www-form-urlencoded', name='example') == '123' + assert utils.validate_body_param({'a': 123}, 'application/x-www-form-urlencoded', name='example') == {'a': '123'} + with pytest.raises(exceptions.ValidationError): + utils.validate_form_param({'a': {'foo': 'bar'}}, 'application/x-www-form-urlencoded', name='example') with pytest.raises(exceptions.ValidationError): - data = datetime.datetime.now() - utils.validate_form_data(data, 'application/json', name='example') + utils.validate_body_param(123, 'application/x-www-form-urlencoded', name='example') - assert utils.validate_form_data(123, 'application/x-www-form-urlencoded', name='example') == '123' + # Multipart + assert utils.validate_form_param(123, 'multipart/form', name='example') == '123' + assert utils.validate_body_param({'a': 123}, 'multipart/form', name='example') == {'a': '123'} + with pytest.raises(exceptions.ValidationError): + utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_body_param(123, 'multipart/form', name='example') + + # Raw upload + with pytest.raises(exceptions.ValidationError): + utils.validate_body_param(123, 'application/octet-stream', name='example') - with pytest.raises(exceptions.InvalidLinkError): - assert utils.validate_form_data(123, 'invalid/media-type', name='example') - with pytest.raises(exceptions.InvalidLinkError): - assert utils.validate_form_data(123, '', name='example') + # Invalid encoding on outgoing request + with pytest.raises(exceptions.TransportError): + assert utils.validate_form_param(123, 'invalid/media-type', name='example') + with pytest.raises(exceptions.TransportError): + assert utils.validate_form_param(123, '', name='example') + with pytest.raises(exceptions.TransportError): + assert utils.validate_body_param(123, 'invalid/media-type', name='example') From 07734407cbda200f2a831e774955f538959cb35f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 1 Sep 2016 12:34:51 +0100 Subject: [PATCH 15/74] Add File namedtuple --- coreapi/compat.py | 8 ++---- coreapi/transports/http.py | 51 +++++++++++++++++----------------- coreapi/utils.py | 57 +++++++++++++++++++++++++++++--------- tests/test_utils.py | 12 ++++++++ 4 files changed, 84 insertions(+), 44 deletions(-) diff --git a/coreapi/compat.py b/coreapi/compat.py index ce896e2..c3e3eb6 100644 --- a/coreapi/compat.py +++ b/coreapi/compat.py @@ -15,12 +15,10 @@ string_types = (basestring,) text_type = unicode + basestring = basestring COMPACT_SEPARATORS = (b',', b':') VERBOSE_SEPARATORS = (b',', b': ') - def is_file(obj): - return isinstance(obj, file) - def b64encode(input_string): # Provide a consistently-as-unicode interface across 2.x and 3.x return base64.b64encode(input_string) @@ -32,12 +30,10 @@ def b64encode(input_string): string_types = (str,) text_type = str + basestring = (str, bytes) COMPACT_SEPARATORS = (',', ':') VERBOSE_SEPARATORS = (',', ': ') - def is_file(obj): - return isinstance(obj, IOBase) - def b64encode(input_string): # Provide a consistently-as-unicode interface across 2.x and 3.x return base64.b64encode(input_string.encode('ascii')).decode('ascii') diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 5af7edc..6866f13 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -2,14 +2,14 @@ from __future__ import unicode_literals from collections import OrderedDict from coreapi import exceptions, utils -from coreapi.compat import is_file, urlparse +from coreapi.compat import urlparse from coreapi.document import Document, Object, Link, Array, Error from coreapi.transports.base import BaseTransport +from coreapi.utils import guess_filename, is_file, File import collections import requests import itypes import mimetypes -import os import uritemplate @@ -119,29 +119,29 @@ def _get_headers(url, decoders, credentials=None): return headers -def _get_upload_content_type(file_obj): +def _get_upload_headers(file_obj): """ - When a raw file upload is made, determine a content-type where possible. + When a raw file upload is made, determine the Content-Type and + Content-Disposition headers to use with the request. """ - name = getattr(file_obj, 'name', None) - if name is not None: + name = guess_filename(file_obj) + content_type = None + content_disposition = None + + # Determine the content type of the upload. + if getattr(file_obj, 'content_type', None): + content_type = file_obj.content_type + elif name: content_type, encoding = mimetypes.guess_type(name) - else: - content_type = 'application/octet-stream' - return content_type + # Determine the content disposition of the upload. + if name: + content_disposition = 'attachment; filename="%s"' % name -def _get_upload_content_disposition(file_obj): - """ - When a raw file upload is made, determine a content-type where possible. - """ - name = getattr(file_obj, 'name', None) - if name is not None: - filename = os.path.basename(file_obj.name) - content_disposition = 'attachment; filename="%s"' % filename - else: - content_disposition = 'attachment' - return content_disposition + return { + 'Content-Type': content_type or 'application/octet-stream', + 'Content-Disposition': content_disposition or 'attachment' + } def _build_http_request(session, url, method, headers=None, encoding=None, params=empty_params): @@ -164,11 +164,12 @@ def _build_http_request(session, url, method, headers=None, encoding=None, param elif encoding == 'application/x-www-form-urlencoded': opts['data'] = params.data elif encoding == 'application/octet-stream': - opts['data'] = params.data - content_type = _get_upload_content_type(params.data) - content_disposition = _get_upload_content_disposition(params.data) - opts['headers']['content-type'] = content_type - opts['headers']['content-disposition'] = content_disposition + if isinstance(params.data, File): + opts['data'] = params.data.content + else: + opts['data'] = params.data + upload_headers = _get_upload_headers(params.data) + opts['headers'].update(upload_headers) request = requests.Request(method, url, **opts) return session.prepare_request(request) diff --git a/coreapi/utils.py b/coreapi/utils.py index 6afd55c..429be5f 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,5 +1,29 @@ from coreapi import exceptions -from coreapi.compat import is_file, string_types, text_type, urlparse +from coreapi.compat import string_types, text_type, urlparse +from collections import namedtuple +import os + + +File = namedtuple('File', 'name content content_type') +File.__new__.__defaults__ = (None,) + + +def is_file(obj): + if isinstance(obj, File): + return True + + if hasattr(obj, '__iter__') and not isinstance(obj, (basestring, list, tuple, dict)): + # A stream object. + return True + + return False + + +def guess_filename(obj): + name = getattr(obj, 'name', None) + if name and isinstance(name, basestring) and name[0] != '<' and name[-1] != '>': + return os.path.basename(name) + return None def determine_transport(transports, url): @@ -72,7 +96,7 @@ def negotiate_encoder(encoders, accept=None): def validate_path_param(value, name): - value = _validate_form_primative(value, name, allow_list=False) + value = _validate_form_field(value, name, allow_list=False) if not value: msg = 'Parameter %s: May not be empty.' raise exceptions.ValidationError(msg % name) @@ -80,7 +104,7 @@ def validate_path_param(value, name): def validate_query_param(value, name): - return _validate_form_primative(value, name) + return _validate_form_field(value, name) def validate_body_param(value, encoding, name): @@ -102,27 +126,31 @@ def validate_form_param(value, encoding, name): if encoding == 'application/json': return _validate_json_data(value, name) elif encoding == 'multipart/form': - return _validate_form_primative(value, name, allow_files=True) + return _validate_form_field(value, name, allow_files=True) elif encoding == 'application/x-www-form-urlencoded': - return _validate_form_primative(value, name) + return _validate_form_field(value, name) msg = 'Unsupported encoding "%s" for outgoing request.' raise exceptions.TransportError(msg % encoding) def _validate_form_object(value, name, allow_files=False): + """ + Ensure that `value` can be encoded as form data or as query parameters. + """ if not isinstance(value, dict): msg = 'Parameter %s: Must be an object.' raise exceptions.ValidationError(msg % name) return { - text_type(item_key): _validate_form_primative(item_val, name, allow_files=allow_files) + text_type(item_key): _validate_form_field(item_val, name, allow_files=allow_files) for item_key, item_val in value.items() } -def _validate_form_primative(value, name, allow_files=False, allow_list=True): +def _validate_form_field(value, name, allow_files=False, allow_list=True): """ - Parameters in query parameters or form data should be basic types, that - have a simple string representation. A list of basic types is also valid. + Ensure that `value` can be encoded as a single form data or a query parameter. + Basic types that has a simple string representation are supported. + A list of basic types is also valid. """ if isinstance(value, string_types): return value @@ -130,10 +158,10 @@ def _validate_form_primative(value, name, allow_files=False, allow_list=True): return {True: 'true', False: 'false', None: ''}[value] elif isinstance(value, (int, float)): return "%s" % value - elif allow_list and isinstance(value, (list, tuple)): + elif allow_list and isinstance(value, (list, tuple)) and not is_file(value): # Only the top-level element may be a list. return [ - _validate_form_primative(item, name, allow_files=False, allow_list=False) + _validate_form_field(item, name, allow_files=False, allow_list=False) for item in value ] elif allow_files and is_file(value): @@ -144,9 +172,12 @@ def _validate_form_primative(value, name, allow_files=False, allow_list=True): def _validate_json_data(value, name): - if (value is None) or isinstance(value, string_types + (bool, int, float)): + """ + Ensure that `value` can be encoded into JSON. + """ + if (value is None) or isinstance(value, (bool, int, float, basestring)): return value - elif isinstance(value, (list, tuple)): + elif isinstance(value, (list, tuple)) and not is_file(value): return [_validate_json_data(item, name) for item in value] elif isinstance(value, dict): return { diff --git a/tests/test_utils.py b/tests/test_utils.py index 576991b..15602c2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -49,6 +49,12 @@ def test_validate_form_data(): with pytest.raises(exceptions.ValidationError): utils.validate_body_param(data, 'application/json', name='example') + data = utils.File('abc.txt', None) + with pytest.raises(exceptions.ValidationError): + utils.validate_form_param(data, 'application/json', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_body_param(data, 'application/json', name='example') + # URL Encoded assert utils.validate_form_param(123, 'application/x-www-form-urlencoded', name='example') == '123' assert utils.validate_body_param({'a': 123}, 'application/x-www-form-urlencoded', name='example') == {'a': '123'} @@ -56,10 +62,16 @@ def test_validate_form_data(): utils.validate_form_param({'a': {'foo': 'bar'}}, 'application/x-www-form-urlencoded', name='example') with pytest.raises(exceptions.ValidationError): utils.validate_body_param(123, 'application/x-www-form-urlencoded', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_form_param(utils.File('abc.txt', None), 'application/x-www-form-urlencoded', name='example') + with pytest.raises(exceptions.ValidationError): + utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'application/x-www-form-urlencoded', name='example') # Multipart assert utils.validate_form_param(123, 'multipart/form', name='example') == '123' + assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form', name='example') == utils.File('abc.txt', None) assert utils.validate_body_param({'a': 123}, 'multipart/form', name='example') == {'a': '123'} + assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form', name='example') == {'a': utils.File('abc.txt', None)} with pytest.raises(exceptions.ValidationError): utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form', name='example') with pytest.raises(exceptions.ValidationError): From 89105ee818e64559d331c4dc5b4546e868bcacc0 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 1 Sep 2016 12:39:08 +0100 Subject: [PATCH 16/74] Tweak for py3 compat --- coreapi/__init__.py | 2 +- coreapi/compat.py | 2 -- coreapi/utils.py | 6 +++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index f894276..2bd47c0 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '1.32.3' +__version__ = '2.0.0' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/compat.py b/coreapi/compat.py index c3e3eb6..476789d 100644 --- a/coreapi/compat.py +++ b/coreapi/compat.py @@ -15,7 +15,6 @@ string_types = (basestring,) text_type = unicode - basestring = basestring COMPACT_SEPARATORS = (b',', b':') VERBOSE_SEPARATORS = (b',', b': ') @@ -30,7 +29,6 @@ def b64encode(input_string): string_types = (str,) text_type = str - basestring = (str, bytes) COMPACT_SEPARATORS = (',', ':') VERBOSE_SEPARATORS = (',', ': ') diff --git a/coreapi/utils.py b/coreapi/utils.py index 429be5f..adfb9e5 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -12,7 +12,7 @@ def is_file(obj): if isinstance(obj, File): return True - if hasattr(obj, '__iter__') and not isinstance(obj, (basestring, list, tuple, dict)): + if hasattr(obj, '__iter__') and not isinstance(obj, (string_types, list, tuple, dict)): # A stream object. return True @@ -21,7 +21,7 @@ def is_file(obj): def guess_filename(obj): name = getattr(obj, 'name', None) - if name and isinstance(name, basestring) and name[0] != '<' and name[-1] != '>': + if name and isinstance(name, string_types) and name[0] != '<' and name[-1] != '>': return os.path.basename(name) return None @@ -175,7 +175,7 @@ def _validate_json_data(value, name): """ Ensure that `value` can be encoded into JSON. """ - if (value is None) or isinstance(value, (bool, int, float, basestring)): + if (value is None) or isinstance(value, (bool, int, float, string_types)): return value elif isinstance(value, (list, tuple)) and not is_file(value): return [_validate_json_data(item, name) for item in value] From ace504afeaa8daa9f7a5d88ad2714ecbca7a0d56 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 6 Sep 2016 16:23:12 +0100 Subject: [PATCH 17/74] Docs & tweaks --- coreapi/client.py | 21 ++--- coreapi/codecs/display.py | 5 +- coreapi/codecs/download.py | 20 ++--- coreapi/document.py | 8 +- coreapi/exceptions.py | 28 +++--- coreapi/transports/http.py | 28 +++--- coreapi/utils.py | 70 +++++++-------- docs/api-guide/client.md | 16 +++- docs/api-guide/codecs.md | 142 ++++++++++++++++++++++++++---- docs/api-guide/document.md | 164 +++++++++++++++++++++++++++++++---- docs/api-guide/exceptions.md | 26 ++++-- docs/api-guide/transports.md | 3 + docs/api-guide/utils.md | 81 ++++++++++++++--- docs/index.md | 60 ++++++++++--- docs/topics/release-notes.md | 1 + mkdocs.yml | 13 ++- tests/test_codecs.py | 6 +- tests/test_transport.py | 8 +- tests/test_utils.py | 76 ++++++++-------- 19 files changed, 576 insertions(+), 200 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index 886478c..da5b031 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -64,19 +64,20 @@ def _validate_parameters(link, parameters): field.name for field in link.fields if not field.required ]) - # Determine any parameter names supplied that are not valid. - unexpected = provided - (optional | required) - unexpected = ['"' + item + '"' for item in sorted(unexpected)] - if unexpected: - prefix = len(unexpected) > 1 and 'parameters: ' or 'parameter: ' - raise exceptions.ValidationError('Unknown ' + prefix + ', '.join(unexpected)) + errors = {} # Determine if any required field names not supplied. missing = required - provided - missing = ['"' + item + '"' for item in sorted(missing)] - if missing: - prefix = len(missing) > 1 and 'parameters: ' or 'parameter: ' - raise exceptions.ValidationError('Missing required ' + prefix + ', '.join(missing)) + for item in missing: + errors[item] = 'This parameter is required.' + + # Determine any parameter names supplied that are not valid. + unexpected = provided - (optional | required) + for item in unexpected: + errors[item] = 'Unknown parameter.' + + if errors: + raise exceptions.ValidationError(errors) def get_default_decoders(): diff --git a/coreapi/codecs/display.py b/coreapi/codecs/display.py index 663ba24..ab2fbd6 100644 --- a/coreapi/codecs/display.py +++ b/coreapi/codecs/display.py @@ -117,5 +117,6 @@ class DisplayCodec(BaseCodec): media_type = 'text/plain' supports = ['encode'] - def encode(self, node, colorize=False, **kwargs): - return _to_plaintext(node, colorize=colorize) + def encode(self, document, **options): + colorize = options.get('colorize', False) + return _to_plaintext(document, colorize=colorize) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 7e8be26..79585ba 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -10,12 +10,13 @@ class DownloadedFile(tempfile._TemporaryFileWrapper): - """ - A wrapper around the returned file object, in order to provide - a clearer interface than simply returning a file handle. - """ - def __init__(self, file, name): - super(DownloadedFile, self).__init__(file, name, delete=False) + def __repr__(self): + state = "closed" if self.close_called else "open" + mode = "" if self.close_called else " '%s'" % self.file.mode + return "" % (self.name, state, mode) + + def __str__(self): + return self.__repr__() def _unique_output_path(path): @@ -111,7 +112,7 @@ def __init__(self, download_dir=None): """ `download_dir` - The path to use for file downloads. """ - self._temporary = False + self._temporary = download_dir is None self._download_dir = download_dir def __del__(self): @@ -121,7 +122,6 @@ def __del__(self): @property def download_dir(self): if self._download_dir is None: - self._temporary = True self._download_dir = tempfile.mkdtemp(prefix='temp-coreapi-download-') return self._download_dir @@ -131,7 +131,7 @@ def decode(self, bytestring, **options): content_disposition = options.get('content_disposition') # Write the download to a temporary .download file. - fd, temp_path = tempfile.mkstemp(dir=self.download_dir, suffix='.download') + fd, temp_path = tempfile.mkstemp(suffix='.download') file_handle = os.fdopen(fd, 'wb') file_handle.write(bytestring) file_handle.close() @@ -149,4 +149,4 @@ def decode(self, bytestring, **options): # Move the temporary download file to the final location. os.rename(temp_path, output_path) output_file = open(output_path, 'rb') - return DownloadedFile(output_file, output_path) + return DownloadedFile(output_file, output_path, delete=self._temporary) diff --git a/coreapi/document.py b/coreapi/document.py index c87daa5..a6862e6 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -50,20 +50,20 @@ class Document(itypes.Dict): and the actions that the client may perform. """ def __init__(self, url=None, title=None, content=None): - data = {} if (content is None) else content + content = {} if (content is None) else content if url is not None and not isinstance(url, string_types): raise TypeError("'url' must be a string.") if title is not None and not isinstance(title, string_types): raise TypeError("'title' must be a string.") - if content is not None and not isinstance(content, dict): + if not isinstance(content, dict): raise TypeError("'content' must be a dict.") - if any([not isinstance(key, string_types) for key in data.keys()]): + if any([not isinstance(key, string_types) for key in content.keys()]): raise TypeError('content keys must be strings.') self._url = '' if (url is None) else url self._title = '' if (title is None) else title - self._data = {key: _to_immutable(value) for key, value in data.items()} + self._data = {key: _to_immutable(value) for key, value in content.items()} def clone(self, data): return self.__class__(self.url, self.title, data) diff --git a/coreapi/exceptions.py b/coreapi/exceptions.py index a88b902..3969142 100644 --- a/coreapi/exceptions.py +++ b/coreapi/exceptions.py @@ -2,53 +2,53 @@ from __future__ import unicode_literals -class ParseError(Exception): +class CoreAPIException(Exception): """ - Raised when an invalid Core API encoding is encountered. + A base class for all `coreapi` exceptions. """ pass -class UnsupportedContentType(Exception): +class ParseError(CoreAPIException): """ - Raised when the media specified in the reponse 'Content-Type' header - is not supported. + Raised when an invalid Core API encoding is encountered. """ pass -class NotAcceptable(Exception): +class NoCodecAvailable(CoreAPIException): """ - Raised when the client 'Accept' header could not be satisfied. + Raised when there is no available codec that can handle the given media. """ pass -class TransportError(Exception): +class NetworkError(CoreAPIException): """ Raised when the transport layer fails to make a request or get a response. """ pass -class LinkLookupError(Exception): +class LinkLookupError(CoreAPIException): """ Raised when `.action` fails to index a link in the document. """ pass -class ValidationError(Exception): +class ValidationError(CoreAPIException): """ Raised when the parameters passed do not match the link fields. - * One or more parameters were passed that do not have a corresponding field. - * One or more required fields did not have a corresponding parameter passed. + * A required field was not included. + * An unknown field was included. + * A field was passed an invalid type for the link location/encoding. """ pass -class ErrorMessage(Exception): +class ErrorMessage(CoreAPIException): """ Raised when the transition returns an error message. """ @@ -56,7 +56,7 @@ def __init__(self, error): self.error = error def __repr__(self): - return 'ErrorMessage(%s)' % repr(self.error) + return '%s(%s)' % (self.__class__.__name__, repr(self.error)) def __str__(self): return str(self.error) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 6866f13..d0eddb6 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -54,6 +54,8 @@ def _get_params(method, encoding, fields, params=None): data = {} files = {} + errors = {} + # Ensure graceful behavior in edge-case where both location='body' and # location='form' fields are present. seen_body = False @@ -69,16 +71,22 @@ def _get_params(method, encoding, fields, params=None): # Raw uploads should always use 'body', not 'form'. location = 'body' - if location == 'path': - path[key] = utils.validate_path_param(value, name=key) - elif location == 'query': - query[key] = utils.validate_query_param(value, name=key) - elif location == 'body': - data = utils.validate_body_param(value, encoding=encoding, name=key) - seen_body = True - elif location == 'form': - if not seen_body: - data[key] = utils.validate_form_param(value, encoding=encoding, name=key) + try: + if location == 'path': + path[key] = utils.validate_path_param(value) + elif location == 'query': + query[key] = utils.validate_query_param(value) + elif location == 'body': + data = utils.validate_body_param(value, encoding=encoding) + seen_body = True + elif location == 'form': + if not seen_body: + data[key] = utils.validate_form_param(value, encoding=encoding) + except exceptions.ValidationError as exc: + errors[key] = exc.message + + if errors: + raise exceptions.ValidationError(errors) if isinstance(data, dict): for key, value in list(data.items()): diff --git a/coreapi/utils.py b/coreapi/utils.py index adfb9e5..215532f 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -35,16 +35,16 @@ def determine_transport(transports, url): netloc = url_components.netloc if not scheme: - raise exceptions.TransportError("URL missing scheme '%s'." % url) + raise exceptions.NetworkError("URL missing scheme '%s'." % url) if not netloc: - raise exceptions.TransportError("URL missing hostname '%s'." % url) + raise exceptions.NetworkError("URL missing hostname '%s'." % url) for transport in transports: if scheme in transport.schemes: return transport - raise exceptions.TransportError("Unsupported URL scheme '%s'." % scheme) + raise exceptions.NetworkError("Unsupported URL scheme '%s'." % scheme) def negotiate_decoder(decoders, content_type=None): @@ -64,7 +64,7 @@ def negotiate_decoder(decoders, content_type=None): return codec msg = "Unsupported media in Content-Type header '%s'" % content_type - raise exceptions.UnsupportedContentType(msg) + raise exceptions.NoCodecAvailable(msg) def negotiate_encoder(encoders, accept=None): @@ -92,61 +92,61 @@ def negotiate_encoder(encoders, accept=None): return encoders[0] msg = "Unsupported media in Accept header '%s'" % accept - raise exceptions.NotAcceptable(msg) + raise exceptions.NoCodecAvailable(msg) -def validate_path_param(value, name): - value = _validate_form_field(value, name, allow_list=False) +def validate_path_param(value): + value = _validate_form_field(value, allow_list=False) if not value: msg = 'Parameter %s: May not be empty.' - raise exceptions.ValidationError(msg % name) + raise exceptions.ValidationError(msg) return value -def validate_query_param(value, name): - return _validate_form_field(value, name) +def validate_query_param(value): + return _validate_form_field(value) -def validate_body_param(value, encoding, name): +def validate_body_param(value, encoding): if encoding == 'application/json': - return _validate_json_data(value, name) + return _validate_json_data(value) elif encoding == 'multipart/form': - return _validate_form_object(value, name, allow_files=True) + return _validate_form_object(value, allow_files=True) elif encoding == 'application/x-www-form-urlencoded': - return _validate_form_object(value, name) + return _validate_form_object(value) elif encoding == 'application/octet-stream': if not is_file(value): - msg = 'Parameter %s: Must be an file upload.' - raise exceptions.ValidationError(msg % name) + msg = 'Must be an file upload.' + raise exceptions.ValidationError(msg) msg = 'Unsupported encoding "%s" for outgoing request.' - raise exceptions.TransportError(msg % encoding) + raise exceptions.NetworkError(msg % encoding) -def validate_form_param(value, encoding, name): +def validate_form_param(value, encoding): if encoding == 'application/json': - return _validate_json_data(value, name) + return _validate_json_data(value) elif encoding == 'multipart/form': - return _validate_form_field(value, name, allow_files=True) + return _validate_form_field(value, allow_files=True) elif encoding == 'application/x-www-form-urlencoded': - return _validate_form_field(value, name) + return _validate_form_field(value) msg = 'Unsupported encoding "%s" for outgoing request.' - raise exceptions.TransportError(msg % encoding) + raise exceptions.NetworkError(msg % encoding) -def _validate_form_object(value, name, allow_files=False): +def _validate_form_object(value, allow_files=False): """ Ensure that `value` can be encoded as form data or as query parameters. """ if not isinstance(value, dict): - msg = 'Parameter %s: Must be an object.' - raise exceptions.ValidationError(msg % name) + msg = 'Must be an object.' + raise exceptions.ValidationError(msg) return { - text_type(item_key): _validate_form_field(item_val, name, allow_files=allow_files) + text_type(item_key): _validate_form_field(item_val, allow_files=allow_files) for item_key, item_val in value.items() } -def _validate_form_field(value, name, allow_files=False, allow_list=True): +def _validate_form_field(value, allow_files=False, allow_list=True): """ Ensure that `value` can be encoded as a single form data or a query parameter. Basic types that has a simple string representation are supported. @@ -161,29 +161,29 @@ def _validate_form_field(value, name, allow_files=False, allow_list=True): elif allow_list and isinstance(value, (list, tuple)) and not is_file(value): # Only the top-level element may be a list. return [ - _validate_form_field(item, name, allow_files=False, allow_list=False) + _validate_form_field(item, allow_files=False, allow_list=False) for item in value ] elif allow_files and is_file(value): return value - msg = 'Parameter %s: Must be a primative type.' - raise exceptions.ValidationError(msg % name) + msg = 'Must be a primative type.' + raise exceptions.ValidationError(msg) -def _validate_json_data(value, name): +def _validate_json_data(value): """ Ensure that `value` can be encoded into JSON. """ if (value is None) or isinstance(value, (bool, int, float, string_types)): return value elif isinstance(value, (list, tuple)) and not is_file(value): - return [_validate_json_data(item, name) for item in value] + return [_validate_json_data(item) for item in value] elif isinstance(value, dict): return { - text_type(item_key): _validate_json_data(item_val, name) + text_type(item_key): _validate_json_data(item_val) for item_key, item_val in value.items() } - msg = 'Parameter %s: Must be a JSON primative.' - raise exceptions.ValidationError(msg % name) + msg = 'Must be a JSON primative.' + raise exceptions.ValidationError(msg) diff --git a/docs/api-guide/client.md b/docs/api-guide/client.md index ee1d654..b88f9f6 100644 --- a/docs/api-guide/client.md +++ b/docs/api-guide/client.md @@ -7,6 +7,8 @@ interactions against the API. An example client session might look something like this: + from coreapi import Client + client = Client() document = client.get('https://api.example.org/') data = client.action(document, ['flights', 'search'], params={ @@ -48,7 +50,8 @@ When no arguments are passed, the following defaults are used: decoders = [ codecs.CoreJSONCodec(), # application/vnd.coreapi+json codecs.JSONCodec(), # application/json - codecs.TextCodec() # text/* + codecs.TextCodec(), # text/* + codecs.DownloadCodec() # */* ] transports = [ @@ -65,17 +68,22 @@ properties on a client instance: ## Making an initial request -* get(url) +* `get(url)` Make a network request to the given URL, and return a decoded `Document`. +**TODO** + --- ## Interacting with an API -* action(self, document, keys, params=None, overrides=None, validate=True) +* `action(self, document, keys, params=None, overrides=None, validate=True)` Effect an interaction against the given document. -* `keys` - A list of strings that index a link within the document. +* `document` - A `Document` instance. +* `keys` - A list of strings that index a `Link` within the document. * `params` - A dictionary of parameters to use for the API interaction. + +**TODO** diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 3466ba8..7bb32fe 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -10,7 +10,7 @@ the bytestring returned in the body of the response. When using a Core API client, HTTP responses are decoded with an appropriate codec, based on the `Content-Type` of the response. -## Using codecs +## Using a Codec All the codecs provided by the `coreapi` library are instantiated without arguments, for example: @@ -21,11 +21,15 @@ A codec will provide either one or both of the `decode` or `encode` methods. ### Decoding -* decode(content, url=None) +* `decode(bytestring, **options)` + +**TODO** ### Encoding -* encode(document, **options) +* `encode(document, **options)` + +**TODO** ### Attributes @@ -36,12 +40,12 @@ The following attributes are available on codec instances: The `supports` option should be one of the four following options: -* `['decode', 'encode']` # Supports both decoding and encoding documents. -* `['decode']` # Supports decoding documents only. -* `['encode']` # Supports encoding documents only. -* `['data']` # Indicates that the codec supports decoding, - # but that it is expected to return plain data, - # rather than a `Document` object. +* `['decode', 'encode']` - Supports both decoding and encoding documents. +* `['decode']` - Supports decoding documents only. +* `['encode']` - Supports encoding documents only. +* `['data']` - Indicates that the codec supports decoding, + but that it is expected to return plain data, + rather than a `Document` object. --- @@ -57,8 +61,9 @@ Supports decoding or encoding the Core JSON format. Example of decoding a Core JSON bytestring into a `Document` instance: - >>> codec = codecs.TextCodec() - >>> content = b'{"_type": "document", ...' + >>> from coreapi import codecs + >>> codec = codecs.CoreJSONCodec() + >>> content = b'{"_type": "document", ...}' >>> document = codec.decode(content) >>> print(document) @@ -72,6 +77,15 @@ Example of encoding a `Document` instance into a Core JSON bytestring: "_type": "document" } +#### Encoding options + +**indent**: Set to `True` for an indented representation. The default is to generate a compact representation. + +#### Decoding options + +**base_url**: The URL from which the document was retrieved. Used to resolve any relative +URLs in the document. + --- ### JSONCodec @@ -84,7 +98,8 @@ Supports decoding JSON data. Example: - >>> codec = codecs.TextCodec() + >>> from coreapi import codecs + >>> codec = codecs.JSONCodec() >>> content = b'{"string": "abc", "boolean": true, "null": null}' >>> data = codec.decode(content) >>> print(data) @@ -102,6 +117,7 @@ Supports decoding plain-text responses. Example: + >>> from coreapi import codecs >>> codec = codecs.TextCodec() >>> data = codec.decode(b'hello, world!') >>> print(data) @@ -109,6 +125,48 @@ Example: --- +### DownloadCodec + +Supports decoding arbitrary media as a download file. Returns a temporary file +that will be deleted once it goes out of scope. + +**.media_type**: `*/*` + +**.supports**: `['data']` + +Example: + + >>> from coreapi import codecs + >>> codec = codecs.DownloadCodec() + >>> data = codec.decode(b'...') + >>> print(data) + + +#### Instantiation + +By default this codec returns a temporary file that will be deleted once it +goes out of scope. If you want to return temporary files that are not +deleted when they go out of scope then you can instantiate the `DownloadCodec` +with a `download_dir` argument. + +For example, to download files to the current working directory: + + >>> import os + >>> codecs.DownloadCodec(download_dir=os.getcwd()) + +#### Decoding options + +**base_url**: The URL from which the document was retrieved. May be used to +generate an output filename if no `Content-Disposition` header exists. + +**content_type**: The response Content-Type header. May be used to determine a +suffix for the output filename if no `Content-Disposition` header exists. + +**content_disposition**: The response Content-Disposition header. May be [used to +indicate the download filename][content-disposition-filename]. + +--- + ### DisplayCodec Supports encoding a `Document` to a display representation. @@ -119,12 +177,18 @@ Supports encoding a `Document` to a display representation. Example: + >>> from coreapi import codecs >>> codec = codecs.DisplayCodec() - >>> content = codec.encode(document, indent=True) + >>> content = codec.encode(document) >>> print(content) 'search': link(from, to, date) +#### Options + +**colorize**: Set to `True` to include ANSI color escapes for terminal representations. +See the Python `click` package [for more details][click-ansi]. + --- ### PythonCodec @@ -137,8 +201,9 @@ Supports encoding a `Document` to an its Python representation. Example: + >>> from coreapi import codecs >>> codec = codecs.PythonCodec() - >>> content = codec.encode(document, indent=True) + >>> content = codec.encode(document) >>> print(content) Document( title='Flight Search API', @@ -166,12 +231,15 @@ of the `decode` or `encode` methods. For example: + from coreapi import codecs + import yaml + class YAMLCodec(codecs.BaseCodec): media_type = 'application/yaml' supports = ['data'] - def decode(content, url=None): - return yaml... + def decode(content, **options): + return yaml.safe_load(content) ### The codec registry @@ -196,6 +264,48 @@ codecs which are registered by the `coreapi` package itself is as follows: 'corejson=coreapi.codecs:CoreJSONCodec', 'json=coreapi.codecs:JSONCodec', 'text=coreapi.codecs:TextCodec', + 'download=coreapi.codecs:DownloadCodec', ] } ) + +--- + +## External packages + +The following third-party packages are available. + +### OpenAPI + +A codec for [OpenAPI][openapi] schemas, also known as "Swagger". Installable [from PyPI][openapi-pypi] as `openapi-codec`, and [available on GitHub][openapi-github]. + +### JSON Hyper-Schema + +A codec for [JSON Hyper-Schema][jsonhyperschema]. Installable [from PyPI][jsonhyperschema-pypi] as `jsonhyperschema-codec`, and [available on GitHub][jsonhyperschema-github]. + +### API Blueprint + +A codec for [API Blueprint][apiblueprint] schemas. Installable [from PyPI][apiblueprint-pypi] as `apiblueprint-codec`, and [available on GitHub][apiblueprint-github]. + +### HAL + +A codec for the [HAL][hal] hypermedia format. Installable [from PyPI][hal-pypi] as `hal-codec`, and [available on GitHub][hal-github]. + +[content-disposition-filename]: https://tools.ietf.org/html/draft-ietf-httpbis-content-disp-00#section-3.3 +[click-ansi]: http://click.pocoo.org/5/utils/#ansi-colors + +[openapi]: https://openapis.org/specification +[openapi-pypi]: https://pypi.python.org/pypi/openapi-codec +[openapi-github]: https://github.com/core-api/python-openapi-codec + +[jsonhyperschema]: http://json-schema.org/latest/json-schema-hypermedia.html +[jsonhyperschema-pypi]: https://pypi.python.org/pypi/jsonhyperschema-codec +[jsonhyperschema-github]: https://github.com/core-api/python-jsonhyperschema-codec + +[apiblueprint]: https://apiblueprint.org/ +[apiblueprint-pypi]: https://pypi.python.org/pypi/apiblueprint-codec +[apiblueprint-github]: https://github.com/core-api/python-apiblueprint-codec + +[hal]: http://stateless.co/hal_specification.html +[hal-pypi]: https://pypi.python.org/pypi/hal-codec +[hal-github]: https://github.com/core-api/python-hal-codec diff --git a/docs/api-guide/document.md b/docs/api-guide/document.md index 37e116b..730d680 100644 --- a/docs/api-guide/document.md +++ b/docs/api-guide/document.md @@ -1,27 +1,155 @@ # Documents -## Document +A CoreAPI document is a primitive that may be used to represent either schema of hypermedia responses. -* url -* title -* content +By including information about the available interactions that an API exposes, +the document allows users to interact with the API at an interface level, rather +than a network level. -## Link +In the schema case a document will include only links. Interactions to the API +endpoints will typically return plain data. -* url -* action -* encoding -* transform -* description +In the hypermedia case a document will include both links and data. interactions +to the API endpoints will typically return a new document. -## Field +--- -* name -* required -* location -* description +## Usage -## Error +### Retrieving a document -* title -* content +Typically a Document will first be obtained by making a request with a +client instance. + + >>> document = client.get('https://api.example.com/users/') + +A document can also be loaded from a raw bytestring, by using a codec instance. + + >>> codec = codecs.CoreJSONCodec() + >>> bytestring = open('document.corejson', 'rb').read() + >>> document = codec.decode(bytestring) + +### Inspecting a document + +A document has some associated metadata that may be inspected. + + >>> document.url + 'https://api.example.com/' + >>> document.title + 'Example API' + +A document may contain content, which may include nested dictionaries and list. +The top level element is always a dictionary. The instance may be accessed using +Python's standard dictionary lookup syntax. + +Schema type documents will contain `Link` instances as the leaf nodes in the content. + + >>> document['users']['create'] + Link(url='https://api.example.com/users/', action='post', fields=[...]) + +Hypermedia documents will also contain `Link` instances, but may also contain +data, or nested `Document` instances. + + >>> document['results']['count'] + 45 + >>> document['results']['items'][0] + Document(url='https://api.example.com/users/0/', content={...}) + >>> document['results']['items'][0]['username'] + 'tomchristie' + +### Interacting with a document + +In order to interact with an API, a document is passed as the first argument to +a client instance. A list of strings indexing into a link in the document is passed +as the second argument. + + >>> data = client.action(document, ['users', 'list']) + +Some links may accept a set of parameters, each of which may be either required or optional. + + >>> data = client.action(document, ['users', 'list'], params={'is_admin': True}) + +A document may be reloaded, by fetching the `document.url` property. + + >>> document = client.get(document.url) # Reload the current document + +--- + +## Document primitives + +When using the `coreapi` library as an API client, you won't typically be instantiating +document instances, but rather retrieving them using the client. + +However, if you're using the `coreapi` library on the server-side, you can use +the document primitives directly, in order to create schema or hypermedia representations. +The document should then be encoded using an available codec in order to form the schema response. + +### Document + +The following are available attributes, and may be passed when instantiating a `Document`: + +* `url` - A string giving the canonical URL for this document. +* `title` - A string describing this document. +* `content` - A dictionary containing all the data and links made available by this document. + +A document instance also supports dictionary-style lookup on it's contents. + + >>> document['results']['count'] + 45 + +The following properties are available on a document instance, and on any +nested dictionaries it contains: + +* `links` - A dictionary-like property including only items that are `Link` instances. +* `data` - A dictionary-like property including only items that are not `Link` instances. + +### Link + +The following are available attributes, and may be passed when instantiating a `Link`: + +* `url` - A string giving the URL against which the request should be made. +* `action` - A string giving the type of outgoing request that should be made. +* `encoding` - A string giving the encoding used for outgoing requests. +* `transform` - A string describing how the response should +* `description` - A string describing this link. +* `fields` - A list of field instances. + +### Field + +The following are available attributes, and may be passed when instantiating a `Field`: + +* `name` - A string describing a short name for the parameter. +* `required` - A boolean indicating if this is a required parameter on the link. +* `location` - A string describing how this parameter should be included in the outgoing request. +* `description` - A string describing this parameter on the link. + +--- + +## Handling errors + +Error responses are similar to Document responses. Both contain a dictionary of +content. However, an error does not represent a network resource, and so does +not have an associated URL, in the same way as a `Document` does. + +When an error response is returned by an API, the `ErrorMessage` exception is raised. +The `Error` instance itself is available on the exception as the `.error` attribute. + + params = { + 'location_code': 'berlin-4353', + 'start_date': '2018-01-03', + 'end_date': '2018-01-07', + 'room_type': 'double', + } + try: + data = client.action(document, ['bookings', 'create'], params=params) + except coreapi.exceptions.ErrorMessage as exc: + print("Error: %s" % exc.error) + else: + print("Success: %s" % data) + +### Error + +The following are available attributes, and may be passed when instantiating an `Error`: + +* `title` - A string describing the error. +* `content` - A dictionary containing all the data or links made available by this error. diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 37db17d..136cc08 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -1,27 +1,37 @@ # Exceptions -## ParseError +Any of the exceptions raised by the `coreapi` library may be imported from the `coreapi.exceptions` module: -A response was returned with malformed content. + from coreapi.exceptions import CoreAPIException -## UnsupportedContentType +## CoreAPIException -A response was returned with a content type that is not handled by any of the available codecs. +A base class for all `coreapi` exceptions. -## NotAcceptable +## ParseError +A response was returned with malformed content. +## NoCodecAvailable -## TransportError +Raised when there is no available codec that can handle the given media. + +## NetworkError An issue occurred with the network request. ## LinkLookupError -The keys passed in a `client.action()` call did not reference a link in the document. +The keys passed in a [`client.action()`][action] call did not reference a link in the document. ## ValidationError -The parameters passed in a `client.action()` call were invalid. +The parameters passed in a [`client.action()`][action] call did not match the set of required and optional fields made available by the link, or if the type of parameters passed could +not be supported by the given encoding on the link. ## ErrorMessage + +The server returned a CoreAPI [Error][error]. + +[action]: /api-guide/client.md#interacting-with-an-api +[error]: /api-guide/document.md#error diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md index e69de29..49b2aaa 100644 --- a/docs/api-guide/transports.md +++ b/docs/api-guide/transports.md @@ -0,0 +1,3 @@ +# Transports + +## HTTPTransport diff --git a/docs/api-guide/utils.md b/docs/api-guide/utils.md index 8a847a8..0c8e943 100644 --- a/docs/api-guide/utils.md +++ b/docs/api-guide/utils.md @@ -1,29 +1,88 @@ # Utilities -## lookup_elements(document, keys, strict_types=True) +The `coreapi.utils` module provides a number of helper functions that +may be useful if writing a custom client or transport class. -Given a document and a list of keys [...] +--- -## determine_transport(transports, url) +## Negotiation utilities + +The following functions are used to determine which of a set of transports +or codecs should be used when performing an API interaction. + +### determine_transport + +**Signature**: `determine_transport(transports, url)` Given a list of transports and a URL, return the appropriate transport for making network requests to that URL. -May raise `TransportError` +May raise `NetworkError`. +### negotiate_decoder -## negotiate_decoder(decoders, content_type=None) +**Signature**: `negotiate_decoder(codecs, content_type=None)` -Given a list of codecs, and the value of an HTTP response 'Content-Type' header, +Given a list of codecs, and the value of an HTTP response `Content-Type` header, return the appropriate codec for decoding the response content. +May raise `NoCodecAvailable`. -May raise `UnsupportedContentType` - +### negotiate_encoder -## negotiate_encoder(encoders, accept=None) +**Signature**: `negotiate_encoder(codecs, accept=None)` -Given a list of codecs, and the value of an incoming HTTP request 'Accept' +Given a list of codecs, and the value of an incoming HTTP request `Accept` header, return the appropriate codec for encoding the outgoing response content. -May raise `NotAcceptable` +Allows server implementations to provide for client-driven content negotiation. + +May raise `NoCodecAvailable`. + +--- + +## Validation utilities + +Different request encodings have different capabilities. For example, `application/json` +supports a range of data primitives, but does not support file uploads. In contrast, +`multipart/form` only supports string primatives and file uploads. + +The following helper functions validate that the types passed to an action are suitable +for use with the given encoding, and ensure that a consistent exception type is raised +if an invalid value is passed. + +### validate_path_param + +**Signature**: `validate_path_param(value)` + +Returns the value, coerced into a string primitive. Validates that the value that is suitable for use in URI-encoded path parameters. Empty strings and composite types such as dictionaries are disallowed. + +May raise `ValidationError`. + +### validate_query_param + +**Signature**: `validate_query_param(value)` + +Returns the value, coerced into a string primitive. Validates that the value is suitable for use in URL query parameters. + +May raise `ValidationError`. + +### validate_body_param + +**Signature**: `validate_body_param(value, encoding)` + +Returns the value, coerced into a primitive that is valid for the given encoding. Validates that the parameter types provided may be used as the body of the outgoing request. + +Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form` and `application/octet-stream`. + +May raise `ValidationError` for an invalid value, or `NetworkError` for an unsupported encoding. + +### validate_form_param + +**Signature**: `validate_body_param(value, encoding)` + +Returns the value, coerced into a primitive that is valid for the given encoding. Validates that the parameter types provided may be used as a key-value item for part of the body of the outgoing request. + +Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form`. + +May raise `ValidationError`, or `NetworkError` for an unsupported encoding. diff --git a/docs/index.md b/docs/index.md index da37213..da3007d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,17 +1,53 @@ -# Welcome to MkDocs +# CoreAPI - Python Client -For full documentation visit [mkdocs.org](http://mkdocs.org). +The Python CoreAPI client allows you to interact with any API that exposes +a supported schema or hypermedia format. -## Commands +## Installation -* `mkdocs new [dir-name]` - Create a new project. -* `mkdocs serve` - Start the live-reloading docs server. -* `mkdocs build` - Build the documentation site. -* `mkdocs help` - Print this help message. +Install [from PyPI][coreapi-pypi], using pip: -## Project layout + $ pip install coreapi - mkdocs.yml # The configuration file. - docs/ - index.md # The documentation homepage. - ... # Other markdown pages, images and other files. +## Quickstart + +**TODO** + +## Supported formats + +The following schema and hypermedia formats are currently supported: + +Name | Description +--------------------|--------------- +CoreJSON | Schema & Hypermedia +OpenAPI ("Swagger") | Schema +API Blueprint | Schema +JSON Hyper-Schema | Schema +HAL | Hypermedia + +## License + +Copyright © 2015-2016, Tom Christie. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +[coreapi-pypi]: https://pypi.python.org/pypi/coreapi diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md index e69de29..38bf0d7 100644 --- a/docs/topics/release-notes.md +++ b/docs/topics/release-notes.md @@ -0,0 +1 @@ +# Release Notes diff --git a/mkdocs.yml b/mkdocs.yml index c97182f..f051c7e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1 +1,12 @@ -site_name: My Docs +site_name: CoreAPI - Python Client +pages: +- Home: index.md +- API Guide: + - Clients: api-guide/client.md + - Documents: api-guide/document.md + - Codecs: api-guide/codecs.md + - Transports: api-guide/transports.md + - Exceptions: api-guide/exceptions.md + - Utilities: api-guide/utils.md +- Topics: + - Release Notes: topics/release-notes.md diff --git a/tests/test_codecs.py b/tests/test_codecs.py index c761420..eb77a68 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -2,7 +2,7 @@ from coreapi.codecs import CoreJSONCodec from coreapi.codecs.corejson import _document_to_primative, _primative_to_document from coreapi.document import Document, Link, Error, Field -from coreapi.exceptions import ParseError, UnsupportedContentType, NotAcceptable +from coreapi.exceptions import ParseError, NoCodecAvailable from coreapi.utils import negotiate_decoder, negotiate_encoder import pytest @@ -215,7 +215,7 @@ def test_get_supported_decoder_with_parameters(): def test_get_unsupported_decoder(): - with pytest.raises(UnsupportedContentType): + with pytest.raises(NoCodecAvailable): negotiate_decoder([CoreJSONCodec()], 'application/csv') @@ -251,7 +251,7 @@ def test_get_underspecified_encoder(): def test_get_unsupported_encoder(): - with pytest.raises(NotAcceptable): + with pytest.raises(NoCodecAvailable): negotiate_encoder([CoreJSONCodec()], 'application/csv') diff --git a/tests/test_transport.py b/tests/test_transport.py index a9aea1d..09cb849 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -2,7 +2,7 @@ from coreapi import Document, Link, Field from coreapi.codecs import CoreJSONCodec from coreapi.compat import force_text -from coreapi.exceptions import TransportError +from coreapi.exceptions import NetworkError from coreapi.transports import HTTPTransport from coreapi.utils import determine_transport import pytest @@ -30,17 +30,17 @@ def __init__(self, content): # Test transport errors. def test_unknown_scheme(): - with pytest.raises(TransportError): + with pytest.raises(NetworkError): determine_transport(transports, 'ftp://example.org') def test_missing_scheme(): - with pytest.raises(TransportError): + with pytest.raises(NetworkError): determine_transport(transports, 'example.org') def test_missing_hostname(): - with pytest.raises(TransportError): + with pytest.raises(NetworkError): determine_transport(transports, 'http://') diff --git a/tests/test_utils.py b/tests/test_utils.py index 15602c2..3a43428 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,28 +4,28 @@ def test_validate_path_param(): - assert utils.validate_path_param(1, name='example') == '1' - assert utils.validate_path_param(True, name='example') == 'true' + assert utils.validate_path_param(1) == '1' + assert utils.validate_path_param(True) == 'true' with pytest.raises(exceptions.ValidationError): - utils.validate_path_param(None, name='example') + utils.validate_path_param(None) with pytest.raises(exceptions.ValidationError): - utils.validate_path_param('', name='example') + utils.validate_path_param('') with pytest.raises(exceptions.ValidationError): - utils.validate_path_param({}, name='example') + utils.validate_path_param({}) with pytest.raises(exceptions.ValidationError): - utils.validate_path_param([], name='example') + utils.validate_path_param([]) def test_validate_query_param(): - assert utils.validate_query_param(1, name='example') == '1' - assert utils.validate_query_param(True, name='example') == 'true' - assert utils.validate_query_param(None, name='example') == '' - assert utils.validate_query_param('', name='example') == '' - assert utils.validate_query_param([1, 2, 3], name='example') == ['1', '2', '3'] + assert utils.validate_query_param(1) == '1' + assert utils.validate_query_param(True) == 'true' + assert utils.validate_query_param(None) == '' + assert utils.validate_query_param('') == '' + assert utils.validate_query_param([1, 2, 3]) == ['1', '2', '3'] with pytest.raises(exceptions.ValidationError): - utils.validate_query_param({}, name='example') + utils.validate_query_param({}) with pytest.raises(exceptions.ValidationError): - utils.validate_query_param([1, 2, {}], name='example') + utils.validate_query_param([1, 2, {}]) def test_validate_form_data(): @@ -39,52 +39,52 @@ def test_validate_form_data(): 'array': [1, 2, 3], 'object': {'a': 1, 'b': 2, 'c': 3} } - assert utils.validate_form_param(data, 'application/json', name='example') == data - assert utils.validate_body_param(data, 'application/json', name='example') == data + assert utils.validate_form_param(data, 'application/json') == data + assert utils.validate_body_param(data, 'application/json') == data # Invalid JSON data = datetime.datetime.now() with pytest.raises(exceptions.ValidationError): - utils.validate_form_param(data, 'application/json', name='example') + utils.validate_form_param(data, 'application/json') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(data, 'application/json', name='example') + utils.validate_body_param(data, 'application/json') data = utils.File('abc.txt', None) with pytest.raises(exceptions.ValidationError): - utils.validate_form_param(data, 'application/json', name='example') + utils.validate_form_param(data, 'application/json') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(data, 'application/json', name='example') + utils.validate_body_param(data, 'application/json') # URL Encoded - assert utils.validate_form_param(123, 'application/x-www-form-urlencoded', name='example') == '123' - assert utils.validate_body_param({'a': 123}, 'application/x-www-form-urlencoded', name='example') == {'a': '123'} + assert utils.validate_form_param(123, 'application/x-www-form-urlencoded') == '123' + assert utils.validate_body_param({'a': 123}, 'application/x-www-form-urlencoded') == {'a': '123'} with pytest.raises(exceptions.ValidationError): - utils.validate_form_param({'a': {'foo': 'bar'}}, 'application/x-www-form-urlencoded', name='example') + utils.validate_form_param({'a': {'foo': 'bar'}}, 'application/x-www-form-urlencoded') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(123, 'application/x-www-form-urlencoded', name='example') + utils.validate_body_param(123, 'application/x-www-form-urlencoded') with pytest.raises(exceptions.ValidationError): - utils.validate_form_param(utils.File('abc.txt', None), 'application/x-www-form-urlencoded', name='example') + utils.validate_form_param(utils.File('abc.txt', None), 'application/x-www-form-urlencoded') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'application/x-www-form-urlencoded', name='example') + utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'application/x-www-form-urlencoded') # Multipart - assert utils.validate_form_param(123, 'multipart/form', name='example') == '123' - assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form', name='example') == utils.File('abc.txt', None) - assert utils.validate_body_param({'a': 123}, 'multipart/form', name='example') == {'a': '123'} - assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form', name='example') == {'a': utils.File('abc.txt', None)} + assert utils.validate_form_param(123, 'multipart/form') == '123' + assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form') == utils.File('abc.txt', None) + assert utils.validate_body_param({'a': 123}, 'multipart/form') == {'a': '123'} + assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form') == {'a': utils.File('abc.txt', None)} with pytest.raises(exceptions.ValidationError): - utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form', name='example') + utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(123, 'multipart/form', name='example') + utils.validate_body_param(123, 'multipart/form') # Raw upload with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(123, 'application/octet-stream', name='example') + utils.validate_body_param(123, 'application/octet-stream') # Invalid encoding on outgoing request - with pytest.raises(exceptions.TransportError): - assert utils.validate_form_param(123, 'invalid/media-type', name='example') - with pytest.raises(exceptions.TransportError): - assert utils.validate_form_param(123, '', name='example') - with pytest.raises(exceptions.TransportError): - assert utils.validate_body_param(123, 'invalid/media-type', name='example') + with pytest.raises(exceptions.NetworkError): + assert utils.validate_form_param(123, 'invalid/media-type') + with pytest.raises(exceptions.NetworkError): + assert utils.validate_form_param(123, '') + with pytest.raises(exceptions.NetworkError): + assert utils.validate_body_param(123, 'invalid/media-type') From 5f2dea6b01ea0d73ee68b8637defb0ad8d735ef2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 13:59:37 +0100 Subject: [PATCH 18/74] Docs --- docs/api-guide/codecs.md | 46 +++++++++++++--------- docs/api-guide/document.md | 9 +++++ docs/api-guide/transports.md | 76 ++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 18 deletions(-) diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 7bb32fe..3cd6408 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -21,31 +21,41 @@ A codec will provide either one or both of the `decode` or `encode` methods. ### Decoding -* `decode(bytestring, **options)` +**Signature**: `decode(bytestring, **options)` -**TODO** +Given a bytestring, returns a decoded `Document`, `Error`, or raw data. + +An example of decoding a document: + + codec = codecs.CoreJSONCodec() + bytestring = open('document.corejson', 'rb').read() + document = codec.decode(bytestring) + +The available `options` keywords depend on the codec. ### Encoding -* `encode(document, **options)` +**Signature**: `encode(document, **options)` + +Given a `Document` or `Error`, return an encoded representation as a bytestring. -**TODO** +An example of encoding a document: + + codec = codecs.CoreJSONCodec() + bytestring = codec.encode(document) + output = open('document.corejson', 'wb') + output.write(bytestring) + output.close() + +The available `options` keywords depend on the codec. ### Attributes The following attributes are available on codec instances: * `media_type` - A string indicating the media type that the codec represents. -* `supports` - A list of strings, indicating the operations that the codec supports. - -The `supports` option should be one of the four following options: - -* `['decode', 'encode']` - Supports both decoding and encoding documents. -* `['decode']` - Supports decoding documents only. -* `['encode']` - Supports encoding documents only. -* `['data']` - Indicates that the codec supports decoding, - but that it is expected to return plain data, - rather than a `Document` object. +* `plain_data` - A boolean. May be set to `True` to indicate that the codec is + expected to return plain data, rather than schemas or hypermedia. --- @@ -57,7 +67,7 @@ Supports decoding or encoding the Core JSON format. **.media_type**: `application/vnd.coreapi+json` -**.supports**: `['decode', 'encode']` +**.plain_data**: `False` Example of decoding a Core JSON bytestring into a `Document` instance: @@ -94,7 +104,7 @@ Supports decoding JSON data. **.media_type**: `application/json` -**.supports**: `['data']` +**.plain_data**: `True` Example: @@ -113,7 +123,7 @@ Supports decoding plain-text responses. **.media_type**: `text/*` -**.supports**: `['data']` +**.plain_data**: `True` Example: @@ -132,7 +142,7 @@ that will be deleted once it goes out of scope. **.media_type**: `*/*` -**.supports**: `['data']` +**.plain_data**: `True` Example: diff --git a/docs/api-guide/document.md b/docs/api-guide/document.md index 730d680..259f1f3 100644 --- a/docs/api-guide/document.md +++ b/docs/api-guide/document.md @@ -114,6 +114,9 @@ The following are available attributes, and may be passed when instantiating a ` * `description` - A string describing this link. * `fields` - A list of field instances. +Note that the behaviour of link attributes is defined at the transport level, +rather than at the document level. See [the `HTTPTransport` documentation for more details][link-behaviour]. + ### Field The following are available attributes, and may be passed when instantiating a `Field`: @@ -123,6 +126,9 @@ The following are available attributes, and may be passed when instantiating a ` * `location` - A string describing how this parameter should be included in the outgoing request. * `description` - A string describing this parameter on the link. +Note that the behaviour of the `location` attribute is defined at the transport level, +rather than at the document level. See [the `HTTPTransport` documentation for more details][link-behaviour]. + --- ## Handling errors @@ -153,3 +159,6 @@ The following are available attributes, and may be passed when instantiating an * `title` - A string describing the error. * `content` - A dictionary containing all the data or links made available by this error. + + +[link-behaviour]: transports.md#link-behaviour diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md index 49b2aaa..80c136f 100644 --- a/docs/api-guide/transports.md +++ b/docs/api-guide/transports.md @@ -1,3 +1,79 @@ # Transports +Transports are responsible for making the actual network requests, and handling +the responses. + +Whenever an action is taken on a link, the scheme of the URL is inspected, and +the responsibility for making a request is passed to an appropriate transport class. + +By default only an HTTP transport implementation is included, but this approach means +that other network protocols can also be supported by Core API, while remaining +transparent to the user of the client library. + ## HTTPTransport + +The `HTTPTransport` class supports the `http` and `https` schemes. + +### Instantiation + +**Signature**: `HTTPTransport(credentials=None, headers=None, session=None)` + +* `credentials` - A dictionary of items that maps domain name to values that should be used in the request `Authorization` header for each domain. +* `headers` - A dictionary of items that should be included in the outgoing request headers. +* `session` - A [requests session instance][sessions] to use when sending requests. This can be used to further customize how HTTP requests and responses are handled, for instance by allowing [transport adapaters][transport-adapters] to be attached to the underlying session. + +### Link behaviour + +The following describes how the various Link and Field properties are used when +making an HTTP network request. + +#### Link.action + +The link `action` property is uppercased and then used to determine the HTTP +method for the request. + +If left blank then the `GET` method is used. + +#### Link.encoding + +The link `encoding` property is used to determine how any `location='form'` or +`location='body'` parameters should be encoded in order to form the body of +the request. + +Supported encodings are: + +* `'application/json'` - Suitable for primitive and composite types. +* `'application/x-www-form-urlencoded'` - Suitable for primitive types. +* `'multipart/form'` - Suitable for primitive types and file uploads. +* `'application/octet-stream'` - Suitable for raw file uploads, with a `location='body'` field. + +If left blank and a request body is included, then `'application/json'` is used. + +#### Link.transform + +The link `transform` property is *only relevant when the link is contained in an +embedded document*. This allows hypermedia documents to effect partial updates. + +* `'new'` - The response document should be returned as the result. +* `'inplace'` - The embedded document should be updated in-place, and the resulting + top-level document returned as the result. + +If left blank and a link in an embedded document is acted on, then `'inplace'` is used for `'PUT'`, `'PATCH'`, and `'DELETE'` requests. For any other request `'new'` is used. + +#### Field.location + +The link `location` property determines how the parameter is used to build the outgoing request. + +* `'path'` - The parameter is included in the URL, with the link + 'url' value acting as a [URI template][uri-template]. +* `'query'` - The parameter is included as a URL query parameter. +* `'body'` - The parameter is encoded and included as the body of the request. +* `'form'` - The parameter is treated as a single key-value item in an + dictionary of items. It should be encoded together with any other form + parameters, and included as the body of the request. + +If left blank, then `'query'` is used for `'GET'` and `'DELETE'` requests. For any other request `'form'` is used. + +[sessions]: http://docs.python-requests.org/en/master/user/advanced/#session-objects +[transport-adapters]: http://docs.python-requests.org/en/master/user/advanced/#transport-adapters +[uri-template]: https://tools.ietf.org/html/rfc6570 From b02af0128410ebfb063cba70328a9dfefbb0967d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 13:59:49 +0100 Subject: [PATCH 19/74] Fallbacks for v1.x interfaces --- coreapi/client.py | 15 ++++++++++++++- coreapi/codecs/base.py | 36 +++++++++++++++++++++++++++++++----- coreapi/codecs/corejson.py | 1 - coreapi/codecs/display.py | 1 - coreapi/codecs/download.py | 2 +- coreapi/codecs/jsondata.py | 2 +- coreapi/codecs/python.py | 1 - coreapi/codecs/text.py | 2 +- coreapi/transports/http.py | 12 ++++++++---- coreapi/utils.py | 7 +++++++ 10 files changed, 63 insertions(+), 16 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index da5b031..c5cb370 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -122,7 +122,20 @@ def get(self, url, force_codec=False): def reload(self, document, force_codec=False): return self.get(document.url, force_codec=force_codec) - def action(self, document, keys, params=None, validate=True, overrides=None): + def action(self, document, keys, params=None, validate=True, overrides=None, **kwargs): + if kwargs: + # Fallback for v1.x overrides. + # Will be removed at some point, most likely in a 2.1 release. + if overrides is None: + overrides = {} + if 'action' in kwargs: + overrides['action'] = kwargs.pop('action') + if 'encoding' in kwargs: + overrides['encoding'] = kwargs.pop('encoding') + if 'transform' in kwargs: + overrides['transform'] = kwargs.pop('transform') + assert not kwargs, 'Unknown keyword argument(s) passed: ' + ', '.join(kwargs.keys()) + if isinstance(keys, string_types): keys = [keys] diff --git a/coreapi/codecs/base.py b/coreapi/codecs/base.py index c21e083..991fd66 100644 --- a/coreapi/codecs/base.py +++ b/coreapi/codecs/base.py @@ -3,10 +3,36 @@ class BaseCodec(itypes.Object): media_type = None - supports = [] # 'encoding', 'decoding', 'data' + plain_data = False - def decode(self, bytestring, **options): - raise NotImplementedError() # pragma: nocover + # We don't implement stubs, to ensure that we can check which of these + # two operations a codec supports. For example: + # `if hasattr(codec, 'decode'): ...` - def encode(self, document, **options): - raise NotImplementedError() # pragma: nocover + # def decode(self, bytestring, **options): + # pass + + # def encode(self, document, **options): + # pass + + # The following will be removed at some point, most likely in a 2.1 release: + def dump(self, *args, **kwargs): + # Fallback for v1.x interface + return self.encode(*args, **kwargs) + + def load(self, *args, **kwargs): + # Fallback for v1.x interface + return self.decode(*args, **kwargs) + + @property + def supports(self): + # Fallback for v1.x interface. + if self.plain_data: + return ['data'] + + ret = [] + if hasattr(self, 'encode'): + ret.append('encoding') + if hasattr(self, 'decode'): + ret.append('decoding') + return ret diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index edc19e7..dd1317c 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -228,7 +228,6 @@ def _primative_to_document(data, base_url=None): class CoreJSONCodec(BaseCodec): media_type = 'application/vnd.coreapi+json' - supports = ['encode', 'decode'] def decode(self, bytestring, **options): """ diff --git a/coreapi/codecs/display.py b/coreapi/codecs/display.py index ab2fbd6..26195cd 100644 --- a/coreapi/codecs/display.py +++ b/coreapi/codecs/display.py @@ -115,7 +115,6 @@ class DisplayCodec(BaseCodec): A plaintext representation of a Document, intended for readability. """ media_type = 'text/plain' - supports = ['encode'] def encode(self, document, **options): colorize = options.get('colorize', False) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 79585ba..81957bb 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -106,7 +106,7 @@ class DownloadCodec(BaseCodec): A codec to handle raw file downloads, such as images and other media. """ media_type = '*/*' - supports = ['data'] + plain_data = True def __init__(self, download_dir=None): """ diff --git a/coreapi/codecs/jsondata.py b/coreapi/codecs/jsondata.py index 10ab488..1f5faca 100644 --- a/coreapi/codecs/jsondata.py +++ b/coreapi/codecs/jsondata.py @@ -7,7 +7,7 @@ class JSONCodec(BaseCodec): media_type = 'application/json' - supports = ['data'] + plain_data = True def decode(self, bytestring, **options): """ diff --git a/coreapi/codecs/python.py b/coreapi/codecs/python.py index 2487a89..40764ca 100644 --- a/coreapi/codecs/python.py +++ b/coreapi/codecs/python.py @@ -64,7 +64,6 @@ class PythonCodec(BaseCodec): A Python representation of a Document, for use with '__repr__'. """ media_type = 'text/python' - supports = ['encode'] def encode(self, document, **options): # Object and Array only have the class name wrapper if they diff --git a/coreapi/codecs/text.py b/coreapi/codecs/text.py index bc44104..78fbe18 100644 --- a/coreapi/codecs/text.py +++ b/coreapi/codecs/text.py @@ -4,7 +4,7 @@ class TextCodec(BaseCodec): media_type = 'text/*' - supports = ['data'] + plain_data = True def decode(self, bytestring, **options): return bytestring.decode('utf-8') diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index d0eddb6..8428767 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -285,8 +285,7 @@ def _handle_inplace_replacements(document, link, link_ancestors): class HTTPTransport(BaseTransport): schemes = ['http', 'https'] - def __init__(self, credentials=None, headers=None, session=None, - request_callback=None, response_callback=None): + def __init__(self, credentials=None, headers=None, session=None, request_callback=None, response_callback=None): if headers: headers = {key.lower(): value for key, value in headers.items()} if session is None: @@ -294,6 +293,9 @@ def __init__(self, credentials=None, headers=None, session=None, self._credentials = itypes.Dict(credentials or {}) self._headers = itypes.Dict(headers or {}) self._session = session + + # Fallback for v1.x overrides. + # Will be removed at some point, most likely in a 2.1 release. self._request_callback = request_callback self._response_callback = response_callback @@ -315,11 +317,13 @@ def transition(self, link, decoders, params=None, link_ancestors=None, force_cod headers.update(self.headers) request = _build_http_request(session, url, method, headers, encoding, params) - if self._request_callback: + + if self._request_callback is not None: self._request_callback(request) response = session.send(request) - if self._response_callback: + + if self._response_callback is not None: self._response_callback(response) result = _decode_result(response, decoders, force_codec) diff --git a/coreapi/utils.py b/coreapi/utils.py index 215532f..bc0b6f7 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -26,6 +26,9 @@ def guess_filename(obj): return None +# Negotiation utilities. USed to determine which codec or transport class +# should be used, given a list of supported instances. + def determine_transport(transports, url): """ Given a URL determine the appropriate transport instance. @@ -95,6 +98,10 @@ def negotiate_encoder(encoders, accept=None): raise exceptions.NoCodecAvailable(msg) +# Validation utilities. Used to ensure that we get consitent validation +# exceptions when invalid types are passed as a parameter, rather than +# an exception occuring when the request is made. + def validate_path_param(value): value = _validate_form_field(value, allow_list=False) if not value: From d5f2962d4c7d446fccbc5e69d544b4518e3a40b7 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 14:24:25 +0100 Subject: [PATCH 20/74] Treat DisplayCodec and PythonCodec as implementation details --- coreapi/client.py | 18 ++++++------- coreapi/codecs/display.py | 3 +++ coreapi/codecs/python.py | 3 +++ docs/api-guide/codecs.md | 56 --------------------------------------- 4 files changed, 15 insertions(+), 65 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index c5cb370..373b01b 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -122,19 +122,19 @@ def get(self, url, force_codec=False): def reload(self, document, force_codec=False): return self.get(document.url, force_codec=force_codec) - def action(self, document, keys, params=None, validate=True, overrides=None, **kwargs): - if kwargs: + def action(self, document, keys, params=None, validate=True, overrides=None, + action=None, encoding=None, transform=None): + if (action is not None) or (encoding is not None) or (transform is not None): # Fallback for v1.x overrides. # Will be removed at some point, most likely in a 2.1 release. if overrides is None: overrides = {} - if 'action' in kwargs: - overrides['action'] = kwargs.pop('action') - if 'encoding' in kwargs: - overrides['encoding'] = kwargs.pop('encoding') - if 'transform' in kwargs: - overrides['transform'] = kwargs.pop('transform') - assert not kwargs, 'Unknown keyword argument(s) passed: ' + ', '.join(kwargs.keys()) + if action is not None: + overrides['action'] = action + if encoding is not None: + overrides['encoding'] = encoding + if transform is not None: + overrides['transform'] = transform if isinstance(keys, string_types): keys = [keys] diff --git a/coreapi/codecs/display.py b/coreapi/codecs/display.py index 26195cd..250e0cc 100644 --- a/coreapi/codecs/display.py +++ b/coreapi/codecs/display.py @@ -1,3 +1,6 @@ +# Note that `DisplayCodec` is deliberately omitted from the documentation, +# as it is considered an implementation detail. +# It may move into a utility function in the future. from __future__ import unicode_literals from coreapi.codecs.base import BaseCodec from coreapi.compat import console_style, string_types diff --git a/coreapi/codecs/python.py b/coreapi/codecs/python.py index 40764ca..ba0b35d 100644 --- a/coreapi/codecs/python.py +++ b/coreapi/codecs/python.py @@ -1,3 +1,6 @@ +# Note that `DisplayCodec` is deliberately omitted from the documentation, +# as it is considered an implementation detail. +# It may move into a utility function in the future. from __future__ import unicode_literals from coreapi.codecs.base import BaseCodec from coreapi.document import Document, Link, Array, Object, Error, Field diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 3cd6408..998a44e 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -177,62 +177,6 @@ indicate the download filename][content-disposition-filename]. --- -### DisplayCodec - -Supports encoding a `Document` to a display representation. - -**.media_type**: `text/plain` - -**.supports**: `['encoding']` - -Example: - - >>> from coreapi import codecs - >>> codec = codecs.DisplayCodec() - >>> content = codec.encode(document) - >>> print(content) - - 'search': link(from, to, date) - -#### Options - -**colorize**: Set to `True` to include ANSI color escapes for terminal representations. -See the Python `click` package [for more details][click-ansi]. - ---- - -### PythonCodec - -Supports encoding a `Document` to an its Python representation. - -**.media_type**: `text/python` - -**.supports**: `['encoding']` - -Example: - - >>> from coreapi import codecs - >>> codec = codecs.PythonCodec() - >>> content = codec.encode(document) - >>> print(content) - Document( - title='Flight Search API', - url='http://api.example.com/', - content={ - 'search': Link( - url='/search/', - action='get', - fields=[ - Field(name='from'), - Field(name='to'), - Field(name='date') - ] - ) - } - ) - ---- - ## Custom Codecs Custom codec classes may be created by inheriting from `BaseCodec`, setting From df6e4f14fb49e3ca7ed3baec0b52fabc84ca16f5 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 15:44:01 +0100 Subject: [PATCH 21/74] Drop plain_data --- coreapi/codecs/base.py | 3 +-- coreapi/codecs/download.py | 1 - coreapi/codecs/jsondata.py | 1 - coreapi/codecs/text.py | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/coreapi/codecs/base.py b/coreapi/codecs/base.py index 991fd66..10182ce 100644 --- a/coreapi/codecs/base.py +++ b/coreapi/codecs/base.py @@ -3,7 +3,6 @@ class BaseCodec(itypes.Object): media_type = None - plain_data = False # We don't implement stubs, to ensure that we can check which of these # two operations a codec supports. For example: @@ -27,7 +26,7 @@ def load(self, *args, **kwargs): @property def supports(self): # Fallback for v1.x interface. - if self.plain_data: + if '+' not in self.media_type: return ['data'] ret = [] diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 81957bb..ed83ba7 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -106,7 +106,6 @@ class DownloadCodec(BaseCodec): A codec to handle raw file downloads, such as images and other media. """ media_type = '*/*' - plain_data = True def __init__(self, download_dir=None): """ diff --git a/coreapi/codecs/jsondata.py b/coreapi/codecs/jsondata.py index 1f5faca..494f975 100644 --- a/coreapi/codecs/jsondata.py +++ b/coreapi/codecs/jsondata.py @@ -7,7 +7,6 @@ class JSONCodec(BaseCodec): media_type = 'application/json' - plain_data = True def decode(self, bytestring, **options): """ diff --git a/coreapi/codecs/text.py b/coreapi/codecs/text.py index 78fbe18..f52b1de 100644 --- a/coreapi/codecs/text.py +++ b/coreapi/codecs/text.py @@ -4,7 +4,6 @@ class TextCodec(BaseCodec): media_type = 'text/*' - plain_data = True def decode(self, bytestring, **options): return bytestring.decode('utf-8') From ee7ebb12afa1d606b84966286dc9a6d8f133cb56 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 15:53:55 +0100 Subject: [PATCH 22/74] Docs --- README.md | 114 ++++++++++------------------------- coreapi/__init__.py | 2 +- docs/api-guide/client.md | 25 ++++++-- docs/api-guide/codecs.md | 38 +++--------- docs/api-guide/document.md | 2 +- docs/api-guide/transports.md | 27 ++++++--- docs/index.md | 53 ++++++++++++---- mkdocs.yml | 4 +- 8 files changed, 127 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 43b274f..481e4e5 100644 --- a/README.md +++ b/README.md @@ -1,108 +1,58 @@ # Python client library -Python client library for [Core API][core-api]. - [![build-status-image]][travis] [![pypi-version]][pypi] -**Requirements**: Python 2.7, 3.3+ - ---- - -### Installation - -To install, use pip. - - pip install coreapi - ---- - -### Retrieving and inspecting documents - -To initially access a Core API interface, create a client, and use the `get()` method. - - >>> import coreapi - >>> client = coreapi.Client() - >>> doc = client.get('http://coreapi.herokuapp.com') - -We can now inspect the returned document. - - >>> print(doc) - - 'notes': [ - - 'complete': False, - 'description': 'Schedule design meeting', - 'delete': link(), - 'edit': link([description], [complete]), - - 'complete: True, - 'description': 'Write release notes', - 'delete': link(), - 'edit': link([description], [complete]) - ], - 'add_note': link(description) - -Documents are key-value pairs, and their elements can be accessed by indexing into them. - - >>> doc['notes'][0]['description'] - 'Schedule design meeting' - -You can also inspect the document URL and title. +Python client library for [Core API][core-api]. - >>> doc.url - 'http://coreapi.herokuapp.com/' - >>> doc.title - 'Notes' +**Requirements**: Python 2.7, 3.3+ --- -### Interacting with documents +## Installation -Documents in the Python Core API library are immutable objects. To perform a transition we use the `.action()` method and assign the resulting new document. +Install from PyPI, using pip: - >>> doc = client.action(doc, ['add_note'], params={'description': 'My new note.'}) + $ pip install coreapi -The arguments to `.action()` are: +## Quickstart -* The existing document. -* A string or list of strings or integers, indexing the link to act on. -* Any parameters to use when the acting on the link. +Create a client instance: -Transitions may update part of the document tree. + from coreapi import Client + client = Client() - >>> doc = client.action(doc, ['notes', 0, 'edit'], params={'complete': True}) - >>> doc['notes'][0]['complete'] - True +Retrieve an API schema: -Or they may remove part of the document tree. + document = client.get('https://api.example.org/') - >>> while doc['notes']: - >>> doc = client.action(doc, ['notes', 0, 'delete']) - >>> len(doc['notes']) - 0 - ---- +Interact with the API: -### Saving and loading documents + data = client.action(document, ['flights', 'search'], params={ + 'from': 'LHR', + 'to': 'PA', + 'date': '2016-10-12' + }) -To save or load documents into raw bytestrings, instantiate a codec and then -use the `dump()` and `load()` methods. +## Supported formats -For example, to save a document to disk. +The following schema and hypermedia formats are currently supported, either +through built-in support, or as a third-party codec: - codec = coreapi.codecs.CoreJSONCodec() - content = codec.dump(doc) - file = open('doc.json', 'wb') - file.write(content) - file.close() +Name | Notes +--------------------|--------------------------------|------------------------------------ +CoreJSON | `application/vnd.coreapi+json` | Supports both Schemas & Hypermedia. +OpenAPI ("Swagger") | `application/openapi+json` | Schema support. +JSON Hyper-Schema | `application/schema+json` | Schema support. +HAL | `application/hal+json` | Hypermedia support. -To load the same document back again. +Additionally, the following plain data content types are supported: - file = open('doc.json', 'rb') - content = file.read() - file.close() - doc = codec.load(content) +Name | Content type | Notes +------------|--------------------|--------------------------------- +JSON | `application/json` | Returns Python primitive types. +Plain text | `text/*` | Returns a Python string instance. +Other media | `*/*` | Returns a temporary download file. --- diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 2bd47c0..76f2e6e 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -8,5 +8,5 @@ __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', - 'codecs', 'exceptions', 'transports', 'utils' + 'codecs', 'exceptions', 'transports', 'utils', ] diff --git a/docs/api-guide/client.md b/docs/api-guide/client.md index b88f9f6..4349b11 100644 --- a/docs/api-guide/client.md +++ b/docs/api-guide/client.md @@ -68,17 +68,22 @@ properties on a client instance: ## Making an initial request -* `get(url)` +**Signature**: `get(url)` -Make a network request to the given URL, and return a decoded `Document`. +Make a network request to the given URL. If fetching an API schema or hypermedia +resource, then this should typically return a decoded `Document`. -**TODO** +* `url` - The URL that should be retrieved. + +For example: + + document = client.get('https://api.example.org/') --- ## Interacting with an API -* `action(self, document, keys, params=None, overrides=None, validate=True)` +**Signature**: `action(self, document, keys, params=None)` Effect an interaction against the given document. @@ -86,4 +91,14 @@ Effect an interaction against the given document. * `keys` - A list of strings that index a `Link` within the document. * `params` - A dictionary of parameters to use for the API interaction. -**TODO** +For example, making a request without any parameters: + + data = client.action(document, ['flights', 'list_airports']) + +Or making a request, with parameters included: + + data = client.action(document, ['flights', 'search'], params={ + 'from': 'LHR', + 'to': 'PA', + 'date': '2016-10-12' + }) diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 998a44e..94dfd56 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -10,16 +10,18 @@ the bytestring returned in the body of the response. When using a Core API client, HTTP responses are decoded with an appropriate codec, based on the `Content-Type` of the response. -## Using a Codec +## Using a codec All the codecs provided by the `coreapi` library are instantiated without arguments, for example: + from coreapi import codecs + codec = codecs.CoreJSONCodec() -A codec will provide either one or both of the `decode` or `encode` methods. +A codec will provide either one or both of the `decode()` or `encode()` methods. -### Decoding +#### Decoding **Signature**: `decode(bytestring, **options)` @@ -27,13 +29,12 @@ Given a bytestring, returns a decoded `Document`, `Error`, or raw data. An example of decoding a document: - codec = codecs.CoreJSONCodec() bytestring = open('document.corejson', 'rb').read() document = codec.decode(bytestring) The available `options` keywords depend on the codec. -### Encoding +#### Encoding **Signature**: `encode(document, **options)` @@ -41,7 +42,6 @@ Given a `Document` or `Error`, return an encoded representation as a bytestring. An example of encoding a document: - codec = codecs.CoreJSONCodec() bytestring = codec.encode(document) output = open('document.corejson', 'wb') output.write(bytestring) @@ -49,17 +49,15 @@ An example of encoding a document: The available `options` keywords depend on the codec. -### Attributes +#### Attributes -The following attributes are available on codec instances: +The following attribute is available on codec instances: * `media_type` - A string indicating the media type that the codec represents. -* `plain_data` - A boolean. May be set to `True` to indicate that the codec is - expected to return plain data, rather than schemas or hypermedia. --- -## Available Codecs +## Available codecs ### CoreJSONCodec @@ -67,8 +65,6 @@ Supports decoding or encoding the Core JSON format. **.media_type**: `application/vnd.coreapi+json` -**.plain_data**: `False` - Example of decoding a Core JSON bytestring into a `Document` instance: >>> from coreapi import codecs @@ -104,8 +100,6 @@ Supports decoding JSON data. **.media_type**: `application/json` -**.plain_data**: `True` - Example: >>> from coreapi import codecs @@ -123,8 +117,6 @@ Supports decoding plain-text responses. **.media_type**: `text/*` -**.plain_data**: `True` - Example: >>> from coreapi import codecs @@ -142,8 +134,6 @@ that will be deleted once it goes out of scope. **.media_type**: `*/*` -**.plain_data**: `True` - Example: >>> from coreapi import codecs @@ -177,7 +167,7 @@ indicate the download filename][content-disposition-filename]. --- -## Custom Codecs +## Custom codecs Custom codec classes may be created by inheriting from `BaseCodec`, setting the `media_type` and `supports` properties, and implementing one or both @@ -237,10 +227,6 @@ A codec for [OpenAPI][openapi] schemas, also known as "Swagger". Installable [fr A codec for [JSON Hyper-Schema][jsonhyperschema]. Installable [from PyPI][jsonhyperschema-pypi] as `jsonhyperschema-codec`, and [available on GitHub][jsonhyperschema-github]. -### API Blueprint - -A codec for [API Blueprint][apiblueprint] schemas. Installable [from PyPI][apiblueprint-pypi] as `apiblueprint-codec`, and [available on GitHub][apiblueprint-github]. - ### HAL A codec for the [HAL][hal] hypermedia format. Installable [from PyPI][hal-pypi] as `hal-codec`, and [available on GitHub][hal-github]. @@ -256,10 +242,6 @@ A codec for the [HAL][hal] hypermedia format. Installable [from PyPI][hal-pypi] [jsonhyperschema-pypi]: https://pypi.python.org/pypi/jsonhyperschema-codec [jsonhyperschema-github]: https://github.com/core-api/python-jsonhyperschema-codec -[apiblueprint]: https://apiblueprint.org/ -[apiblueprint-pypi]: https://pypi.python.org/pypi/apiblueprint-codec -[apiblueprint-github]: https://github.com/core-api/python-apiblueprint-codec - [hal]: http://stateless.co/hal_specification.html [hal-pypi]: https://pypi.python.org/pypi/hal-codec [hal-github]: https://github.com/core-api/python-hal-codec diff --git a/docs/api-guide/document.md b/docs/api-guide/document.md index 259f1f3..8f3fb4a 100644 --- a/docs/api-guide/document.md +++ b/docs/api-guide/document.md @@ -161,4 +161,4 @@ The following are available attributes, and may be passed when instantiating an * `content` - A dictionary containing all the data or links made available by this error. -[link-behaviour]: transports.md#link-behaviour +[link-behaviour]: transports.md#making-requests diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md index 80c136f..2c5d62e 100644 --- a/docs/api-guide/transports.md +++ b/docs/api-guide/transports.md @@ -10,31 +10,33 @@ By default only an HTTP transport implementation is included, but this approach that other network protocols can also be supported by Core API, while remaining transparent to the user of the client library. -## HTTPTransport +## Available transports + +### HTTPTransport The `HTTPTransport` class supports the `http` and `https` schemes. -### Instantiation +#### Instantiation **Signature**: `HTTPTransport(credentials=None, headers=None, session=None)` * `credentials` - A dictionary of items that maps domain name to values that should be used in the request `Authorization` header for each domain. * `headers` - A dictionary of items that should be included in the outgoing request headers. -* `session` - A [requests session instance][sessions] to use when sending requests. This can be used to further customize how HTTP requests and responses are handled, for instance by allowing [transport adapaters][transport-adapters] to be attached to the underlying session. +* `session` - A [requests session instance][sessions] to use when sending requests. This can be used to further customize how HTTP requests and responses are handled, for instance by allowing [transport adapters][transport-adapters] to be attached to the underlying session. -### Link behaviour +#### Making requests The following describes how the various Link and Field properties are used when making an HTTP network request. -#### Link.action +**Link.action** The link `action` property is uppercased and then used to determine the HTTP method for the request. If left blank then the `GET` method is used. -#### Link.encoding +**Link.encoding** The link `encoding` property is used to determine how any `location='form'` or `location='body'` parameters should be encoded in order to form the body of @@ -49,7 +51,7 @@ Supported encodings are: If left blank and a request body is included, then `'application/json'` is used. -#### Link.transform +**Link.transform** The link `transform` property is *only relevant when the link is contained in an embedded document*. This allows hypermedia documents to effect partial updates. @@ -60,7 +62,7 @@ embedded document*. This allows hypermedia documents to effect partial updates. If left blank and a link in an embedded document is acted on, then `'inplace'` is used for `'PUT'`, `'PATCH'`, and `'DELETE'` requests. For any other request `'new'` is used. -#### Field.location +**Field.location** The link `location` property determines how the parameter is used to build the outgoing request. @@ -74,6 +76,15 @@ The link `location` property determines how the parameter is used to build the o If left blank, then `'query'` is used for `'GET'` and `'DELETE'` requests. For any other request `'form'` is used. +## Custom transports + +The transport interface is not yet finalized, as it may still be subject to minor +changes in a future release. + +## External packages + +No third party transport classes are currently available. + [sessions]: http://docs.python-requests.org/en/master/user/advanced/#session-objects [transport-adapters]: http://docs.python-requests.org/en/master/user/advanced/#transport-adapters [uri-template]: https://tools.ietf.org/html/rfc6570 diff --git a/docs/index.md b/docs/index.md index da3007d..b0715f2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,8 @@ -# CoreAPI - Python Client +# Core API - Python Client -The Python CoreAPI client allows you to interact with any API that exposes -a supported schema or hypermedia format. +Python client library for [Core API][core-api]. + +Allows you to interact with any API that exposes a supported schema or hypermedia format. ## Installation @@ -11,19 +12,44 @@ Install [from PyPI][coreapi-pypi], using pip: ## Quickstart -**TODO** +Create a client instance: + + from coreapi import Client + client = Client() + +Retrieve an API schema: + + document = client.get('https://api.example.org/') + +Interact with the API: + + data = client.action(document, ['flights', 'search'], params={ + 'from': 'LHR', + 'to': 'PA', + 'date': '2016-10-12' + }) ## Supported formats -The following schema and hypermedia formats are currently supported: +The following schema and hypermedia formats are currently supported, either +through [built-in support][built-in-codecs], or as a [third-party codec][third-party-codecs]: + +Name | Notes +--------------------|--------------------------------|------------------------------------ +CoreJSON | `application/vnd.coreapi+json` | Supports both Schemas & Hypermedia. +OpenAPI ("Swagger") | `application/openapi+json` | Schema support. +JSON Hyper-Schema | `application/schema+json` | Schema support. +HAL | `application/hal+json` | Hypermedia support. + +Additionally, the following plain data content types [are supported][built-in-codecs]: + +Name | Content type | Notes +------------|--------------------|--------------------------------- +JSON | `application/json` | Returns Python primitive types. +Plain text | `text/*` | Returns a Python string instance. +Other media | `*/*` | Returns a temporary download file. -Name | Description ---------------------|--------------- -CoreJSON | Schema & Hypermedia -OpenAPI ("Swagger") | Schema -API Blueprint | Schema -JSON Hyper-Schema | Schema -HAL | Hypermedia +--- ## License @@ -50,4 +76,7 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +[core-api]: http://www.coreapi.org/ +[built-in-codecs]: api-guide/codecs.md#available-codecs +[third-party-codecs]: api-guide/codecs.md#external-packages [coreapi-pypi]: https://pypi.python.org/pypi/coreapi diff --git a/mkdocs.yml b/mkdocs.yml index f051c7e..683babe 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: CoreAPI - Python Client +site_name: Core API - Python Client pages: - Home: index.md - API Guide: @@ -10,3 +10,5 @@ pages: - Utilities: api-guide/utils.md - Topics: - Release Notes: topics/release-notes.md + +repo_url: https://github.com/core-api/python-client/ From 91460f5087a8532d11a031e71f46a96cb912cb4d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 16:29:05 +0100 Subject: [PATCH 23/74] Added 'Field.type' attribute --- coreapi/codecs/corejson.py | 3 +++ coreapi/document.py | 4 ++-- docs/api-guide/document.md | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index dd1317c..988a60f 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -152,6 +152,8 @@ def _document_to_primative(node, base_url=None): ret['required'] = node.required if node.location: ret['location'] = node.location + if node.type: + ret['type'] = node.type if node.description: ret['description'] = node.description return ret @@ -203,6 +205,7 @@ def _primative_to_document(data, base_url=None): name=_get_string(item, 'name'), required=_get_bool(item, 'required'), location=_get_string(item, 'location'), + type=_get_string(item, 'type'), description=_get_string(item, 'description') ) for item in fields if isinstance(item, dict) diff --git a/coreapi/document.py b/coreapi/document.py index a6862e6..2e4edfb 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -36,8 +36,8 @@ def _key_sorting(item): # The field class, as used by Link objects: -Field = namedtuple('Field', ['name', 'required', 'location', 'description']) -Field.__new__.__defaults__ = (False, '', '') +Field = namedtuple('Field', ['name', 'required', 'location', 'type', 'description']) +Field.__new__.__defaults__ = (False, '', '', '') # The Core API primatives: diff --git a/docs/api-guide/document.md b/docs/api-guide/document.md index 8f3fb4a..3886937 100644 --- a/docs/api-guide/document.md +++ b/docs/api-guide/document.md @@ -124,6 +124,7 @@ The following are available attributes, and may be passed when instantiating a ` * `name` - A string describing a short name for the parameter. * `required` - A boolean indicating if this is a required parameter on the link. * `location` - A string describing how this parameter should be included in the outgoing request. +* `type` - A string describing the kind of [input control][html5-input-control] this parameter represents. * `description` - A string describing this parameter on the link. Note that the behaviour of the `location` attribute is defined at the transport level, @@ -160,5 +161,5 @@ The following are available attributes, and may be passed when instantiating an * `title` - A string describing the error. * `content` - A dictionary containing all the data or links made available by this error. - [link-behaviour]: transports.md#making-requests +[html5-input-control]: https://www.w3.org/TR/html-markup/input.html From 5ce2df38612536cac0c04230fe3a89d433d9c610 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 8 Sep 2016 16:50:32 +0100 Subject: [PATCH 24/74] application/vnd.coreapi+json -> application/coreapi+json --- README.md | 14 +++++++------- coreapi/codecs/base.py | 7 +++++++ coreapi/codecs/corejson.py | 5 ++++- coreapi/transports/http.py | 2 +- coreapi/utils.py | 15 +++++++++------ docs/api-guide/codecs.md | 2 +- docs/index.md | 14 +++++++------- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 481e4e5..ebde295 100644 --- a/README.md +++ b/README.md @@ -39,16 +39,16 @@ Interact with the API: The following schema and hypermedia formats are currently supported, either through built-in support, or as a third-party codec: -Name | Notes ---------------------|--------------------------------|------------------------------------ -CoreJSON | `application/vnd.coreapi+json` | Supports both Schemas & Hypermedia. -OpenAPI ("Swagger") | `application/openapi+json` | Schema support. -JSON Hyper-Schema | `application/schema+json` | Schema support. -HAL | `application/hal+json` | Hypermedia support. +Name | Media type | Notes +--------------------|----------------------------|------------------------------------ +CoreJSON | `application/coreapi+json` | Supports both Schemas & Hypermedia. +OpenAPI ("Swagger") | `application/openapi+json` | Schema support. +JSON Hyper-Schema | `application/schema+json` | Schema support. +HAL | `application/hal+json` | Hypermedia support. Additionally, the following plain data content types are supported: -Name | Content type | Notes +Name | Media type | Notes ------------|--------------------|--------------------------------- JSON | `application/json` | Returns Python primitive types. Plain text | `text/*` | Returns a Python string instance. diff --git a/coreapi/codecs/base.py b/coreapi/codecs/base.py index 10182ce..7b99e56 100644 --- a/coreapi/codecs/base.py +++ b/coreapi/codecs/base.py @@ -35,3 +35,10 @@ def supports(self): if hasattr(self, 'decode'): ret.append('decoding') return ret + + def get_media_types(self): + # Fallback, while transitioning from `application/vnd.coreapi+json` + # to simply `application/coreapi+json`. + if hasattr(self, 'media_types'): + return self.media_types + return [self.media_type] diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 988a60f..5c7045f 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -230,7 +230,10 @@ def _primative_to_document(data, base_url=None): class CoreJSONCodec(BaseCodec): - media_type = 'application/vnd.coreapi+json' + media_type = 'application/coreapi+json' + + # The following is due to be deprecated... + media_types = ['application/coreapi+json', 'application/vnd.coreapi+json'] def decode(self, bytestring, **options): """ diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 8428767..87c32aa 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -110,7 +110,7 @@ def _get_headers(url, decoders, credentials=None): """ Return a dictionary of HTTP headers to use in the outgoing request. """ - accept = '%s, */*' % decoders[0].media_type + accept = '%s, */*' % ', '.join(decoders[0].get_media_types()) headers = { 'accept': accept, diff --git a/coreapi/utils.py b/coreapi/utils.py index bc0b6f7..fbc72c6 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -63,8 +63,9 @@ def negotiate_decoder(decoders, content_type=None): wildcard_type = '*/*' for codec in decoders: - if codec.media_type in (content_type, main_type, wildcard_type): - return codec + for media_type in codec.get_media_types(): + if media_type in (content_type, main_type, wildcard_type): + return codec msg = "Unsupported media in Content-Type header '%s'" % content_type raise exceptions.NoCodecAvailable(msg) @@ -84,12 +85,14 @@ def negotiate_encoder(encoders, accept=None): ]) for codec in encoders: - if codec.media_type in acceptable: - return codec + for media_type in codec.get_media_types(): + if media_type in acceptable: + return codec for codec in encoders: - if codec.media_type.split('/')[0] + '/*' in acceptable: - return codec + for media_type in codec.get_media_types(): + if codec.media_type.split('/')[0] + '/*' in acceptable: + return codec if '*/*' in acceptable: return encoders[0] diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 94dfd56..2179658 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -63,7 +63,7 @@ The following attribute is available on codec instances: Supports decoding or encoding the Core JSON format. -**.media_type**: `application/vnd.coreapi+json` +**.media_type**: `application/coreapi+json` Example of decoding a Core JSON bytestring into a `Document` instance: diff --git a/docs/index.md b/docs/index.md index b0715f2..efb36bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,16 +34,16 @@ Interact with the API: The following schema and hypermedia formats are currently supported, either through [built-in support][built-in-codecs], or as a [third-party codec][third-party-codecs]: -Name | Notes ---------------------|--------------------------------|------------------------------------ -CoreJSON | `application/vnd.coreapi+json` | Supports both Schemas & Hypermedia. -OpenAPI ("Swagger") | `application/openapi+json` | Schema support. -JSON Hyper-Schema | `application/schema+json` | Schema support. -HAL | `application/hal+json` | Hypermedia support. +Name | Media type | Notes +--------------------|----------------------------|------------------------------------ +CoreJSON | `application/coreapi+json` | Supports both Schemas & Hypermedia. +OpenAPI ("Swagger") | `application/openapi+json` | Schema support. +JSON Hyper-Schema | `application/schema+json` | Schema support. +HAL | `application/hal+json` | Hypermedia support. Additionally, the following plain data content types [are supported][built-in-codecs]: -Name | Content type | Notes +Name | Media type | Notes ------------|--------------------|--------------------------------- JSON | `application/json` | Returns Python primitive types. Plain text | `text/*` | Returns a Python string instance. From 55ad1ae8b1b63b08bc8d16eefb1a2de6a4019dc0 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 11:07:02 +0100 Subject: [PATCH 25/74] Fix multipart encoding content-type --- coreapi/client.py | 1 + coreapi/utils.py | 5 +++-- docs/api-guide/transports.md | 2 +- docs/api-guide/utils.md | 6 +++--- tests/test_utils.py | 12 ++++++------ 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index 373b01b..b408fb4 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -120,6 +120,7 @@ def get(self, url, force_codec=False): return transport.transition(link, self.decoders, force_codec=force_codec) def reload(self, document, force_codec=False): + # Fallback for v1.x. To be removed in favour of explict `get` style. return self.get(document.url, force_codec=force_codec) def action(self, document, keys, params=None, validate=True, overrides=None, diff --git a/coreapi/utils.py b/coreapi/utils.py index fbc72c6..d0b934b 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -120,7 +120,7 @@ def validate_query_param(value): def validate_body_param(value, encoding): if encoding == 'application/json': return _validate_json_data(value) - elif encoding == 'multipart/form': + elif encoding == 'multipart/form-data': return _validate_form_object(value, allow_files=True) elif encoding == 'application/x-www-form-urlencoded': return _validate_form_object(value) @@ -128,6 +128,7 @@ def validate_body_param(value, encoding): if not is_file(value): msg = 'Must be an file upload.' raise exceptions.ValidationError(msg) + return value msg = 'Unsupported encoding "%s" for outgoing request.' raise exceptions.NetworkError(msg % encoding) @@ -135,7 +136,7 @@ def validate_body_param(value, encoding): def validate_form_param(value, encoding): if encoding == 'application/json': return _validate_json_data(value) - elif encoding == 'multipart/form': + elif encoding == 'multipart/form-data': return _validate_form_field(value, allow_files=True) elif encoding == 'application/x-www-form-urlencoded': return _validate_form_field(value) diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md index 2c5d62e..2830e93 100644 --- a/docs/api-guide/transports.md +++ b/docs/api-guide/transports.md @@ -46,7 +46,7 @@ Supported encodings are: * `'application/json'` - Suitable for primitive and composite types. * `'application/x-www-form-urlencoded'` - Suitable for primitive types. -* `'multipart/form'` - Suitable for primitive types and file uploads. +* `'multipart/form-data'` - Suitable for primitive types and file uploads. * `'application/octet-stream'` - Suitable for raw file uploads, with a `location='body'` field. If left blank and a request body is included, then `'application/json'` is used. diff --git a/docs/api-guide/utils.md b/docs/api-guide/utils.md index 0c8e943..9694991 100644 --- a/docs/api-guide/utils.md +++ b/docs/api-guide/utils.md @@ -45,7 +45,7 @@ May raise `NoCodecAvailable`. Different request encodings have different capabilities. For example, `application/json` supports a range of data primitives, but does not support file uploads. In contrast, -`multipart/form` only supports string primatives and file uploads. +`multipart/form-data` only supports string primitives and file uploads. The following helper functions validate that the types passed to an action are suitable for use with the given encoding, and ensure that a consistent exception type is raised @@ -73,7 +73,7 @@ May raise `ValidationError`. Returns the value, coerced into a primitive that is valid for the given encoding. Validates that the parameter types provided may be used as the body of the outgoing request. -Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form` and `application/octet-stream`. +Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form-data` and `application/octet-stream`. May raise `ValidationError` for an invalid value, or `NetworkError` for an unsupported encoding. @@ -83,6 +83,6 @@ May raise `ValidationError` for an invalid value, or `NetworkError` for an unsup Returns the value, coerced into a primitive that is valid for the given encoding. Validates that the parameter types provided may be used as a key-value item for part of the body of the outgoing request. -Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form`. +Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form-data`. May raise `ValidationError`, or `NetworkError` for an unsupported encoding. diff --git a/tests/test_utils.py b/tests/test_utils.py index 3a43428..b348841 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -68,14 +68,14 @@ def test_validate_form_data(): utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'application/x-www-form-urlencoded') # Multipart - assert utils.validate_form_param(123, 'multipart/form') == '123' - assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form') == utils.File('abc.txt', None) - assert utils.validate_body_param({'a': 123}, 'multipart/form') == {'a': '123'} - assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form') == {'a': utils.File('abc.txt', None)} + assert utils.validate_form_param(123, 'multipart/form-data') == '123' + assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form-data') == utils.File('abc.txt', None) + assert utils.validate_body_param({'a': 123}, 'multipart/form-data') == {'a': '123'} + assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form-data') == {'a': utils.File('abc.txt', None)} with pytest.raises(exceptions.ValidationError): - utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form') + utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form-data') with pytest.raises(exceptions.ValidationError): - utils.validate_body_param(123, 'multipart/form') + utils.validate_body_param(123, 'multipart/form-data') # Raw upload with pytest.raises(exceptions.ValidationError): From df52bb3d34606db343f762a99bbbcbf8d85f485c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 13:30:56 +0100 Subject: [PATCH 26/74] Tweak DownloadCodec tempfile behavior slightly --- coreapi/codecs/download.py | 27 +++++++++++++++------------ coreapi/transports/http.py | 1 + docs/api-guide/codecs.md | 22 ++++++++++++++++++---- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index ed83ba7..c491a37 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -5,11 +5,12 @@ import mimetypes import os import posixpath -import shutil import tempfile class DownloadedFile(tempfile._TemporaryFileWrapper): + basename = None + def __repr__(self): state = "closed" if self.close_called else "open" mode = "" if self.close_called else " '%s'" % self.file.mode @@ -98,6 +99,8 @@ def _get_filename(base_url=None, content_type=None, content_disposition=None): filename = _get_filename_from_content_disposition(content_disposition) if base_url and not filename: filename = _get_filename_from_url(base_url, content_type) + if not filename: + return None # Ensure empty filenames return as `None` for consistency. return filename @@ -111,17 +114,11 @@ def __init__(self, download_dir=None): """ `download_dir` - The path to use for file downloads. """ - self._temporary = download_dir is None + self._delete_on_close = download_dir is None self._download_dir = download_dir - def __del__(self): - if self._temporary and self._download_dir: - shutil.rmtree(self._download_dir) - @property def download_dir(self): - if self._download_dir is None: - self._download_dir = tempfile.mkdtemp(prefix='temp-coreapi-download-') return self._download_dir def decode(self, bytestring, **options): @@ -137,15 +134,21 @@ def decode(self, bytestring, **options): # Determine the output filename. output_filename = _get_filename(base_url, content_type, content_disposition) - if not output_filename: - # Fallback if no output filename could be determined. + if output_filename is None: output_filename = os.path.basename(temp_path) + # Determine the output directory. + output_dir = self._download_dir + if output_dir is None: + output_dir = os.path.dirname(temp_path) + # Determine the full output path. - output_path = os.path.join(self.download_dir, output_filename) + output_path = os.path.join(output_dir, output_filename) output_path = _unique_output_path(output_path) # Move the temporary download file to the final location. os.rename(temp_path, output_path) output_file = open(output_path, 'rb') - return DownloadedFile(output_file, output_path, delete=self._temporary) + downloaded = DownloadedFile(output_file, output_path, delete=self._delete_on_close) + downloaded.basename = output_filename + return downloaded diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 87c32aa..7d3623f 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -88,6 +88,7 @@ def _get_params(method, encoding, fields, params=None): if errors: raise exceptions.ValidationError(errors) + # Move any files from 'data' into 'files'. if isinstance(data, dict): for key, value in list(data.items()): if is_file(data[key]): diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index 2179658..ac1c6eb 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -129,18 +129,31 @@ Example: ### DownloadCodec -Supports decoding arbitrary media as a download file. Returns a temporary file +Supports decoding arbitrary media as a download file. Returns a [temporary file][tempfile] that will be deleted once it goes out of scope. **.media_type**: `*/*` Example: - >>> from coreapi import codecs >>> codec = codecs.DownloadCodec() - >>> data = codec.decode(b'...') - >>> print(data) + >>> download = codec.decode(b'abc...xyz') + >>> print(download) + >>> content = download.read() + >>> print(content) + abc...xyz + +The download filename will be determined by either the `Content-Disposition` +header, or based on the request URL and the `Content-Type` header. Download +collisions are avoided by using incrementing filenames where required. +The original name used for the download file can be inspected using `.basename`. + + >>> download = codec.decode(b'abc...xyz', content_type='image/png', base_url='http://example.com/download/') + >>> download.name + '/var/folders/2k/qjf3np5s28zf2f58963pz2k40000gn/T/download.png' + >>> download.basename + 'download.png' #### Instantiation @@ -233,6 +246,7 @@ A codec for the [HAL][hal] hypermedia format. Installable [from PyPI][hal-pypi] [content-disposition-filename]: https://tools.ietf.org/html/draft-ietf-httpbis-content-disp-00#section-3.3 [click-ansi]: http://click.pocoo.org/5/utils/#ansi-colors +[tempfile]: https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryFile [openapi]: https://openapis.org/specification [openapi-pypi]: https://pypi.python.org/pypi/openapi-codec From 46bec50217b6e33a08338e365dda242b96f238b1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 15:52:42 +0100 Subject: [PATCH 27/74] Docs --- coreapi/client.py | 4 +-- coreapi/codecs/download.py | 13 +------- coreapi/exceptions.py | 2 +- coreapi/transports/http.py | 4 +-- coreapi/utils.py | 23 +++++++++++---- docs/api-guide/exceptions.md | 4 +-- docs/api-guide/utils.md | 57 +++++++++++++++++++++++++++++++++--- docs/topics/release-notes.md | 24 +++++++++++++++ tests/test_utils.py | 34 ++++++++++----------- 9 files changed, 120 insertions(+), 45 deletions(-) diff --git a/coreapi/client.py b/coreapi/client.py index b408fb4..f5c1f62 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -54,7 +54,7 @@ def _lookup_link(document, keys): def _validate_parameters(link, parameters): """ Ensure that parameters passed to the link are correct. - Raises a `ValidationError` if any parameters do not validate. + Raises a `ParameterError` if any parameters do not validate. """ provided = set(parameters.keys()) required = set([ @@ -77,7 +77,7 @@ def _validate_parameters(link, parameters): errors[item] = 'Unknown parameter.' if errors: - raise exceptions.ValidationError(errors) + raise exceptions.ParameterError(errors) def get_default_decoders(): diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index c491a37..3e8bed1 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -1,6 +1,7 @@ # coding: utf-8 from coreapi.codecs.base import BaseCodec from coreapi.compat import urlparse +from coreapi.utils import DownloadedFile import cgi import mimetypes import os @@ -8,18 +9,6 @@ import tempfile -class DownloadedFile(tempfile._TemporaryFileWrapper): - basename = None - - def __repr__(self): - state = "closed" if self.close_called else "open" - mode = "" if self.close_called else " '%s'" % self.file.mode - return "" % (self.name, state, mode) - - def __str__(self): - return self.__repr__() - - def _unique_output_path(path): """ Given a path like '/a/b/c.txt' diff --git a/coreapi/exceptions.py b/coreapi/exceptions.py index 3969142..fee52f0 100644 --- a/coreapi/exceptions.py +++ b/coreapi/exceptions.py @@ -37,7 +37,7 @@ class LinkLookupError(CoreAPIException): pass -class ValidationError(CoreAPIException): +class ParameterError(CoreAPIException): """ Raised when the parameters passed do not match the link fields. diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 7d3623f..dffc048 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -82,11 +82,11 @@ def _get_params(method, encoding, fields, params=None): elif location == 'form': if not seen_body: data[key] = utils.validate_form_param(value, encoding=encoding) - except exceptions.ValidationError as exc: + except exceptions.ParameterError as exc: errors[key] = exc.message if errors: - raise exceptions.ValidationError(errors) + raise exceptions.ParameterError(errors) # Move any files from 'data' into 'files'. if isinstance(data, dict): diff --git a/coreapi/utils.py b/coreapi/utils.py index d0b934b..200d83c 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -2,6 +2,7 @@ from coreapi.compat import string_types, text_type, urlparse from collections import namedtuple import os +import tempfile File = namedtuple('File', 'name content content_type') @@ -26,6 +27,18 @@ def guess_filename(obj): return None +class DownloadedFile(tempfile._TemporaryFileWrapper): + basename = None + + def __repr__(self): + state = "closed" if self.close_called else "open" + mode = "" if self.close_called else " '%s'" % self.file.mode + return "" % (self.name, state, mode) + + def __str__(self): + return self.__repr__() + + # Negotiation utilities. USed to determine which codec or transport class # should be used, given a list of supported instances. @@ -109,7 +122,7 @@ def validate_path_param(value): value = _validate_form_field(value, allow_list=False) if not value: msg = 'Parameter %s: May not be empty.' - raise exceptions.ValidationError(msg) + raise exceptions.ParameterError(msg) return value @@ -127,7 +140,7 @@ def validate_body_param(value, encoding): elif encoding == 'application/octet-stream': if not is_file(value): msg = 'Must be an file upload.' - raise exceptions.ValidationError(msg) + raise exceptions.ParameterError(msg) return value msg = 'Unsupported encoding "%s" for outgoing request.' raise exceptions.NetworkError(msg % encoding) @@ -150,7 +163,7 @@ def _validate_form_object(value, allow_files=False): """ if not isinstance(value, dict): msg = 'Must be an object.' - raise exceptions.ValidationError(msg) + raise exceptions.ParameterError(msg) return { text_type(item_key): _validate_form_field(item_val, allow_files=allow_files) for item_key, item_val in value.items() @@ -179,7 +192,7 @@ def _validate_form_field(value, allow_files=False, allow_list=True): return value msg = 'Must be a primative type.' - raise exceptions.ValidationError(msg) + raise exceptions.ParameterError(msg) def _validate_json_data(value): @@ -197,4 +210,4 @@ def _validate_json_data(value): } msg = 'Must be a JSON primative.' - raise exceptions.ValidationError(msg) + raise exceptions.ParameterError(msg) diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 136cc08..346a7af 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -24,14 +24,14 @@ An issue occurred with the network request. The keys passed in a [`client.action()`][action] call did not reference a link in the document. -## ValidationError +## ParameterError The parameters passed in a [`client.action()`][action] call did not match the set of required and optional fields made available by the link, or if the type of parameters passed could not be supported by the given encoding on the link. ## ErrorMessage -The server returned a CoreAPI [Error][error]. +The server returned a CoreAPI [Error][error] document. [action]: /api-guide/client.md#interacting-with-an-api [error]: /api-guide/document.md#error diff --git a/docs/api-guide/utils.md b/docs/api-guide/utils.md index 9694991..d59888f 100644 --- a/docs/api-guide/utils.md +++ b/docs/api-guide/utils.md @@ -5,6 +5,52 @@ may be useful if writing a custom client or transport class. --- +## File utilities + +The following classes are used to indicate upload and download file content. + +### File + +May be used as a parameter with links that require a file input. + +**Signature**: `File(name, content, content_type=None)` + +* `name` - The filename. +* `content` - A string, bytestring, or stream object. +* `content_type` - An optional string representing the content type of the file. + +An open file or other stream may also be used directly as a parameter, instead +of a `File` instance, but the `File` instance makes it easier to specify the +filename and content in code. + +Example: + + >>> from coreapi.utils import File + >>> upload = File('example.csv', 'a,b,c\n1,2,3\n4,5,6\n') + >>> data = client.action(document, ['store', 'upload_media'], params={'upload': upload}) + +### DownloadedFile + +A temporary file instance, used to represent downloaded media. + +Available attributes: + +* `name` - The full filename, including the path. +* `basename` - The filename as determined at the point of download. + +Example: + + >>> download = client.action(document, ['user', 'get_profile_image']) + >>> download.basename + 'avatar.png' + >>> download.read() + b'...' + +By default the file will be deleted when this object goes out of scope. See +[the `DownloadCodec` documentation][download-codec] for more details. + +--- + ## Negotiation utilities The following functions are used to determine which of a set of transports @@ -57,7 +103,7 @@ if an invalid value is passed. Returns the value, coerced into a string primitive. Validates that the value that is suitable for use in URI-encoded path parameters. Empty strings and composite types such as dictionaries are disallowed. -May raise `ValidationError`. +May raise `ParameterError`. ### validate_query_param @@ -65,7 +111,7 @@ May raise `ValidationError`. Returns the value, coerced into a string primitive. Validates that the value is suitable for use in URL query parameters. -May raise `ValidationError`. +May raise `ParameterError`. ### validate_body_param @@ -75,7 +121,7 @@ Returns the value, coerced into a primitive that is valid for the given encoding Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form-data` and `application/octet-stream`. -May raise `ValidationError` for an invalid value, or `NetworkError` for an unsupported encoding. +May raise `ParameterError` for an invalid value, or `NetworkError` for an unsupported encoding. ### validate_form_param @@ -85,4 +131,7 @@ Returns the value, coerced into a primitive that is valid for the given encoding Valid encodings are `application/json`, `x-www-form-urlencoded`, `multipart/form-data`. -May raise `ValidationError`, or `NetworkError` for an unsupported encoding. +May raise `ParameterError`, or `NetworkError` for an unsupported encoding. + + +[download-codec]: codecs.md#downloadcodec diff --git a/docs/topics/release-notes.md b/docs/topics/release-notes.md index 38bf0d7..357ab09 100644 --- a/docs/topics/release-notes.md +++ b/docs/topics/release-notes.md @@ -1 +1,25 @@ # Release Notes + +## 2.0 + +* Upload and download support. +* Media type changes from `application/vnd.coreapi+json` to `application/coreapi+json`. + For backwards compatibility, either are currently accepted. +* Codec methods `dump()`/`load()` become `encode()`/`decode()`. The old style + methods currently continue to work for backward compatibility. +* The client instance validates that passed parameters match the available parameter names. + Fails if unknown parameters are included, or required parameters are not included. +* `.action()` now accepts a `validate=False` argument, to turn off parameter validation. +* Parameter values are validated against the encoding used on the link to ensure + that they can be represented in the request. +* `type` annotation added to `Field` instances. +* `multipart/form-data` is now consistently used on multipart links, even when + no file arguments are passed. +* `action`, `encoding`, and `transform` parameters to `.action()` now replaced with a + single `overrides` argument. The old style arguments currently continue to work for + backward compatibility. +* The `supports` attribute is no longer used when defining codec classes. A + `supports` property currently exists on the base class, to provide backwards + compatibility for `coreapi-cli`. + +The various backwards compatibility shims are planned to be removed in the 2.1 release. diff --git a/tests/test_utils.py b/tests/test_utils.py index b348841..53550e9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,13 +6,13 @@ def test_validate_path_param(): assert utils.validate_path_param(1) == '1' assert utils.validate_path_param(True) == 'true' - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_path_param(None) - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_path_param('') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_path_param({}) - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_path_param([]) @@ -22,9 +22,9 @@ def test_validate_query_param(): assert utils.validate_query_param(None) == '' assert utils.validate_query_param('') == '' assert utils.validate_query_param([1, 2, 3]) == ['1', '2', '3'] - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_query_param({}) - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_query_param([1, 2, {}]) @@ -44,27 +44,27 @@ def test_validate_form_data(): # Invalid JSON data = datetime.datetime.now() - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_form_param(data, 'application/json') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param(data, 'application/json') data = utils.File('abc.txt', None) - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_form_param(data, 'application/json') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param(data, 'application/json') # URL Encoded assert utils.validate_form_param(123, 'application/x-www-form-urlencoded') == '123' assert utils.validate_body_param({'a': 123}, 'application/x-www-form-urlencoded') == {'a': '123'} - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_form_param({'a': {'foo': 'bar'}}, 'application/x-www-form-urlencoded') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param(123, 'application/x-www-form-urlencoded') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_form_param(utils.File('abc.txt', None), 'application/x-www-form-urlencoded') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'application/x-www-form-urlencoded') # Multipart @@ -72,13 +72,13 @@ def test_validate_form_data(): assert utils.validate_form_param(utils.File('abc.txt', None), 'multipart/form-data') == utils.File('abc.txt', None) assert utils.validate_body_param({'a': 123}, 'multipart/form-data') == {'a': '123'} assert utils.validate_body_param({'a': utils.File('abc.txt', None)}, 'multipart/form-data') == {'a': utils.File('abc.txt', None)} - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_form_param({'a': {'foo': 'bar'}}, 'multipart/form-data') - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param(123, 'multipart/form-data') # Raw upload - with pytest.raises(exceptions.ValidationError): + with pytest.raises(exceptions.ParameterError): utils.validate_body_param(123, 'application/octet-stream') # Invalid encoding on outgoing request From 71b5738ebffd7063ce1a69e0e04dbd293ac63986 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 16:07:43 +0100 Subject: [PATCH 28/74] Docs --- docs/api-guide/exceptions.md | 44 +++++++++++++++++++++++++++--------- mkdocs.yml | 2 ++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/api-guide/exceptions.md b/docs/api-guide/exceptions.md index 346a7af..8d12ea5 100644 --- a/docs/api-guide/exceptions.md +++ b/docs/api-guide/exceptions.md @@ -4,34 +4,56 @@ Any of the exceptions raised by the `coreapi` library may be imported from the ` from coreapi.exceptions import CoreAPIException -## CoreAPIException +## The base class + +#### CoreAPIException A base class for all `coreapi` exceptions. -## ParseError +--- -A response was returned with malformed content. +## Server errors -## NoCodecAvailable +The following exception occurs when the server returns an error response. -Raised when there is no available codec that can handle the given media. +#### ErrorMessage -## NetworkError +The server returned a CoreAPI [Error][error] document. -An issue occurred with the network request. +--- -## LinkLookupError +## Client errors + +The following exceptions indicate that an incorrect interaction was attempted using the client. + +#### LinkLookupError The keys passed in a [`client.action()`][action] call did not reference a link in the document. -## ParameterError +#### ParameterError The parameters passed in a [`client.action()`][action] call did not match the set of required and optional fields made available by the link, or if the type of parameters passed could not be supported by the given encoding on the link. -## ErrorMessage +--- + +## Request errors + +The following exceptions indicate that an error occurred when handling +some aspect of the API request. + +#### ParseError + +A response was returned with malformed content. + +#### NoCodecAvailable + +Raised when there is no available codec that can handle the given media. + +#### NetworkError + +An issue occurred with the network request. -The server returned a CoreAPI [Error][error] document. [action]: /api-guide/client.md#interacting-with-an-api [error]: /api-guide/document.md#error diff --git a/mkdocs.yml b/mkdocs.yml index 683babe..3e99c5a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,5 @@ site_name: Core API - Python Client + pages: - Home: index.md - API Guide: @@ -12,3 +13,4 @@ pages: - Release Notes: topics/release-notes.md repo_url: https://github.com/core-api/python-client/ +copyright: Copyright © 2015, Tom Christie. From b150e77dac576de29612e47c0bdedb54a45d5170 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 16:28:20 +0100 Subject: [PATCH 29/74] Add 'coreapi.transports' entry point. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 5571b8e..9550203 100755 --- a/setup.py +++ b/setup.py @@ -73,6 +73,9 @@ def get_package_data(package): 'json=coreapi.codecs:JSONCodec', 'text=coreapi.codecs:TextCodec', 'download=coreapi.codecs:DownloadCodec', + ], + 'coreapi.transports': [ + 'http=coreapi.transports:HTTPTransport', ] }, classifiers=[ From 5371fd4cbc74c532506009315a33de17bdb91049 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Sep 2016 16:41:20 +0100 Subject: [PATCH 30/74] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ebde295..249b288 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Python client library +# [Python client library][docs] [![build-status-image]][travis] [![pypi-version]][pypi] @@ -81,6 +81,7 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +[docs]: http://core-api.github.io/python-client/ [core-api]: https://github.com/core-api/core-api/ [build-status-image]: https://secure.travis-ci.org/core-api/python-client.svg?branch=master [travis]: http://travis-ci.org/core-api/python-client?branch=master From 6a8f0303e0430b1efa4ba01c330305a970380886 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Sep 2016 11:23:27 +0100 Subject: [PATCH 31/74] Fix bug with DownloadedFile representation in Python 3 --- coreapi/__init__.py | 2 +- coreapi/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 76f2e6e..dbf233b 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.0' +__version__ = '2.0.1' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/utils.py b/coreapi/utils.py index 200d83c..3907723 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -31,8 +31,8 @@ class DownloadedFile(tempfile._TemporaryFileWrapper): basename = None def __repr__(self): - state = "closed" if self.close_called else "open" - mode = "" if self.close_called else " '%s'" % self.file.mode + state = "closed" if self.closed else "open" + mode = "" if self.closed else " '%s'" % self.file.mode return "" % (self.name, state, mode) def __str__(self): From e6185e465771f7104dee60b29ab555f41c8bff5b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Sep 2016 11:43:13 +0100 Subject: [PATCH 32/74] Minor improvements to download behavior --- coreapi/__init__.py | 2 +- coreapi/codecs/download.py | 7 +++++-- coreapi/transports/http.py | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index dbf233b..3e75412 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.1' +__version__ = '2.0.2' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 3e8bed1..636bafa 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -133,10 +133,13 @@ def decode(self, bytestring, **options): # Determine the full output path. output_path = os.path.join(output_dir, output_filename) - output_path = _unique_output_path(output_path) # Move the temporary download file to the final location. - os.rename(temp_path, output_path) + if output_path != temp_path: + output_path = _unique_output_path(output_path) + os.rename(temp_path, output_path) + + # Open the file and return the file object. output_file = open(output_path, 'rb') downloaded = DownloadedFile(output_file, output_path, delete=self._delete_on_close) downloaded.basename = output_filename diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index dffc048..5d1030c 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -111,10 +111,12 @@ def _get_headers(url, decoders, credentials=None): """ Return a dictionary of HTTP headers to use in the outgoing request. """ - accept = '%s, */*' % ', '.join(decoders[0].get_media_types()) + accept_media_types = decoders[0].get_media_types() + if '*/*' not in accept_media_types: + accept_media_types.append('*/*') headers = { - 'accept': accept, + 'accept': ', '.join(media_types), 'user-agent': 'coreapi' } From 62556a66a69c74e9354c2f09b7f96abfd0799c68 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Sep 2016 11:45:16 +0100 Subject: [PATCH 33/74] Fix variable name --- coreapi/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 5d1030c..6618a27 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -116,7 +116,7 @@ def _get_headers(url, decoders, credentials=None): accept_media_types.append('*/*') headers = { - 'accept': ', '.join(media_types), + 'accept': ', '.join(accept_media_types), 'user-agent': 'coreapi' } From 1062bcc2fff64b5e9208996718721bf5ea031ca8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Sep 2016 12:21:10 +0100 Subject: [PATCH 34/74] More graceful download filenames --- coreapi/__init__.py | 2 +- coreapi/codecs/download.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 3e75412..983ce96 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.2' +__version__ = '2.0.3' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 636bafa..af7133b 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -70,13 +70,16 @@ def _get_filename_from_url(url, content_type=None): parsed = urlparse.urlparse(url) final_path_component = posixpath.basename(parsed.path.rstrip('/')) filename = _safe_filename(final_path_component) - if filename and ('.' not in filename) and (content_type is not None): - # If no extension exists then attempt to add one, - # based on the content type. - ext = mimetypes.guess_extension(content_type) - if ext: - filename = filename + ext - return filename + suffix = mimetypes.guess_extension(content_type or '') + + if filename: + if '.' not in filename: + return filename + suffix + return filename + elif suffix: + return 'download' + suffix + + return None def _get_filename(base_url=None, content_type=None, content_disposition=None): From c02124ee4efabd203347c66db006828d73bcb7cc Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 20 Sep 2016 13:08:27 +0100 Subject: [PATCH 35/74] Better mimetype guesses --- coreapi/__init__.py | 2 +- coreapi/codecs/download.py | 5 +-- coreapi/utils.py | 89 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 983ce96..89ec880 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.3' +__version__ = '2.0.4' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index af7133b..16e5a13 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -1,9 +1,8 @@ # coding: utf-8 from coreapi.codecs.base import BaseCodec from coreapi.compat import urlparse -from coreapi.utils import DownloadedFile +from coreapi.utils import DownloadedFile, guess_extension import cgi -import mimetypes import os import posixpath import tempfile @@ -70,7 +69,7 @@ def _get_filename_from_url(url, content_type=None): parsed = urlparse.urlparse(url) final_path_component = posixpath.basename(parsed.path.rstrip('/')) filename = _safe_filename(final_path_component) - suffix = mimetypes.guess_extension(content_type or '') + suffix = guess_extension(content_type or '') if filename: if '.' not in filename: diff --git a/coreapi/utils.py b/coreapi/utils.py index 3907723..a254123 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -27,6 +27,95 @@ def guess_filename(obj): return None +def guess_extension(content_type): + """ + Python's `mimetypes.guess_extension` is no use because it simply returns + the first of an unordered set. We use the same set of media types here, + but take a reasonable preference on what extension to map to. + """ + return { + 'application/javascript': '.js', + 'application/msword': '.doc', + 'application/octet-stream': '.bin', + 'application/oda': '.oda', + 'application/pdf': '.pdf', + 'application/pkcs7-mime': '.p7c', + 'application/postscript': '.ps', + 'application/vnd.apple.mpegurl': '.m3u', + 'application/vnd.ms-excel': '.xls', + 'application/vnd.ms-powerpoint': '.ppt', + 'application/x-bcpio': '.bcpio', + 'application/x-cpio': '.cpio', + 'application/x-csh': '.csh', + 'application/x-dvi': '.dvi', + 'application/x-gtar': '.gtar', + 'application/x-hdf': '.hdf', + 'application/x-latex': '.latex', + 'application/x-mif': '.mif', + 'application/x-netcdf': '.nc', + 'application/x-pkcs12': '.p12', + 'application/x-pn-realaudio': '.ram', + 'application/x-python-code': '.pyc', + 'application/x-sh': '.sh', + 'application/x-shar': '.shar', + 'application/x-shockwave-flash': '.swf', + 'application/x-sv4cpio': '.sv4cpio', + 'application/x-sv4crc': '.sv4crc', + 'application/x-tar': '.tar', + 'application/x-tcl': '.tcl', + 'application/x-tex': '.tex', + 'application/x-texinfo': '.texinfo', + 'application/x-troff': '.tr', + 'application/x-troff-man': '.man', + 'application/x-troff-me': '.me', + 'application/x-troff-ms': '.ms', + 'application/x-ustar': '.ustar', + 'application/x-wais-source': '.src', + 'application/xml': '.xml', + 'application/zip': '.zip', + 'audio/basic': '.au', + 'audio/mpeg': '.mp3', + 'audio/x-aiff': '.aif', + 'audio/x-pn-realaudio': '.ra', + 'audio/x-wav': '.wav', + 'image/gif': '.gif', + 'image/ief': '.ief', + 'image/jpeg': '.jpe', + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/tiff': '.tiff', + 'image/vnd.microsoft.icon': '.ico', + 'image/x-cmu-raster': '.ras', + 'image/x-ms-bmp': '.bmp', + 'image/x-portable-anymap': '.pnm', + 'image/x-portable-bitmap': '.pbm', + 'image/x-portable-graymap': '.pgm', + 'image/x-portable-pixmap': '.ppm', + 'image/x-rgb': '.rgb', + 'image/x-xbitmap': '.xbm', + 'image/x-xpixmap': '.xpm', + 'image/x-xwindowdump': '.xwd', + 'message/rfc822': '.eml', + 'text/css': '.css', + 'text/csv': '.csv', + 'text/html': '.html', + 'text/plain': '.txt', + 'text/richtext': '.rtx', + 'text/tab-separated-values': '.tsv', + 'text/x-python': '.py', + 'text/x-setext': '.etx', + 'text/x-sgml': '.sgml', + 'text/x-vcard': '.vcf', + 'text/xml': '.xml', + 'video/mp4': '.mp4', + 'video/mpeg': '.mpeg', + 'video/quicktime': '.mov', + 'video/webm': '.webm', + 'video/x-msvideo': '.avi', + 'video/x-sgi-movie': '.movie' + }.get(content_type, '') + + class DownloadedFile(tempfile._TemporaryFileWrapper): basename = None From 08389a2cb3039841198e9f877386b65046dd53b9 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Sep 2016 10:07:25 +0100 Subject: [PATCH 36/74] Minor fixes --- coreapi/__init__.py | 2 +- coreapi/codecs/base.py | 2 +- coreapi/codecs/python.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 89ec880..993c7a8 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.4' +__version__ = '2.0.5' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/base.py b/coreapi/codecs/base.py index 7b99e56..6f20044 100644 --- a/coreapi/codecs/base.py +++ b/coreapi/codecs/base.py @@ -40,5 +40,5 @@ def get_media_types(self): # Fallback, while transitioning from `application/vnd.coreapi+json` # to simply `application/coreapi+json`. if hasattr(self, 'media_types'): - return self.media_types + return list(self.media_types) return [self.media_type] diff --git a/coreapi/codecs/python.py b/coreapi/codecs/python.py index ba0b35d..d0bebf6 100644 --- a/coreapi/codecs/python.py +++ b/coreapi/codecs/python.py @@ -44,6 +44,8 @@ def _to_repr(node): args += ", encoding=%s" % repr(node.encoding) if node.transform: args += ", transform=%s" % repr(node.transform) + if node.description: + args += ", description=%s" % repr(node.description) if node.fields: fields_repr = ', '.join(_to_repr(item) for item in node.fields) args += ", fields=[%s]" % fields_repr @@ -57,6 +59,8 @@ def _to_repr(node): args += ', required=True' if node.location: args += ', location=%s' % repr(node.location) + if node.description: + args += ', description=%s' % repr(node.description) return 'Field(%s)' % args return repr(node) From 86fb56da7092f4c459250238a4b9a06e4d645e09 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Sep 2016 10:37:34 +0100 Subject: [PATCH 37/74] Use backwards compatible load method from transport --- coreapi/__init__.py | 2 +- coreapi/transports/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 993c7a8..60e4531 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.5' +__version__ = '2.0.6' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 6618a27..1b3aeba 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -246,7 +246,7 @@ def _decode_result(response, decoders, force_codec=False): if 'content-disposition' in response.headers: options['content_disposition'] = response.headers['content-disposition'] - result = codec.decode(response.content, **options) + result = codec.load(response.content, **options) else: # No content returned in response. result = None From 07382cb1f1f74808c05282327818c7639d379c12 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 23 Sep 2016 11:24:32 +0100 Subject: [PATCH 38/74] Fix parameter errors --- coreapi/__init__.py | 2 +- coreapi/transports/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 60e4531..038795a 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.6' +__version__ = '2.0.7' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 1b3aeba..2973914 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -83,7 +83,7 @@ def _get_params(method, encoding, fields, params=None): if not seen_body: data[key] = utils.validate_form_param(value, encoding=encoding) except exceptions.ParameterError as exc: - errors[key] = exc.message + errors[key] = "%s" % exc if errors: raise exceptions.ParameterError(errors) From 5bd115fa6576647122172bfd1eaeb68d3b9ad4fc Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 26 Sep 2016 14:48:27 +0100 Subject: [PATCH 39/74] Add format argument --- coreapi/__init__.py | 2 +- coreapi/client.py | 14 ++++++++++---- coreapi/codecs/corejson.py | 1 + coreapi/codecs/download.py | 1 + coreapi/codecs/jsondata.py | 1 + coreapi/codecs/text.py | 1 + docs/api-guide/client.md | 1 + docs/api-guide/codecs.md | 8 ++++++-- 8 files changed, 22 insertions(+), 7 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 038795a..71395a5 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.7' +__version__ = '2.0.8' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/client.py b/coreapi/client.py index f5c1f62..1275f4b 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -112,16 +112,22 @@ def decoders(self): def transports(self): return self._transports - def get(self, url, force_codec=False): + def get(self, url, format=None, force_codec=False): link = Link(url, action='get') + decoders = self.decoders + if format: + decoders = [decoder for decoder in self.decoders if decoder.format==format] + if not decoders: + raise ValueError("No decoder available with format='%s'" % format) + # Perform the action, and return a new document. transport = determine_transport(self.transports, link.url) - return transport.transition(link, self.decoders, force_codec=force_codec) + return transport.transition(link, decoders, force_codec=force_codec) - def reload(self, document, force_codec=False): + def reload(self, document, format=None, force_codec=False): # Fallback for v1.x. To be removed in favour of explict `get` style. - return self.get(document.url, force_codec=force_codec) + return self.get(document.url, format=format, force_codec=force_codec) def action(self, document, keys, params=None, validate=True, overrides=None, action=None, encoding=None, transform=None): diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 5c7045f..819a482 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -231,6 +231,7 @@ def _primative_to_document(data, base_url=None): class CoreJSONCodec(BaseCodec): media_type = 'application/coreapi+json' + format = 'corejson' # The following is due to be deprecated... media_types = ['application/coreapi+json', 'application/vnd.coreapi+json'] diff --git a/coreapi/codecs/download.py b/coreapi/codecs/download.py index 16e5a13..0995690 100644 --- a/coreapi/codecs/download.py +++ b/coreapi/codecs/download.py @@ -100,6 +100,7 @@ class DownloadCodec(BaseCodec): A codec to handle raw file downloads, such as images and other media. """ media_type = '*/*' + format = 'download' def __init__(self, download_dir=None): """ diff --git a/coreapi/codecs/jsondata.py b/coreapi/codecs/jsondata.py index 494f975..9fa1732 100644 --- a/coreapi/codecs/jsondata.py +++ b/coreapi/codecs/jsondata.py @@ -7,6 +7,7 @@ class JSONCodec(BaseCodec): media_type = 'application/json' + format = 'json' def decode(self, bytestring, **options): """ diff --git a/coreapi/codecs/text.py b/coreapi/codecs/text.py index f52b1de..7760498 100644 --- a/coreapi/codecs/text.py +++ b/coreapi/codecs/text.py @@ -4,6 +4,7 @@ class TextCodec(BaseCodec): media_type = 'text/*' + format = 'text' def decode(self, bytestring, **options): return bytestring.decode('utf-8') diff --git a/docs/api-guide/client.md b/docs/api-guide/client.md index 4349b11..ea07c80 100644 --- a/docs/api-guide/client.md +++ b/docs/api-guide/client.md @@ -74,6 +74,7 @@ Make a network request to the given URL. If fetching an API schema or hypermedia resource, then this should typically return a decoded `Document`. * `url` - The URL that should be retrieved. +* `format` - Optional. Force the given codec to be used when decoding the response. For example: diff --git a/docs/api-guide/codecs.md b/docs/api-guide/codecs.md index ac1c6eb..ee85bfa 100644 --- a/docs/api-guide/codecs.md +++ b/docs/api-guide/codecs.md @@ -64,6 +64,7 @@ The following attribute is available on codec instances: Supports decoding or encoding the Core JSON format. **.media_type**: `application/coreapi+json` +**.format**: `openapi` Example of decoding a Core JSON bytestring into a `Document` instance: @@ -99,6 +100,7 @@ URLs in the document. Supports decoding JSON data. **.media_type**: `application/json` +**.format**: `json` Example: @@ -116,6 +118,7 @@ Example: Supports decoding plain-text responses. **.media_type**: `text/*` +**.format**: `text` Example: @@ -133,6 +136,7 @@ Supports decoding arbitrary media as a download file. Returns a [temporary file] that will be deleted once it goes out of scope. **.media_type**: `*/*` +**.format**: `download` Example: @@ -183,7 +187,7 @@ indicate the download filename][content-disposition-filename]. ## Custom codecs Custom codec classes may be created by inheriting from `BaseCodec`, setting -the `media_type` and `supports` properties, and implementing one or both +the `media_type` and `format` properties, and implementing one or both of the `decode` or `encode` methods. For example: @@ -193,7 +197,7 @@ For example: class YAMLCodec(codecs.BaseCodec): media_type = 'application/yaml' - supports = ['data'] + format = 'yaml' def decode(content, **options): return yaml.safe_load(content) From 0f7db6cf9fc4039e310731962b1dfb26b2fc55ce Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 26 Sep 2016 14:59:31 +0100 Subject: [PATCH 40/74] Pep8 cleanup --- coreapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/client.py b/coreapi/client.py index 1275f4b..fab9c37 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -117,7 +117,7 @@ def get(self, url, format=None, force_codec=False): decoders = self.decoders if format: - decoders = [decoder for decoder in self.decoders if decoder.format==format] + decoders = [decoder for decoder in self.decoders if decoder.format == format] if not decoders: raise ValueError("No decoder available with format='%s'" % format) From 028298898d22fe540f5f75a0657e62917311d7fe Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 1 Nov 2016 14:26:53 +0000 Subject: [PATCH 41/74] Introduce a NamedTemporaryFile fallback. --- coreapi/__init__.py | 2 +- coreapi/compat.py | 6 ++++++ coreapi/utils.py | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 71395a5..a5a1680 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.8' +__version__ = '2.0.9' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/compat.py b/coreapi/compat.py index 476789d..6d06c2f 100644 --- a/coreapi/compat.py +++ b/coreapi/compat.py @@ -55,3 +55,9 @@ def force_text(string): except ImportError: def console_style(text, **kwargs): return text + + +try: + from tempfile import _TemporaryFileWrapper +except ImportError: + _TemporaryFileWrapper = None diff --git a/coreapi/utils.py b/coreapi/utils.py index a254123..fef0fdb 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -1,5 +1,5 @@ from coreapi import exceptions -from coreapi.compat import string_types, text_type, urlparse +from coreapi.compat import string_types, text_type, urlparse, _TemporaryFileWrapper from collections import namedtuple import os import tempfile @@ -116,16 +116,23 @@ def guess_extension(content_type): }.get(content_type, '') -class DownloadedFile(tempfile._TemporaryFileWrapper): - basename = None +if _TemporaryFileWrapper: + # Ideally we subclass this so that we can present a custom representation. + class DownloadedFile(_TemporaryFileWrapper): + basename = None - def __repr__(self): - state = "closed" if self.closed else "open" - mode = "" if self.closed else " '%s'" % self.file.mode - return "" % (self.name, state, mode) + def __repr__(self): + state = "closed" if self.closed else "open" + mode = "" if self.closed else " '%s'" % self.file.mode + return "" % (self.name, state, mode) - def __str__(self): - return self.__repr__() + def __str__(self): + return self.__repr__() +else: + # On some platforms (eg GAE) the private _TemporaryFileWrapper may not be + # available, just use the standard `NamedTemporaryFile` function + # in this case. + DownloadedFile = tempfile.NamedTemporaryFile # Negotiation utilities. USed to determine which codec or transport class From bc4c8c7bb01938c773950140271178d3995bb97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Padilla?= Date: Tue, 1 Nov 2016 11:03:45 -0400 Subject: [PATCH 42/74] Add wheel support --- requirements.txt | 3 +++ setup.cfg | 2 ++ setup.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 setup.cfg diff --git a/requirements.txt b/requirements.txt index 926da25..4a7d2a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,6 @@ uritemplate coverage flake8 pytest + +# Packaging requirements +wheel diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5e40900 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = 1 diff --git a/setup.py b/setup.py index 9550203..671e60e 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_package_data(package): if sys.argv[-1] == 'publish': - os.system("python setup.py sdist upload") + os.system("python setup.py sdist bdist_wheel upload") print("You probably want to also tag the version now:") print(" git tag -a %s -m 'version %s'" % (version, version)) print(" git push --tags") From b0d9124edda9e3eef2b270d1561d0cff1ef5ef79 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 29 Nov 2016 11:58:32 +0000 Subject: [PATCH 43/74] Add document description and link titles --- coreapi/__init__.py | 2 +- coreapi/codecs/corejson.py | 10 ++++++++-- coreapi/document.py | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index a5a1680..2743607 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.9' +__version__ = '2.0.10' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 819a482..9d4a2e1 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -102,6 +102,8 @@ def _document_to_primative(node, base_url=None): meta['url'] = url if node.title: meta['title'] = node.title + if node.description: + meta['description'] = node.description if meta: ret['_meta'] = meta @@ -138,6 +140,8 @@ def _document_to_primative(node, base_url=None): ret['encoding'] = node.encoding if node.transform: ret['transform'] = node.transform + if node.title: + ret['title'] = node.title if node.description: ret['description'] = node.description if node.fields: @@ -181,8 +185,9 @@ def _primative_to_document(data, base_url=None): url = _get_string(meta, 'url') url = urlparse.urljoin(base_url, url) title = _get_string(meta, 'title') + description = _get_string(meta, 'description') content = _get_content(data, base_url=url) - return Document(url=url, title=title, content=content) + return Document(url=url, title=title, description=description, content=content) if isinstance(data, dict) and data.get('_type') == 'error': # Error @@ -198,6 +203,7 @@ def _primative_to_document(data, base_url=None): action = _get_string(data, 'action') encoding = _get_string(data, 'encoding') transform = _get_string(data, 'transform') + title = _get_string(data, 'title') description = _get_string(data, 'description') fields = _get_list(data, 'fields') fields = [ @@ -212,7 +218,7 @@ def _primative_to_document(data, base_url=None): ] return Link( url=url, action=action, encoding=encoding, transform=transform, - description=description, fields=fields + title=title, description=description, fields=fields ) elif isinstance(data, dict): diff --git a/coreapi/document.py b/coreapi/document.py index 2e4edfb..4822b43 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -49,13 +49,15 @@ class Document(itypes.Dict): Expresses the data that the client may access, and the actions that the client may perform. """ - def __init__(self, url=None, title=None, content=None): + def __init__(self, url=None, title=None, description=None, content=None): content = {} if (content is None) else content if url is not None and not isinstance(url, string_types): raise TypeError("'url' must be a string.") if title is not None and not isinstance(title, string_types): raise TypeError("'title' must be a string.") + if description is not None and not isinstance(description, string_types): + raise TypeError("'description' must be a string.") if not isinstance(content, dict): raise TypeError("'content' must be a dict.") if any([not isinstance(key, string_types) for key in content.keys()]): @@ -63,6 +65,7 @@ def __init__(self, url=None, title=None, content=None): self._url = '' if (url is None) else url self._title = '' if (title is None) else title + self._description = '' if (description is None) else description self._data = {key: _to_immutable(value) for key, value in content.items()} def clone(self, data): @@ -95,6 +98,10 @@ def url(self): def title(self): return self._title + @property + def description(self): + return self._description + @property def data(self): return OrderedDict([ @@ -163,7 +170,7 @@ class Link(itypes.Object): """ Links represent the actions that a client may perform. """ - def __init__(self, url=None, action=None, encoding=None, transform=None, description=None, fields=None): + def __init__(self, url=None, action=None, encoding=None, transform=None, title=None, description=None, fields=None): if (url is not None) and (not isinstance(url, string_types)): raise TypeError("Argument 'url' must be a string.") if (action is not None) and (not isinstance(action, string_types)): @@ -172,6 +179,8 @@ def __init__(self, url=None, action=None, encoding=None, transform=None, descrip raise TypeError("Argument 'encoding' must be a string.") if (transform is not None) and (not isinstance(transform, string_types)): raise TypeError("Argument 'transform' must be a string.") + if (title is not None) and (not isinstance(title, string_types)): + raise TypeError("Argument 'title' must be a string.") if (description is not None) and (not isinstance(description, string_types)): raise TypeError("Argument 'description' must be a string.") if (fields is not None) and (not isinstance(fields, (list, tuple))): @@ -186,6 +195,7 @@ def __init__(self, url=None, action=None, encoding=None, transform=None, descrip self._action = '' if (action is None) else action self._encoding = '' if (encoding is None) else encoding self._transform = '' if (transform is None) else transform + self._title = '' if (title is None) else title self._description = '' if (description is None) else description self._fields = () if (fields is None) else tuple([ item if isinstance(item, Field) else Field(item, required=False, location='') @@ -208,6 +218,10 @@ def encoding(self): def transform(self): return self._transform + @property + def title(self): + return self._title + @property def description(self): return self._description From f0e68549bfb9b65a87207d6dc4747ba658b44fa4 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 29 Nov 2016 11:59:09 +0000 Subject: [PATCH 44/74] Version bump --- coreapi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 2743607..f115ada 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.0.10' +__version__ = '2.1.0' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', From 44e2ffe731b2c57274c99774145c83617c8347b5 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 29 Nov 2016 12:04:10 +0000 Subject: [PATCH 45/74] Add Document.media_type --- coreapi/codecs/corejson.py | 8 +++++++- coreapi/document.py | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 9d4a2e1..a48531f 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -187,7 +187,13 @@ def _primative_to_document(data, base_url=None): title = _get_string(meta, 'title') description = _get_string(meta, 'description') content = _get_content(data, base_url=url) - return Document(url=url, title=title, description=description, content=content) + return Document( + url=url, + title=title, + description=description, + media_type='application/coreapi+json', + content=content + ) if isinstance(data, dict) and data.get('_type') == 'error': # Error diff --git a/coreapi/document.py b/coreapi/document.py index 4822b43..eb06e3d 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -49,7 +49,7 @@ class Document(itypes.Dict): Expresses the data that the client may access, and the actions that the client may perform. """ - def __init__(self, url=None, title=None, description=None, content=None): + def __init__(self, url=None, title=None, description=None, media_type=None, content=None): content = {} if (content is None) else content if url is not None and not isinstance(url, string_types): @@ -58,6 +58,8 @@ def __init__(self, url=None, title=None, description=None, content=None): raise TypeError("'title' must be a string.") if description is not None and not isinstance(description, string_types): raise TypeError("'description' must be a string.") + if media_type is not None and not isinstance(media_type, string_types): + raise TypeError("'media_type' must be a string.") if not isinstance(content, dict): raise TypeError("'content' must be a dict.") if any([not isinstance(key, string_types) for key in content.keys()]): @@ -66,6 +68,7 @@ def __init__(self, url=None, title=None, description=None, content=None): self._url = '' if (url is None) else url self._title = '' if (title is None) else title self._description = '' if (description is None) else description + self._media_type = '' if (media_type is None) else media_type self._data = {key: _to_immutable(value) for key, value in content.items()} def clone(self, data): @@ -102,6 +105,10 @@ def title(self): def description(self): return self._description + @property + def media_type(self): + return self._media_type + @property def data(self): return OrderedDict([ From 3817a5ac0f5396bdfdfdef45a95febc5a61b3069 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 29 Nov 2016 12:08:59 +0000 Subject: [PATCH 46/74] Add Field.example --- coreapi/codecs/corejson.py | 3 ++- coreapi/document.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index a48531f..f683950 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -218,7 +218,8 @@ def _primative_to_document(data, base_url=None): required=_get_bool(item, 'required'), location=_get_string(item, 'location'), type=_get_string(item, 'type'), - description=_get_string(item, 'description') + description=_get_string(item, 'description'), + example=item.get('example') ) for item in fields if isinstance(item, dict) ] diff --git a/coreapi/document.py b/coreapi/document.py index eb06e3d..6ecb19e 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -36,8 +36,8 @@ def _key_sorting(item): # The field class, as used by Link objects: -Field = namedtuple('Field', ['name', 'required', 'location', 'type', 'description']) -Field.__new__.__defaults__ = (False, '', '', '') +Field = namedtuple('Field', ['name', 'required', 'location', 'type', 'description', 'example']) +Field.__new__.__defaults__ = (False, '', '', '', None) # The Core API primatives: From ddeb2fc14f8760a9ed383f13eba80c8e5492bcb2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 29 Nov 2016 12:26:14 +0000 Subject: [PATCH 47/74] Fix Document.clone() method --- coreapi/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/document.py b/coreapi/document.py index 6ecb19e..73b9eeb 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -72,7 +72,7 @@ def __init__(self, url=None, title=None, description=None, media_type=None, cont self._data = {key: _to_immutable(value) for key, value in content.items()} def clone(self, data): - return self.__class__(self.url, self.title, data) + return self.__class__(self.url, self.title, self.description, self.media_type, data) def __iter__(self): items = sorted(self._data.items(), key=_key_sorting) From aa2ca4c44b0e167ceafe64568d3cc250cb3037b1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 1 Dec 2016 11:26:28 +0000 Subject: [PATCH 48/74] Allow format='...' to automatically load an installed codec --- coreapi/__init__.py | 2 +- coreapi/client.py | 9 +++++++-- coreapi/utils.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index f115ada..e97122d 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.1.0' +__version__ = '2.1.1' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/client.py b/coreapi/client.py index fab9c37..e6ef9ce 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -1,7 +1,7 @@ from coreapi import codecs, exceptions, transports from coreapi.compat import string_types from coreapi.document import Document, Link -from coreapi.utils import determine_transport +from coreapi.utils import determine_transport, get_installed_codecs import collections import itypes @@ -117,9 +117,14 @@ def get(self, url, format=None, force_codec=False): decoders = self.decoders if format: + force_codec = True decoders = [decoder for decoder in self.decoders if decoder.format == format] if not decoders: - raise ValueError("No decoder available with format='%s'" % format) + installed_codecs = get_installed_codecs() + if format in installed_codecs: + decoders = [installed_codecs[format]] + else: + raise ValueError("No decoder available with format='%s'" % format) # Perform the action, and return a new document. transport = determine_transport(self.transports, link.url) diff --git a/coreapi/utils.py b/coreapi/utils.py index fef0fdb..2b049d5 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -2,9 +2,22 @@ from coreapi.compat import string_types, text_type, urlparse, _TemporaryFileWrapper from collections import namedtuple import os +import pkg_resources import tempfile +def get_installed_codecs(): + packages = [ + (package, package.load()) for package in + pkg_resources.iter_entry_points(group='coreapi.codecs') + ] + return { + package.name: cls() for (package, cls) in packages + } + + +# File utilities for upload and download support. + File = namedtuple('File', 'name content content_type') File.__new__.__defaults__ = (None,) From 6dd8ede4857435aed427183cb75bfa9b3d5304f5 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 24 Jan 2017 10:51:56 +0000 Subject: [PATCH 49/74] Add 'schema' to fields --- coreapi/codecs/corejson.py | 10 +++------- coreapi/codecs/python.py | 4 ++-- coreapi/document.py | 4 ++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index f683950..6854ab8 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -156,10 +156,8 @@ def _document_to_primative(node, base_url=None): ret['required'] = node.required if node.location: ret['location'] = node.location - if node.type: - ret['type'] = node.type - if node.description: - ret['description'] = node.description + if node.schema: + ret['schema'] = node.schema return ret elif isinstance(node, Object): @@ -217,9 +215,7 @@ def _primative_to_document(data, base_url=None): name=_get_string(item, 'name'), required=_get_bool(item, 'required'), location=_get_string(item, 'location'), - type=_get_string(item, 'type'), - description=_get_string(item, 'description'), - example=item.get('example') + schema=item.get('schema', None) ) for item in fields if isinstance(item, dict) ] diff --git a/coreapi/codecs/python.py b/coreapi/codecs/python.py index d0bebf6..6265a28 100644 --- a/coreapi/codecs/python.py +++ b/coreapi/codecs/python.py @@ -59,8 +59,8 @@ def _to_repr(node): args += ', required=True' if node.location: args += ', location=%s' % repr(node.location) - if node.description: - args += ', description=%s' % repr(node.description) + if node.schema: + args += ', schema=%s' % repr(node.schema) return 'Field(%s)' % args return repr(node) diff --git a/coreapi/document.py b/coreapi/document.py index 73b9eeb..8fa489e 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -36,8 +36,8 @@ def _key_sorting(item): # The field class, as used by Link objects: -Field = namedtuple('Field', ['name', 'required', 'location', 'type', 'description', 'example']) -Field.__new__.__defaults__ = (False, '', '', '', None) +Field = namedtuple('Field', ['name', 'required', 'location', 'schema']) +Field.__new__.__defaults__ = (False, '', None) # The Core API primatives: From 337a22cf56e0fcdb04a3a31d39fc9ad434e3177e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 1 Feb 2017 11:39:03 +0000 Subject: [PATCH 50/74] Don't serialize broken schemas --- coreapi/codecs/corejson.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 6854ab8..b998ec8 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -156,8 +156,8 @@ def _document_to_primative(node, base_url=None): ret['required'] = node.required if node.location: ret['location'] = node.location - if node.schema: - ret['schema'] = node.schema + #if node.schema: + # ret['schema'] = node.schema return ret elif isinstance(node, Object): From 1e08ecc27d2e1b6f3b0508a02a383b76a4329571 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 1 Feb 2017 14:44:31 +0000 Subject: [PATCH 51/74] Sorted links --- coreapi/document.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/coreapi/document.py b/coreapi/document.py index 8fa489e..198e368 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -26,11 +26,19 @@ def _str(node): def _key_sorting(item): """ Document and Object sorting. - Regular attributes sorted alphabetically, then links sorted alphabetically. + Regular attributes sorted alphabetically. + Links are sorted based on their URL and action. """ key, value = item if isinstance(value, Link): - return (1, key) + action_priority = { + 'get': 0, + 'post': 1, + 'put': 2, + 'patch': 3, + 'delete': 4 + }.get(value.action, 5) + return (1, (value.url, action_priority)) return (0, key) From 2dd9ab8b820ebfc134cbbe6e8e2c9af03ad1bd17 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Feb 2017 10:37:15 +0000 Subject: [PATCH 52/74] Add coreschema to requirements --- .travis.yml | 1 + coreapi/__init__.py | 2 +- requirements.txt | 1 + setup.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c572eae..af709cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +cache: pip python: - "2.7" diff --git a/coreapi/__init__.py b/coreapi/__init__.py index e97122d..e783de8 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.1.1' +__version__ = '2.2.0' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/requirements.txt b/requirements.txt index 4a7d2a8..71ddd11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # Package requirements +coreschema itypes requests uritemplate diff --git a/setup.py b/setup.py index 671e60e..ef91e7f 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ def get_package_data(package): packages=get_packages('coreapi'), package_data=get_package_data('coreapi'), install_requires=[ + 'coreschema', 'requests', 'itypes', 'uritemplate' From c391e628db96b528c06f1404649d55aecfea13f2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Feb 2017 10:38:30 +0000 Subject: [PATCH 53/74] Fix comment --- coreapi/codecs/corejson.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index b998ec8..b4f722b 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -156,8 +156,6 @@ def _document_to_primative(node, base_url=None): ret['required'] = node.required if node.location: ret['location'] = node.location - #if node.schema: - # ret['schema'] = node.schema return ret elif isinstance(node, Object): From eb9a181ac5123df68ec0bf738913f2de331d9405 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Feb 2017 12:38:59 +0000 Subject: [PATCH 54/74] Tweak equality operator --- coreapi/__init__.py | 2 +- coreapi/document.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index e783de8..8b9d54a 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.2.0' +__version__ = '2.2.1' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/document.py b/coreapi/document.py index 198e368..c0c51f6 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -253,7 +253,7 @@ def __eq__(self, other): self.encoding == other.encoding and self.transform == other.transform and self.description == other.description and - set(self.fields) == set(other.fields) + self.fields == other.fields ) def __repr__(self): From 02e6346adff7c81c971e086e308d5dca3e417c6b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Feb 2017 13:06:54 +0000 Subject: [PATCH 55/74] Version 2.2.2 --- coreapi/__init__.py | 2 +- coreapi/document.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 8b9d54a..aac2737 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.2.1' +__version__ = '2.2.2' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/document.py b/coreapi/document.py index c0c51f6..614c400 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -253,7 +253,7 @@ def __eq__(self, other): self.encoding == other.encoding and self.transform == other.transform and self.description == other.description and - self.fields == other.fields + sorted(self.fields, key=lambda f: f.name) == sorted(other.fields, key=lambda f: f.name) ) def __repr__(self): From c376e61fc3713443d25630b1c8779b6d9a9c07c8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Feb 2017 14:13:23 +0000 Subject: [PATCH 56/74] Allow deprecated arguments on Field. --- coreapi/__init__.py | 2 +- coreapi/document.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index aac2737..8f106de 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.2.2' +__version__ = '2.2.3' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/document.py b/coreapi/document.py index 614c400..da9044a 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -44,8 +44,10 @@ def _key_sorting(item): # The field class, as used by Link objects: -Field = namedtuple('Field', ['name', 'required', 'location', 'schema']) -Field.__new__.__defaults__ = (False, '', None) +# NOTE: 'type', 'description' and 'examaple' are now deprecated, +# in favor of 'schema'. +Field = namedtuple('Field', ['name', 'required', 'location', 'schema', 'description', 'type', 'example']) +Field.__new__.__defaults__ = (False, '', None, None, None, None) # The Core API primatives: From d463f967ad1ff5a30b08f017e48b8a1686f50d7a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 10 Feb 2017 13:08:53 +0000 Subject: [PATCH 57/74] Support schemas in corejson --- coreapi/__init__.py | 2 +- coreapi/codecs/corejson.py | 51 +++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 8f106de..bfed1a4 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.2.3' +__version__ = '2.2.4' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index b4f722b..5261618 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -5,13 +5,60 @@ from coreapi.compat import COMPACT_SEPARATORS, VERBOSE_SEPARATORS from coreapi.document import Document, Link, Array, Object, Error, Field from coreapi.exceptions import ParseError +import coreschema import json +# Schema encoding and decoding. +# Just a naive first-pass at this point. + +SCHEMA_CLASS_TO_TYPE_ID = { + coreschema.Object: 'object', + coreschema.Array: 'array', + coreschema.Number: 'number', + coreschema.Integer: 'integer', + coreschema.String: 'string', + coreschema.Boolean: 'boolean', + coreschema.Null: 'null', + coreschema.Enum: 'enum', + coreschema.Anything: 'anything' +} + +TYPE_ID_TO_SCHEMA_CLASS = { + value: key + for key, value + in SCHEMA_CLASS_TO_TYPE_ID.items() +} + + +def encode_schema_to_corejson(schema): + type_id = SCHEMA_CLASS_TO_TYPE_ID.get(schema.__class__, 'anything') + return { + '_type': type_id, + 'title': schema.title, + 'description': schema.description + } + + +def decode_schema_from_corejson(data): + type_id = _get_string(data, '_type') + title = _get_string(data, 'title') + description = _get_string(data, 'description') + schema_cls = TYPE_ID_TO_SCHEMA_CLASS.get(type_id, coreschema.Anything) + return schema_cls(title=title, description=description) + + # Robust dictionary lookups, that always return an item of the correct # type, using an empty default if an incorrect type exists. # Useful for liberal parsing of inputs. +def _get_schema(item, key): + schema_data = _get_dict(item, key) + if schema_data: + return decode_schema_from_corejson(schema_data) + return None + + def _get_string(item, key): value = item.get(key) if isinstance(value, string_types): @@ -156,6 +203,8 @@ def _document_to_primative(node, base_url=None): ret['required'] = node.required if node.location: ret['location'] = node.location + if node.schema: + ret['schema'] = encode_schema_to_corejson(node.schema) return ret elif isinstance(node, Object): @@ -213,7 +262,7 @@ def _primative_to_document(data, base_url=None): name=_get_string(item, 'name'), required=_get_bool(item, 'required'), location=_get_string(item, 'location'), - schema=item.get('schema', None) + schema=_get_schema(item, 'schema') ) for item in fields if isinstance(item, dict) ] From a91c76503fb1d325f1b1acbe222e05184f19571e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Mar 2017 13:35:35 +0000 Subject: [PATCH 58/74] Add support for auth schemes --- coreapi/__init__.py | 6 +- coreapi/auth.py | 69 +++++++++++++++++++++++ coreapi/client.py | 13 +++-- coreapi/compat.py | 2 + coreapi/transports/http.py | 111 ++++++++++++++++++++++++++----------- coreapi/utils.py | 14 +++++ docs/api-guide/auth.md | 84 ++++++++++++++++++++++++++++ docs/api-guide/client.md | 23 +++++--- docs/index.md | 7 ++- mkdocs.yml | 1 + 10 files changed, 283 insertions(+), 47 deletions(-) create mode 100644 coreapi/auth.py create mode 100644 docs/api-guide/auth.md diff --git a/coreapi/__init__.py b/coreapi/__init__.py index bfed1a4..5f069ba 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -1,12 +1,12 @@ # coding: utf-8 -from coreapi import codecs, exceptions, transports, utils +from coreapi import auth, codecs, exceptions, transports, utils from coreapi.client import Client from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.2.4' +__version__ = '2.3.0' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', - 'codecs', 'exceptions', 'transports', 'utils', + 'auth', 'codecs', 'exceptions', 'transports', 'utils', ] diff --git a/coreapi/auth.py b/coreapi/auth.py new file mode 100644 index 0000000..c3fee4a --- /dev/null +++ b/coreapi/auth.py @@ -0,0 +1,69 @@ +from coreapi.utils import domain_matches +from requests.auth import AuthBase, HTTPBasicAuth + + +class BasicAuthentication(HTTPBasicAuth): + allow_cookies = False + + def __init__(self, username, password, domain=None): + self.domain = domain + super(BasicAuthentication, self).__init__(username, password) + + def __call__(self, request): + if not domain_matches(request, self.domain): + return request + + return super(BasicAuthentication, self).__call__(request) + + +class TokenAuthentication(AuthBase): + allow_cookies = False + prefix = 'Bearer' + + def __init__(self, token, prefix=None, domain=None): + """ + * Use an unauthenticated client, and make a request to obtain a token. + * Create an authenticated client using eg. `TokenAuthentication(token="")` + """ + self.token = token + self.domain = domain + if prefix is not None: + self.prefix = prefix + + def __call__(self, request): + if not domain_matches(request, self.domain): + return request + + request.headers['Authorization'] = '%s %s' % (self.prefix, self.token) + return request + + +class SessionAuthentication(AuthBase): + """ + Enables session based login. + + * Make an initial request to obtain a CSRF token. + * Make a login request. + """ + allow_cookies = True + safe_methods = ('GET', 'HEAD', 'OPTIONS', 'TRACE') + + def __init__(self, csrf_cookie_name=None, csrf_header_name=None, domain=None): + self.csrf_cookie_name = csrf_cookie_name + self.csrf_header_name = csrf_header_name + self.csrf_token = None + self.domain = domain + + def store_csrf_token(self, response, **kwargs): + if self.csrf_cookie_name in response.cookies: + self.csrf_token = response.cookies[self.csrf_cookie_name] + + def __call__(self, request): + if not domain_matches(request, self.domain): + return request + + if self.csrf_token and self.csrf_header_name is not None and (request.method not in self.safe_methods): + request.headers[self.csrf_header_name] = self.csrf_token + if self.csrf_cookie_name is not None: + request.register_hook('response', self.store_csrf_token) + return request diff --git a/coreapi/client.py b/coreapi/client.py index e6ef9ce..d02a59c 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -89,18 +89,23 @@ def get_default_decoders(): ] -def get_default_transports(): +def get_default_transports(auth=None, session=None): return [ - transports.HTTPTransport() + transports.HTTPTransport(auth=auth, session=session) ] class Client(itypes.Object): - def __init__(self, decoders=None, transports=None): + def __init__(self, decoders=None, transports=None, auth=None, session=None): + assert transports is None or auth is None, ( + "Cannot specify both 'auth' and 'transports'. " + "When specifying transport instances explicitly you should set " + "the authentication directly on the transport." + ) if decoders is None: decoders = get_default_decoders() if transports is None: - transports = get_default_transports() + transports = get_default_transports(auth=auth) self._decoders = itypes.List(decoders) self._transports = itypes.List(transports) diff --git a/coreapi/compat.py b/coreapi/compat.py index 6d06c2f..9890ec1 100644 --- a/coreapi/compat.py +++ b/coreapi/compat.py @@ -12,6 +12,7 @@ try: # Python 2 import urlparse + import cookielib as cookiejar string_types = (basestring,) text_type = unicode @@ -26,6 +27,7 @@ def b64encode(input_string): # Python 3 import urllib.parse as urlparse from io import IOBase + from http import cookiejar string_types = (str,) text_type = str diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 2973914..694b731 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from collections import OrderedDict from coreapi import exceptions, utils -from coreapi.compat import urlparse +from coreapi.compat import cookiejar, urlparse from coreapi.document import Document, Object, Link, Array, Error from coreapi.transports.base import BaseTransport from coreapi.utils import guess_filename, is_file, File @@ -11,6 +11,7 @@ import itypes import mimetypes import uritemplate +import warnings Params = collections.namedtuple('Params', ['path', 'query', 'data', 'files']) @@ -18,9 +19,11 @@ class ForceMultiPartDict(dict): - # A dictionary that always evaluates as True. - # Allows us to force requests to use multipart encoding, even when no - # file parameters are passed. + """ + A dictionary that always evaluates as True. + Allows us to force requests to use multipart encoding, even when no + file parameters are passed. + """ def __bool__(self): return True @@ -28,6 +31,55 @@ def __nonzero__(self): return True +class BlockAll(cookiejar.CookiePolicy): + """ + A cookie policy that rejects all cookies. + Used to override the default `requests` behavior. + """ + return_ok = set_ok = domain_return_ok = path_return_ok = lambda self, *args, **kwargs: False + netscape = True + rfc2965 = hide_cookie2 = False + + +class DomainCredentials(requests.auth.AuthBase): + """ + Custom auth class to support deprecated 'credentials' argument. + """ + allow_cookies = False + credentials = None + + def __init__(self, credentials=None): + self.credentials = credentials + + def __call__(self, request): + if not self.credentials: + return request + + # Include any authorization credentials relevant to this domain. + url_components = urlparse.urlparse(request.url) + host = url_components.hostname + if host in self.credentials: + request.headers['Authorization'] = self.credentials[host] + return request + + +class CallbackAdapter(requests.adapters.HTTPAdapter): + """ + Custom requests HTTP adapter, to support deprecated callback arguments. + """ + def __init__(self, request_callback=None, response_callback=None): + self.request_callback = request_callback + self.response_callback = response_callback + + def send(self, request, **kwargs): + if self.request_callback is not None: + self.request_callback(request) + response = super(CallbackAdapter, self).send(request, **kwargs) + if self.response_callback is not None: + self.response_callback(response) + return response + + def _get_method(action): if not action: return 'GET' @@ -107,7 +159,7 @@ def _get_url(url, path_params): return url -def _get_headers(url, decoders, credentials=None): +def _get_headers(url, decoders): """ Return a dictionary of HTTP headers to use in the outgoing request. """ @@ -120,13 +172,6 @@ def _get_headers(url, decoders, credentials=None): 'user-agent': 'coreapi' } - if credentials: - # Include any authorization credentials relevant to this domain. - url_components = urlparse.urlparse(url) - host = url_components.hostname - if host in credentials: - headers['authorization'] = credentials[host] - return headers @@ -288,24 +333,34 @@ def _handle_inplace_replacements(document, link, link_ancestors): class HTTPTransport(BaseTransport): schemes = ['http', 'https'] - def __init__(self, credentials=None, headers=None, session=None, request_callback=None, response_callback=None): + def __init__(self, credentials=None, headers=None, auth=None, session=None, request_callback=None, response_callback=None): if headers: headers = {key.lower(): value for key, value in headers.items()} if session is None: session = requests.Session() - self._credentials = itypes.Dict(credentials or {}) + if auth is not None: + session.auth = auth + if not getattr(session.auth, 'allow_cookies', False): + session.cookies.set_policy(BlockAll()) + + if credentials is not None: + warnings.warn( + "The 'credentials' argument is now deprecated in favor of 'auth'.", + DeprecationWarning + ) + auth = DomainCredentials(credentials) + if request_callback is not None or response_callback is not None: + warnings.warn( + "The 'request_callback' and 'response_callback' arguments are now deprecated. " + "Use a custom 'session' instance instead.", + DeprecationWarning + ) + session.mount('https://', CallbackAdapter(request_callback, response_callback)) + session.mount('http://', CallbackAdapter(request_callback, response_callback)) + self._headers = itypes.Dict(headers or {}) self._session = session - # Fallback for v1.x overrides. - # Will be removed at some point, most likely in a 2.1 release. - self._request_callback = request_callback - self._response_callback = response_callback - - @property - def credentials(self): - return self._credentials - @property def headers(self): return self._headers @@ -316,19 +371,11 @@ def transition(self, link, decoders, params=None, link_ancestors=None, force_cod encoding = _get_encoding(link.encoding) params = _get_params(method, encoding, link.fields, params) url = _get_url(link.url, params.path) - headers = _get_headers(url, decoders, self.credentials) + headers = _get_headers(url, decoders) headers.update(self.headers) request = _build_http_request(session, url, method, headers, encoding, params) - - if self._request_callback is not None: - self._request_callback(request) - response = session.send(request) - - if self._response_callback is not None: - self._response_callback(response) - result = _decode_result(response, decoders, force_codec) if isinstance(result, Document) and link_ancestors: diff --git a/coreapi/utils.py b/coreapi/utils.py index 2b049d5..4283583 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -6,6 +6,20 @@ import tempfile +def domain_matches(request, domain): + """ + Domain string matching against an outgoing request. + Patterns starting with '*' indicate a wildcard domain. + """ + if (domain is None) or (domain == '*'): + return True + + host = urlparse.urlparse(request.url).hostname + if domain.startswith('*'): + return host.endswith(domain[1:]) + return host == domain + + def get_installed_codecs(): packages = [ (package, package.load()) for package in diff --git a/docs/api-guide/auth.md b/docs/api-guide/auth.md new file mode 100644 index 0000000..9ebc68f --- /dev/null +++ b/docs/api-guide/auth.md @@ -0,0 +1,84 @@ +# Authentication + +Authentication instances are responsible for handling the network authentication. + +## Using authentication + +Typically, you'll provide authentication configuration by passing an authentication instance to the client. + + import coreapi + + auth = coreapi.auth.BasicAuthentication(username='...', password='...') + coreapi.Client(auth=auth) + +It's recommended that you limit authentication scheme to only provide credentials to endpoints that match the expected domain. + + auth = coreapi.auth.BasicAuthentication( + username='...', + password='...', + domain='api.example.com' + ) + +You can also provide wildcard domains: + + auth = coreapi.auth.BasicAuthentication( + username='...', + password='...', + domain='*.example.com' + ) + +--- + +## Available authentication schemes + +The following authentication schemes are provided as built-in options... + +### BasicAuthentication + +Uses [HTTP Basic Authentication][basic-auth]. + +**Signature**: `BasicAuthentication(username, password, domain='*')` + +### TokenAuthentication + +Uses [HTTP Bearer token authentication][bearer-auth], and can be used for OAuth 2, JWT, and custom token authentication schemes. + +Outgoing requests will include the provided token in the request`Authorization` headers, in the following format: + + Authorization: Bearer xxxx-xxxxxxxx-xxxx + +The prefix may be customized if required, in order to support HTTP authentication schemes that are not [officially registered][http-auth-schemes]. + +A typical authentication flow using `TokenAuthentication` would be: + +* Using an unauthenticated client make a request providing the users credentials to an endpoint to that returns an API token. +* Instantiate an authenticated client using the returned token, and use this for all future requests. + +**Signature**: `TokenAuthentication(token, prefix='Bearer', domain='*')` + +### SessionAuthentication + +This authentication scheme enables cookies in order to allow a session cookie to be saved and maintained throughout the client's session. + +In order to support CSRF protected sessions, this scheme also supports saving CSRF tokens in the incoming response cookies, and mirroring those tokens back to the server by using a CSRF header in any subsequent outgoing requests. + +A typical authentication flow using `SessionAuthentication` would be: + +* Using an unauthenticated client make an initial request to an endpoint that returns a CSRF cookie. +* Use the unauthenticated client to make a request to a login endpoint, providing the users credentials. +* Subsequent requests by the client will now be authenticated. + +**Signature**: `SessionAuthentication(csrf_cookie_name=None, csrf_header_name=None, domain='*')` + +--- + +## Custom authentication + +Custom authentication classes may be created by subclassing `requests.AuthBase`, and implmenting the following: + +* Set the `allow_cookies` class attribute to either `True` or `False`. +* Provide a `__call__(self, request)` method, which should return an authenticated request instance. + +[basic-auth]: https://tools.ietf.org/html/rfc7617 +[bearer-auth]: https://tools.ietf.org/html/rfc6750 +[http-auth-schemes]: https://www.iana.org/assignments/http-authschemes/ \ No newline at end of file diff --git a/docs/api-guide/client.md b/docs/api-guide/client.md index ea07c80..e08926f 100644 --- a/docs/api-guide/client.md +++ b/docs/api-guide/client.md @@ -30,20 +30,29 @@ A client instance holds the configuration about which transports are available for making network requests, and which codecs are available for decoding the content of network responses. -This configuration is set by passing either or both of the `decoders` and -`transports` arguments. The signature of the `Client` class is: +The signature of the `Client` class is: - Client(decoders=None, transports=None) + Client(decoders=None, transports=None, auth=None, session=None) -For example the following would instantiate a client that is capable of -decoding either Core JSON schema responses, or decoding plain JSON +Arguments: + +* `decoders` - A list of decoder instances for decoding the content of responses. +* `transports` - A list of transport instances available for making network requests. +* `auth` - A authentication instance. Used when instantiating the default HTTP transport. +* `session` - A `requests` session instance. Used when instantiating the default HTTP transport. + +For example the following would instantiate a client, authenticated using HTTP basic auth, that is capable of decoding either Core JSON schema responses, or decoding plain JSON data responses: + from coreapi import codecs + from coreapi.auth import BasicAuthentication + decoders = [ codecs.CoreJSONCodec(), codecs.JSONCodec() ] - client = Client(decoders=decoders) + auth = BasicAuthentication(domain='*', username='example', password='xxx') + client = Client(decoders=decoders, auth=auth) When no arguments are passed, the following defaults are used: @@ -55,7 +64,7 @@ When no arguments are passed, the following defaults are used: ] transports = [ - transports.HTTPTransport() # http, https + transports.HTTPTransport(auth=auth, session=session) # http, https ] The configured decoders and transports are made available as read-only diff --git a/docs/index.md b/docs/index.md index efb36bd..51886bb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,11 @@ Interact with the API: 'date': '2016-10-12' }) +Creating an authenticated client instance: + + auth = coreapi.auth.TokenAuthentication(token='xxxx-xxxxxxxx-xxxx') + client = Client(auth=auth) + ## Supported formats The following schema and hypermedia formats are currently supported, either @@ -53,7 +58,7 @@ Other media | `*/*` | Returns a temporary download file. ## License -Copyright © 2015-2016, Tom Christie. +Copyright © 2015-2017, Tom Christie. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/mkdocs.yml b/mkdocs.yml index 3e99c5a..3575753 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,7 @@ pages: - API Guide: - Clients: api-guide/client.md - Documents: api-guide/document.md + - Authentication: api-guide/auth.md - Codecs: api-guide/codecs.md - Transports: api-guide/transports.md - Exceptions: api-guide/exceptions.md From f719334dfda8c0200dac47d3ef44eb8f5b94fde3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 8 Mar 2017 13:39:58 +0000 Subject: [PATCH 59/74] Improve error titles --- coreapi/transports/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index 694b731..a548024 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -299,7 +299,8 @@ def _decode_result(response, decoders, force_codec=False): # Coerce 4xx and 5xx codes into errors. is_error = response.status_code >= 400 and response.status_code <= 599 if is_error and not isinstance(result, Error): - result = _coerce_to_error(result, default_title=response.reason) + default_title = '%d %s' % (response.status_code, response.reason) + result = _coerce_to_error(result, default_title=default_title) return result From 704337e5c67ad73526d80e8849528a8f381fb9e8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 9 Mar 2017 09:18:38 +0000 Subject: [PATCH 60/74] Use 'scheme' not 'prefix' for TokenAuthentication. --- coreapi/auth.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/coreapi/auth.py b/coreapi/auth.py index c3fee4a..e110547 100644 --- a/coreapi/auth.py +++ b/coreapi/auth.py @@ -18,23 +18,23 @@ def __call__(self, request): class TokenAuthentication(AuthBase): allow_cookies = False - prefix = 'Bearer' + scheme = 'Bearer' - def __init__(self, token, prefix=None, domain=None): + def __init__(self, token, scheme=None, domain=None): """ * Use an unauthenticated client, and make a request to obtain a token. * Create an authenticated client using eg. `TokenAuthentication(token="")` """ self.token = token self.domain = domain - if prefix is not None: - self.prefix = prefix + if scheme is not None: + self.scheme = scheme def __call__(self, request): if not domain_matches(request, self.domain): return request - request.headers['Authorization'] = '%s %s' % (self.prefix, self.token) + request.headers['Authorization'] = '%s %s' % (self.scheme, self.token) return request From 8cfad1d30845b1e6f99785c1ddede4a283725701 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 9 Mar 2017 09:19:48 +0000 Subject: [PATCH 61/74] Tweak docs --- docs/api-guide/auth.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api-guide/auth.md b/docs/api-guide/auth.md index 9ebc68f..c4210e4 100644 --- a/docs/api-guide/auth.md +++ b/docs/api-guide/auth.md @@ -6,7 +6,7 @@ Authentication instances are responsible for handling the network authentication Typically, you'll provide authentication configuration by passing an authentication instance to the client. - import coreapi + import coreapi auth = coreapi.auth.BasicAuthentication(username='...', password='...') coreapi.Client(auth=auth) @@ -43,18 +43,18 @@ Uses [HTTP Basic Authentication][basic-auth]. Uses [HTTP Bearer token authentication][bearer-auth], and can be used for OAuth 2, JWT, and custom token authentication schemes. -Outgoing requests will include the provided token in the request`Authorization` headers, in the following format: +Outgoing requests will include the provided token in the request `Authorization` headers, in the following format: Authorization: Bearer xxxx-xxxxxxxx-xxxx -The prefix may be customized if required, in order to support HTTP authentication schemes that are not [officially registered][http-auth-schemes]. +The scheme name may be customized if required, in order to support HTTP authentication schemes that are not [officially registered][http-auth-schemes]. A typical authentication flow using `TokenAuthentication` would be: * Using an unauthenticated client make a request providing the users credentials to an endpoint to that returns an API token. * Instantiate an authenticated client using the returned token, and use this for all future requests. -**Signature**: `TokenAuthentication(token, prefix='Bearer', domain='*')` +**Signature**: `TokenAuthentication(token, scheme='Bearer', domain='*')` ### SessionAuthentication @@ -81,4 +81,4 @@ Custom authentication classes may be created by subclassing `requests.AuthBase`, [basic-auth]: https://tools.ietf.org/html/rfc7617 [bearer-auth]: https://tools.ietf.org/html/rfc6750 -[http-auth-schemes]: https://www.iana.org/assignments/http-authschemes/ \ No newline at end of file +[http-auth-schemes]: https://www.iana.org/assignments/http-authschemes/ From 7db1aede281c536998208a593ddeff7b8d37c8a4 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 9 Mar 2017 12:53:03 +0000 Subject: [PATCH 62/74] Update transport docs to add 'auth', and deprecate 'credentials' --- docs/api-guide/transports.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api-guide/transports.md b/docs/api-guide/transports.md index 2830e93..08c3639 100644 --- a/docs/api-guide/transports.md +++ b/docs/api-guide/transports.md @@ -18,9 +18,9 @@ The `HTTPTransport` class supports the `http` and `https` schemes. #### Instantiation -**Signature**: `HTTPTransport(credentials=None, headers=None, session=None)` +**Signature**: `HTTPTransport(auth=None, headers=None, session=None)` -* `credentials` - A dictionary of items that maps domain name to values that should be used in the request `Authorization` header for each domain. +* `auth` - An authentication instance, or None. * `headers` - A dictionary of items that should be included in the outgoing request headers. * `session` - A [requests session instance][sessions] to use when sending requests. This can be used to further customize how HTTP requests and responses are handled, for instance by allowing [transport adapters][transport-adapters] to be attached to the underlying session. From 5197840c89630ac35622b00e416aec22df0d8dc3 Mon Sep 17 00:00:00 2001 From: Tommy Beadle Date: Wed, 5 Apr 2017 10:57:55 -0400 Subject: [PATCH 63/74] Fix spelling of primitive. --- coreapi/codecs/corejson.py | 26 +++++++++++++------------- coreapi/document.py | 2 +- coreapi/utils.py | 4 ++-- tests/test_codecs.py | 20 ++++++++++---------- tests/test_document.py | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 5261618..091e7a3 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -128,15 +128,15 @@ def _get_content(item, base_url=None): Return a dictionary of content, for documents, objects and errors. """ return { - _unescape_key(key): _primative_to_document(value, base_url) + _unescape_key(key): _primitive_to_document(value, base_url) for key, value in item.items() if key not in ('_type', '_meta') } -def _document_to_primative(node, base_url=None): +def _document_to_primitive(node, base_url=None): """ - Take a Core API document and return Python primatives + Take a Core API document and return Python primitives ready to be rendered into the JSON style encoding. """ if isinstance(node, Document): @@ -156,7 +156,7 @@ def _document_to_primative(node, base_url=None): # Fill in key-value content. ret.update([ - (_escape_key(key), _document_to_primative(value, base_url=url)) + (_escape_key(key), _document_to_primitive(value, base_url=url)) for key, value in node.items() ]) return ret @@ -170,7 +170,7 @@ def _document_to_primative(node, base_url=None): # Fill in key-value content. ret.update([ - (_escape_key(key), _document_to_primative(value, base_url=base_url)) + (_escape_key(key), _document_to_primitive(value, base_url=base_url)) for key, value in node.items() ]) return ret @@ -193,7 +193,7 @@ def _document_to_primative(node, base_url=None): ret['description'] = node.description if node.fields: ret['fields'] = [ - _document_to_primative(field) for field in node.fields + _document_to_primitive(field) for field in node.fields ] return ret @@ -209,19 +209,19 @@ def _document_to_primative(node, base_url=None): elif isinstance(node, Object): return OrderedDict([ - (_escape_key(key), _document_to_primative(value, base_url=base_url)) + (_escape_key(key), _document_to_primitive(value, base_url=base_url)) for key, value in node.items() ]) elif isinstance(node, Array): - return [_document_to_primative(value) for value in node] + return [_document_to_primitive(value) for value in node] return node -def _primative_to_document(data, base_url=None): +def _primitive_to_document(data, base_url=None): """ - Take Python primatives as returned from parsing JSON content, + Take Python primitives as returned from parsing JSON content, and return a Core API document. """ if isinstance(data, dict) and data.get('_type') == 'document': @@ -278,7 +278,7 @@ def _primative_to_document(data, base_url=None): elif isinstance(data, list): # Array - content = [_primative_to_document(item, base_url) for item in data] + content = [_primitive_to_document(item, base_url) for item in data] return Array(content) # String, Integer, Number, Boolean, null. @@ -303,7 +303,7 @@ def decode(self, bytestring, **options): except ValueError as exc: raise ParseError('Malformed JSON. %s' % exc) - doc = _primative_to_document(data, base_url) + doc = _primitive_to_document(data, base_url) if isinstance(doc, Object): doc = Document(content=dict(doc)) @@ -331,5 +331,5 @@ def encode(self, document, **options): 'separators': COMPACT_SEPARATORS } - data = _document_to_primative(document) + data = _document_to_primitive(document) return force_bytes(json.dumps(data, **kwargs)) diff --git a/coreapi/document.py b/coreapi/document.py index da9044a..0fb62a7 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -50,7 +50,7 @@ def _key_sorting(item): Field.__new__.__defaults__ = (False, '', None, None, None, None) -# The Core API primatives: +# The Core API primitives: class Document(itypes.Dict): """ diff --git a/coreapi/utils.py b/coreapi/utils.py index 4283583..0b27d46 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -314,7 +314,7 @@ def _validate_form_field(value, allow_files=False, allow_list=True): elif allow_files and is_file(value): return value - msg = 'Must be a primative type.' + msg = 'Must be a primitive type.' raise exceptions.ParameterError(msg) @@ -332,5 +332,5 @@ def _validate_json_data(value): for item_key, item_val in value.items() } - msg = 'Must be a JSON primative.' + msg = 'Must be a JSON primitive.' raise exceptions.ParameterError(msg) diff --git a/tests/test_codecs.py b/tests/test_codecs.py index eb77a68..8ebbd63 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -1,6 +1,6 @@ # coding: utf-8 from coreapi.codecs import CoreJSONCodec -from coreapi.codecs.corejson import _document_to_primative, _primative_to_document +from coreapi.codecs.corejson import _document_to_primitive, _primitive_to_document from coreapi.document import Document, Link, Error, Field from coreapi.exceptions import ParseError, NoCodecAvailable from coreapi.utils import negotiate_decoder, negotiate_encoder @@ -27,10 +27,10 @@ def doc(): }) -# Documents have a mapping to python primatives in JSON style. +# Documents have a mapping to python primitives in JSON style. -def test_document_to_primative(doc): - data = _document_to_primative(doc) +def test_document_to_primitive(doc): + data = _document_to_primitive(doc) assert data == { '_type': 'document', '_meta': { @@ -46,7 +46,7 @@ def test_document_to_primative(doc): } -def test_primative_to_document(doc): +def test_primitive_to_document(doc): data = { '_type': 'document', '_meta': { @@ -60,27 +60,27 @@ def test_primative_to_document(doc): 'nested': {'child': {'_type': 'link', 'url': 'http://example.org/123'}}, '__type': 'needs escaping' } - assert _primative_to_document(data) == doc + assert _primitive_to_document(data) == doc -def test_error_to_primative(): +def test_error_to_primitive(): error = Error(title='Failure', content={'messages': ['failed']}) data = { '_type': 'error', '_meta': {'title': 'Failure'}, 'messages': ['failed'] } - assert _document_to_primative(error) == data + assert _document_to_primitive(error) == data -def test_primative_to_error(): +def test_primitive_to_error(): error = Error(title='Failure', content={'messages': ['failed']}) data = { '_type': 'error', '_meta': {'title': 'Failure'}, 'messages': ['failed'] } - assert _primative_to_document(data) == error + assert _primitive_to_document(data) == error # Codecs can load a document successfully. diff --git a/tests/test_document.py b/tests/test_document.py index 289a83d..db1d1c1 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -129,7 +129,7 @@ def test_error_does_not_support_property_assignment(): error.integer = 456 -# Children in documents are immutable primatives. +# Children in documents are immutable primitives. def test_document_dictionaries_coerced_to_objects(doc): assert isinstance(doc['dict'], Object) From 82afc12e48ad2efa24cf0c06de731b774f243c37 Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Thu, 27 Apr 2017 16:23:42 +0200 Subject: [PATCH 64/74] Fix typos --- coreapi/document.py | 2 +- coreapi/utils.py | 2 +- runtests | 4 ++-- tests/test_document.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/coreapi/document.py b/coreapi/document.py index 0fb62a7..c6c9ceb 100644 --- a/coreapi/document.py +++ b/coreapi/document.py @@ -44,7 +44,7 @@ def _key_sorting(item): # The field class, as used by Link objects: -# NOTE: 'type', 'description' and 'examaple' are now deprecated, +# NOTE: 'type', 'description' and 'example' are now deprecated, # in favor of 'schema'. Field = namedtuple('Field', ['name', 'required', 'location', 'schema', 'description', 'type', 'example']) Field.__new__.__defaults__ = (False, '', None, None, None, None) diff --git a/coreapi/utils.py b/coreapi/utils.py index 0b27d46..fb7ade4 100644 --- a/coreapi/utils.py +++ b/coreapi/utils.py @@ -237,7 +237,7 @@ def negotiate_encoder(encoders, accept=None): raise exceptions.NoCodecAvailable(msg) -# Validation utilities. Used to ensure that we get consitent validation +# Validation utilities. Used to ensure that we get consistent validation # exceptions when invalid types are passed as a parameter, rather than # an exception occuring when the request is made. diff --git a/runtests b/runtests index 54a984d..8b260d9 100755 --- a/runtests +++ b/runtests @@ -35,10 +35,10 @@ def flake8_main(args): def report_coverage(cov, fail_if_not_100=False): - precent_covered = cov.report( + percent_covered = cov.report( file=NullFile(), **COVERAGE_OPTIONS ) - if precent_covered == 100: + if percent_covered == 100: print('100% coverage') return if fail_if_not_100: diff --git a/tests/test_document.py b/tests/test_document.py index db1d1c1..6b06de7 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -50,7 +50,7 @@ def error(): def _dedent(string): """ - Convience function for dedenting multiline strings, + Convenience function for dedenting multiline strings, for string comparison purposes. """ lines = string.splitlines() From dbec97a44b332c4f8e40019a4c96a06dcf11d7b8 Mon Sep 17 00:00:00 2001 From: Tommy Beadle Date: Wed, 5 Apr 2017 13:06:11 -0400 Subject: [PATCH 65/74] Fix the encoding/decoding of Enum types. The 'enum' argument is a required parameter to creating an Enum. Store it in an 'extra' dict that can be passed as **kwargs during its creation. --- coreapi/codecs/corejson.py | 8 ++++-- tests/test_codecs.py | 55 +++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 091e7a3..37d312c 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -33,19 +33,23 @@ def encode_schema_to_corejson(schema): type_id = SCHEMA_CLASS_TO_TYPE_ID.get(schema.__class__, 'anything') - return { + retval = { '_type': type_id, 'title': schema.title, 'description': schema.description } + if isinstance(schema, coreschema.Enum): + retval['extra'] = {'enum': schema.enum} + return retval def decode_schema_from_corejson(data): type_id = _get_string(data, '_type') title = _get_string(data, 'title') description = _get_string(data, 'description') + extra = _get_dict(data, 'extra') schema_cls = TYPE_ID_TO_SCHEMA_CLASS.get(type_id, coreschema.Anything) - return schema_cls(title=title, description=description) + return schema_cls(title=title, description=description, **extra) # Robust dictionary lookups, that always return an item of the correct diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 8ebbd63..7529aa9 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -4,6 +4,7 @@ from coreapi.document import Document, Link, Error, Field from coreapi.exceptions import ParseError, NoCodecAvailable from coreapi.utils import negotiate_decoder, negotiate_encoder +from coreschema import Enum, String import pytest @@ -21,7 +22,13 @@ def doc(): 'integer': 123, 'dict': {'key': 'value'}, 'list': [1, 2, 3], - 'link': Link(url='http://example.org/', fields=[Field(name='example')]), + 'link': Link( + url='http://example.org/', + fields=[ + Field(name='noschema'), + Field(name='string_example', schema=String()), + Field(name='enum_example', schema=Enum(['a', 'b', 'c'])), + ]), 'nested': {'child': Link(url='http://example.org/123')}, '_type': 'needs escaping' }) @@ -40,7 +47,26 @@ def test_document_to_primitive(doc): 'integer': 123, 'dict': {'key': 'value'}, 'list': [1, 2, 3], - 'link': {'_type': 'link', 'fields': [{'name': 'example'}]}, + 'link': {'_type': 'link', 'fields': [ + {'name': 'noschema'}, + { + 'name': 'string_example', + 'schema': { + '_type': 'string', + 'title': '', + 'description': '', + }, + }, + { + 'name': 'enum_example', + 'schema': { + '_type': 'enum', + 'title': '', + 'description': '', + 'extra': {'enum': ['a', 'b', 'c']}, + }, + }, + ]}, 'nested': {'child': {'_type': 'link', 'url': '/123'}}, '__type': 'needs escaping' } @@ -56,7 +82,30 @@ def test_primitive_to_document(doc): 'integer': 123, 'dict': {'key': 'value'}, 'list': [1, 2, 3], - 'link': {'_type': 'link', 'url': 'http://example.org/', 'fields': [{'name': 'example'}]}, + 'link': { + '_type': 'link', + 'url': 'http://example.org/', + 'fields': [ + {'name': 'noschema'}, + { + 'name': 'string_example', + 'schema': { + '_type': 'string', + 'title': '', + 'description': '', + }, + }, + { + 'name': 'enum_example', + 'schema': { + '_type': 'enum', + 'title': '', + 'description': '', + 'extra': {'enum': ['a', 'b', 'c']}, + }, + }, + ], + }, 'nested': {'child': {'_type': 'link', 'url': 'http://example.org/123'}}, '__type': 'needs escaping' } From 13fa37f58d08c87028a4a87149b15b5e833582f1 Mon Sep 17 00:00:00 2001 From: Jesse Riggins Date: Tue, 23 May 2017 10:46:52 -0700 Subject: [PATCH 66/74] Fix the encoding/decoding of Enum types. This is an update from tbeadle/enum branch with changes that tomchriste suggested. --- coreapi/codecs/corejson.py | 10 +++++++--- tests/test_codecs.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index 37d312c..fd88a58 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -39,7 +39,7 @@ def encode_schema_to_corejson(schema): 'description': schema.description } if isinstance(schema, coreschema.Enum): - retval['extra'] = {'enum': schema.enum} + retval['enum'] = schema.enum return retval @@ -47,9 +47,13 @@ def decode_schema_from_corejson(data): type_id = _get_string(data, '_type') title = _get_string(data, 'title') description = _get_string(data, 'description') - extra = _get_dict(data, 'extra') + + kwargs = {} + if type_id == 'enum': + kwargs['enum'] = _get_list(data, 'enum') + schema_cls = TYPE_ID_TO_SCHEMA_CLASS.get(type_id, coreschema.Anything) - return schema_cls(title=title, description=description, **extra) + return schema_cls(title=title, description=description, **kwargs) # Robust dictionary lookups, that always return an item of the correct diff --git a/tests/test_codecs.py b/tests/test_codecs.py index 7529aa9..32458cd 100644 --- a/tests/test_codecs.py +++ b/tests/test_codecs.py @@ -63,7 +63,7 @@ def test_document_to_primitive(doc): '_type': 'enum', 'title': '', 'description': '', - 'extra': {'enum': ['a', 'b', 'c']}, + 'enum': ['a', 'b', 'c'], }, }, ]}, @@ -101,7 +101,7 @@ def test_primitive_to_document(doc): '_type': 'enum', 'title': '', 'description': '', - 'extra': {'enum': ['a', 'b', 'c']}, + 'enum': ['a', 'b', 'c'], }, }, ], From 24d59f4b796a8b30764f54ce539e1e6eee793e2a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 23 May 2017 12:08:59 -0700 Subject: [PATCH 67/74] Version 2.3.1 --- coreapi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index 5f069ba..af3be54 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.3.0' +__version__ = '2.3.1' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', From b550a2dc69497b80960fd3cb2acbc67c9b6ecfcf Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Mon, 7 Aug 2017 12:54:57 +0100 Subject: [PATCH 68/74] Test against Python 3.5 and 3.6 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index af709cd..f132d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" + - "3.6" install: - pip install -r requirements.txt From 64fd5f1dd37789b6635b83a9790983f013adfd00 Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Mon, 7 Aug 2017 12:55:25 +0100 Subject: [PATCH 69/74] Add Python 2/3 trove classifiers to setup.py Fixes #130. --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index ef91e7f..7dabead 100755 --- a/setup.py +++ b/setup.py @@ -86,6 +86,13 @@ def get_package_data(package): 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', ] ) From 27f4f6ed71703fe4787364090158a542ca954eb1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 5 Oct 2017 15:02:10 +0100 Subject: [PATCH 70/74] Version 2.3.2 (Compat with API Star schemas) --- coreapi/__init__.py | 2 +- coreapi/codecs/corejson.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index af3be54..c368812 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.3.1' +__version__ = '2.3.2' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', diff --git a/coreapi/codecs/corejson.py b/coreapi/codecs/corejson.py index fd88a58..f025533 100644 --- a/coreapi/codecs/corejson.py +++ b/coreapi/codecs/corejson.py @@ -32,13 +32,16 @@ def encode_schema_to_corejson(schema): - type_id = SCHEMA_CLASS_TO_TYPE_ID.get(schema.__class__, 'anything') + if hasattr(schema, 'typename'): + type_id = schema.typename + else: + type_id = SCHEMA_CLASS_TO_TYPE_ID.get(schema.__class__, 'anything') retval = { '_type': type_id, 'title': schema.title, 'description': schema.description } - if isinstance(schema, coreschema.Enum): + if hasattr(schema, 'enum'): retval['enum'] = schema.enum return retval From 2685e5dc7e2d3a56fffb4d680f58de6e9446a5ab Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 5 Oct 2017 15:04:25 +0100 Subject: [PATCH 71/74] Version 2.3.3 (Clean up PyPI package) --- coreapi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/__init__.py b/coreapi/__init__.py index c368812..92ac890 100644 --- a/coreapi/__init__.py +++ b/coreapi/__init__.py @@ -4,7 +4,7 @@ from coreapi.document import Array, Document, Link, Object, Error, Field -__version__ = '2.3.2' +__version__ = '2.3.3' __all__ = [ 'Array', 'Document', 'Link', 'Object', 'Error', 'Field', 'Client', From 687542679f64eafea4120de2fdeb92eefce5feb9 Mon Sep 17 00:00:00 2001 From: Damien Toma Date: Tue, 16 Jan 2018 14:36:15 +0800 Subject: [PATCH 72/74] Merge environment settings for SSL requests When using requests' builder, we bypass the environment settings, which include information about SSL certificates. This makes requests to a SSL endpoint fail. This commit adds a call to `session.merge_environment_settings` to fix this issue, and updates the test suite accordingly. --- coreapi/transports/http.py | 3 ++- tests/test_integration.py | 8 ++++---- tests/test_transport.py | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/coreapi/transports/http.py b/coreapi/transports/http.py index a548024..7338e61 100644 --- a/coreapi/transports/http.py +++ b/coreapi/transports/http.py @@ -376,7 +376,8 @@ def transition(self, link, decoders, params=None, link_ancestors=None, force_cod headers.update(self.headers) request = _build_http_request(session, url, method, headers, encoding, params) - response = session.send(request) + settings = session.merge_environment_settings(request.url, None, None, None, None) + response = session.send(request, **settings) result = _decode_result(response, decoders, force_codec) if isinstance(result, Document) and link_ancestors: diff --git a/tests/test_integration.py b/tests/test_integration.py index b38de9a..0a2e602 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -41,7 +41,7 @@ def test_dump(document): def test_get(monkeypatch): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'{"_type": "document", "example": 123}') monkeypatch.setattr(requests.Session, 'send', mockreturn) @@ -52,7 +52,7 @@ def mockreturn(self, request): def test_follow(monkeypatch, document): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'{"_type": "document", "example": 123}') monkeypatch.setattr(requests.Session, 'send', mockreturn) @@ -63,7 +63,7 @@ def mockreturn(self, request): def test_reload(monkeypatch): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'{"_type": "document", "example": 123}') monkeypatch.setattr(requests.Session, 'send', mockreturn) @@ -75,7 +75,7 @@ def mockreturn(self, request): def test_error(monkeypatch, document): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'{"_type": "error", "message": ["failed"]}') monkeypatch.setattr(requests.Session, 'send', mockreturn) diff --git a/tests/test_transport.py b/tests/test_transport.py index 09cb849..eeeb17a 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -47,7 +47,7 @@ def test_missing_hostname(): # Test basic transition types. def test_get(monkeypatch, http): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'{"_type": "document", "example": 123}') monkeypatch.setattr(requests.Session, 'send', mockreturn) @@ -58,7 +58,7 @@ def mockreturn(self, request): def test_get_with_parameters(monkeypatch, http): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): insert = request.path_url.encode('utf-8') return MockResponse( b'{"_type": "document", "url": "' + insert + b'"}' @@ -72,7 +72,7 @@ def mockreturn(self, request): def test_get_with_path_parameter(monkeypatch, http): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): insert = request.url.encode('utf-8') return MockResponse( b'{"_type": "document", "example": "' + insert + b'"}' @@ -90,7 +90,7 @@ def mockreturn(self, request): def test_post(monkeypatch, http): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): codec = CoreJSONCodec() body = force_text(request.body) content = codec.encode(Document(content={'data': json.loads(body)})) @@ -104,7 +104,7 @@ def mockreturn(self, request): def test_delete(monkeypatch, http): - def mockreturn(self, request): + def mockreturn(self, request, *args, **kwargs): return MockResponse(b'') monkeypatch.setattr(requests.Session, 'send', mockreturn) From c4000c26fcaca35f72ae3274049f76f24903d487 Mon Sep 17 00:00:00 2001 From: Saeed Esfandi Date: Tue, 30 Jan 2018 12:48:51 +0330 Subject: [PATCH 73/74] use input session for get default transports --- coreapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreapi/client.py b/coreapi/client.py index d02a59c..00b0057 100644 --- a/coreapi/client.py +++ b/coreapi/client.py @@ -105,7 +105,7 @@ def __init__(self, decoders=None, transports=None, auth=None, session=None): if decoders is None: decoders = get_default_decoders() if transports is None: - transports = get_default_transports(auth=auth) + transports = get_default_transports(auth=auth, session=session) self._decoders = itypes.List(decoders) self._transports = itypes.List(transports) From d264fd277a878ea40f9f75755e461f8a9418671f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 4 Jun 2018 09:19:12 +0100 Subject: [PATCH 74/74] Create LICENSE.md --- LICENSE.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..e92b342 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,29 @@ +# License + +Copyright © 2017-present, Tom Christie. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.