diff --git a/README.rst b/README.rst index 2744d96..d81fd05 100644 --- a/README.rst +++ b/README.rst @@ -1,24 +1,26 @@ Syncano ======= -Python QuickStart Guide ------------------------ +Build Status +------------ -You can find quick start on installing and using Syncano's Python library in our [documentation](http://docs.syncano.com/docs/python). +**Master** -For more detailed information on how to use Syncano and its features - our [Developer Manual](http://docs.syncano.com/docs/getting-started-with-syncano) should be very helpful. +.. image:: https://circleci.com/gh/Syncano/syncano-python/tree/master.svg?style=svg&circle-token=738c379fd91cc16b82758e6be89d0c21926655e0 + :target: https://circleci.com/gh/Syncano/syncano-python/tree/master -In case you need help working with the library - email us at libraries@syncano.com - we will be happy to help! +**Develop** -You can also find library reference hosted on GitHub pages [here](http://syncano.github.io/syncano-python/). +.. image:: https://circleci.com/gh/Syncano/syncano-python/tree/develop.svg?style=svg&circle-token=738c379fd91cc16b82758e6be89d0c21926655e0 + :target: https://circleci.com/gh/Syncano/syncano-python/tree/develop -Backwards incompatible changes ------------------------------- +Python QuickStart Guide +----------------------- + +You can find quick start on installing and using Syncano's Python library in our `documentation `_. -Version 4.x and 5.x is designed for new release of Syncano platform and -is **not compatible** with any previous releases. +For more detailed information on how to use Syncano and its features - our `Developer Manual `_ should be very helpful. -Code from `0.6.x` release is avalable on [stable/0.6.x](https://github.com/Syncano/syncano-python/tree/stable/0.6.x) branch -and it can be installed directly from pip via: +In case you need help working with the library - email us at libraries@syncano.com - we will be happy to help! -``pip install syncano==0.6.2 --pre`` +You can also find library reference hosted on GitHub pages `here `_. diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst new file mode 100644 index 0000000..f61c811 --- /dev/null +++ b/docs/source/custom_sockets.rst @@ -0,0 +1,276 @@ +.. _custom-sockets: + +========================= +Custom Sockets in Syncano +========================= + +``Syncano`` gives its users the ability to create Custom Sockets. What this means is that users can define very specific +endpoints in their Syncano application, and use them exactly like they would any other Syncano +module (Classes, Scripts, etc), using standard API calls. +Currently, Custom Sockets allow only one dependency - Scripts. Under the hood, +each API call executes a Script, and the result of this execution is returned as a result of the +API call. + +Creating a custom Socket +------------------------ + +To create a custom Socket follow these steps:: + + import syncano + from syncano.models import CustomSocket, Endpoint, ScriptCall, ScriptDependency, RuntimeChoices + from syncano.connection import Connection + + # 1. Initialize a custom Socket. + custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do API call) + + # 2. Define endpoints. + my_endpoint = Endpoint(name='my_endpoint') # no API call here + my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) + my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) + + # What happened here: + # - We defined a new endpoint that will be visible under the name `my_endpoint` + # - You will be able to call this endpoint (execute attached `call`), + # by sending a request, using any defined method to the following API route: + # :///instances//endpoints/sockets/my_endpoint/ + # - To get details for that endpoint, you need to send a GET request to following API route: + # :///instances//sockets/my_custom_socket/endpoints/my_endpoint/ + # + # Following the example above - we defined two calls on our endpoint with the `add_call` method + # The first one means that using a GET method will call the `custom_script` Script, + # and second one means that using a POST method will call the `another_custom_script` Script. + # At the moment, only Scripts are available as endpoint calls. + # + # As a general rule - to get endpoint details (but not call them), use following API route: + # :///instances//sockets/my_custom_socket/endpoints// + # and to run your endpoints (e.g. execute Script connected to them), use following API route: + # :///instances//endpoints/sockets// + + # 3. After creation of the endpoint, add it to your custom_socket. + custom_socket.add_endpoint(my_endpoint) + + # 4. Define dependency. + # 4.1 Using a new Script - define a new source code. + custom_socket.add_dependency( + ScriptDependency( + Script( + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print("custom_script")' + ), + name='custom_script' + ) + ) + # 4.2 Using an existing Script. + another_custom_script = Script.please.get(id=2) + custom_socket.add_dependency( + ScriptDependency( + another_custom_script, + name='another_custom_script', + ) + ) + + # 4.3 Using an existing ScriptEndpoint. + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint + ) + + # 5. Install custom_socket. + custom_socket.install() # this will make an API call and create a script; + +It may take some time to set up the Socket, so you can check the status. +It's possible to check the custom Socket status:: + + # Reload will refresh object using Syncano API. + custom_socket.reload() + print(custom_socket.status) + # and + print(custom_socket.status_info) + +Updating the custom Socket +-------------------------- + +To update custom Socket, use:: + + custom_socket = CustomSocket.please.get(name='my_custom_socket') + + # to remove endpoint/dependency + + custom_socket.remove_endpoint(endpoint_name='my_endpoint') + custom_socket.remove_dependency(dependency_name='custom_script') + + # or to add a new endpoint/dependency: + + custom_socket.add_endpoint(new_endpoint) # see above code for endpoint examples; + custom_socket.add_dependency(new_dependency) # see above code for dependency examples; + + # save changes on Syncano + + custom_socket.update() + + +Running custom Socket +------------------------- + +To run a custom Socket use:: + + # this will run `my_endpoint` - and call `custom_script` using GET method; + result = custom_socket.run(method='GET', endpoint_name='my_endpoint') + + +Read all endpoints in a custom Socket +----------------------------------- + +To get the all defined endpoints in a custom Socket run:: + + endpoints = custom_socket.get_endpoints() + + for endpoint in endpoints: + print(endpoint.name) + print(endpoint.calls) + +To run a particular endpoint:: + + endpoint.run(method='GET') + # or: + endpoint.run(method='POST', data={'name': 'test_name'}) + +Data will be passed to the API call in the request body. + +Read all endpoints +------------------ + +To get all endpoints that are defined in all custom Sockets:: + + socket_endpoint_list = SocketEndpoint.get_all_endpoints() + +Above code will return a list with SocketEndpoint objects. To run an endpoint, +choose one endpoint first, e.g.: + + endpoint = socket_endpoint_list[0] + +and now run it:: + + endpoint.run(method='GET') + # or: + endpoint.run(method='POST', data={'custom_data': 1}) + +Custom Sockets endpoints +------------------------ + +Each custom socket requires defining at least one endpoint. This endpoint is defined by name and +a list of calls. Each call is defined by its name and a list of methods. `name` is used as an +identification for the dependency, eg. if `name` is equal to 'my_script' - the ScriptEndpoint with name 'my_script' +will be used (if it exists and Script source and passed runtime match) -- otherwise a new one will be created. +There's a special wildcard method: `methods=['*']` - this allows you to execute the provided custom Socket +with any request method (GET, POST, PATCH, etc.). + +To add an endpoint to a chosen custom_socket use:: + + my_endpoint = Endpoint(name='my_endpoint') # no API call here + my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) + my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) + + custom_socket.add_endpoint(my_endpoint) + +Custom Socket dependency +------------------------ + +Each custom socket has a dependency -- meta information for an endpoint: which resource +should be used to return the API call results. These dependencies are bound to the endpoints call object. +Currently the only supported dependency is a Script. + +**Using new Script** + +:: + + custom_socket.add_dependency( + ScriptDependency( + Script( + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print("custom_script")' + ), + name='custom_script' + ) + ) + + +**Using defined Script** + +:: + + another_custom_script = Script.please.get(id=2) + custom_socket.add_dependency( + ScriptDependency( + another_custom_script, + name='another_custom_script' + ) + ) + +**Using defined Script endpoint** + +:: + + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint + ) + +You can overwrite the Script name in the following way:: + + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint, + name='custom_name' + ) + +Custom Socket recheck +--------------------- + +The creation of a Socket can fail - this can happen, for example, when an endpoint name is already taken by another +custom Socket. To check the creation status use:: + + print(custom_socket.status) + print(custom_socket.status_info) + +You can also re-check a Socket. This mean that all dependencies will be checked - if some of them are missing +(e.g. some were deleted by mistake), they will be created again. If the endpoints and dependencies do not meet +the criteria - an error will be returned in the status field. + +Custom Socket - install from url +-------------------------------- + +To install a socket from url use:: + + CustomSocket(name='new_socket_name').install_from_url(url='https://...') + +If instance name was not provided in connection arguments, do:: + + CustomSocket(name='new_socket_name').install_from_url(url='https://...', instance_name='instance_name') + +Custom Socket - raw format +-------------------------- + +If you prefer raw JSON format for creating Sockets, the Python library allows you to do so:::: + + CustomSocket.please.create( + name='my_custom_socket_3', + endpoints={ + "my_endpoint_3": { + "calls": + [ + {"type": "script", "name": "my_script_3", "methods": ["POST"]} + ] + } + }, + dependencies=[ + { + "type": "script", + "runtime_name": "python_library_v5.0", + "name": "my_script_3", + "source": "print(3)" + } + ] + ) + +The disadvantage of this method is that the internal structure of the JSON file must be known by the developer. diff --git a/docs/source/index.rst b/docs/source/index.rst index e54ea4a..1aaa32c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: getting_started interacting + custom_sockets refs/syncano diff --git a/docs/source/refs/syncano.models.custom_sockets.rst b/docs/source/refs/syncano.models.custom_sockets.rst new file mode 100644 index 0000000..3cecb71 --- /dev/null +++ b/docs/source/refs/syncano.models.custom_sockets.rst @@ -0,0 +1,7 @@ +syncano.models.custom_sockets +============================= + +.. automodule:: syncano.models.custom_sockets + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.custom_sockets_utils.rst b/docs/source/refs/syncano.models.custom_sockets_utils.rst new file mode 100644 index 0000000..dec7aba --- /dev/null +++ b/docs/source/refs/syncano.models.custom_sockets_utils.rst @@ -0,0 +1,7 @@ +syncano.models.custom_sockets_utils +=================================== + +.. automodule:: syncano.models.custom_sockets_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.geo.rst b/docs/source/refs/syncano.models.geo.rst new file mode 100644 index 0000000..d9eee0a --- /dev/null +++ b/docs/source/refs/syncano.models.geo.rst @@ -0,0 +1,7 @@ +syncano.models.geo +================== + +.. automodule:: syncano.models.geo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.hosting.rst b/docs/source/refs/syncano.models.hosting.rst new file mode 100644 index 0000000..48a9639 --- /dev/null +++ b/docs/source/refs/syncano.models.hosting.rst @@ -0,0 +1,7 @@ +syncano.models.hosting +====================== + +.. automodule:: syncano.models.hosting + :members: + :undoc-members: + :show-inheritance: diff --git a/setup.py b/setup.py index 2551e61..66d524b 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,8 @@ def readme(): version=__version__, description='Python Library for syncano.com api', long_description=readme(), - author='Daniel Kopka', - author_email='daniel.kopka@syncano.com', + author='Syncano', + author_email='support@syncano.io', url='http://syncano.com', packages=find_packages(exclude=['tests']), zip_safe=False, diff --git a/syncano/__init__.py b/syncano/__init__.py index 93b0df2..e0c54af 100644 --- a/syncano/__init__.py +++ b/syncano/__init__.py @@ -2,7 +2,7 @@ import os __title__ = 'Syncano Python' -__version__ = '5.3.0' +__version__ = '5.4.0' __author__ = "Daniel Kopka, Michal Kobus, and Sebastian Opalczynski" __credits__ = ["Daniel Kopka", "Michal Kobus", diff --git a/syncano/connection.py b/syncano/connection.py index 4efff69..3fcf6ad 100644 --- a/syncano/connection.py +++ b/syncano/connection.py @@ -1,4 +1,5 @@ import json +import time from copy import deepcopy import requests @@ -267,6 +268,11 @@ def make_request(self, method_name, path, **kwargs): url = self.build_url(path) response = method(url, **params) + + while response.status_code == 429: # throttling; + retry_after = response.headers.get('retry-after', 1) + time.sleep(float(retry_after)) + response = method(url, **params) content = self.get_response_content(url, response) if files: diff --git a/syncano/models/base.py b/syncano/models/base.py index 38c24b5..d8c4db9 100644 --- a/syncano/models/base.py +++ b/syncano/models/base.py @@ -12,4 +12,6 @@ from .geo import * # NOQA from .backups import * # NOQA from .hosting import * # NOQA -from .data_views import DataEndpoint as EndpointData # NOQA \ No newline at end of file +from .data_views import DataEndpoint as EndpointData # NOQA +from .custom_sockets import * # NOQA +from .custom_sockets_utils import Endpoint, ScriptCall, ScriptDependency # NOQA diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py new file mode 100644 index 0000000..441d59c --- /dev/null +++ b/syncano/models/custom_sockets.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +from syncano.exceptions import SyncanoValueError +from syncano.models.custom_sockets_utils import DependencyMetadataMixin, EndpointMetadataMixin + +from . import fields +from .base import Instance, Model + + +class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): + """ + OO wrapper around instance custom sockets. + Look at the custom socket documentation for more details. + + :ivar name: :class:`~syncano.models.fields.StringField` + :ivar endpoints: :class:`~syncano.models.fields.JSONField` + :ivar dependencies: :class:`~syncano.models.fields.JSONField` + :ivar metadata: :class:`~syncano.models.fields.JSONField` + :ivar status: :class:`~syncano.models.fields.StringField` + :ivar status_info: :class:`~syncano.models.fields.StringField` + :ivar links: :class:`~syncano.models.fields.LinksField` + """ + + name = fields.StringField(max_length=64, primary_key=True) + description = fields.StringField(required=False) + endpoints = fields.JSONField() + dependencies = fields.JSONField() + metadata = fields.JSONField(required=False) + status = fields.StringField(read_only=True, required=False) + status_info = fields.StringField(read_only=True, required=False) + created_at = fields.DateTimeField(read_only=True, required=False) + updated_at = fields.DateTimeField(read_only=True, required=False) + links = fields.LinksField() + + class Meta: + parent = Instance + endpoints = { + 'detail': { + 'methods': ['get', 'put', 'patch', 'delete'], + 'path': '/sockets/{name}/', + }, + 'list': { + 'methods': ['post', 'get'], + 'path': '/sockets/', + } + } + + def get_endpoints(self): + return SocketEndpoint.get_all_endpoints(instance_name=self.instance_name) + + def run(self, endpoint_name, method='GET', data=None): + endpoint = self._find_endpoint(endpoint_name) + return endpoint.run(method=method, data=data or {}) + + def _find_endpoint(self, endpoint_name): + endpoints = self.get_endpoints() + for endpoint in endpoints: + if '{}/{}'.format(self.name, endpoint_name) == endpoint.name: + return endpoint + raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) + + def install_from_url(self, url, instance_name=None): + instance_name = self.__class__.please.properties.get('instance_name') or instance_name + instance = Instance.please.get(name=instance_name) + + install_path = instance.links.sockets_install + connection = self._get_connection() + response = connection.request('POST', install_path, data={'name': self.name, 'install_url': url}) + + return response + + def install(self): + if not self.is_new(): + raise SyncanoValueError('Custom socket already installed.') + + created_socket = self.__class__.please.create( + name=self.name, + endpoints=self.endpoints_data, + dependencies=self.dependencies_data + ) + + created_socket._raw_data['links'] = created_socket._raw_data['links'].links_dict + self.to_python(created_socket._raw_data) + return self + + def update(self): + if self.is_new(): + raise SyncanoValueError('Install socket first.') + + update_socket = self.__class__.please.update( + name=self.name, + endpoints=self.endpoints_data, + dependencies=self.dependencies_data + ) + + update_socket._raw_data['links'] = update_socket._raw_data['links'].links_dict + self.to_python(update_socket._raw_data) + return self + + def recheck(self): + recheck_path = self.links.recheck + connection = self._get_connection() + rechecked_socket = connection.request('POST', recheck_path) + self.to_python(rechecked_socket) + return self + + +class SocketEndpoint(Model): + """ + OO wrapper around endpoints defined in CustomSocket instance. + Look at the custom socket documentation for more details. + + :ivar name: :class:`~syncano.models.fields.StringField` + :ivar calls: :class:`~syncano.models.fields.JSONField` + :ivar links: :class:`~syncano.models.fields.LinksField` + """ + name = fields.StringField(max_length=64, primary_key=True) + allowed_methods = fields.JSONField() + links = fields.LinksField() + + class Meta: + parent = CustomSocket + endpoints = { + 'detail': { + 'methods': ['get'], + 'path': '/endpoints/{name}/' + }, + 'list': { + 'methods': ['get'], + 'path': '/endpoints/' + } + } + + def run(self, method='GET', data=None): + endpoint_path = self.links.self + connection = self._get_connection() + if not self._validate_method(method): + raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) + method = method.lower() + if method in ['get', 'delete']: + response = connection.request(method, endpoint_path) + elif method in ['post', 'put', 'patch']: + response = connection.request(method, endpoint_path, data=data or {}) + else: + raise SyncanoValueError('Method: {} not supported.'.format(method)) + return response + + @classmethod + def get_all_endpoints(cls, instance_name=None): + connection = cls._meta.connection + all_endpoints_path = Instance._meta.resolve_endpoint( + 'endpoints', + {'name': cls.please.properties.get('instance_name') or instance_name} + ) + response = connection.request('GET', all_endpoints_path) + return [cls(**endpoint) for endpoint in response['objects']] + + def _validate_method(self, method): + if '*' in self.allowed_methods or method in self.allowed_methods: + return True + return False diff --git a/syncano/models/custom_sockets_utils.py b/syncano/models/custom_sockets_utils.py new file mode 100644 index 0000000..96eaf6e --- /dev/null +++ b/syncano/models/custom_sockets_utils.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +import six +from syncano.exceptions import SyncanoValueError + +from .incentives import Script, ScriptEndpoint + + +class CallType(object): + """ + The type of the call object used in the custom socket; + """ + SCRIPT = 'script' + + +class DependencyType(object): + """ + The type of the dependency object used in the custom socket; + """ + SCRIPT = 'script' + + +class BaseCall(object): + """ + Base class for call object. + """ + + call_type = None + + def __init__(self, name, methods): + self.name = name + self.methods = methods + + def to_dict(self): + if self.call_type is None: + raise SyncanoValueError('call_type not set.') + return { + 'type': self.call_type, + 'name': self.name, + 'methods': self.methods + } + + +class ScriptCall(BaseCall): + """ + Script call object. + + The JSON format is as follows (to_dict in the base class):: + + { + 'type': 'script', + 'name': ', + 'methods': [], + } + + methods can be as follows: + * ['GET'] + * ['*'] - which will do a call on every request method; + """ + call_type = CallType.SCRIPT + + +class Endpoint(object): + """ + The object which stores metadata about endpoints in custom socket; + + The JSON format is as follows:: + + { + '': { + 'calls': [ + + ] + } + } + + """ + def __init__(self, name): + self.name = name + self.calls = [] + + def add_call(self, call): + self.calls.append(call) + + def to_endpoint_data(self): + return { + self.name: { + 'calls': [call.to_dict() for call in self.calls] + } + } + + +class BaseDependency(object): + """ + Base dependency object; + + On the base of the fields attribute - the JSON format of the dependency is returned. + The fields are taken from the dependency object - which can be Script (supported now). + """ + + fields = [] + dependency_type = None + name = None + + def to_dependency_data(self): + if self.dependency_type is None: + raise SyncanoValueError('dependency_type not set.') + dependency_data = {'type': self.dependency_type} + dependency_data.update(self.get_dependency_data()) + return dependency_data + + def get_name(self): + raise NotImplementedError() + + def get_dependency_data(self): + raise NotImplementedError() + + def create_from_raw_data(self, raw_data): + raise NotImplementedError() + + +class ScriptDependency(BaseDependency): + """ + Script dependency object; + + The JSON format is as follows:: + { + 'type': 'script', + 'runtime_name': '', + 'source': '', + 'name': '' + } + """ + + dependency_type = DependencyType.SCRIPT + fields = [ + 'runtime_name', + 'source' + ] + + def __init__(self, script_or_script_endpoint, name=None): + if not isinstance(script_or_script_endpoint, (Script, ScriptEndpoint)): + raise SyncanoValueError('Script or ScriptEndpoint expected.') + + if isinstance(script_or_script_endpoint, Script) and not name: + raise SyncanoValueError('Name should be provided.') + + self.dependency_object = script_or_script_endpoint + self.name = name + + def get_name(self): + if self.name is not None: + return {'name': self.name} + return {'name': self.dependency_object.name} + + def get_dependency_data(self): + + if isinstance(self.dependency_object, ScriptEndpoint): + script = Script.please.get(id=self.dependency_object.script, + instance_name=self.dependency_object.instance_name) + else: + script = self.dependency_object + + dependency_data = self.get_name() + dependency_data.update({ + field_name: getattr(script, field_name) for field_name in self.fields + }) + return dependency_data + + @classmethod + def create_from_raw_data(cls, raw_data): + return cls(**{ + 'script_or_script_endpoint': Script(source=raw_data['source'], runtime_name=raw_data['runtime_name']), + 'name': raw_data['name'], + }) + + +class EndpointMetadataMixin(object): + """ + A mixin which allows to collect Endpoints objects and transform them to the appropriate JSON format. + """ + + def __init__(self, *args, **kwargs): + self._endpoints = [] + super(EndpointMetadataMixin, self).__init__(*args, **kwargs) + if self.endpoints: + self.update_endpoints() + + def update_endpoints(self): + for raw_endpoint_name, raw_endpoint in six.iteritems(self.endpoints): + endpoint = Endpoint( + name=raw_endpoint_name, + ) + for call in raw_endpoint['calls']: + call_class = self._get_call_class(call['type']) + call_instance = call_class(name=call['name'], methods=call['methods']) + endpoint.add_call(call_instance) + + self.add_endpoint(endpoint) + + @classmethod + def _get_call_class(cls, call_type): + if call_type == CallType.SCRIPT: + return ScriptCall + + def add_endpoint(self, endpoint): + self._endpoints.append(endpoint) + + def remove_endpoint(self, endpoint_name): + for index, endpoint in enumerate(self._endpoints): + if endpoint.name == endpoint_name: + self._endpoints.pop(index) + break + + @property + def endpoints_data(self): + endpoints = {} + for endpoint in self._endpoints: + endpoints.update(endpoint.to_endpoint_data()) + return endpoints + + +class DependencyMetadataMixin(object): + """ + A mixin which allows to collect Dependencies objects and transform them to the appropriate JSON format. + """ + + def __init__(self, *args, **kwargs): + self._dependencies = [] + super(DependencyMetadataMixin, self).__init__(*args, **kwargs) + if self.dependencies: + self.update_dependencies() + + def update_dependencies(self): + for raw_depedency in self.dependencies: + depedency_class = self._get_depedency_klass(raw_depedency['type']) + self.add_dependency(depedency_class.create_from_raw_data(raw_depedency)) + + @classmethod + def _get_depedency_klass(cls, depedency_type): + if depedency_type == DependencyType.SCRIPT: + return ScriptDependency + + def add_dependency(self, depedency): + self._dependencies.append(depedency) + + def remove_dependency(self, dependency_name): + for index, dependency in enumerate(self._dependencies): + if dependency_name == getattr(dependency.dependency_object, dependency.id_name, None): + self._dependencies.pop(index) + break + + @property + def dependencies_data(self): + return [dependency.to_dependency_data() for dependency in self._dependencies] diff --git a/syncano/models/data_views.py b/syncano/models/data_views.py index ae0384d..69dd488 100644 --- a/syncano/models/data_views.py +++ b/syncano/models/data_views.py @@ -127,3 +127,6 @@ def _get_response_template_name(self, response_template): 'Invalid response_template. Must be template\'s name or ResponseTemplate object.' ) return name + + def add_object(self, **kwargs): + return Object(instance_name=self.instance_name, class_name=self.class_name, **kwargs).save() diff --git a/syncano/models/fields.py b/syncano/models/fields.py index f6b96eb..8e4087e 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -473,7 +473,7 @@ def to_python(self, value): return LinksWrapper(value, self.IGNORED_LINKS) def to_native(self, value): - return value + return value.to_native() class ModelField(Field): @@ -508,7 +508,7 @@ def validate(self, value, model_instance): if not isinstance(value, (self.rel, dict)) and not self.is_data_object_mixin: raise self.ValidationError('Value needs to be a {0} instance.'.format(self.rel.__name__)) - if (self.required and isinstance(value, self.rel))or \ + if (self.required and isinstance(value, self.rel)) or \ (self.is_data_object_mixin and hasattr(value, 'validate')): value.validate() diff --git a/syncano/models/incentives.py b/syncano/models/incentives.py index 12a0d4d..84ef2e4 100644 --- a/syncano/models/incentives.py +++ b/syncano/models/incentives.py @@ -45,7 +45,7 @@ class Script(Model): >>> Script.please.run('instance-name', 1234) >>> Script.please.run('instance-name', 1234, payload={'variable_one': 1, 'variable_two': 2}) - >>> Script.please.run('instance-name', 1234, payload="{\"variable_one\": 1, \"variable_two\": 2}") + >>> Script.please.run('instance-name', 1234, payload='{"variable_one": 1, "variable_two": 2}') or via instance:: @@ -54,7 +54,7 @@ class Script(Model): >>> s.run(variable_one=1, variable_two=2) """ - label = fields.StringField(max_length=80) + label = fields.StringField(max_length=80, required=False) description = fields.StringField(required=False) source = fields.StringField() runtime_name = fields.StringField() diff --git a/syncano/models/instances.py b/syncano/models/instances.py index f4f128a..788ba05 100644 --- a/syncano/models/instances.py +++ b/syncano/models/instances.py @@ -82,6 +82,10 @@ class Meta: 'config': { 'methods': ['put', 'get'], 'path': '/v1.1/instances/{name}/snippets/config/', + }, + 'endpoints': { + 'methods': ['get'], + 'path': '/v1.1/instances/{name}/endpoints/sockets/' } } diff --git a/tests/integration_test.py b/tests/integration_test.py index 622722c..2acfcf3 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -478,7 +478,7 @@ def test_source_run(self): ) trace = script.run() - while trace.status == 'pending': + while trace.status in ['pending', 'processing']: sleep(1) trace.reload() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py new file mode 100644 index 0000000..2ddd2f3 --- /dev/null +++ b/tests/integration_test_custom_socket.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +import time + +from syncano.models import ( + CustomSocket, + Endpoint, + RuntimeChoices, + Script, + ScriptCall, + ScriptDependency, + ScriptEndpoint, + SocketEndpoint +) +from tests.integration_test import InstanceMixin, IntegrationTest + + +class CustomSocketTest(InstanceMixin, IntegrationTest): + + def test_install_custom_socket(self): + # this tests new ScriptEndpoint dependency create; + self.assert_custom_socket('installing', self._define_dependencies_new_script_endpoint) + + def test_dependencies_new_script(self): + self.assert_custom_socket('new_script_installing', self._define_dependencies_new_script) + + def test_dependencies_existing_script(self): + self.assert_custom_socket('existing_script_installing', self._define_dependencies_existing_script) + + def test_dependencies_existing_script_endpoint(self): + self.assert_custom_socket('existing_script_e_installing', + self._define_dependencies_existing_script_endpoint) + + def test_creating_raw_data(self): + custom_socket = CustomSocket.please.create( + name='my_custom_socket_123', + endpoints={ + "my_custom_endpoint_123": { + "calls": [{"type": "script", "name": "script_123", "methods": ["GET", "POST"]}] + } + }, + dependencies=[ + { + "type": "script", + "runtime_name": "python_library_v5.0", + "name": "script_123", + "source": "print(123)" + } + ] + ) + + self.assertTrue(custom_socket.name) + + def test_custom_socket_run(self): + suffix = 'default' + custom_socket = self._create_custom_socket(suffix, self._define_dependencies_new_script_endpoint) + self._assert_custom_socket(custom_socket) + results = custom_socket.run('my_endpoint_{}'.format(suffix)) + self.assertEqual(results['result']['stdout'], suffix) + + def test_custom_socket_recheck(self): + suffix = 'recheck' + custom_socket = self._create_custom_socket(suffix, self._define_dependencies_new_script_endpoint) + self._assert_custom_socket(custom_socket) + custom_socket = custom_socket.recheck() + self._assert_custom_socket(custom_socket) + + def test_fetching_all_endpoints(self): + all_endpoints = SocketEndpoint.get_all_endpoints() + self.assertTrue(isinstance(all_endpoints, list)) + self.assertTrue(len(all_endpoints) >= 1) + self.assertTrue(all_endpoints[0].name) + + def test_endpoint_run(self): + script_endpoint = SocketEndpoint.get_all_endpoints()[0] + result = script_endpoint.run() + self.assertIsInstance(result, dict) + self.assertTrue(result['result']['stdout']) + + def test_custom_socket_update(self): + socket_to_update = self._create_custom_socket('to_update', self._define_dependencies_new_script_endpoint) + socket_to_update.remove_endpoint(endpoint_name='my_endpoint_to_update') + + new_endpoint = Endpoint(name='my_endpoint_new_to_update') + new_endpoint.add_call( + ScriptCall(name='script_default', methods=['GET']) + ) + + socket_to_update.add_endpoint(new_endpoint) + socket_to_update.update() + time.sleep(2) # wait for custom socket setup; + socket_to_update.reload() + self.assertIn('my_endpoint_new_to_update', socket_to_update.endpoints) + + def assert_custom_socket(self, suffix, dependency_method): + custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) + self._assert_custom_socket(custom_socket) + + def _assert_custom_socket(self, custom_socket): + self._wait_till_socket_process(custom_socket) + self.assertTrue(custom_socket.name) + self.assertTrue(custom_socket.created_at) + self.assertTrue(custom_socket.updated_at) + + @classmethod + def _create_custom_socket(cls, suffix, dependency_method): + custom_socket = CustomSocket(name='my_custom_socket_{}'.format(suffix)) + + cls._define_endpoints(suffix, custom_socket) + dependency_method(suffix, custom_socket) + + custom_socket.install() + return custom_socket + + @classmethod + def _define_endpoints(cls, suffix, custom_socket): + endpoint = Endpoint(name='my_endpoint_{}'.format(suffix)) + endpoint.add_call( + ScriptCall( + name='script_endpoint_{}'.format(suffix), + methods=['GET', 'POST'] + ) + ) + custom_socket.add_endpoint(endpoint) + + @classmethod + def _define_dependencies_new_script_endpoint(cls, suffix, custom_socket): + script = cls._create_script(suffix) + script_endpoint = ScriptEndpoint( + name='script_endpoint_{}'.format(suffix), + script=script.id + ) + custom_socket.add_dependency( + ScriptDependency( + script_endpoint + ) + ) + + @classmethod + def _define_dependencies_new_script(cls, suffix, custom_socket): + custom_socket.add_dependency( + ScriptDependency( + Script( + source='print("{}")'.format(suffix), + runtime_name=RuntimeChoices.PYTHON_V5_0 + ), + name='script_endpoint_{}'.format(suffix), + ) + ) + + @classmethod + def _define_dependencies_existing_script(cls, suffix, custom_socket): + # create Script first: + cls._create_script(suffix) + custom_socket.add_dependency( + ScriptDependency( + Script.please.first(), + name='script_endpoint_{}'.format(suffix), + ) + ) + + @classmethod + def _define_dependencies_existing_script_endpoint(cls, suffix, custom_socket): + script = cls._create_script(suffix) + ScriptEndpoint.please.create( + name='script_endpoint_{}'.format(suffix), + script=script.id + ) + custom_socket.add_dependency( + ScriptDependency( + ScriptEndpoint.please.first() + ) + ) + + @classmethod + def _create_script(cls, suffix): + return Script.please.create( + label='script_{}'.format(suffix), + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print("{}")'.format(suffix) + ) + + @classmethod + def _wait_till_socket_process(cls, custom_socket): + while custom_socket.status == 'checking': + custom_socket.reload() diff --git a/tests/integration_test_data_endpoint.py b/tests/integration_test_data_endpoint.py index 4cccd9f..6ce9c1e 100644 --- a/tests/integration_test_data_endpoint.py +++ b/tests/integration_test_data_endpoint.py @@ -7,76 +7,93 @@ class DataEndpointTest(InstanceMixin, IntegrationTest): - schema = [ - { - 'name': 'title', - 'type': 'string', - 'order_index': True, - 'filter_index': True - } - ] + @classmethod + def setUpClass(cls): + super(DataEndpointTest, cls).setUpClass() - template_content = ''' - {% if action == 'list' %} - {% set objects = response.objects %} - {% elif action == 'retrieve' %} - {% set objects = [response] %} - {% else %} - {% set objects = [] %} - {% endif %} - {% if objects %} - + schema = [ + { + 'name': 'title', + 'type': 'string', + 'order_index': True, + 'filter_index': True + } + ] - - {% for key in objects[0] if key not in fields_to_skip %} - {{ key }} - {% endfor %} - - {% for object in objects %} - - {% for key, value in object.iteritems() if key not in fields_to_skip %} - {{ value }} - {% endfor %} + template_content = ''' + {% if action == 'list' %} + {% set objects = response.objects %} + {% elif action == 'retrieve' %} + {% set objects = [response] %} + {% else %} + {% set objects = [] %} + {% endif %} + {% if objects %} + + + + {% for key in objects[0] if key not in fields_to_skip %} + {{ key }} + {% endfor %} - {% endfor %} - - {% endif %} - ''' + {% for object in objects %} + + {% for key, value in object.iteritems() if key not in fields_to_skip %} + {{ value }} + {% endfor %} + + {% endfor %} + + {% endif %} + ''' - template_context = { - "tr_header_classes": "", - "th_header_classes": "", - "tr_row_classes": "", - "table_classes": "", - "td_row_classes": "", - "fields_to_skip": [ - "id", - "channel", - "channel_room", - "group", - "links", - "group_permissions", - "owner_permissions", - "other_permissions", - "owner", - "revision", - "updated_at", - "created_at" - ] - } + template_context = { + "tr_header_classes": "", + "th_header_classes": "", + "tr_row_classes": "", + "table_classes": "", + "td_row_classes": "", + "fields_to_skip": [ + "id", + "channel", + "channel_room", + "group", + "links", + "group_permissions", + "owner_permissions", + "other_permissions", + "owner", + "revision", + "updated_at", + "created_at" + ] + } - def test_template_response(self): - Class(name='test_class', schema=self.schema).save() - Object(class_name='test_class', title='test_title').save() - template = ResponseTemplate( + cls.klass = Class(name='test_class', schema=schema).save() + cls.template = ResponseTemplate( name='test_template', - content=self.template_content, + content=template_content, content_type='text/html', - context=self.template_context + context=template_context ).save() - data_endpoint = DataEndpoint(name='test_endpoint', class_name='test_class').save() + cls.data_endpoint = DataEndpoint(name='test_endpoint', class_name='test_class').save() - response = list(data_endpoint.get(response_template=template)) + def setUp(self): + for obj in self.instance.classes.get(name='test_class').objects.all(): + obj.delete() + + def test_template_response(self): + Object(class_name=self.klass.name, title='test_title').save() + response = list(self.data_endpoint.get(response_template=self.template)) self.assertEqual(len(response), 1, 'Data endpoint should return 1 element if queried with response_template.') data = re.sub('[\s+]', '', response[0]) self.assertEqual(data, '
title
test_title
') + + def test_create_object(self): + objects_count = len(list(self.data_endpoint.get())) + self.assertEqual(objects_count, 0) + self.data_endpoint.add_object(title='another title') + objects_count = len(list(self.data_endpoint.get())) + self.assertEqual(objects_count, 1, 'New object should have been created.') + obj = next(self.data_endpoint.get()) + self.assertEqual(obj['title'], 'another title', 'Created object should have proper title.')