From 90c4333e8030b20f13030770a184098f86039b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 9 Aug 2016 15:57:13 +0200 Subject: [PATCH 01/35] [LIB-821] add retry-after handling; --- syncano/connection.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/syncano/connection.py b/syncano/connection.py index 4efff69..b5ad82c 100644 --- a/syncano/connection.py +++ b/syncano/connection.py @@ -3,6 +3,8 @@ import requests import six +import time + import syncano from syncano.exceptions import RevisionMismatchException, SyncanoRequestError, SyncanoValueError @@ -267,6 +269,12 @@ def make_request(self, method_name, path, **kwargs): url = self.build_url(path) response = method(url, **params) + + while response.status_code == 429: # throttling; + retry_after = response.headers.get('retry-after', 1) + time.sleep(float(retry_after)) + response = method(url, **params) + content = self.get_response_content(url, response) if files: From 3801a26fd806bd3e0a2947857da6991ab64b60d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 9 Aug 2016 16:00:02 +0200 Subject: [PATCH 02/35] [LIB-821] correct isort issues; --- syncano/connection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/syncano/connection.py b/syncano/connection.py index b5ad82c..c4c7706 100644 --- a/syncano/connection.py +++ b/syncano/connection.py @@ -1,10 +1,9 @@ import json +import time from copy import deepcopy import requests import six -import time - import syncano from syncano.exceptions import RevisionMismatchException, SyncanoRequestError, SyncanoValueError From 12cda74d7fe6260e84b87270fcccfe32f3d31b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Wed, 10 Aug 2016 18:01:15 +0200 Subject: [PATCH 03/35] [LIB-837][WIP] working on custom sockets in LIB --- syncano/models/base.py | 3 +- syncano/models/custom_sockets.py | 220 +++++++++++++++++++++++++++++++ syncano/models/fields.py | 2 +- 3 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 syncano/models/custom_sockets.py diff --git a/syncano/models/base.py b/syncano/models/base.py index 38c24b5..4b73a1d 100644 --- a/syncano/models/base.py +++ b/syncano/models/base.py @@ -12,4 +12,5 @@ from .geo import * # NOQA from .backups import * # NOQA from .hosting import * # NOQA -from .data_views import DataEndpoint as EndpointData # NOQA \ No newline at end of file +from .data_views import DataEndpoint as EndpointData # NOQA +from .custom_sockets import * # NOQA \ No newline at end of file diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py new file mode 100644 index 0000000..305d716 --- /dev/null +++ b/syncano/models/custom_sockets.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +from .base import Instance, Model +from . import fields + +from syncano.exceptions import SyncanoValueError + + +class CallTypeE(object): + SCRIPT = 'script' + + +class DependencyTypeE(object): + SCRIPT = 'script' + + +class Call(object): + + def __init__(self, name, methods, call_type=None): + call_type = call_type or CallTypeE.SCRIPT + self.type = call_type + self.name = name + self.methods = methods + + def to_dict(self): + return { + 'type': self.type, + 'name': self.name, + 'methods': self.methods + } + + +class Endpoint(object): + + def __init__(self, name): + self.name = name + self.calls = [] + + def add_call(self, call): + self.calls.append(call) + + def to_endpoint_data(self): + return { + self.name: { + 'calls': [call.to_dict() for call in self.calls] + } + } + + +class BaseDependency(object): + + fields = [] + dependency_type = None + field_mapping = {} + + def __init__(self, dependency_object): + self.dependency_object = dependency_object + + def to_dependency_data(self): + if self.dependency_type is None: + raise SyncanoValueError('dependency_type not set.') + dependency_data = {'type': self.dependency_type} + dependency_data.update({field_name: getattr( + self.dependency_object, + self.field_mapping.get(field_name, field_name) + ) for field_name in self.fields}) + return dependency_data + + +class ScriptDependency(BaseDependency): + dependency_type = DependencyTypeE.SCRIPT + fields = [ + 'runtime_name', + 'name', + 'source' + ] + + field_mapping = {'name': 'label'} + + +class EndpointMetadataMixin(object): + + def __init__(self, *args, **kwargs): + self._endpoints = [] + super(EndpointMetadataMixin, self).__init__(*args, **kwargs) + + def add_endpoint(self, endpoint): + self._endpoints.append(endpoint) + + @property + def endpoints_data(self): + endpoints = {} + for endpoint in self._endpoints: + endpoints.update(endpoint.to_endpoint_data()) + return endpoints + + +class DependencyMetadataMixin(object): + + def __init__(self, *args, **kwargs): + self._dependencies = [] + super(DependencyMetadataMixin, self).__init__(*args, **kwargs) + + def add_dependency(self, depedency): + self._dependencies.append(depedency) + + @property + def dependencies_data(self): + return [dependency.to_dependency_data() for dependency in self._dependencies] + + +class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): + """ + OO wrapper around instance custom sockets. + + :ivar name: :class:`~syncano.models.fields.StringField` + :ivar endpoints: :class:`~syncano.models.fields.JSONField` + :ivar dependencies: :class:`~syncano.models.fields.JSONField` + :ivar metadata: :class:`~syncano.models.fields.JSONField` + :ivar links: :class:`~syncano.models.fields.LinksField` + """ + + name = fields.StringField(max_length=64) + endpoints = fields.JSONField() + dependencies = fields.JSONField() + metadata = fields.JSONField(read_only=True, required=False) + links = fields.LinksField() + + class Meta: + parent = Instance + endpoints = { + 'detail': { + 'methods': ['get', 'put', 'patch', 'delete'], + 'path': '/sockets/{name}/', + }, + 'list': { + 'methods': ['post', 'get'], + 'path': '/sockets/', + } + } + + def get_endpoints(self): + endpoints_path = self.links.endpoints + connection = self._get_connection() + response = connection.request('GET', endpoints_path) + endpoints = [] + for endpoint in response['objects']: + endpoints.append(SocketEndpoint(**endpoint)) + return endpoints + + def run(self, method, endpoint_name, data={}): + endpoint = self._find_endpoint(endpoint_name) + return endpoint.run(method, data=data) + + def _find_endpoint(self, endpoint_name): + endpoints = self.get_endpoints() + for endpoint in endpoints: + print(endpoint.name, endpoint_name) + if endpoint_name == endpoint.name: + return endpoint + raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) + + def publish(self): + created_socket = self.__class__.please.create( + name=self.name, + endpoints=self.endpoints_data, + dependencies=self.dependencies_data + ) + raw_data = created_socket._raw_data + raw_data['links'] = raw_data['links'].links_dict + self.to_python(raw_data) + return self + + +class SocketEndpoint(Model): + """ + OO wrapper around endpoints defined in CustomSocket instance. + + :ivar name: :class:`~syncano.models.fields.StringField` + :ivar calls: :class:`~syncano.models.fields.JSONField` + :ivar links: :class:`~syncano.models.fields.LinksField` + """ + name = fields.StringField(max_length=64, primary_key=True) + calls = fields.JSONField() + links = fields.LinksField() + + class Meta: + parent = CustomSocket + endpoints = { + 'detail': { + 'methods': ['get'], + 'path': '/endpoints/{name}/' + }, + 'list': { + 'methods': ['get'], + 'path': '/endpoints/' + } + } + + def run(self, method='GET', data={}): + endpoint_path = self.links.endpoint + connection = self._get_connection() + if not self._validate_method(method): + raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) + + if method == ['GET', 'DELETE']: + response = connection.request(method, endpoint_path) + elif method in ['POST', 'PUT', 'PATCH']: + response = connection.request(method, endpoint_path, data=data) + else: + raise SyncanoValueError('Method: {} not supported.'.format(method)) + return response + + def _validate_method(self, method): + + methods = [] + for call in self.calls: + methods.extend(call['methods']) + if '*' in methods or method in methods: + return True + return False diff --git a/syncano/models/fields.py b/syncano/models/fields.py index f6b96eb..feb70a8 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -473,7 +473,7 @@ def to_python(self, value): return LinksWrapper(value, self.IGNORED_LINKS) def to_native(self, value): - return value + return value.to_native() class ModelField(Field): From 460006f22610a843b63ffcc982da804ac3c427bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Thu, 11 Aug 2016 15:30:30 +0200 Subject: [PATCH 04/35] [LIB-837] Add missing docs about custom_sockets; move utils to another file, add possibility to recheck and update custom socket; add possibility to remove endpoints and dependencies; --- docs/source/custom_sockets.rst | 214 +++++++++++++++++ docs/source/index.rst | 1 + .../refs/syncano.models.custom_sockets.rst | 7 + .../syncano.models.custom_sockets_utils.rst | 7 + docs/source/refs/syncano.models.geo.rst | 7 + docs/source/refs/syncano.models.hosting.rst | 7 + syncano/models/base.py | 3 +- syncano/models/custom_sockets.py | 147 +++--------- syncano/models/custom_sockets_utils.py | 222 ++++++++++++++++++ tests/integration_test_custom_socket.py | 6 + 10 files changed, 509 insertions(+), 112 deletions(-) create mode 100644 docs/source/custom_sockets.rst create mode 100644 docs/source/refs/syncano.models.custom_sockets.rst create mode 100644 docs/source/refs/syncano.models.custom_sockets_utils.rst create mode 100644 docs/source/refs/syncano.models.geo.rst create mode 100644 docs/source/refs/syncano.models.hosting.rst create mode 100644 syncano/models/custom_sockets_utils.py create mode 100644 tests/integration_test_custom_socket.py diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst new file mode 100644 index 0000000..976063e --- /dev/null +++ b/docs/source/custom_sockets.rst @@ -0,0 +1,214 @@ +.. _custom-sockets: + +========================= +Custom Sockets in Syncano +========================= + +``Syncano`` provides possibility of creating the custom sockets. It means that there's a possibility +to define a very specific endpoints in syncano application and use them as normal api calls. +Currently custom sockets allow only one dependency - script. This mean that on the backend side +each time the api is called - the script is executed and result from this script is returned as a result of the +api call. + +Creating a custom socket +------------------------ + +There are two methods of creating the custom socket. First: use the helpers objects defined in Python Libray. +Second: use the raw format - this is described below. + +To create a custom socket follow the steps:: + + import syncano + from syncano.models import CustomSocket, Endpoint, ScriptCall, ScriptDependency, RuntimeChoices + from syncano.connection import Connection + + custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do api call) + + # define endpoints + my_endpoint = Endpoint(name='my_endpoint') # again - no api call here + my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) + my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) + + # explanation for the above lines: + # The endpoint will be seen under `my_endpoint` name: + # On this syncano api endpoint the above endpoint will be called (after custom socket creation) + # :///instances//endpoints/sockets/my_endpoint/ + # On this syncano api endpoint the details of the defined endpoint will be returned + # :///instances//sockets/my_custom_socket/endpoints/my_endpoint/ + # For the above endpoint - the two calls are defined, one uses GET method - the custom_script will be executed + # there, second uses the POST method and then the another_custom_script will be called; + # Currently only script are available for calls; + + # After the creation of the endpoint, add them to custom_socket: + custom_socket.add_endpoint(my_endpoint) + + # define dependency now; + # using a new script - defining new source code; + custom_socket.add_dependency( + ScriptDependency( + Script( + label='custom_script', + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print("custom_script")' + ) + ) + ) + # using a existing script: + another_custom_script = Script.please.get(id=2) + custom_socket.add_dependency( + ScriptDependency( + another_custom_script + ) + ) + + # now it is time to publish custom_socket; + custom_socket.publish() # this will do an api call and will create script; + +Some time is needed to setup the environment for this custom socket. +There is possibility to check the custom socket status:: + + print(custom_socket.status) + # and + print(custom_socket.status_info) + + # to reload object (read it again from syncano api) use: + custom_socket.reload() + + + +Updating the custom socket +-------------------------- + +To update custom socket, use:: + + custom_socket = CustomSocket.please.get(name='my_custom_socket') + + custom_socket.remove_endpoint(endpoint_name='my_endpoint') + custom_socket.remove_dependency(dependency_name='custom_script') + + # or add new: + + custom_socket.add_endpoint(new_endpoint) # see above code for endpoint examples; + custom_socket.add_dependency(new_dependency) # see above code for dependency examples; + + custom_socket.update() + + +Running the custom socket +------------------------- + +To run custom socket use:: + + # this will run the my_endpoint - and call the custom_script (method is GET); + result = custom_socket.run(method='GET', endpoint_name='my_endpoint') + + +Read all endpoints +------------------ + +To get the all defined endpoints in custom socket run:: + + endpoints = custom_socket.get_endpoints() + + for endpoint in endpoints: + print(endpoint.name) + print(endpoint.calls) + +To run particular endpoint:: + + endpoint.run(method='GET') + # or: + endpoint.run(method='POST', data={'name': 'test_name'}) + +The data will be passed to the api call in the request body. + +Custom sockets endpoints +------------------------ + +Each custom socket is created from at least one endpoint. The endpoint is characterized by name and +defined calls. Calls is characterized by name and methods. The name is a identification for dependency, eg. +if it's equal to 'my_script' - the Script with label 'my_script' will be used (if exist and the source match), +or new one will be created. +There's a special wildcard method: `methods=['*']` - this mean that any request with +any method will be executed in this endpoint. + +To add endpoint to the custom_socket use:: + + my_endpoint = Endpoint(name='my_endpoint') # again - no api call here + my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) + my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) + + custom_socket.add_endpoint(my_endpoint) + +Custom socket dependency +------------------------ + +Each custom socket has dependency - this is a meta information for endpoint: which resource +should be used to return the api call results. The dependencies are bind to the endpoints call objects. +Currently supported dependency in only script. + +**Using new script** + +:: + + custom_socket.add_dependency( + ScriptDependency( + Script( + label='custom_script', + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print("custom_script")' + ) + ) + ) + + +**Using defined script** + +:: + + another_custom_script = Script.please.get(id=2) + custom_socket.add_dependency( + ScriptDependency( + another_custom_script + ) + ) + + +Custom socket recheck +--------------------- + +The creation of the socket can fail - this happen, eg. when endpoint name is already taken by another +custom socket. To check the statuses use:: + + print(custom_socket.status) + print(custom_socket.status_info) + +There is a possibility to re-check socket - this mean that if conditions are met - the socket will be +`created` again and available to use - if not the error will be returned in status field. + +Custom socket - raw format +-------------------------- + +There is a possibility to create a custom socket from the raw JSON format:: + + CustomSocket.please.create( + name='my_custom_socket_3', + endpoints={ + "my_endpoint_3": { + "calls": + [ + {"type": "script", "name": "my_script_3", "methods": ["POST"]} + ] + } + }, + dependencies=[ + { + "type": "script", + "runtime_name": "python_library_v5.0", + "name": "my_script_3", + "source": "print(3)" + } + ] + ) + +The disadvantage of this method is that - the JSON internal structure must be known by developer. diff --git a/docs/source/index.rst b/docs/source/index.rst index e54ea4a..1aaa32c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: getting_started interacting + custom_sockets refs/syncano diff --git a/docs/source/refs/syncano.models.custom_sockets.rst b/docs/source/refs/syncano.models.custom_sockets.rst new file mode 100644 index 0000000..3cecb71 --- /dev/null +++ b/docs/source/refs/syncano.models.custom_sockets.rst @@ -0,0 +1,7 @@ +syncano.models.custom_sockets +============================= + +.. automodule:: syncano.models.custom_sockets + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.custom_sockets_utils.rst b/docs/source/refs/syncano.models.custom_sockets_utils.rst new file mode 100644 index 0000000..dec7aba --- /dev/null +++ b/docs/source/refs/syncano.models.custom_sockets_utils.rst @@ -0,0 +1,7 @@ +syncano.models.custom_sockets_utils +=================================== + +.. automodule:: syncano.models.custom_sockets_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.geo.rst b/docs/source/refs/syncano.models.geo.rst new file mode 100644 index 0000000..d9eee0a --- /dev/null +++ b/docs/source/refs/syncano.models.geo.rst @@ -0,0 +1,7 @@ +syncano.models.geo +================== + +.. automodule:: syncano.models.geo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/refs/syncano.models.hosting.rst b/docs/source/refs/syncano.models.hosting.rst new file mode 100644 index 0000000..48a9639 --- /dev/null +++ b/docs/source/refs/syncano.models.hosting.rst @@ -0,0 +1,7 @@ +syncano.models.hosting +====================== + +.. automodule:: syncano.models.hosting + :members: + :undoc-members: + :show-inheritance: diff --git a/syncano/models/base.py b/syncano/models/base.py index 4b73a1d..d8c4db9 100644 --- a/syncano/models/base.py +++ b/syncano/models/base.py @@ -13,4 +13,5 @@ from .backups import * # NOQA from .hosting import * # NOQA from .data_views import DataEndpoint as EndpointData # NOQA -from .custom_sockets import * # NOQA \ No newline at end of file +from .custom_sockets import * # NOQA +from .custom_sockets_utils import Endpoint, ScriptCall, ScriptDependency # NOQA diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 305d716..9c58f52 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -1,116 +1,15 @@ # -*- coding: utf-8 -*- -from .base import Instance, Model -from . import fields - from syncano.exceptions import SyncanoValueError +from syncano.models.custom_sockets_utils import DependencyMetadataMixin, EndpointMetadataMixin - -class CallTypeE(object): - SCRIPT = 'script' - - -class DependencyTypeE(object): - SCRIPT = 'script' - - -class Call(object): - - def __init__(self, name, methods, call_type=None): - call_type = call_type or CallTypeE.SCRIPT - self.type = call_type - self.name = name - self.methods = methods - - def to_dict(self): - return { - 'type': self.type, - 'name': self.name, - 'methods': self.methods - } - - -class Endpoint(object): - - def __init__(self, name): - self.name = name - self.calls = [] - - def add_call(self, call): - self.calls.append(call) - - def to_endpoint_data(self): - return { - self.name: { - 'calls': [call.to_dict() for call in self.calls] - } - } - - -class BaseDependency(object): - - fields = [] - dependency_type = None - field_mapping = {} - - def __init__(self, dependency_object): - self.dependency_object = dependency_object - - def to_dependency_data(self): - if self.dependency_type is None: - raise SyncanoValueError('dependency_type not set.') - dependency_data = {'type': self.dependency_type} - dependency_data.update({field_name: getattr( - self.dependency_object, - self.field_mapping.get(field_name, field_name) - ) for field_name in self.fields}) - return dependency_data - - -class ScriptDependency(BaseDependency): - dependency_type = DependencyTypeE.SCRIPT - fields = [ - 'runtime_name', - 'name', - 'source' - ] - - field_mapping = {'name': 'label'} - - -class EndpointMetadataMixin(object): - - def __init__(self, *args, **kwargs): - self._endpoints = [] - super(EndpointMetadataMixin, self).__init__(*args, **kwargs) - - def add_endpoint(self, endpoint): - self._endpoints.append(endpoint) - - @property - def endpoints_data(self): - endpoints = {} - for endpoint in self._endpoints: - endpoints.update(endpoint.to_endpoint_data()) - return endpoints - - -class DependencyMetadataMixin(object): - - def __init__(self, *args, **kwargs): - self._dependencies = [] - super(DependencyMetadataMixin, self).__init__(*args, **kwargs) - - def add_dependency(self, depedency): - self._dependencies.append(depedency) - - @property - def dependencies_data(self): - return [dependency.to_dependency_data() for dependency in self._dependencies] +from . import fields +from .base import Instance, Model class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): """ OO wrapper around instance custom sockets. + Look at the custom socket documentation for more details. :ivar name: :class:`~syncano.models.fields.StringField` :ivar endpoints: :class:`~syncano.models.fields.JSONField` @@ -119,10 +18,12 @@ class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): :ivar links: :class:`~syncano.models.fields.LinksField` """ - name = fields.StringField(max_length=64) + name = fields.StringField(max_length=64, primary_key=True) endpoints = fields.JSONField() dependencies = fields.JSONField() metadata = fields.JSONField(read_only=True, required=False) + status = fields.StringField(read_only=True, required=False) + status_info = fields.StringField(read_only=True, required=False) links = fields.LinksField() class Meta: @@ -154,26 +55,50 @@ def run(self, method, endpoint_name, data={}): def _find_endpoint(self, endpoint_name): endpoints = self.get_endpoints() for endpoint in endpoints: - print(endpoint.name, endpoint_name) if endpoint_name == endpoint.name: return endpoint raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) def publish(self): + if not self.is_new(): + raise SyncanoValueError('Can not publish already defined custom socket.') + created_socket = self.__class__.please.create( name=self.name, endpoints=self.endpoints_data, dependencies=self.dependencies_data ) - raw_data = created_socket._raw_data - raw_data['links'] = raw_data['links'].links_dict - self.to_python(raw_data) + + created_socket._raw_data['links'] = created_socket._raw_data['links'].links_dict + self.to_python(created_socket._raw_data) + return self + + def update(self): + if self.is_new(): + raise SyncanoValueError('Publish socket first.') + + update_socket = self.__class__.please.update( + name=self.name, + endpoints=self.endpoints_data, + dependencies=self.dependencies_data + ) + + update_socket._raw_data['links'] = update_socket._raw_data['links'].links_dict + self.to_python(update_socket._raw_data) + return self + + def recheck(self): + recheck_path = self.links.recheck + connection = self._get_connection() + rechecked_socket = connection.request('POST', recheck_path) + self.to_python(rechecked_socket) return self class SocketEndpoint(Model): """ OO wrapper around endpoints defined in CustomSocket instance. + Look at the custom socket documentation for more details. :ivar name: :class:`~syncano.models.fields.StringField` :ivar calls: :class:`~syncano.models.fields.JSONField` @@ -202,7 +127,7 @@ def run(self, method='GET', data={}): if not self._validate_method(method): raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) - if method == ['GET', 'DELETE']: + if method in ['GET', 'DELETE']: response = connection.request(method, endpoint_path) elif method in ['POST', 'PUT', 'PATCH']: response = connection.request(method, endpoint_path, data=data) diff --git a/syncano/models/custom_sockets_utils.py b/syncano/models/custom_sockets_utils.py new file mode 100644 index 0000000..34c3256 --- /dev/null +++ b/syncano/models/custom_sockets_utils.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +import six +from syncano.exceptions import SyncanoValueError + +from .incentives import Script + + +class CallTypeE(object): + """ + The type of the call object used in the custom socket; + """ + SCRIPT = 'script' + + +class DependencyTypeE(object): + """ + The type of the dependency object used in the custom socket; + """ + SCRIPT = 'script' + + +class BaseCall(object): + """ + Base class for call object. + """ + + call_type = None + + def __init__(self, name, methods): + self.name = name + self.methods = methods + + def to_dict(self): + if self.call_type is None: + raise SyncanoValueError('call_type not set.') + return { + 'type': self.call_type, + 'name': self.name, + 'methods': self.methods + } + + +class ScriptCall(BaseCall): + """ + Script call object. + + The JSON format is as follows (to_dict in the base class):: + + { + 'type': 'script', + 'name': ', + 'methods': [], + } + + methods can be as follows: + * ['GET'] + * ['*'] - which will do a call on every request method; + """ + call_type = CallTypeE.SCRIPT + + +class Endpoint(object): + """ + The object which stores metadata about endpoints in custom socket; + + The JSON format is as follows:: + + { + ': { + 'calls': [ + + ] + } + } + + """ + def __init__(self, name): + self.name = name + self.calls = [] + + def add_call(self, call): + self.calls.append(call) + + def to_endpoint_data(self): + return { + self.name: { + 'calls': [call.to_dict() for call in self.calls] + } + } + + +class BaseDependency(object): + """ + Base dependency object; + + On the base of the fields attribute - the JSON format of the dependency is returned. + The fields are taken from the dependency object - which can be Script (supported now). + """ + + fields = [] + dependency_type = None + field_mapping = {} + + def __init__(self, dependency_object): + self.dependency_object = dependency_object + + def to_dependency_data(self): + if self.dependency_type is None: + raise SyncanoValueError('dependency_type not set.') + dependency_data = {'type': self.dependency_type} + dependency_data.update({field_name: getattr( + self.dependency_object, + self.field_mapping.get(field_name, field_name) + ) for field_name in self.fields}) + return dependency_data + + +class ScriptDependency(BaseDependency): + """ + Script dependency object; + + The JSON format is as follows:: + { + 'type': 'script', + 'runtime_name': '', + } + """ + dependency_type = DependencyTypeE.SCRIPT + fields = [ + 'runtime_name', + 'name', + 'source' + ] + + field_mapping = {'name': 'label'} + id_name = 'label' + + +class EndpointMetadataMixin(object): + """ + A mixin which allows to collect Endpoints objects and transform them to the appropriate JSON format. + """ + + def __init__(self, *args, **kwargs): + self._endpoints = [] + super(EndpointMetadataMixin, self).__init__(*args, **kwargs) + if self.endpoints: + self.update_endpoints() + + def update_endpoints(self): + for raw_endpoint_name, raw_endpoint in six.iteritems(self.endpoints): + endpoint = Endpoint( + name=raw_endpoint_name, + ) + for call in raw_endpoint['calls']: + call_class = self._get_call_class(call['type']) + call_instance = call_class(name=call['name'], methods=call['methods']) + endpoint.add_call(call_instance) + + self.add_endpoint(endpoint) + + @classmethod + def _get_call_class(cls, call_type): + if call_type == CallTypeE.SCRIPT: + return ScriptCall + + def add_endpoint(self, endpoint): + self._endpoints.append(endpoint) + + def remove_endpoint(self, endpoint_name): + for index, endpoint in enumerate(self._endpoints): + if endpoint.name == endpoint_name: + self._endpoints.pop(index) + break + + @property + def endpoints_data(self): + endpoints = {} + for endpoint in self._endpoints: + endpoints.update(endpoint.to_endpoint_data()) + return endpoints + + +class DependencyMetadataMixin(object): + """ + A mixin which allows to collect Dependencies objects and transform them to the appropriate JSON format. + """ + + def __init__(self, *args, **kwargs): + self._dependencies = [] + super(DependencyMetadataMixin, self).__init__(*args, **kwargs) + if self.dependencies: + self.update_dependencies() + + def update_dependencies(self): + for raw_depedency in self.dependencies: + depedency_class, object_class = self._get_depedency_klass(raw_depedency['type']) + + self.add_dependency(depedency_class( + object_class(**{ + depedency_class.field_mapping.get(field_name, field_name): raw_depedency.get(field_name) + for field_name in depedency_class.fields + }) + )) + + @classmethod + def _get_depedency_klass(cls, depedency_type): + if depedency_type == DependencyTypeE.SCRIPT: + return ScriptDependency, Script + + def add_dependency(self, depedency): + self._dependencies.append(depedency) + + def remove_dependency(self, dependency_name): + for index, dependency in enumerate(self._dependencies): + if dependency_name == getattr(dependency.dependency_object, dependency.id_name, None): + self._dependencies.pop(index) + break + + @property + def dependencies_data(self): + return [dependency.to_dependency_data() for dependency in self._dependencies] diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py new file mode 100644 index 0000000..5218d9d --- /dev/null +++ b/tests/integration_test_custom_socket.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from tests.integration_test import InstanceMixin, IntegrationTest + + +class CustomSocketTest(InstanceMixin, IntegrationTest): + pass From 5eca90660f6f169bfd811fc5ca8e976d143b316a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Thu, 11 Aug 2016 15:34:11 +0200 Subject: [PATCH 05/35] [LIB-837] add missing fields in doc string; --- syncano/models/custom_sockets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 9c58f52..59a93b5 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -15,6 +15,8 @@ class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): :ivar endpoints: :class:`~syncano.models.fields.JSONField` :ivar dependencies: :class:`~syncano.models.fields.JSONField` :ivar metadata: :class:`~syncano.models.fields.JSONField` + :ivar status: :class:`~syncano.models.fields.StringField` + :ivar status_info: :class:`~syncano.models.fields.StringField` :ivar links: :class:`~syncano.models.fields.LinksField` """ From cb5f0ab06204c77aa7bc6021abb6bf6a4221f12e Mon Sep 17 00:00:00 2001 From: Robert Kopaczewski Date: Fri, 12 Aug 2016 13:44:03 +0200 Subject: [PATCH 06/35] [readme_changes] Some readme changes Removed info about 0.6.x version as it may be confusing for newer users. Fixed urls to work for rst --- README.rst | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 2744d96..396f7df 100644 --- a/README.rst +++ b/README.rst @@ -4,21 +4,10 @@ Syncano Python QuickStart Guide ----------------------- -You can find quick start on installing and using Syncano's Python library in our [documentation](http://docs.syncano.com/docs/python). +You can find quick start on installing and using Syncano's Python library in our `documentation `_. -For more detailed information on how to use Syncano and its features - our [Developer Manual](http://docs.syncano.com/docs/getting-started-with-syncano) should be very helpful. +For more detailed information on how to use Syncano and its features - our `Developer Manual `_ should be very helpful. In case you need help working with the library - email us at libraries@syncano.com - we will be happy to help! -You can also find library reference hosted on GitHub pages [here](http://syncano.github.io/syncano-python/). - -Backwards incompatible changes ------------------------------- - -Version 4.x and 5.x is designed for new release of Syncano platform and -is **not compatible** with any previous releases. - -Code from `0.6.x` release is avalable on [stable/0.6.x](https://github.com/Syncano/syncano-python/tree/stable/0.6.x) branch -and it can be installed directly from pip via: - -``pip install syncano==0.6.2 --pre`` +You can also find library reference hosted on GitHub pages `here `_. From c0f75cbe59acc493a796e685bb678fbd3f846f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 12 Aug 2016 14:58:08 +0200 Subject: [PATCH 07/35] [LIB-837] correct documentation; add possibility to read all endpoints; correct ScriptDependency to handle ScriptEndpoint; make Script a model field in ScriptEndpoint; overall fixes; --- docs/source/custom_sockets.rst | 115 ++++++++++------- setup.py | 4 +- syncano/models/custom_sockets.py | 14 +- syncano/models/custom_sockets_utils.py | 82 ++++++++---- syncano/models/fields.py | 4 + syncano/models/incentives.py | 6 +- syncano/models/instances.py | 4 + tests/integration_test_custom_socket.py | 162 +++++++++++++++++++++++- 8 files changed, 310 insertions(+), 81 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 976063e..41efde8 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -4,78 +4,80 @@ Custom Sockets in Syncano ========================= -``Syncano`` provides possibility of creating the custom sockets. It means that there's a possibility -to define a very specific endpoints in syncano application and use them as normal api calls. +``Syncano`` provides possibility of creating custom sockets. It means that there's a possibility +to define a very specific endpoints in syncano application and use them as normal API calls. Currently custom sockets allow only one dependency - script. This mean that on the backend side -each time the api is called - the script is executed and result from this script is returned as a result of the -api call. +each time the API is called - the script is executed and result from this script is returned as a result of the +API call. Creating a custom socket ------------------------ -There are two methods of creating the custom socket. First: use the helpers objects defined in Python Libray. -Second: use the raw format - this is described below. - -To create a custom socket follow the steps:: +To create a custom socket follow these steps:: import syncano from syncano.models import CustomSocket, Endpoint, ScriptCall, ScriptDependency, RuntimeChoices from syncano.connection import Connection - custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do api call) + # 1. Initialize the custom socket. + custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do API call) - # define endpoints - my_endpoint = Endpoint(name='my_endpoint') # again - no api call here + # 2. Define endpoints. + my_endpoint = Endpoint(name='my_endpoint') # again - no API call here my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) # explanation for the above lines: # The endpoint will be seen under `my_endpoint` name: - # On this syncano api endpoint the above endpoint will be called (after custom socket creation) + # On this syncano API endpoint the above endpoint will be called (after custom socket creation) # :///instances//endpoints/sockets/my_endpoint/ - # On this syncano api endpoint the details of the defined endpoint will be returned + # On this syncano API endpoint the details of the defined endpoint will be returned # :///instances//sockets/my_custom_socket/endpoints/my_endpoint/ # For the above endpoint - the two calls are defined, one uses GET method - the custom_script will be executed # there, second uses the POST method and then the another_custom_script will be called; # Currently only script are available for calls; - # After the creation of the endpoint, add them to custom_socket: + # 3. After the creation of the endpoint, add them to custom_socket. custom_socket.add_endpoint(my_endpoint) - # define dependency now; - # using a new script - defining new source code; + # 4. Define dependency now. + # 4.1 using a new script - defining new source code. custom_socket.add_dependency( ScriptDependency( - Script( - label='custom_script', + name='custom_script' + script=Script( runtime_name=RuntimeChoices.PYTHON_V5_0, source='print("custom_script")' ) ) ) - # using a existing script: + # 4.2 using an existing script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( - another_custom_script + name='another_custom_script', + script=another_custom_script ) ) - # now it is time to publish custom_socket; - custom_socket.publish() # this will do an api call and will create script; + # 4.3 using an existing ScriptEndpoint. + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint=script_endpoint + ) + + # 5. Publish custom_socket. + custom_socket.publish() # this will do an API call and will create script; Some time is needed to setup the environment for this custom socket. There is possibility to check the custom socket status:: + # Reload will refresh object using syncano API. + custom_socket.reload() print(custom_socket.status) # and print(custom_socket.status_info) - # to reload object (read it again from syncano api) use: - custom_socket.reload() - - - Updating the custom socket -------------------------- @@ -103,8 +105,8 @@ To run custom socket use:: result = custom_socket.run(method='GET', endpoint_name='my_endpoint') -Read all endpoints ------------------- +Read all endpoints in custom socket +----------------------------------- To get the all defined endpoints in custom socket run:: @@ -114,27 +116,40 @@ To get the all defined endpoints in custom socket run:: print(endpoint.name) print(endpoint.calls) -To run particular endpoint:: +To run a particular endpoint:: endpoint.run(method='GET') # or: endpoint.run(method='POST', data={'name': 'test_name'}) -The data will be passed to the api call in the request body. +The data will be passed to the API call in the request body. + +Read all endpoints +------------------ + +To get all endpoints that are defined in all custom sockets:: + + socket_endpoint_list = SocketEndpoint.get_all_endpoints() + +Above code will return a list with SocketEndpoint objects. To run such endpoint, use:: + + socket_endpoint_list.run(method='GET') + # or: + socket_endpoint_list.run(method='POST', data={'custom_data': 1}) Custom sockets endpoints ------------------------ -Each custom socket is created from at least one endpoint. The endpoint is characterized by name and -defined calls. Calls is characterized by name and methods. The name is a identification for dependency, eg. -if it's equal to 'my_script' - the Script with label 'my_script' will be used (if exist and the source match), -or new one will be created. +Each custom socket requires to define at least one endpoint. The endpoint is defined by name and +a list of calls. Each call is defined by a name and a list of methods. The name is a identification for dependency, eg. +if it's equal to 'my_script' - the ScriptEndpoint with name 'my_script' will be used +(if it exists and Script source and runtime matches) or a new one will be created. There's a special wildcard method: `methods=['*']` - this mean that any request with any method will be executed in this endpoint. -To add endpoint to the custom_socket use:: +To add an endpoint to the custom_socket use:: - my_endpoint = Endpoint(name='my_endpoint') # again - no api call here + my_endpoint = Endpoint(name='my_endpoint') # again - no API call here my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) @@ -144,8 +159,8 @@ Custom socket dependency ------------------------ Each custom socket has dependency - this is a meta information for endpoint: which resource -should be used to return the api call results. The dependencies are bind to the endpoints call objects. -Currently supported dependency in only script. +should be used to return the API call results. The dependencies are bind to the endpoints call objects. +Currently the only supported dependency is script. **Using new script** @@ -153,8 +168,8 @@ Currently supported dependency in only script. custom_socket.add_dependency( ScriptDependency( - Script( - label='custom_script', + name='custom_script' + script=Script( runtime_name=RuntimeChoices.PYTHON_V5_0, source='print("custom_script")' ) @@ -169,10 +184,19 @@ Currently supported dependency in only script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( - another_custom_script + name='another_custom_script', + script=another_custom_script ) ) +**Using defined script endpoint** + +:: + + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint=script_endpoint + ) Custom socket recheck --------------------- @@ -183,13 +207,14 @@ custom socket. To check the statuses use:: print(custom_socket.status) print(custom_socket.status_info) -There is a possibility to re-check socket - this mean that if conditions are met - the socket will be -`created` again and available to use - if not the error will be returned in status field. +There is a possibility to re-check socket - this mean that if conditions are met - the socket endpoints and dependencies +will be checked - and if some of them are missing (eg. mistake deletion), they will be created again. +If the endpoints and dependencies do not met the criteria - the error will be returned in the status field. Custom socket - raw format -------------------------- -There is a possibility to create a custom socket from the raw JSON format:: +If you prefer raw JSON format for creating sockets, you can resort to use it in python library as well:::: CustomSocket.please.create( name='my_custom_socket_3', diff --git a/setup.py b/setup.py index 2551e61..66d524b 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,8 @@ def readme(): version=__version__, description='Python Library for syncano.com api', long_description=readme(), - author='Daniel Kopka', - author_email='daniel.kopka@syncano.com', + author='Syncano', + author_email='support@syncano.io', url='http://syncano.com', packages=find_packages(exclude=['tests']), zip_safe=False, diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 59a93b5..f7a4fda 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -128,15 +128,23 @@ def run(self, method='GET', data={}): connection = self._get_connection() if not self._validate_method(method): raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) - - if method in ['GET', 'DELETE']: + method = method.lower() + if method in ['get', 'delete']: response = connection.request(method, endpoint_path) - elif method in ['POST', 'PUT', 'PATCH']: + elif method in ['post', 'put', 'patch']: response = connection.request(method, endpoint_path, data=data) else: raise SyncanoValueError('Method: {} not supported.'.format(method)) return response + @classmethod + def get_all_endpoints(cls): + connection = cls._meta.connection + all_endpoints_path = Instance._meta.resolve_endpoint('endpoints', + {'name': cls.please.properties.get('instance_name')}) + response = connection.request('GET', all_endpoints_path) + return [cls(**endpoint) for endpoint in response['objects']] + def _validate_method(self, method): methods = [] diff --git a/syncano/models/custom_sockets_utils.py b/syncano/models/custom_sockets_utils.py index 34c3256..450a40e 100644 --- a/syncano/models/custom_sockets_utils.py +++ b/syncano/models/custom_sockets_utils.py @@ -2,17 +2,17 @@ import six from syncano.exceptions import SyncanoValueError -from .incentives import Script +from .incentives import Script, ScriptEndpoint -class CallTypeE(object): +class CallType(object): """ The type of the call object used in the custom socket; """ SCRIPT = 'script' -class DependencyTypeE(object): +class DependencyType(object): """ The type of the dependency object used in the custom socket; """ @@ -56,7 +56,7 @@ class ScriptCall(BaseCall): * ['GET'] * ['*'] - which will do a call on every request method; """ - call_type = CallTypeE.SCRIPT + call_type = CallType.SCRIPT class Endpoint(object): @@ -99,21 +99,23 @@ class BaseDependency(object): fields = [] dependency_type = None - field_mapping = {} - - def __init__(self, dependency_object): - self.dependency_object = dependency_object def to_dependency_data(self): if self.dependency_type is None: raise SyncanoValueError('dependency_type not set.') dependency_data = {'type': self.dependency_type} - dependency_data.update({field_name: getattr( - self.dependency_object, - self.field_mapping.get(field_name, field_name) - ) for field_name in self.fields}) + dependency_data.update(self.get_dependency_data()) return dependency_data + def get_name(self): + raise NotImplementedError() + + def get_dependency_data(self): + raise NotImplementedError() + + def create_from_raw_data(self, raw_data): + raise NotImplementedError() + class ScriptDependency(BaseDependency): """ @@ -123,17 +125,49 @@ class ScriptDependency(BaseDependency): { 'type': 'script', 'runtime_name': '', + 'source': '', + 'name': '' } """ - dependency_type = DependencyTypeE.SCRIPT + + dependency_type = DependencyType.SCRIPT fields = [ 'runtime_name', - 'name', 'source' ] - field_mapping = {'name': 'label'} - id_name = 'label' + def __init__(self, name=None, script=None, script_endpoint=None): + if name and script and script_endpoint: + raise SyncanoValueError("Usage: ScriptDependency(name='', script=Script(...)) or " + "ScriptDependency(ScriptEndpoint(...))") + if (name and not script) or (not name and script): + raise SyncanoValueError("Usage: ScriptDependency(name='', script=Script(...))") + + if script and not isinstance(script, Script): + raise SyncanoValueError("Expected Script type object.") + + if script_endpoint and not isinstance(script_endpoint, ScriptEndpoint): + raise SyncanoValueError("Expected ScriptEndpoint type object.") + + if not script_endpoint: + self.dependency_object = ScriptEndpoint(name=name, script=script) + else: + self.dependency_object = script_endpoint + + def get_name(self): + return {'name': self.dependency_object.name} + + def get_dependency_data(self): + dependency_data = self.get_name() + dependency_data.update({ + field_name: getattr(self.dependency_object.script, field_name) for field_name in self.fields + }) + return dependency_data + + @classmethod + def create_from_raw_data(cls, raw_data): + return cls(**{'script_endpoint': ScriptEndpoint(name=raw_data['name'], script=Script( + source=raw_data['source'], runtime_name=raw_data['runtime_name']))}) class EndpointMetadataMixin(object): @@ -161,7 +195,7 @@ def update_endpoints(self): @classmethod def _get_call_class(cls, call_type): - if call_type == CallTypeE.SCRIPT: + if call_type == CallType.SCRIPT: return ScriptCall def add_endpoint(self, endpoint): @@ -194,19 +228,13 @@ def __init__(self, *args, **kwargs): def update_dependencies(self): for raw_depedency in self.dependencies: - depedency_class, object_class = self._get_depedency_klass(raw_depedency['type']) - - self.add_dependency(depedency_class( - object_class(**{ - depedency_class.field_mapping.get(field_name, field_name): raw_depedency.get(field_name) - for field_name in depedency_class.fields - }) - )) + depedency_class = self._get_depedency_klass(raw_depedency['type']) + self.add_dependency(depedency_class.create_from_raw_data(raw_depedency)) @classmethod def _get_depedency_klass(cls, depedency_type): - if depedency_type == DependencyTypeE.SCRIPT: - return ScriptDependency, Script + if depedency_type == DependencyType.SCRIPT: + return ScriptDependency def add_dependency(self, depedency): self._dependencies.append(depedency) diff --git a/syncano/models/fields.py b/syncano/models/fields.py index feb70a8..cf74253 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -523,6 +523,10 @@ def to_python(self, value): if isinstance(value, dict): return self.rel(**value) + # try to fetch object; + if isinstance(value, int): + return self.rel.please.get(id=value) + raise self.ValidationError("'{0}' has unsupported format.".format(value)) def to_native(self, value): diff --git a/syncano/models/incentives.py b/syncano/models/incentives.py index 12a0d4d..435433a 100644 --- a/syncano/models/incentives.py +++ b/syncano/models/incentives.py @@ -45,7 +45,7 @@ class Script(Model): >>> Script.please.run('instance-name', 1234) >>> Script.please.run('instance-name', 1234, payload={'variable_one': 1, 'variable_two': 2}) - >>> Script.please.run('instance-name', 1234, payload="{\"variable_one\": 1, \"variable_two\": 2}") + >>> Script.please.run('instance-name', 1234, payload='{"variable_one": 1, "variable_two": 2}') or via instance:: @@ -54,7 +54,7 @@ class Script(Model): >>> s.run(variable_one=1, variable_two=2) """ - label = fields.StringField(max_length=80) + label = fields.StringField(max_length=80, required=False) description = fields.StringField(required=False) source = fields.StringField() runtime_name = fields.StringField() @@ -222,7 +222,7 @@ class ScriptEndpoint(Model): """ name = fields.SlugField(max_length=50, primary_key=True) - script = fields.IntegerField(label='script id') + script = fields.ModelField('Script', label='script id') public = fields.BooleanField(required=False, default=False) public_link = fields.ChoiceField(required=False, read_only=True) links = fields.LinksField() diff --git a/syncano/models/instances.py b/syncano/models/instances.py index f4f128a..788ba05 100644 --- a/syncano/models/instances.py +++ b/syncano/models/instances.py @@ -82,6 +82,10 @@ class Meta: 'config': { 'methods': ['put', 'get'], 'path': '/v1.1/instances/{name}/snippets/config/', + }, + 'endpoints': { + 'methods': ['get'], + 'path': '/v1.1/instances/{name}/endpoints/sockets/' } } diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 5218d9d..7dc3ecc 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -1,6 +1,166 @@ # -*- coding: utf-8 -*- +import time + +from syncano.models import CustomSocket, Endpoint, RuntimeChoices, Script, ScriptCall, ScriptDependency, ScriptEndpoint from tests.integration_test import InstanceMixin, IntegrationTest class CustomSocketTest(InstanceMixin, IntegrationTest): - pass + + def setUp(self): + self.custom_socket = self._create_custom_socket('default', self._define_dependencies_new_script_endpoint) + + def test_publish_custom_socket(self): + # this test new ScriptEndpoint dependency create; + self.assert_custom_socket('publishing', self._define_dependencies_new_script_endpoint) + + def test_dependencies_new_script(self): + self.assert_custom_socket('new_script_publishing', self._define_dependencies_new_script) + + def test_dependencies_existing_script(self): + self.assert_custom_socket('existing_script_publishing', self._define_dependencies_existing_script) + + def test_dependencies_existing_script_endpoint(self): + self.assert_custom_socket('existing_script_endpoint_publishing', + self._define_dependencies_existing_script_endpoint) + + def test_creating_raw_data(self): + custom_socket = CustomSocket.please.create( + name='my_custom_socket_123', + endpoints={ + "my_custom_endpoint_123": { + "calls": [{"type": "script", "name": "script_123", "methods": ["POST"]}] + } + }, + dependencies=[ + { + "type": "script", + "runtime_name": "python_library_v5.0", + "name": "script_123", + "source": "print(123)" + } + ] + ) + + self.assertTrue(custom_socket.id) + + def test_custom_socket_run(self): + results = self.custom_socket.run('GET', 'my_endpoint_default') + self.assertEqual(results['stdout'], 'script_default') + + def test_custom_socket_recheck(self): + custom_socket = self.custom_socket.recheck() + self.assertTrue(custom_socket.id) + + def test_fetching_all_endpoints(self): + all_endpoints = ScriptEndpoint.get_all_endpoints() + self.assertTrue(isinstance(all_endpoints, list)) + self.assertTrue(len(all_endpoints) >= 1) + self.assertTrue(all_endpoints[0].name) + + def test_endpoint_run(self): + script_endpoint = ScriptEndpoint.please.first() + result = script_endpoint.run('GET') + suffix = script_endpoint.name.split('_')[-1] + self.assertTrue(result['stdout'].endswith(suffix)) + + def test_custom_socket_update(self): + socket_to_update = self._create_custom_socket('to_update', self._define_dependencies_new_script_endpoint) + socket_to_update.remove_endpoint(endpoint_name='my_endpoint_to_update') + + new_endpoint = Endpoint(name='my_endpoint_new_to_update') + new_endpoint.add_call( + ScriptCall(name='script_default', methods=['GET']) + ) + + self.custom_socket.update() + time.sleep(2) # wait for custom socket setup; + self.custom_socket.reload() + self.assertIn('my_endpoint_new_default', self.custom_socket.endpoints) + + def assert_custom_socket(self, suffix, dependency_method): + custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) + self.assertTrue(custom_socket.id) + + @classmethod + def _create_custom_socket(cls, suffix, dependency_method): + custom_socket = CustomSocket(name='my_custom_socket_{}'.format(suffix)) + + cls._define_endpoints(suffix, custom_socket) + dependency_method(suffix, custom_socket) + + custom_socket.publish() + return custom_socket + + @classmethod + def _initialize_socket(cls, suffix): + return CustomSocket(name='my_custom_socket_{}'.format(suffix)) + + @classmethod + def _define_endpoints(cls, suffix, custom_socket): + endpoint = Endpoint(name='my_endpoint_{}'.format(suffix)) + endpoint.add_call( + ScriptCall( + name='script_{}'.format(suffix), + methods=['GET', 'POST'] + ) + ) + custom_socket.add_endpoint(endpoint) + + @classmethod + def _define_dependencies_new_script_endpoint(cls, suffix, custom_socket): + custom_socket.add_dependency( + ScriptDependency( + script_endpoint=ScriptEndpoint( + name='script_endpoint_{}'.format(suffix), + script=Script( + source='print({})'.format(suffix), + runtime_name=RuntimeChoices.PYTHON_V5_0 + ) + ) + ) + ) + + @classmethod + def _define_dependencies_new_script(cls, suffix, custom_socket): + custom_socket.add_dependency( + ScriptDependency( + name='script_endpoint_{}'.format(suffix), + script=Script( + source='print({})'.format(suffix), + runtime_name=RuntimeChoices.PYTHON_V5_0 + ) + ) + ) + + @classmethod + def _define_dependencies_existing_script(cls, suffix, custom_socket): + # create Script first: + cls._create_script(suffix) + custom_socket.add_dependency( + ScriptDependency( + name='script_endpoint_{}'.format(suffix), + script=Script.please.first() + ) + ) + + @classmethod + def _define_dependencies_existing_script_endpoint(cls, suffix, custom_socket): + script = cls._create_script(suffix) + ScriptEndpoint.please.create( + name='script_endpoint_{}'.format(suffix), + script=script + ) + custom_socket.add_dependency( + ScriptDependency( + script_endpoint=ScriptEndpoint.please.first() + ) + ) + + @classmethod + def _create_script(cls, suffix): + return Script.please.create( + label='script_{}'.format(suffix), + runtime_name=RuntimeChoices.PYTHON_V5_0, + source='print({})'.format(suffix) + ) From e108b5a7806f55cbbfbe00a06abd4aedab8e8c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 12 Aug 2016 15:24:42 +0200 Subject: [PATCH 08/35] [LIB-837] Correct after tests; --- syncano/models/incentives.py | 2 +- tests/integration_test_custom_socket.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/syncano/models/incentives.py b/syncano/models/incentives.py index 435433a..585a03a 100644 --- a/syncano/models/incentives.py +++ b/syncano/models/incentives.py @@ -222,7 +222,7 @@ class ScriptEndpoint(Model): """ name = fields.SlugField(max_length=50, primary_key=True) - script = fields.ModelField('Script', label='script id') + script = fields.ModelField('Script', just_pk=True) public = fields.BooleanField(required=False, default=False) public_link = fields.ChoiceField(required=False, read_only=True) links = fields.LinksField() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 7dc3ecc..c7884a8 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -80,7 +80,7 @@ def test_custom_socket_update(self): def assert_custom_socket(self, suffix, dependency_method): custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) - self.assertTrue(custom_socket.id) + self.assertTrue(custom_socket.name) @classmethod def _create_custom_socket(cls, suffix, dependency_method): @@ -92,10 +92,6 @@ def _create_custom_socket(cls, suffix, dependency_method): custom_socket.publish() return custom_socket - @classmethod - def _initialize_socket(cls, suffix): - return CustomSocket(name='my_custom_socket_{}'.format(suffix)) - @classmethod def _define_endpoints(cls, suffix, custom_socket): endpoint = Endpoint(name='my_endpoint_{}'.format(suffix)) From 5eab4640a047792885b26cd8223fe975f464f290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 12 Aug 2016 15:54:40 +0200 Subject: [PATCH 09/35] [LIB-837] another portion of corrects; --- syncano/models/fields.py | 4 +++- tests/integration_test_custom_socket.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/syncano/models/fields.py b/syncano/models/fields.py index cf74253..ead0dea 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -478,6 +478,8 @@ def to_native(self, value): class ModelField(Field): + read_only = False + def __init__(self, rel, *args, **kwargs): self.rel = rel self.just_pk = kwargs.pop('just_pk', True) @@ -508,7 +510,7 @@ def validate(self, value, model_instance): if not isinstance(value, (self.rel, dict)) and not self.is_data_object_mixin: raise self.ValidationError('Value needs to be a {0} instance.'.format(self.rel.__name__)) - if (self.required and isinstance(value, self.rel))or \ + if (self.required and isinstance(value, self.rel)) or \ (self.is_data_object_mixin and hasattr(value, 'validate')): value.validate() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index c7884a8..146dce3 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -7,8 +7,10 @@ class CustomSocketTest(InstanceMixin, IntegrationTest): - def setUp(self): - self.custom_socket = self._create_custom_socket('default', self._define_dependencies_new_script_endpoint) + @classmethod + def setUpClass(cls): + super(CustomSocketTest, cls).setUpClass() + cls.custom_socket = cls._create_custom_socket('default', cls._define_dependencies_new_script_endpoint) def test_publish_custom_socket(self): # this test new ScriptEndpoint dependency create; @@ -42,7 +44,7 @@ def test_creating_raw_data(self): ] ) - self.assertTrue(custom_socket.id) + self.assertTrue(custom_socket.name) def test_custom_socket_run(self): results = self.custom_socket.run('GET', 'my_endpoint_default') From 1afcd9f983907ed6c512cd2baef71d58cd4a3010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 12 Aug 2016 16:09:47 +0200 Subject: [PATCH 10/35] [LIB-837] correct tests again; --- tests/integration_test_custom_socket.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 146dce3..b69d955 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -1,7 +1,16 @@ # -*- coding: utf-8 -*- import time -from syncano.models import CustomSocket, Endpoint, RuntimeChoices, Script, ScriptCall, ScriptDependency, ScriptEndpoint +from syncano.models import ( + CustomSocket, + Endpoint, + RuntimeChoices, + Script, + ScriptCall, + ScriptDependency, + ScriptEndpoint, + SocketEndpoint +) from tests.integration_test import InstanceMixin, IntegrationTest @@ -23,7 +32,7 @@ def test_dependencies_existing_script(self): self.assert_custom_socket('existing_script_publishing', self._define_dependencies_existing_script) def test_dependencies_existing_script_endpoint(self): - self.assert_custom_socket('existing_script_endpoint_publishing', + self.assert_custom_socket('existing_script_e_publishing', self._define_dependencies_existing_script_endpoint) def test_creating_raw_data(self): @@ -48,14 +57,14 @@ def test_creating_raw_data(self): def test_custom_socket_run(self): results = self.custom_socket.run('GET', 'my_endpoint_default') - self.assertEqual(results['stdout'], 'script_default') + self.assertEqual(results.result['stdout'], 'script_default') def test_custom_socket_recheck(self): custom_socket = self.custom_socket.recheck() - self.assertTrue(custom_socket.id) + self.assertTrue(custom_socket.name) def test_fetching_all_endpoints(self): - all_endpoints = ScriptEndpoint.get_all_endpoints() + all_endpoints = SocketEndpoint.get_all_endpoints() self.assertTrue(isinstance(all_endpoints, list)) self.assertTrue(len(all_endpoints) >= 1) self.assertTrue(all_endpoints[0].name) @@ -64,17 +73,18 @@ def test_endpoint_run(self): script_endpoint = ScriptEndpoint.please.first() result = script_endpoint.run('GET') suffix = script_endpoint.name.split('_')[-1] - self.assertTrue(result['stdout'].endswith(suffix)) + self.assertTrue(result.result['stdout'].endswith(suffix)) def test_custom_socket_update(self): socket_to_update = self._create_custom_socket('to_update', self._define_dependencies_new_script_endpoint) - socket_to_update.remove_endpoint(endpoint_name='my_endpoint_to_update') + socket_to_update.remove_endpoint(endpoint_name='my_endpoint_default') new_endpoint = Endpoint(name='my_endpoint_new_to_update') new_endpoint.add_call( ScriptCall(name='script_default', methods=['GET']) ) + self.custom_socket.add_endpoint(Endpoint) self.custom_socket.update() time.sleep(2) # wait for custom socket setup; self.custom_socket.reload() From 48eb08eb673d140b01cb14d680a7f2e612da7caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 12 Aug 2016 16:53:39 +0200 Subject: [PATCH 11/35] [LIB-837] change tests instead of doing a magic in ModelField; --- syncano/models/backups.py | 2 +- syncano/models/fields.py | 4 ---- tests/integration_test.py | 6 +++--- tests/integration_test_cache.py | 2 +- tests/integration_test_custom_socket.py | 8 ++++---- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/syncano/models/backups.py b/syncano/models/backups.py index bf77353..3e92342 100644 --- a/syncano/models/backups.py +++ b/syncano/models/backups.py @@ -28,7 +28,7 @@ class Backup(Model): size = fields.IntegerField(read_only=True) status = fields.StringField(read_only=True) status_info = fields.StringField(read_only=True) - author = fields.ModelField('Admin') + author = fields.ModelField('Admin', read_only=True) details = fields.JSONField(read_only=True) updated_at = fields.DateTimeField(read_only=True, required=False) diff --git a/syncano/models/fields.py b/syncano/models/fields.py index ead0dea..c8ba8a1 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -525,10 +525,6 @@ def to_python(self, value): if isinstance(value, dict): return self.rel(**value) - # try to fetch object; - if isinstance(value, int): - return self.rel.please.get(id=value) - raise self.ValidationError("'{0}' has unsupported format.".format(value)) def to_native(self, value): diff --git a/tests/integration_test.py b/tests/integration_test.py index 622722c..816ea7a 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -548,7 +548,7 @@ def test_list(self): def test_create(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.script.id, + script=self.script, name='wh%s' % self.generate_hash()[:10], ) @@ -557,7 +557,7 @@ def test_create(self): def test_script_run(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.script.id, + script=self.script, name='wh%s' % self.generate_hash()[:10], ) @@ -569,7 +569,7 @@ def test_script_run(self): def test_custom_script_run(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.custom_script.id, + script=self.custom_script, name='wh%s' % self.generate_hash()[:10], ) diff --git a/tests/integration_test_cache.py b/tests/integration_test_cache.py index 3aff7ad..ba45bc8 100644 --- a/tests/integration_test_cache.py +++ b/tests/integration_test_cache.py @@ -51,7 +51,7 @@ def setUpClass(cls): cls.script_endpoint = cls.instance.script_endpoints.create( name='test_script_endpoint', - script=cls.script.id + script=cls.script ) def test_cache_request(self): diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index b69d955..b805ad9 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -84,11 +84,11 @@ def test_custom_socket_update(self): ScriptCall(name='script_default', methods=['GET']) ) - self.custom_socket.add_endpoint(Endpoint) - self.custom_socket.update() + socket_to_update.add_endpoint(new_endpoint) + socket_to_update.update() time.sleep(2) # wait for custom socket setup; - self.custom_socket.reload() - self.assertIn('my_endpoint_new_default', self.custom_socket.endpoints) + socket_to_update.reload() + self.assertIn('my_endpoint_new_default', socket_to_update.endpoints) def assert_custom_socket(self, suffix, dependency_method): custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) From 4e2ee332c17d09d8bddd4e1855783ac02cfbd54c Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Fri, 12 Aug 2016 18:41:24 -0400 Subject: [PATCH 12/35] Language updates --- docs/source/custom_sockets.rst | 79 ++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 41efde8..0dd81d8 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -4,10 +4,11 @@ Custom Sockets in Syncano ========================= -``Syncano`` provides possibility of creating custom sockets. It means that there's a possibility -to define a very specific endpoints in syncano application and use them as normal API calls. -Currently custom sockets allow only one dependency - script. This mean that on the backend side -each time the API is called - the script is executed and result from this script is returned as a result of the +``Syncano`` gives its users an ability to create custom sockets. It means, that users can define +a very specific endpoints in their Syncano application, and use them as other Syncano +modules (Classes, Scripts, etc), using standard API calls. +Currently, custom sockets allow only one dependency - Scripts. It means that on the backend side, +each API call executes a Script and result of the execution, is returned as a result of the API call. Creating a custom socket @@ -19,16 +20,16 @@ To create a custom socket follow these steps:: from syncano.models import CustomSocket, Endpoint, ScriptCall, ScriptDependency, RuntimeChoices from syncano.connection import Connection - # 1. Initialize the custom socket. + # 1. Initialize a custom socket. custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do API call) # 2. Define endpoints. - my_endpoint = Endpoint(name='my_endpoint') # again - no API call here + my_endpoint = Endpoint(name='my_endpoint') # no API call here my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) - # explanation for the above lines: - # The endpoint will be seen under `my_endpoint` name: + # Explanation of the above lines: + # Defined endpoint will be visible under `my_endpoint` name: # On this syncano API endpoint the above endpoint will be called (after custom socket creation) # :///instances//endpoints/sockets/my_endpoint/ # On this syncano API endpoint the details of the defined endpoint will be returned @@ -37,11 +38,11 @@ To create a custom socket follow these steps:: # there, second uses the POST method and then the another_custom_script will be called; # Currently only script are available for calls; - # 3. After the creation of the endpoint, add them to custom_socket. + # 3. After creation of the endpoint, add it to your custom_socket. custom_socket.add_endpoint(my_endpoint) - # 4. Define dependency now. - # 4.1 using a new script - defining new source code. + # 4. Define dependency. + # 4.1 Using a new script - define a new source code. custom_socket.add_dependency( ScriptDependency( name='custom_script' @@ -51,7 +52,7 @@ To create a custom socket follow these steps:: ) ) ) - # 4.2 using an existing script. + # 4.2 Using an existing script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( @@ -60,19 +61,19 @@ To create a custom socket follow these steps:: ) ) - # 4.3 using an existing ScriptEndpoint. + # 4.3 Using an existing ScriptEndpoint. script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') custom_socket.add_dependency( script_endpoint=script_endpoint ) # 5. Publish custom_socket. - custom_socket.publish() # this will do an API call and will create script; + custom_socket.publish() # this will make an API call and create a script; -Some time is needed to setup the environment for this custom socket. -There is possibility to check the custom socket status:: +Sometimes, it's needed to set up the environment for the custom socket. +It's possible to check the custom socket status:: - # Reload will refresh object using syncano API. + # Reload will refresh object using Syncano API. custom_socket.reload() print(custom_socket.status) # and @@ -85,30 +86,34 @@ To update custom socket, use:: custom_socket = CustomSocket.please.get(name='my_custom_socket') + # to remove endpoint/dependency + custom_socket.remove_endpoint(endpoint_name='my_endpoint') custom_socket.remove_dependency(dependency_name='custom_script') - # or add new: + # or to add a new endpoint/dependency: custom_socket.add_endpoint(new_endpoint) # see above code for endpoint examples; custom_socket.add_dependency(new_dependency) # see above code for dependency examples; + # save changes on Syncano + custom_socket.update() -Running the custom socket +Running custom socket ------------------------- -To run custom socket use:: +To run a custom socket use:: - # this will run the my_endpoint - and call the custom_script (method is GET); + # this will run `my_endpoint` - and call `custom_script` (using GET method); result = custom_socket.run(method='GET', endpoint_name='my_endpoint') -Read all endpoints in custom socket +Read all endpoints in a custom socket ----------------------------------- -To get the all defined endpoints in custom socket run:: +To get the all defined endpoints in a custom socket run:: endpoints = custom_socket.get_endpoints() @@ -122,7 +127,7 @@ To run a particular endpoint:: # or: endpoint.run(method='POST', data={'name': 'test_name'}) -The data will be passed to the API call in the request body. +Data will be passed to the API call in the request body. Read all endpoints ------------------ @@ -140,16 +145,16 @@ Above code will return a list with SocketEndpoint objects. To run such endpoint, Custom sockets endpoints ------------------------ -Each custom socket requires to define at least one endpoint. The endpoint is defined by name and -a list of calls. Each call is defined by a name and a list of methods. The name is a identification for dependency, eg. +Each custom socket requires a definition of at least one endpoint. This endpoint is defined by name and +a list of calls. Each call is defined by its name and a list of methods. Name is used in identification for dependency, eg. if it's equal to 'my_script' - the ScriptEndpoint with name 'my_script' will be used -(if it exists and Script source and runtime matches) or a new one will be created. -There's a special wildcard method: `methods=['*']` - this mean that any request with +(if it exists and Script source and passed runtime match) -- otherwise a new one will be created. +There's a special wildcard method: `methods=['*']` - it means that any request with any method will be executed in this endpoint. -To add an endpoint to the custom_socket use:: +To add an endpoint to a chosen custom_socket use:: - my_endpoint = Endpoint(name='my_endpoint') # again - no API call here + my_endpoint = Endpoint(name='my_endpoint') # no API call here my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) @@ -158,9 +163,9 @@ To add an endpoint to the custom_socket use:: Custom socket dependency ------------------------ -Each custom socket has dependency - this is a meta information for endpoint: which resource -should be used to return the API call results. The dependencies are bind to the endpoints call objects. -Currently the only supported dependency is script. +Each custom socket has a dependency -- meta information for an endpoint: which resource +should be used to return the API call results. These dependencies are bound to the endpoints call objects. +Currently the only supported dependency is a Script. **Using new script** @@ -201,15 +206,15 @@ Currently the only supported dependency is script. Custom socket recheck --------------------- -The creation of the socket can fail - this happen, eg. when endpoint name is already taken by another +The creation of the socket can fail - this can happen, e.g. when an endpoint name is already taken by another custom socket. To check the statuses use:: print(custom_socket.status) print(custom_socket.status_info) There is a possibility to re-check socket - this mean that if conditions are met - the socket endpoints and dependencies -will be checked - and if some of them are missing (eg. mistake deletion), they will be created again. -If the endpoints and dependencies do not met the criteria - the error will be returned in the status field. +will be checked - and if some of them are missing (e.g. some were deleted by mistake), they will be created again. +If the endpoints and dependencies do not meet the criteria - an error will be returned in the status field. Custom socket - raw format -------------------------- @@ -236,4 +241,4 @@ If you prefer raw JSON format for creating sockets, you can resort to use it in ] ) -The disadvantage of this method is that - the JSON internal structure must be known by developer. +The disadvantage of this method is that internal structure of the JSON file must be known by developer. From fe52e7a88af493a4d2b74ccd30e5fd7203583710 Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Fri, 12 Aug 2016 18:47:03 -0400 Subject: [PATCH 13/35] [ci skip] readme --- docs/source/custom_sockets.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 0dd81d8..9c15956 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -4,11 +4,11 @@ Custom Sockets in Syncano ========================= -``Syncano`` gives its users an ability to create custom sockets. It means, that users can define -a very specific endpoints in their Syncano application, and use them as other Syncano -modules (Classes, Scripts, etc), using standard API calls. -Currently, custom sockets allow only one dependency - Scripts. It means that on the backend side, -each API call executes a Script and result of the execution, is returned as a result of the +``Syncano`` gives its users an ability to create Custom Sockets. What it means is that users can define +a very specific endpoints in their Syncano application, and use them exactly like they would any other Syncano +module (Classes, Scripts, etc), using standard API calls. +Currently, Custom Sockets allow only one dependency - Scripts. It means that under the hood, +each API call executes a Script, and result of this execution is returned as a result of the API call. Creating a custom socket From 7bc974a80e2a0bc3e8c4113a4f6638ffd5c7abb8 Mon Sep 17 00:00:00 2001 From: Mariusz Wisniewski Date: Mon, 15 Aug 2016 16:27:50 -0400 Subject: [PATCH 14/35] [ci skip] Endpoints readme update --- docs/source/custom_sockets.rst | 35 +++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 9c15956..e0b070f 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -28,15 +28,23 @@ To create a custom socket follow these steps:: my_endpoint.add_call(ScriptCall(name='custom_script'), methods=['GET']) my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) - # Explanation of the above lines: - # Defined endpoint will be visible under `my_endpoint` name: - # On this syncano API endpoint the above endpoint will be called (after custom socket creation) - # :///instances//endpoints/sockets/my_endpoint/ - # On this syncano API endpoint the details of the defined endpoint will be returned + # What happened here: + # - We defined a new endpoint, that will be visible under `my_endpoint` name. + # - You will be able to call this endpoint (execute attached `call`), + # by sending a reuqest, using any defined method to following API route: + # :///instances//endpoints/sockets/my_endpoint/ + # - To get details on that endpoint, you need to send a GET request to following API route: # :///instances//sockets/my_custom_socket/endpoints/my_endpoint/ - # For the above endpoint - the two calls are defined, one uses GET method - the custom_script will be executed - # there, second uses the POST method and then the another_custom_script will be called; - # Currently only script are available for calls; + # + # Following example above - we defined two calls on our endpoint. + # First one means that using GET method will call the `custom_script` script, + # and second one means that using POST method will call the `another_custom_script` script. + # At the moment, only scripts are available as endpoint calls. + # + # As a general rule - to get endpoints details (but not call them), use following API route: + # :///instances//sockets/my_custom_socket/endpoints// + # and to run your endpoints (e.g. execute Script connected to them(, use following API route: + # :///instances//endpoints/sockets// # 3. After creation of the endpoint, add it to your custom_socket. custom_socket.add_endpoint(my_endpoint) @@ -136,11 +144,16 @@ To get all endpoints that are defined in all custom sockets:: socket_endpoint_list = SocketEndpoint.get_all_endpoints() -Above code will return a list with SocketEndpoint objects. To run such endpoint, use:: +Above code will return a list with SocketEndpoint objects. To run an endpoint, +choose one endpoint first, e.g.: - socket_endpoint_list.run(method='GET') + endpoint = socket_endpoint_list[0] + +and now run it:: + + endpoint.run(method='GET') # or: - socket_endpoint_list.run(method='POST', data={'custom_data': 1}) + endpoint.run(method='POST', data={'custom_data': 1}) Custom sockets endpoints ------------------------ From bc71ad14e0cde7788290fc0526c2183d4704ecc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 09:53:09 +0200 Subject: [PATCH 15/35] [LIB-837] revert ModelField on script in script endpoint model; --- syncano/models/backups.py | 2 +- syncano/models/incentives.py | 2 +- tests/integration_test.py | 6 +++--- tests/integration_test_cache.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/syncano/models/backups.py b/syncano/models/backups.py index 3e92342..bf77353 100644 --- a/syncano/models/backups.py +++ b/syncano/models/backups.py @@ -28,7 +28,7 @@ class Backup(Model): size = fields.IntegerField(read_only=True) status = fields.StringField(read_only=True) status_info = fields.StringField(read_only=True) - author = fields.ModelField('Admin', read_only=True) + author = fields.ModelField('Admin') details = fields.JSONField(read_only=True) updated_at = fields.DateTimeField(read_only=True, required=False) diff --git a/syncano/models/incentives.py b/syncano/models/incentives.py index 585a03a..84ef2e4 100644 --- a/syncano/models/incentives.py +++ b/syncano/models/incentives.py @@ -222,7 +222,7 @@ class ScriptEndpoint(Model): """ name = fields.SlugField(max_length=50, primary_key=True) - script = fields.ModelField('Script', just_pk=True) + script = fields.IntegerField(label='script id') public = fields.BooleanField(required=False, default=False) public_link = fields.ChoiceField(required=False, read_only=True) links = fields.LinksField() diff --git a/tests/integration_test.py b/tests/integration_test.py index 816ea7a..622722c 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -548,7 +548,7 @@ def test_list(self): def test_create(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.script, + script=self.script.id, name='wh%s' % self.generate_hash()[:10], ) @@ -557,7 +557,7 @@ def test_create(self): def test_script_run(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.script, + script=self.script.id, name='wh%s' % self.generate_hash()[:10], ) @@ -569,7 +569,7 @@ def test_script_run(self): def test_custom_script_run(self): script_endpoint = self.model.please.create( instance_name=self.instance.name, - script=self.custom_script, + script=self.custom_script.id, name='wh%s' % self.generate_hash()[:10], ) diff --git a/tests/integration_test_cache.py b/tests/integration_test_cache.py index ba45bc8..3aff7ad 100644 --- a/tests/integration_test_cache.py +++ b/tests/integration_test_cache.py @@ -51,7 +51,7 @@ def setUpClass(cls): cls.script_endpoint = cls.instance.script_endpoints.create( name='test_script_endpoint', - script=cls.script + script=cls.script.id ) def test_cache_request(self): From 327a3fc993cefd7bfd2113891251c210b9fc41d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 09:55:01 +0200 Subject: [PATCH 16/35] [LIB-837] correct data passing on run methods; --- syncano/models/custom_sockets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index f7a4fda..e5a6909 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -50,9 +50,9 @@ def get_endpoints(self): endpoints.append(SocketEndpoint(**endpoint)) return endpoints - def run(self, method, endpoint_name, data={}): + def run(self, method, endpoint_name, data=None): endpoint = self._find_endpoint(endpoint_name) - return endpoint.run(method, data=data) + return endpoint.run(method, data=data or {}) def _find_endpoint(self, endpoint_name): endpoints = self.get_endpoints() @@ -123,7 +123,7 @@ class Meta: } } - def run(self, method='GET', data={}): + def run(self, method='GET', data=None): endpoint_path = self.links.endpoint connection = self._get_connection() if not self._validate_method(method): @@ -132,7 +132,7 @@ def run(self, method='GET', data={}): if method in ['get', 'delete']: response = connection.request(method, endpoint_path) elif method in ['post', 'put', 'patch']: - response = connection.request(method, endpoint_path, data=data) + response = connection.request(method, endpoint_path, data=data or {}) else: raise SyncanoValueError('Method: {} not supported.'.format(method)) return response From d50344b8f706160b7865aa3f20c783cc1b1619ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 10:31:41 +0200 Subject: [PATCH 17/35] [LIB-837] correct custom socket behaviour; --- syncano/models/custom_sockets.py | 2 +- syncano/models/custom_sockets_utils.py | 40 ++++++++++++++----------- tests/integration_test_custom_socket.py | 23 +++++++------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index e5a6909..1bd177a 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -50,7 +50,7 @@ def get_endpoints(self): endpoints.append(SocketEndpoint(**endpoint)) return endpoints - def run(self, method, endpoint_name, data=None): + def run(self, endpoint_name, method='GET', data=None): endpoint = self._find_endpoint(endpoint_name) return endpoint.run(method, data=data or {}) diff --git a/syncano/models/custom_sockets_utils.py b/syncano/models/custom_sockets_utils.py index 450a40e..4b143b2 100644 --- a/syncano/models/custom_sockets_utils.py +++ b/syncano/models/custom_sockets_utils.py @@ -99,6 +99,7 @@ class BaseDependency(object): fields = [] dependency_type = None + name = None def to_dependency_data(self): if self.dependency_type is None: @@ -136,38 +137,41 @@ class ScriptDependency(BaseDependency): 'source' ] - def __init__(self, name=None, script=None, script_endpoint=None): - if name and script and script_endpoint: - raise SyncanoValueError("Usage: ScriptDependency(name='', script=Script(...)) or " - "ScriptDependency(ScriptEndpoint(...))") - if (name and not script) or (not name and script): - raise SyncanoValueError("Usage: ScriptDependency(name='', script=Script(...))") + def __init__(self, script_or_script_endpoint, name=None): + if not isinstance(script_or_script_endpoint, (Script, ScriptEndpoint)): + raise SyncanoValueError('Script or ScriptEndpoint expected') - if script and not isinstance(script, Script): - raise SyncanoValueError("Expected Script type object.") + if isinstance(script_or_script_endpoint, Script) and not name: + raise SyncanoValueError('Name should be provided') - if script_endpoint and not isinstance(script_endpoint, ScriptEndpoint): - raise SyncanoValueError("Expected ScriptEndpoint type object.") - - if not script_endpoint: - self.dependency_object = ScriptEndpoint(name=name, script=script) - else: - self.dependency_object = script_endpoint + self.dependency_object = script_or_script_endpoint + self.name = name def get_name(self): + if self.name is not None: + return {'name': self.name} return {'name': self.dependency_object.name} def get_dependency_data(self): + + if isinstance(self.dependency_object, ScriptEndpoint): + script = Script.please.get(id=self.dependency_object.script, + instance_name=self.dependency_object.instance_name) + else: + script = self.dependency_object + dependency_data = self.get_name() dependency_data.update({ - field_name: getattr(self.dependency_object.script, field_name) for field_name in self.fields + field_name: getattr(script, field_name) for field_name in self.fields }) return dependency_data @classmethod def create_from_raw_data(cls, raw_data): - return cls(**{'script_endpoint': ScriptEndpoint(name=raw_data['name'], script=Script( - source=raw_data['source'], runtime_name=raw_data['runtime_name']))}) + return cls(**{ + 'script_or_script_endpoint': Script(source=raw_data['source'], runtime_name=raw_data['runtime_name']), + 'name': raw_data['name'], + }) class EndpointMetadataMixin(object): diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index b805ad9..9550237 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -117,15 +117,14 @@ def _define_endpoints(cls, suffix, custom_socket): @classmethod def _define_dependencies_new_script_endpoint(cls, suffix, custom_socket): + script = cls.__create_script(suffix) + script_endpoint = ScriptEndpoint( + name='script_endpoint_{}'.format(suffix), + script=script.id + ) custom_socket.add_dependency( ScriptDependency( - script_endpoint=ScriptEndpoint( - name='script_endpoint_{}'.format(suffix), - script=Script( - source='print({})'.format(suffix), - runtime_name=RuntimeChoices.PYTHON_V5_0 - ) - ) + script_endpoint ) ) @@ -133,11 +132,11 @@ def _define_dependencies_new_script_endpoint(cls, suffix, custom_socket): def _define_dependencies_new_script(cls, suffix, custom_socket): custom_socket.add_dependency( ScriptDependency( - name='script_endpoint_{}'.format(suffix), - script=Script( + Script( source='print({})'.format(suffix), runtime_name=RuntimeChoices.PYTHON_V5_0 - ) + ), + name='script_endpoint_{}'.format(suffix), ) ) @@ -147,8 +146,8 @@ def _define_dependencies_existing_script(cls, suffix, custom_socket): cls._create_script(suffix) custom_socket.add_dependency( ScriptDependency( + Script.please.first(), name='script_endpoint_{}'.format(suffix), - script=Script.please.first() ) ) @@ -161,7 +160,7 @@ def _define_dependencies_existing_script_endpoint(cls, suffix, custom_socket): ) custom_socket.add_dependency( ScriptDependency( - script_endpoint=ScriptEndpoint.please.first() + ScriptEndpoint.please.first() ) ) From 42e4373ca31430ce5d41d0320237347be81ac562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 10:50:17 +0200 Subject: [PATCH 18/35] [LIB-837] corrects after qa; --- syncano/models/fields.py | 2 -- tests/integration_test.py | 2 +- tests/integration_test_custom_socket.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/syncano/models/fields.py b/syncano/models/fields.py index c8ba8a1..8e4087e 100644 --- a/syncano/models/fields.py +++ b/syncano/models/fields.py @@ -478,8 +478,6 @@ def to_native(self, value): class ModelField(Field): - read_only = False - def __init__(self, rel, *args, **kwargs): self.rel = rel self.just_pk = kwargs.pop('just_pk', True) diff --git a/tests/integration_test.py b/tests/integration_test.py index 622722c..8f99229 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -478,7 +478,7 @@ def test_source_run(self): ) trace = script.run() - while trace.status == 'pending': + while trace.status == ['pending', 'processing']: sleep(1) trace.reload() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 9550237..4f7d4ec 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -117,7 +117,7 @@ def _define_endpoints(cls, suffix, custom_socket): @classmethod def _define_dependencies_new_script_endpoint(cls, suffix, custom_socket): - script = cls.__create_script(suffix) + script = cls._create_script(suffix) script_endpoint = ScriptEndpoint( name='script_endpoint_{}'.format(suffix), script=script.id From 04c5ef6687dc631982edc34ad30696165e48e472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 11:04:16 +0200 Subject: [PATCH 19/35] [LIB-837] corrects after qa; --- tests/integration_test.py | 2 +- tests/integration_test_custom_socket.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 8f99229..2acfcf3 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -478,7 +478,7 @@ def test_source_run(self): ) trace = script.run() - while trace.status == ['pending', 'processing']: + while trace.status in ['pending', 'processing']: sleep(1) trace.reload() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 4f7d4ec..e9453ce 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -56,7 +56,7 @@ def test_creating_raw_data(self): self.assertTrue(custom_socket.name) def test_custom_socket_run(self): - results = self.custom_socket.run('GET', 'my_endpoint_default') + results = self.custom_socket.run('my_endpoint_default') self.assertEqual(results.result['stdout'], 'script_default') def test_custom_socket_recheck(self): @@ -70,14 +70,14 @@ def test_fetching_all_endpoints(self): self.assertTrue(all_endpoints[0].name) def test_endpoint_run(self): - script_endpoint = ScriptEndpoint.please.first() - result = script_endpoint.run('GET') + script_endpoint = SocketEndpoint.please.first() + result = script_endpoint.run() suffix = script_endpoint.name.split('_')[-1] self.assertTrue(result.result['stdout'].endswith(suffix)) def test_custom_socket_update(self): socket_to_update = self._create_custom_socket('to_update', self._define_dependencies_new_script_endpoint) - socket_to_update.remove_endpoint(endpoint_name='my_endpoint_default') + socket_to_update.remove_endpoint(endpoint_name='my_endpoint_to_update') new_endpoint = Endpoint(name='my_endpoint_new_to_update') new_endpoint.add_call( @@ -88,7 +88,7 @@ def test_custom_socket_update(self): socket_to_update.update() time.sleep(2) # wait for custom socket setup; socket_to_update.reload() - self.assertIn('my_endpoint_new_default', socket_to_update.endpoints) + self.assertIn('my_endpoint_new_to_update', socket_to_update.endpoints) def assert_custom_socket(self, suffix, dependency_method): custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) @@ -156,7 +156,7 @@ def _define_dependencies_existing_script_endpoint(cls, suffix, custom_socket): script = cls._create_script(suffix) ScriptEndpoint.please.create( name='script_endpoint_{}'.format(suffix), - script=script + script=script.id ) custom_socket.add_dependency( ScriptDependency( From 9f5adba3afaa90a3eae2e0e9a038098a8fe48530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 11:47:59 +0200 Subject: [PATCH 20/35] [LIB-837] correct docs examples, polish the tests; --- docs/source/custom_sockets.rst | 30 ++++++++++++++++--------- syncano/models/custom_sockets.py | 4 +++- tests/integration_test_custom_socket.py | 11 +++++++-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index e0b070f..14d78f1 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -53,26 +53,26 @@ To create a custom socket follow these steps:: # 4.1 Using a new script - define a new source code. custom_socket.add_dependency( ScriptDependency( - name='custom_script' - script=Script( + Script( runtime_name=RuntimeChoices.PYTHON_V5_0, source='print("custom_script")' - ) + ), + name='custom_script' ) ) # 4.2 Using an existing script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( + another_custom_script, name='another_custom_script', - script=another_custom_script ) ) # 4.3 Using an existing ScriptEndpoint. script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') custom_socket.add_dependency( - script_endpoint=script_endpoint + script_endpoint ) # 5. Publish custom_socket. @@ -186,11 +186,11 @@ Currently the only supported dependency is a Script. custom_socket.add_dependency( ScriptDependency( - name='custom_script' - script=Script( + Script( runtime_name=RuntimeChoices.PYTHON_V5_0, source='print("custom_script")' - ) + ), + name='custom_script' ) ) @@ -202,8 +202,8 @@ Currently the only supported dependency is a Script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( - name='another_custom_script', - script=another_custom_script + another_custom_script, + name='another_custom_script' ) ) @@ -213,7 +213,15 @@ Currently the only supported dependency is a Script. script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') custom_socket.add_dependency( - script_endpoint=script_endpoint + script_endpoint + ) + +You can overwrite the name in the following way:: + + script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') + custom_socket.add_dependency( + script_endpoint, + name='custom_name' ) Custom socket recheck diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 1bd177a..98923fe 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -26,6 +26,8 @@ class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): metadata = fields.JSONField(read_only=True, required=False) status = fields.StringField(read_only=True, required=False) status_info = fields.StringField(read_only=True, required=False) + created_at = fields.DateTimeField(read_only=True, required=False) + updated_at = fields.DateTimeField(read_only=True, required=False) links = fields.LinksField() class Meta: @@ -52,7 +54,7 @@ def get_endpoints(self): def run(self, endpoint_name, method='GET', data=None): endpoint = self._find_endpoint(endpoint_name) - return endpoint.run(method, data=data or {}) + return endpoint.run(method=method, data=data or {}) def _find_endpoint(self, endpoint_name): endpoints = self.get_endpoints() diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index e9453ce..e9b1efa 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -20,6 +20,7 @@ class CustomSocketTest(InstanceMixin, IntegrationTest): def setUpClass(cls): super(CustomSocketTest, cls).setUpClass() cls.custom_socket = cls._create_custom_socket('default', cls._define_dependencies_new_script_endpoint) + cls._assert_custom_socket(cls.custom_socket) def test_publish_custom_socket(self): # this test new ScriptEndpoint dependency create; @@ -70,7 +71,7 @@ def test_fetching_all_endpoints(self): self.assertTrue(all_endpoints[0].name) def test_endpoint_run(self): - script_endpoint = SocketEndpoint.please.first() + script_endpoint = SocketEndpoint.get_all_endpoints()[0] result = script_endpoint.run() suffix = script_endpoint.name.split('_')[-1] self.assertTrue(result.result['stdout'].endswith(suffix)) @@ -92,7 +93,13 @@ def test_custom_socket_update(self): def assert_custom_socket(self, suffix, dependency_method): custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) - self.assertTrue(custom_socket.name) + self._assert_custom_socket(custom_socket) + + @classmethod + def _assert_custom_socket(cls, custom_socket): + cls.assertTrue(custom_socket.name) + cls.assertTrue(custom_socket.created_at) + cls.assertTrue(custom_socket.updated_at) @classmethod def _create_custom_socket(cls, suffix, dependency_method): From 69f3a11f58000099a5442e476b4e088abf20aab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 11:53:50 +0200 Subject: [PATCH 21/35] [LIB-837] remove default custom socket in setupclass; --- tests/integration_test_custom_socket.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index e9b1efa..b435dd0 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -16,12 +16,6 @@ class CustomSocketTest(InstanceMixin, IntegrationTest): - @classmethod - def setUpClass(cls): - super(CustomSocketTest, cls).setUpClass() - cls.custom_socket = cls._create_custom_socket('default', cls._define_dependencies_new_script_endpoint) - cls._assert_custom_socket(cls.custom_socket) - def test_publish_custom_socket(self): # this test new ScriptEndpoint dependency create; self.assert_custom_socket('publishing', self._define_dependencies_new_script_endpoint) @@ -57,12 +51,18 @@ def test_creating_raw_data(self): self.assertTrue(custom_socket.name) def test_custom_socket_run(self): - results = self.custom_socket.run('my_endpoint_default') - self.assertEqual(results.result['stdout'], 'script_default') + suffix = 'default' + custom_socket = self._create_custom_socket(suffix) + self._assert_custom_socket(custom_socket) + results = custom_socket.run('my_endpoint_{}'.format(suffix)) + self.assertEqual(results.result['stdout'], 'script_{}'.format(suffix)) def test_custom_socket_recheck(self): - custom_socket = self.custom_socket.recheck() - self.assertTrue(custom_socket.name) + suffix = 'recheck' + custom_socket = self._create_custom_socket(suffix) + self._assert_custom_socket(custom_socket) + custom_socket = custom_socket.recheck() + self._assert_custom_socket(custom_socket) def test_fetching_all_endpoints(self): all_endpoints = SocketEndpoint.get_all_endpoints() From 3d1f4ddf78812cb8f5721b25168187c4bd87707d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 11:58:29 +0200 Subject: [PATCH 22/35] [LIB-837] remove default custom socket in setupclass; --- tests/integration_test_custom_socket.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index b435dd0..7b1e579 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -52,14 +52,14 @@ def test_creating_raw_data(self): def test_custom_socket_run(self): suffix = 'default' - custom_socket = self._create_custom_socket(suffix) + custom_socket = self._create_custom_socket(suffix, self._define_dependencies_new_script_endpoint) self._assert_custom_socket(custom_socket) results = custom_socket.run('my_endpoint_{}'.format(suffix)) self.assertEqual(results.result['stdout'], 'script_{}'.format(suffix)) def test_custom_socket_recheck(self): suffix = 'recheck' - custom_socket = self._create_custom_socket(suffix) + custom_socket = self._create_custom_socket(suffix, self._define_dependencies_new_script_endpoint) self._assert_custom_socket(custom_socket) custom_socket = custom_socket.recheck() self._assert_custom_socket(custom_socket) @@ -95,11 +95,10 @@ def assert_custom_socket(self, suffix, dependency_method): custom_socket = self._create_custom_socket(suffix, dependency_method=dependency_method) self._assert_custom_socket(custom_socket) - @classmethod - def _assert_custom_socket(cls, custom_socket): - cls.assertTrue(custom_socket.name) - cls.assertTrue(custom_socket.created_at) - cls.assertTrue(custom_socket.updated_at) + def _assert_custom_socket(self, custom_socket): + self.assertTrue(custom_socket.name) + self.assertTrue(custom_socket.created_at) + self.assertTrue(custom_socket.updated_at) @classmethod def _create_custom_socket(cls, suffix, dependency_method): From d982fd03f6926e4b248c91eedb621aa082ab0692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 16 Aug 2016 13:15:37 +0200 Subject: [PATCH 23/35] [LIB-837] final test correction; --- syncano/connection.py | 1 - tests/integration_test_custom_socket.py | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/syncano/connection.py b/syncano/connection.py index c4c7706..3fcf6ad 100644 --- a/syncano/connection.py +++ b/syncano/connection.py @@ -273,7 +273,6 @@ def make_request(self, method_name, path, **kwargs): retry_after = response.headers.get('retry-after', 1) time.sleep(float(retry_after)) response = method(url, **params) - content = self.get_response_content(url, response) if files: diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 7b1e579..6819235 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -35,7 +35,7 @@ def test_creating_raw_data(self): name='my_custom_socket_123', endpoints={ "my_custom_endpoint_123": { - "calls": [{"type": "script", "name": "script_123", "methods": ["POST"]}] + "calls": [{"type": "script", "name": "script_123", "methods": ["GET", "POST"]}] } }, dependencies=[ @@ -55,7 +55,7 @@ def test_custom_socket_run(self): custom_socket = self._create_custom_socket(suffix, self._define_dependencies_new_script_endpoint) self._assert_custom_socket(custom_socket) results = custom_socket.run('my_endpoint_{}'.format(suffix)) - self.assertEqual(results.result['stdout'], 'script_{}'.format(suffix)) + self.assertEqual(results['result']['stdout'], suffix) def test_custom_socket_recheck(self): suffix = 'recheck' @@ -73,8 +73,8 @@ def test_fetching_all_endpoints(self): def test_endpoint_run(self): script_endpoint = SocketEndpoint.get_all_endpoints()[0] result = script_endpoint.run() - suffix = script_endpoint.name.split('_')[-1] - self.assertTrue(result.result['stdout'].endswith(suffix)) + self.assertIsInstance(result, dict) + self.assertTrue(result['result']['stdout']) def test_custom_socket_update(self): socket_to_update = self._create_custom_socket('to_update', self._define_dependencies_new_script_endpoint) @@ -96,6 +96,7 @@ def assert_custom_socket(self, suffix, dependency_method): self._assert_custom_socket(custom_socket) def _assert_custom_socket(self, custom_socket): + self._wait_till_socket_process(custom_socket) self.assertTrue(custom_socket.name) self.assertTrue(custom_socket.created_at) self.assertTrue(custom_socket.updated_at) @@ -115,7 +116,7 @@ def _define_endpoints(cls, suffix, custom_socket): endpoint = Endpoint(name='my_endpoint_{}'.format(suffix)) endpoint.add_call( ScriptCall( - name='script_{}'.format(suffix), + name='script_endpoint_{}'.format(suffix), methods=['GET', 'POST'] ) ) @@ -139,7 +140,7 @@ def _define_dependencies_new_script(cls, suffix, custom_socket): custom_socket.add_dependency( ScriptDependency( Script( - source='print({})'.format(suffix), + source='print("{}")'.format(suffix), runtime_name=RuntimeChoices.PYTHON_V5_0 ), name='script_endpoint_{}'.format(suffix), @@ -175,5 +176,10 @@ def _create_script(cls, suffix): return Script.please.create( label='script_{}'.format(suffix), runtime_name=RuntimeChoices.PYTHON_V5_0, - source='print({})'.format(suffix) + source='print("{}")'.format(suffix) ) + + @classmethod + def _wait_till_socket_process(cls, custom_socket): + while custom_socket.status == 'checking': + custom_socket.reload() From 8662e428f098e2e684472b76387e50fc4fb0fac9 Mon Sep 17 00:00:00 2001 From: Devin Visslailli Date: Tue, 16 Aug 2016 11:24:19 -0400 Subject: [PATCH 24/35] Update language --- docs/source/custom_sockets.rst | 100 ++++++++++++++++----------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 14d78f1..ac6fd63 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -4,23 +4,23 @@ Custom Sockets in Syncano ========================= -``Syncano`` gives its users an ability to create Custom Sockets. What it means is that users can define -a very specific endpoints in their Syncano application, and use them exactly like they would any other Syncano +``Syncano`` gives its users the ability to create Custom Sockets. What this means is that users can define very specific +endpoints in their Syncano application, and use them exactly like they would any other Syncano module (Classes, Scripts, etc), using standard API calls. -Currently, Custom Sockets allow only one dependency - Scripts. It means that under the hood, -each API call executes a Script, and result of this execution is returned as a result of the +Currently, Custom Sockets allow only one dependency - Scripts. Under the hood, +each API call executes a Script, and the result of this execution is returned as a result of the API call. -Creating a custom socket +Creating a custom Socket ------------------------ -To create a custom socket follow these steps:: +To create a custom Socket follow these steps:: import syncano from syncano.models import CustomSocket, Endpoint, ScriptCall, ScriptDependency, RuntimeChoices from syncano.connection import Connection - # 1. Initialize a custom socket. + # 1. Initialize a custom Socket. custom_socket = CustomSocket(name='my_custom_socket') # this will create an object in place (do API call) # 2. Define endpoints. @@ -29,28 +29,28 @@ To create a custom socket follow these steps:: my_endpoint.add_call(ScriptCall(name='another_custom_script'), methods=['POST']) # What happened here: - # - We defined a new endpoint, that will be visible under `my_endpoint` name. + # - We defined a new endpoint that will be visible under the name `my_endpoint` # - You will be able to call this endpoint (execute attached `call`), - # by sending a reuqest, using any defined method to following API route: + # by sending a request, using any defined method to the following API route: # :///instances//endpoints/sockets/my_endpoint/ - # - To get details on that endpoint, you need to send a GET request to following API route: + # - To get details for that endpoint, you need to send a GET request to following API route: # :///instances//sockets/my_custom_socket/endpoints/my_endpoint/ # - # Following example above - we defined two calls on our endpoint. - # First one means that using GET method will call the `custom_script` script, - # and second one means that using POST method will call the `another_custom_script` script. - # At the moment, only scripts are available as endpoint calls. + # Following the example above - we defined two calls on our endpoint with the `add_call` method + # The first one means that using a GET method will call the `custom_script` Script, + # and second one means that using a POST method will call the `another_custom_script` Script. + # At the moment, only Scripts are available as endpoint calls. # - # As a general rule - to get endpoints details (but not call them), use following API route: + # As a general rule - to get endpoint details (but not call them), use following API route: # :///instances//sockets/my_custom_socket/endpoints// - # and to run your endpoints (e.g. execute Script connected to them(, use following API route: + # and to run your endpoints (e.g. execute Script connected to them), use following API route: # :///instances//endpoints/sockets// # 3. After creation of the endpoint, add it to your custom_socket. custom_socket.add_endpoint(my_endpoint) # 4. Define dependency. - # 4.1 Using a new script - define a new source code. + # 4.1 Using a new Script - define a new source code. custom_socket.add_dependency( ScriptDependency( Script( @@ -60,7 +60,7 @@ To create a custom socket follow these steps:: name='custom_script' ) ) - # 4.2 Using an existing script. + # 4.2 Using an existing Script. another_custom_script = Script.please.get(id=2) custom_socket.add_dependency( ScriptDependency( @@ -78,8 +78,8 @@ To create a custom socket follow these steps:: # 5. Publish custom_socket. custom_socket.publish() # this will make an API call and create a script; -Sometimes, it's needed to set up the environment for the custom socket. -It's possible to check the custom socket status:: +It may take some time to set up the Socket, so you can check the status. +It's possible to check the custom Socket status:: # Reload will refresh object using Syncano API. custom_socket.reload() @@ -87,10 +87,10 @@ It's possible to check the custom socket status:: # and print(custom_socket.status_info) -Updating the custom socket +Updating the custom Socket -------------------------- -To update custom socket, use:: +To update custom Socket, use:: custom_socket = CustomSocket.please.get(name='my_custom_socket') @@ -109,19 +109,19 @@ To update custom socket, use:: custom_socket.update() -Running custom socket +Running custom Socket ------------------------- -To run a custom socket use:: +To run a custom Socket use:: - # this will run `my_endpoint` - and call `custom_script` (using GET method); + # this will run `my_endpoint` - and call `custom_script` using GET method; result = custom_socket.run(method='GET', endpoint_name='my_endpoint') -Read all endpoints in a custom socket +Read all endpoints in a custom Socket ----------------------------------- -To get the all defined endpoints in a custom socket run:: +To get the all defined endpoints in a custom Socket run:: endpoints = custom_socket.get_endpoints() @@ -140,7 +140,7 @@ Data will be passed to the API call in the request body. Read all endpoints ------------------ -To get all endpoints that are defined in all custom sockets:: +To get all endpoints that are defined in all custom Sockets:: socket_endpoint_list = SocketEndpoint.get_all_endpoints() @@ -155,15 +155,15 @@ and now run it:: # or: endpoint.run(method='POST', data={'custom_data': 1}) -Custom sockets endpoints +Custom Sockets endpoints ------------------------ -Each custom socket requires a definition of at least one endpoint. This endpoint is defined by name and -a list of calls. Each call is defined by its name and a list of methods. Name is used in identification for dependency, eg. -if it's equal to 'my_script' - the ScriptEndpoint with name 'my_script' will be used -(if it exists and Script source and passed runtime match) -- otherwise a new one will be created. -There's a special wildcard method: `methods=['*']` - it means that any request with -any method will be executed in this endpoint. +Each custom socket requires defining at least one endpoint. This endpoint is defined by name and +a list of calls. Each call is defined by its name and a list of methods. `name` is used as an +identification for the dependency, eg. if `name` is equal to 'my_script' - the ScriptEndpoint with name 'my_script' +will be used (if it exists and Script source and passed runtime match) -- otherwise a new one will be created. +There's a special wildcard method: `methods=['*']` - this allows you to execute the provided custom Socket +with any request method (GET, POST, PATCH, etc.). To add an endpoint to a chosen custom_socket use:: @@ -173,14 +173,14 @@ To add an endpoint to a chosen custom_socket use:: custom_socket.add_endpoint(my_endpoint) -Custom socket dependency +Custom Socket dependency ------------------------ Each custom socket has a dependency -- meta information for an endpoint: which resource -should be used to return the API call results. These dependencies are bound to the endpoints call objects. +should be used to return the API call results. These dependencies are bound to the endpoints call object. Currently the only supported dependency is a Script. -**Using new script** +**Using new Script** :: @@ -195,7 +195,7 @@ Currently the only supported dependency is a Script. ) -**Using defined script** +**Using defined Script** :: @@ -207,7 +207,7 @@ Currently the only supported dependency is a Script. ) ) -**Using defined script endpoint** +**Using defined Script endpoint** :: @@ -216,7 +216,7 @@ Currently the only supported dependency is a Script. script_endpoint ) -You can overwrite the name in the following way:: +You can overwrite the Script name in the following way:: script_endpoint = ScriptEndpoint.please.get(name='script_endpoint_name') custom_socket.add_dependency( @@ -224,23 +224,23 @@ You can overwrite the name in the following way:: name='custom_name' ) -Custom socket recheck +Custom Socket recheck --------------------- -The creation of the socket can fail - this can happen, e.g. when an endpoint name is already taken by another -custom socket. To check the statuses use:: +The creation of a Socket can fail - this can happen, for example, when an endpoint name is already taken by another +custom Socket. To check the creation status use:: print(custom_socket.status) print(custom_socket.status_info) -There is a possibility to re-check socket - this mean that if conditions are met - the socket endpoints and dependencies -will be checked - and if some of them are missing (e.g. some were deleted by mistake), they will be created again. -If the endpoints and dependencies do not meet the criteria - an error will be returned in the status field. +You can also re-check a Socket. This mean that all dependencies will be checked - if some of them are missing +(e.g. some were deleted by mistake), they will be created again. If the endpoints and dependencies do not meet +the criteria - an error will be returned in the status field. -Custom socket - raw format +Custom Socket - raw format -------------------------- -If you prefer raw JSON format for creating sockets, you can resort to use it in python library as well:::: +If you prefer raw JSON format for creating Sockets, the Python library allows you to do so:::: CustomSocket.please.create( name='my_custom_socket_3', @@ -262,4 +262,4 @@ If you prefer raw JSON format for creating sockets, you can resort to use it in ] ) -The disadvantage of this method is that internal structure of the JSON file must be known by developer. +The disadvantage of this method is that the internal structure of the JSON file must be known by the developer. From 2ff6c68f3093882007e42c9d75023d7122e6882d Mon Sep 17 00:00:00 2001 From: Marcin Skiba Date: Thu, 18 Aug 2016 08:36:23 +0200 Subject: [PATCH 25/35] [LIB-819] - Add support for creating objects through DataEndpoint --- syncano/models/data_views.py | 3 + tests/integration_test_data_endpoint.py | 139 +++++++++++++----------- 2 files changed, 81 insertions(+), 61 deletions(-) diff --git a/syncano/models/data_views.py b/syncano/models/data_views.py index ae0384d..69dd488 100644 --- a/syncano/models/data_views.py +++ b/syncano/models/data_views.py @@ -127,3 +127,6 @@ def _get_response_template_name(self, response_template): 'Invalid response_template. Must be template\'s name or ResponseTemplate object.' ) return name + + def add_object(self, **kwargs): + return Object(instance_name=self.instance_name, class_name=self.class_name, **kwargs).save() diff --git a/tests/integration_test_data_endpoint.py b/tests/integration_test_data_endpoint.py index 4cccd9f..6ce9c1e 100644 --- a/tests/integration_test_data_endpoint.py +++ b/tests/integration_test_data_endpoint.py @@ -7,76 +7,93 @@ class DataEndpointTest(InstanceMixin, IntegrationTest): - schema = [ - { - 'name': 'title', - 'type': 'string', - 'order_index': True, - 'filter_index': True - } - ] + @classmethod + def setUpClass(cls): + super(DataEndpointTest, cls).setUpClass() - template_content = ''' - {% if action == 'list' %} - {% set objects = response.objects %} - {% elif action == 'retrieve' %} - {% set objects = [response] %} - {% else %} - {% set objects = [] %} - {% endif %} - {% if objects %} - + schema = [ + { + 'name': 'title', + 'type': 'string', + 'order_index': True, + 'filter_index': True + } + ] - - {% for key in objects[0] if key not in fields_to_skip %} - {{ key }} - {% endfor %} - - {% for object in objects %} - - {% for key, value in object.iteritems() if key not in fields_to_skip %} - {{ value }} - {% endfor %} + template_content = ''' + {% if action == 'list' %} + {% set objects = response.objects %} + {% elif action == 'retrieve' %} + {% set objects = [response] %} + {% else %} + {% set objects = [] %} + {% endif %} + {% if objects %} + + + + {% for key in objects[0] if key not in fields_to_skip %} + {{ key }} + {% endfor %} - {% endfor %} - - {% endif %} - ''' + {% for object in objects %} + + {% for key, value in object.iteritems() if key not in fields_to_skip %} + {{ value }} + {% endfor %} + + {% endfor %} + + {% endif %} + ''' - template_context = { - "tr_header_classes": "", - "th_header_classes": "", - "tr_row_classes": "", - "table_classes": "", - "td_row_classes": "", - "fields_to_skip": [ - "id", - "channel", - "channel_room", - "group", - "links", - "group_permissions", - "owner_permissions", - "other_permissions", - "owner", - "revision", - "updated_at", - "created_at" - ] - } + template_context = { + "tr_header_classes": "", + "th_header_classes": "", + "tr_row_classes": "", + "table_classes": "", + "td_row_classes": "", + "fields_to_skip": [ + "id", + "channel", + "channel_room", + "group", + "links", + "group_permissions", + "owner_permissions", + "other_permissions", + "owner", + "revision", + "updated_at", + "created_at" + ] + } - def test_template_response(self): - Class(name='test_class', schema=self.schema).save() - Object(class_name='test_class', title='test_title').save() - template = ResponseTemplate( + cls.klass = Class(name='test_class', schema=schema).save() + cls.template = ResponseTemplate( name='test_template', - content=self.template_content, + content=template_content, content_type='text/html', - context=self.template_context + context=template_context ).save() - data_endpoint = DataEndpoint(name='test_endpoint', class_name='test_class').save() + cls.data_endpoint = DataEndpoint(name='test_endpoint', class_name='test_class').save() - response = list(data_endpoint.get(response_template=template)) + def setUp(self): + for obj in self.instance.classes.get(name='test_class').objects.all(): + obj.delete() + + def test_template_response(self): + Object(class_name=self.klass.name, title='test_title').save() + response = list(self.data_endpoint.get(response_template=self.template)) self.assertEqual(len(response), 1, 'Data endpoint should return 1 element if queried with response_template.') data = re.sub('[\s+]', '', response[0]) self.assertEqual(data, '
title
test_title
') + + def test_create_object(self): + objects_count = len(list(self.data_endpoint.get())) + self.assertEqual(objects_count, 0) + self.data_endpoint.add_object(title='another title') + objects_count = len(list(self.data_endpoint.get())) + self.assertEqual(objects_count, 1, 'New object should have been created.') + obj = next(self.data_endpoint.get()) + self.assertEqual(obj['title'], 'another title', 'Created object should have proper title.') From 086416324dae1cbcae07fb64f4d2186e5338de68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Thu, 18 Aug 2016 11:42:18 +0200 Subject: [PATCH 26/35] [LIB-837] add possibility to directly point instance name when getting all endpoints; --- syncano/models/custom_sockets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 98923fe..54156e3 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -140,10 +140,12 @@ def run(self, method='GET', data=None): return response @classmethod - def get_all_endpoints(cls): + def get_all_endpoints(cls, instance_name=None): connection = cls._meta.connection - all_endpoints_path = Instance._meta.resolve_endpoint('endpoints', - {'name': cls.please.properties.get('instance_name')}) + all_endpoints_path = Instance._meta.resolve_endpoint( + 'endpoints', + {'name': cls.please.properties.get('instance_name') or instance_name} + ) response = connection.request('GET', all_endpoints_path) return [cls(**endpoint) for endpoint in response['objects']] From 9246b98a74b49869259b55ea6924cd99102a573c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 19 Aug 2016 14:55:52 +0200 Subject: [PATCH 27/35] [LIB-837] change interface: publish to install in custom socket context; --- docs/source/custom_sockets.rst | 4 ++-- syncano/models/custom_sockets.py | 8 ++++---- tests/integration_test_custom_socket.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index 14d78f1..7751515 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -75,8 +75,8 @@ To create a custom socket follow these steps:: script_endpoint ) - # 5. Publish custom_socket. - custom_socket.publish() # this will make an API call and create a script; + # 5. Install custom_socket. + custom_socket.install() # this will make an API call and create a script; Sometimes, it's needed to set up the environment for the custom socket. It's possible to check the custom socket status:: diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index 54156e3..a3a9339 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -23,7 +23,7 @@ class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): name = fields.StringField(max_length=64, primary_key=True) endpoints = fields.JSONField() dependencies = fields.JSONField() - metadata = fields.JSONField(read_only=True, required=False) + metadata = fields.JSONField(required=False) status = fields.StringField(read_only=True, required=False) status_info = fields.StringField(read_only=True, required=False) created_at = fields.DateTimeField(read_only=True, required=False) @@ -63,9 +63,9 @@ def _find_endpoint(self, endpoint_name): return endpoint raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) - def publish(self): + def install(self): if not self.is_new(): - raise SyncanoValueError('Can not publish already defined custom socket.') + raise SyncanoValueError('Can not install already defined custom socket.') created_socket = self.__class__.please.create( name=self.name, @@ -79,7 +79,7 @@ def publish(self): def update(self): if self.is_new(): - raise SyncanoValueError('Publish socket first.') + raise SyncanoValueError('Install socket first.') update_socket = self.__class__.please.update( name=self.name, diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 6819235..4f1f14d 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -16,18 +16,18 @@ class CustomSocketTest(InstanceMixin, IntegrationTest): - def test_publish_custom_socket(self): + def test_install_custom_socket(self): # this test new ScriptEndpoint dependency create; - self.assert_custom_socket('publishing', self._define_dependencies_new_script_endpoint) + self.assert_custom_socket('installing', self._define_dependencies_new_script_endpoint) def test_dependencies_new_script(self): - self.assert_custom_socket('new_script_publishing', self._define_dependencies_new_script) + self.assert_custom_socket('new_script_installing', self._define_dependencies_new_script) def test_dependencies_existing_script(self): - self.assert_custom_socket('existing_script_publishing', self._define_dependencies_existing_script) + self.assert_custom_socket('existing_script_installing', self._define_dependencies_existing_script) def test_dependencies_existing_script_endpoint(self): - self.assert_custom_socket('existing_script_e_publishing', + self.assert_custom_socket('existing_script_e_installing', self._define_dependencies_existing_script_endpoint) def test_creating_raw_data(self): @@ -108,7 +108,7 @@ def _create_custom_socket(cls, suffix, dependency_method): cls._define_endpoints(suffix, custom_socket) dependency_method(suffix, custom_socket) - custom_socket.publish() + custom_socket.install() return custom_socket @classmethod From 14f6ff7e0ae34a489519ec1cb82ac43649c647c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Fri, 19 Aug 2016 15:04:59 +0200 Subject: [PATCH 28/35] [LIB-837] add status badges to README --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index 2744d96..73e82a8 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,19 @@ Syncano ======= +Build Status +------------ + +**Master** + +.. image:: https://circleci.com/gh/Syncano/syncano-python/tree/master.svg?style=svg&circle-token=738c379fd91cc16b82758e6be89d0c21926655e0 + :target: https://circleci.com/gh/Syncano/syncano-python/tree/master + +**Develop** + +.. image:: https://circleci.com/gh/Syncano/syncano-python/tree/develop.svg?style=svg&circle-token=738c379fd91cc16b82758e6be89d0c21926655e0 + :target: https://circleci.com/gh/Syncano/syncano-python/tree/develop + Python QuickStart Guide ----------------------- From 2372986679ebee9c1df107615d662e20a213914f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Mon, 22 Aug 2016 12:51:32 +0200 Subject: [PATCH 29/35] [LIB-837] small changes after CORE change; --- syncano/models/custom_sockets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index a3a9339..e52a6e4 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -109,7 +109,7 @@ class SocketEndpoint(Model): :ivar links: :class:`~syncano.models.fields.LinksField` """ name = fields.StringField(max_length=64, primary_key=True) - calls = fields.JSONField() + allowed_methods = fields.JSONField() links = fields.LinksField() class Meta: @@ -126,7 +126,7 @@ class Meta: } def run(self, method='GET', data=None): - endpoint_path = self.links.endpoint + endpoint_path = self.links.self connection = self._get_connection() if not self._validate_method(method): raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) From 515e50b83e582be4ea83702f0797f3eb000ebcb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Mon, 22 Aug 2016 13:17:52 +0200 Subject: [PATCH 30/35] [LIB-837] correct SocketEndpoint behaviour (run mainly); --- syncano/models/custom_sockets.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index e52a6e4..d9fc55b 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -59,7 +59,7 @@ def run(self, endpoint_name, method='GET', data=None): def _find_endpoint(self, endpoint_name): endpoints = self.get_endpoints() for endpoint in endpoints: - if endpoint_name == endpoint.name: + if '{}/{}'.format(self.name, endpoint_name) == endpoint.name: return endpoint raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) @@ -127,14 +127,15 @@ class Meta: def run(self, method='GET', data=None): endpoint_path = self.links.self + _, endpoint_name = self.name.split('/', 1) connection = self._get_connection() if not self._validate_method(method): raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) method = method.lower() if method in ['get', 'delete']: - response = connection.request(method, endpoint_path) + response = connection.request(method, "{}/{}".format(endpoint_path, endpoint_name)) elif method in ['post', 'put', 'patch']: - response = connection.request(method, endpoint_path, data=data or {}) + response = connection.request(method, "{}/{}".format(endpoint_path, endpoint_name), data=data or {}) else: raise SyncanoValueError('Method: {} not supported.'.format(method)) return response @@ -150,10 +151,6 @@ def get_all_endpoints(cls, instance_name=None): return [cls(**endpoint) for endpoint in response['objects']] def _validate_method(self, method): - - methods = [] - for call in self.calls: - methods.extend(call['methods']) - if '*' in methods or method in methods: + if '*' in self.allowed_methods or method in self.allowed_methods: return True return False From 16795dd92c858abbaa6422b8af7cb48110b4c996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Mon, 22 Aug 2016 13:53:19 +0200 Subject: [PATCH 31/35] [LIB-837] correct run in CustomSocket; --- syncano/models/custom_sockets.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index d9fc55b..bd4d566 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -44,13 +44,7 @@ class Meta: } def get_endpoints(self): - endpoints_path = self.links.endpoints - connection = self._get_connection() - response = connection.request('GET', endpoints_path) - endpoints = [] - for endpoint in response['objects']: - endpoints.append(SocketEndpoint(**endpoint)) - return endpoints + return SocketEndpoint.get_all_endpoints(instance_name=self.instance_name) def run(self, endpoint_name, method='GET', data=None): endpoint = self._find_endpoint(endpoint_name) @@ -127,15 +121,14 @@ class Meta: def run(self, method='GET', data=None): endpoint_path = self.links.self - _, endpoint_name = self.name.split('/', 1) connection = self._get_connection() if not self._validate_method(method): raise SyncanoValueError('Method: {} not specified in calls for this custom socket.'.format(method)) method = method.lower() if method in ['get', 'delete']: - response = connection.request(method, "{}/{}".format(endpoint_path, endpoint_name)) + response = connection.request(method, endpoint_path) elif method in ['post', 'put', 'patch']: - response = connection.request(method, "{}/{}".format(endpoint_path, endpoint_name), data=data or {}) + response = connection.request(method, endpoint_path, data=data or {}) else: raise SyncanoValueError('Method: {} not supported.'.format(method)) return response From b8c252f84fc903328349d2f2e8916335d7558214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 23 Aug 2016 13:55:28 +0200 Subject: [PATCH 32/35] [LIB-837] add possiblity to install from url; --- syncano/models/custom_sockets.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index bd4d566..c1ae7dd 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -21,6 +21,7 @@ class CustomSocket(EndpointMetadataMixin, DependencyMetadataMixin, Model): """ name = fields.StringField(max_length=64, primary_key=True) + description = fields.StringField(required=False) endpoints = fields.JSONField() dependencies = fields.JSONField() metadata = fields.JSONField(required=False) @@ -57,6 +58,16 @@ def _find_endpoint(self, endpoint_name): return endpoint raise SyncanoValueError('Endpoint {} not found.'.format(endpoint_name)) + def install_from_url(self, url, instance_name=None): + instance_name = self.__class__.please.properties.get('instance_name') or instance_name + instance = Instance.please.get(name=instance_name) + + install_path = instance.links.sockets_install + connection = self._get_connection() + response = connection.request('POST', install_path, data={'name': self.name, 'install_url': url}) + + return response + def install(self): if not self.is_new(): raise SyncanoValueError('Can not install already defined custom socket.') From 3cf2465593f565f7d4f23f8f8b267ccabb75baa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Tue, 23 Aug 2016 13:58:26 +0200 Subject: [PATCH 33/35] [LIB-837] update documentation for installing socket from url; --- docs/source/custom_sockets.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index c584ff3..b7fefe7 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -237,6 +237,17 @@ You can also re-check a Socket. This mean that all dependencies will be checked (e.g. some were deleted by mistake), they will be created again. If the endpoints and dependencies do not meet the criteria - an error will be returned in the status field. +Custom Socket - install from url +-------------------------------- + +To install a socket from url use:: + + CustomSocket(name='new_socket_name').install_from_url(url='https://...') + +If instance name was not provided in connection arguments, do:: + + CustomSocket(name='new_socket_name').install_from_url(url='https://...', instance_name='instance_name') + Custom Socket - raw format -------------------------- From ef39be6ace6ac145f84dd6a003a6a7483d061837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Wed, 24 Aug 2016 12:54:00 +0200 Subject: [PATCH 34/35] [LIB-837] fixes after qa; --- docs/source/custom_sockets.rst | 2 +- syncano/models/custom_sockets.py | 2 +- syncano/models/custom_sockets_utils.py | 6 +++--- tests/integration_test_custom_socket.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/custom_sockets.rst b/docs/source/custom_sockets.rst index b7fefe7..f61c811 100644 --- a/docs/source/custom_sockets.rst +++ b/docs/source/custom_sockets.rst @@ -247,7 +247,7 @@ To install a socket from url use:: If instance name was not provided in connection arguments, do:: CustomSocket(name='new_socket_name').install_from_url(url='https://...', instance_name='instance_name') - + Custom Socket - raw format -------------------------- diff --git a/syncano/models/custom_sockets.py b/syncano/models/custom_sockets.py index c1ae7dd..441d59c 100644 --- a/syncano/models/custom_sockets.py +++ b/syncano/models/custom_sockets.py @@ -70,7 +70,7 @@ def install_from_url(self, url, instance_name=None): def install(self): if not self.is_new(): - raise SyncanoValueError('Can not install already defined custom socket.') + raise SyncanoValueError('Custom socket already installed.') created_socket = self.__class__.please.create( name=self.name, diff --git a/syncano/models/custom_sockets_utils.py b/syncano/models/custom_sockets_utils.py index 4b143b2..96eaf6e 100644 --- a/syncano/models/custom_sockets_utils.py +++ b/syncano/models/custom_sockets_utils.py @@ -66,7 +66,7 @@ class Endpoint(object): The JSON format is as follows:: { - ': { + '': { 'calls': [ ] @@ -139,10 +139,10 @@ class ScriptDependency(BaseDependency): def __init__(self, script_or_script_endpoint, name=None): if not isinstance(script_or_script_endpoint, (Script, ScriptEndpoint)): - raise SyncanoValueError('Script or ScriptEndpoint expected') + raise SyncanoValueError('Script or ScriptEndpoint expected.') if isinstance(script_or_script_endpoint, Script) and not name: - raise SyncanoValueError('Name should be provided') + raise SyncanoValueError('Name should be provided.') self.dependency_object = script_or_script_endpoint self.name = name diff --git a/tests/integration_test_custom_socket.py b/tests/integration_test_custom_socket.py index 4f1f14d..2ddd2f3 100644 --- a/tests/integration_test_custom_socket.py +++ b/tests/integration_test_custom_socket.py @@ -17,7 +17,7 @@ class CustomSocketTest(InstanceMixin, IntegrationTest): def test_install_custom_socket(self): - # this test new ScriptEndpoint dependency create; + # this tests new ScriptEndpoint dependency create; self.assert_custom_socket('installing', self._define_dependencies_new_script_endpoint) def test_dependencies_new_script(self): From 21ddeaf3835840bdd4781dab5ffba739adbd1fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Opa=C5=82czy=C5=84ski?= Date: Wed, 24 Aug 2016 14:25:16 +0200 Subject: [PATCH 35/35] [RELEASE v5.4] bump the version; --- syncano/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncano/__init__.py b/syncano/__init__.py index 93b0df2..e0c54af 100644 --- a/syncano/__init__.py +++ b/syncano/__init__.py @@ -2,7 +2,7 @@ import os __title__ = 'Syncano Python' -__version__ = '5.3.0' +__version__ = '5.4.0' __author__ = "Daniel Kopka, Michal Kobus, and Sebastian Opalczynski" __credits__ = ["Daniel Kopka", "Michal Kobus",