Repository navigation
Expand file tree
/
Copy pathflow.py
More file actions
124 lines (98 loc) · 3.95 KB
/
Copy pathflow.py
File metadata and controls
124 lines (98 loc) · 3.95 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from collections import OrderedDict
import xmltodict
import sklearn
from .._api_calls import _perform_api_call
from .functions import _check_flow_exists
class OpenMLFlow(object):
"""OpenML Flow. Stores machine learning models.
Parameters
----------
model : scikit-learn compatible model
The model the flow consists of. The model needs to have fit and predict methods.
description : string
Description of the flow (free text).
creator : string
FIXME
contributor : string
FIXME
tag : string
FIXME
flow_id : int, optional
Flow ID. Assigned by the server (fixme shouldn't be here?)
uploader : string, optional
User uploading the model (fixme shouldn't be here?). Assigned by the server.
"""
def __init__(self, model, flow_id=None, uploader=None,
description='Flow generated by run_task', creator=None,
contributor=None, tag=None):
self.flow_id = flow_id
self.upoader = uploader
self.description = description
self.creator = creator
self.tag = tag
self.model = model
self.source = "FIXME DEFINE PYTHON FLOW"
self.name = (model.__module__ + "." +
model.__class__.__name__)
self.external_version = 'sklearn_' + sklearn.__version__
def _generate_flow_xml(self):
"""Generate xml representation of self for upload to server.
Returns
-------
flow_xml : string
Flow represented as XML string.
"""
model = self.model
flow_dict = OrderedDict()
flow_dict['oml:flow'] = OrderedDict()
flow_dict['oml:flow']['@xmlns:oml'] = 'http://openml.org/openml'
flow_dict['oml:flow']['oml:name'] = self._get_name()
flow_dict['oml:flow']['oml:external_version'] = self.external_version
flow_dict['oml:flow']['oml:description'] = self.description
clf_params = model.get_params()
flow_parameters = []
for k, v in clf_params.items():
# data_type, default_value, description, recommendedRange
# type = v.__class__.__name__ Not using this because it doesn't conform standards
# eg. int instead of integer
param_dict = {'oml:name': k}
flow_parameters.append(param_dict)
flow_dict['oml:flow']['oml:parameter'] = flow_parameters
flow_xml = xmltodict.unparse(flow_dict, pretty=True)
# A flow may not be uploaded with the encoding specification..
flow_xml = flow_xml.split('\n', 1)[-1]
return flow_xml
def publish(self):
"""Publish flow to OpenML server.
Returns
-------
self : OpenMLFlow
"""
xml_description = self._generate_flow_xml()
file_elements = {'description': xml_description}
return_code, return_value = _perform_api_call(
"flow/", file_elements=file_elements)
self.flow_id = int(xmltodict.parse(return_value)['oml:upload_flow']['oml:id'])
return self
def _ensure_flow_exists(self):
""" Checks if a flow exists for the given model and possibly creates it.
If the given flow exists on the server, the flow-id will simply
be returned. Otherwise it will be uploaded to the server.
Returns
-------
flow_id : int
Flow id on the server.
"""
import sklearn
flow_version = 'sklearn_' + sklearn.__version__
_, _, flow_id = _check_flow_exists(self._get_name(), flow_version)
# TODO add numpy and scipy version!
if int(flow_id) == -1:
return_code, response_xml = self.publish()
response_dict = xmltodict.parse(response_xml)
flow_id = response_dict['oml:upload_flow']['oml:id']
return int(flow_id)
return int(flow_id)
def _get_name(self):
"""Helper function. Can be mocked for testing."""
return self.name