This repository was archived by the owner on Aug 25, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 135
Expand file tree
/
Copy pathplugins.py
More file actions
152 lines (130 loc) · 4.12 KB
/
plugins.py
File metadata and controls
152 lines (130 loc) · 4.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
"""
This file is imported by the top level setup.py and therefore must remain as
independent as possible (no relative imports)
"""
import os
import platform
import contextlib
import importlib.util
def inpath(binary):
return any(
list(
map(
lambda dirname: os.path.isfile(os.path.join(dirname, binary)),
os.environ.get("PATH", "").split(":"),
)
)
)
# List of plugins
CORE_PLUGINS = [
("configloader", "yaml"),
("configloader", "image"),
("model", "scratch"),
("model", "scikit"),
("model", "tensorflow"),
("model", "tensorflow_hub"),
("model", "vowpalWabbit"),
("model", "xgboost"),
("model", "pytorch"),
("model", "spacy"),
("model", "daal4py"),
]
# Models which currently don't support Windows or MacOS
if platform.system() not in {"Windows", "Darwin"}:
CORE_PLUGINS += [
("model", "autosklearn"),
]
CORE_PLUGINS += [
("examples", "shouldi"),
("feature", "git"),
("feature", "auth"),
("operations", "binsec"),
("operations", "data"),
("operations", "deploy"),
("operations", "image"),
("operations", "nlp"),
("operations", "innersource"),
("entities", "alice"),
("service", "http"),
("source", "mysql"),
]
def python_package_installed(module_name: str) -> bool:
"""
Check if a Python package is installed and can be imported
"""
spec = None
with contextlib.suppress(ModuleNotFoundError):
spec = importlib.util.find_spec(module_name)
return bool(spec is not None)
# Dependencies of plugins and how to check if they exist on the system or not
CORE_PLUGIN_DEPS = {
("model", "autosklearn"): {
"swig": lambda: inpath("swig"),
"cython": lambda: inpath("cython"),
}
if platform.system() not in {"Windows", "Darwin"}
and not python_package_installed("autosklearn")
else {},
}
# All packages under configloader/ are really named dffml-config-{name}
ALTERNATIVES = {"configloader": "config"}
# Build a dict of plugin_type_name (aka model, config): list(package_names)
def package_names_by_plugin(validation=None):
by_plugin = {
(plugin_type + ("s" if not plugin_type.endswith("s") else "")): [
"dffml-%s-%s"
% (
ALTERNATIVES.get(plugin_type, plugin_type),
name.replace("_", "-"),
)
for sub_plugin_type, name in CORE_PLUGINS
if sub_plugin_type == plugin_type
and (not validation or validation(sub_plugin_type, name))
]
for plugin_type, plugin_name in CORE_PLUGINS
if plugin_type not in ("examples", "entities")
}
# Operations used to be named features
by_plugin["operations"].extend(by_plugin["features"])
del by_plugin["features"]
# All packages
by_plugin["all"] = [
"dffml-%s-%s"
% (ALTERNATIVES.get(plugin_type, plugin_type), name.replace("_", "-"),)
for plugin_type, name in CORE_PLUGINS
if plugin_type not in ("examples", "entities")
and (not validation or validation(plugin_type, name))
]
return by_plugin
PACKAGE_NAMES_BY_PLUGIN = package_names_by_plugin()
# Same as PACKAGE_NAMES_BY_PLUGIN but only with plugins that have all their
# pre-install dependencies met
PACKAGE_NAMES_BY_PLUGIN_INSTALLABLE = package_names_by_plugin(
lambda plugin_type, plugin_name: all(
map(
lambda check: check(),
CORE_PLUGIN_DEPS.get((plugin_type, plugin_name), {}).values(),
)
)
)
def package_names_to_directory(validation=None):
pkgs = {}
for plugin_type, name in CORE_PLUGINS:
if plugin_type in ("examples", "entities"):
pkg = name
else:
pkg = "dffml-%s-%s" % (
ALTERNATIVES.get(plugin_type, plugin_type),
name.replace("_", "-"),
)
pkgs[pkg] = (
plugin_type,
name,
)
return pkgs
PACKAGE_NAMES_TO_DIRECTORY = package_names_to_directory()
PACKAGE_DIRECTORY_TO_NAME = dict(
zip(
PACKAGE_NAMES_TO_DIRECTORY.values(), PACKAGE_NAMES_TO_DIRECTORY.keys(),
)
)