-
-
Notifications
You must be signed in to change notification settings - Fork 12.4k
Expand file tree
/
Copy pathgenerate_numpy_api.py
More file actions
330 lines (277 loc) · 9.64 KB
/
generate_numpy_api.py
File metadata and controls
330 lines (277 loc) · 9.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
#!/usr/bin/env python3
import argparse
import os
from . import genapi, numpy_api
from .genapi import BoolValuesApi, FunctionApi, GlobalVarApi, TypeApi
# use annotated api when running under cpychecker
h_template = r"""
#if defined(_MULTIARRAYMODULE) || defined(WITH_CPYCHECKER_STEALS_REFERENCE_TO_ARG_ATTRIBUTE)
typedef struct {
PyObject_HEAD
npy_bool obval;
} PyBoolScalarObject;
extern NPY_NO_EXPORT PyTypeObject PyArrayNeighborhoodIter_Type;
extern NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[2];
%s
#else
#if defined(PY_ARRAY_UNIQUE_SYMBOL)
#define PyArray_API PY_ARRAY_UNIQUE_SYMBOL
#define _NPY_VERSION_CONCAT_HELPER2(x, y) x ## y
#define _NPY_VERSION_CONCAT_HELPER(arg) \
_NPY_VERSION_CONCAT_HELPER2(arg, PyArray_RUNTIME_VERSION)
#define PyArray_RUNTIME_VERSION \
_NPY_VERSION_CONCAT_HELPER(PY_ARRAY_UNIQUE_SYMBOL)
#endif
/* By default do not export API in an .so (was never the case on windows) */
#ifndef NPY_API_SYMBOL_ATTRIBUTE
#define NPY_API_SYMBOL_ATTRIBUTE NPY_VISIBILITY_HIDDEN
#endif
#if defined(NO_IMPORT) || defined(NO_IMPORT_ARRAY)
extern NPY_API_SYMBOL_ATTRIBUTE void **PyArray_API;
extern NPY_API_SYMBOL_ATTRIBUTE int PyArray_RUNTIME_VERSION;
#else
#if defined(PY_ARRAY_UNIQUE_SYMBOL)
NPY_API_SYMBOL_ATTRIBUTE void **PyArray_API;
NPY_API_SYMBOL_ATTRIBUTE int PyArray_RUNTIME_VERSION;
#else
static void **PyArray_API = NULL;
static int PyArray_RUNTIME_VERSION = 0;
#endif
#endif
%s
/*
* The DType classes are inconvenient for the Python generation so exposed
* manually in the header below (may be moved).
*/
#include "numpy/_public_dtype_api_table.h"
#if !defined(NO_IMPORT_ARRAY) && !defined(NO_IMPORT)
static inline int
_import_array(void)
{
int st;
PyObject *numpy = PyImport_ImportModule("numpy._core._multiarray_umath");
PyObject *c_api;
if (numpy == NULL && PyErr_ExceptionMatches(PyExc_ModuleNotFoundError)) {
PyErr_Clear();
numpy = PyImport_ImportModule("numpy.core._multiarray_umath");
}
if (numpy == NULL) {
return -1;
}
c_api = PyObject_GetAttrString(numpy, "_ARRAY_API");
Py_DECREF(numpy);
if (c_api == NULL) {
return -1;
}
if (!PyCapsule_CheckExact(c_api)) {
PyErr_SetString(PyExc_RuntimeError, "_ARRAY_API is not PyCapsule object");
Py_DECREF(c_api);
return -1;
}
PyArray_API = (void **)PyCapsule_GetPointer(c_api, NULL);
Py_DECREF(c_api);
if (PyArray_API == NULL) {
PyErr_SetString(PyExc_RuntimeError, "_ARRAY_API is NULL pointer");
return -1;
}
/*
* On exceedingly few platforms these sizes may not match, in which case
* We do not support older NumPy versions at all.
*/
if (sizeof(Py_ssize_t) != sizeof(Py_intptr_t) &&
PyArray_RUNTIME_VERSION < NPY_2_0_API_VERSION) {
PyErr_Format(PyExc_RuntimeError,
"module compiled against NumPy 2.0 but running on NumPy 1.x. "
"Unfortunately, this is not supported on niche platforms where "
"`sizeof(size_t) != sizeof(inptr_t)`.");
}
/*
* Perform runtime check of C API version. As of now NumPy 2.0 is ABI
* backwards compatible (in the exposed feature subset!) for all practical
* purposes.
*/
if (NPY_VERSION < PyArray_GetNDArrayCVersion()) {
PyErr_Format(PyExc_RuntimeError, "module compiled against "\
"ABI version 0x%%x but this version of numpy is 0x%%x", \
(int) NPY_VERSION, (int) PyArray_GetNDArrayCVersion());
return -1;
}
PyArray_RUNTIME_VERSION = (int)PyArray_GetNDArrayCFeatureVersion();
if (NPY_FEATURE_VERSION > PyArray_RUNTIME_VERSION) {
PyErr_Format(PyExc_RuntimeError,
"module was compiled against NumPy C-API version 0x%%x "
"(NumPy " NPY_FEATURE_VERSION_STRING ") "
"but the running NumPy has C-API version 0x%%x. "
"Check the section C-API incompatibility at the "
"Troubleshooting ImportError section at "
"https://numpy.org/devdocs/user/troubleshooting-importerror.html"
"#c-api-incompatibility "
"for indications on how to solve this problem.",
(int)NPY_FEATURE_VERSION, PyArray_RUNTIME_VERSION);
return -1;
}
/*
* Perform runtime check of endianness and check it matches the one set by
* the headers (npy_endian.h) as a safeguard
*/
st = PyArray_GetEndianness();
if (st == NPY_CPU_UNKNOWN_ENDIAN) {
PyErr_SetString(PyExc_RuntimeError,
"FATAL: module compiled as unknown endian");
return -1;
}
#if NPY_BYTE_ORDER == NPY_BIG_ENDIAN
if (st != NPY_CPU_BIG) {
PyErr_SetString(PyExc_RuntimeError,
"FATAL: module compiled as big endian, but "
"detected different endianness at runtime");
return -1;
}
#elif NPY_BYTE_ORDER == NPY_LITTLE_ENDIAN
if (st != NPY_CPU_LITTLE) {
PyErr_SetString(PyExc_RuntimeError,
"FATAL: module compiled as little endian, but "
"detected different endianness at runtime");
return -1;
}
#endif
return 0;
}
#if (SWIG_VERSION < 0x040400)
#define _RETURN_VALUE NULL
#else
#define _RETURN_VALUE 0
#endif
#define import_array() { \
if (_import_array() < 0) { \
PyErr_Print(); \
PyErr_SetString( \
PyExc_ImportError, \
"numpy._core.multiarray failed to import" \
); \
return _RETURN_VALUE; \
} \
}
#define import_array1(ret) { \
if (_import_array() < 0) { \
PyErr_Print(); \
PyErr_SetString( \
PyExc_ImportError, \
"numpy._core.multiarray failed to import" \
); \
return ret; \
} \
}
#define import_array2(msg, ret) { \
if (_import_array() < 0) { \
PyErr_Print(); \
PyErr_SetString(PyExc_ImportError, msg); \
return ret; \
} \
}
#endif
#endif
""" # noqa: E501
c_template = r"""
/* These pointers will be stored in the C-object for use in other
extension modules
*/
void *PyArray_API[] = {
%s
};
"""
def generate_api(output_dir, force=False):
basename = 'multiarray_api'
h_file = os.path.join(output_dir, f'__{basename}.h')
c_file = os.path.join(output_dir, f'__{basename}.c')
targets = (h_file, c_file)
sources = numpy_api.multiarray_api
do_generate_api(targets, sources)
return targets
def do_generate_api(targets, sources):
header_file = targets[0]
c_file = targets[1]
global_vars = sources[0]
scalar_bool_values = sources[1]
types_api = sources[2]
multiarray_funcs = sources[3]
multiarray_api = sources[:]
module_list = []
extension_list = []
init_list = []
# Check multiarray api indexes
multiarray_api_index = genapi.merge_api_dicts(multiarray_api)
unused_index_max = max(multiarray_api_index.get("__unused_indices__", 0))
genapi.check_api_dict(multiarray_api_index)
numpyapi_list = genapi.get_api_functions('NUMPY_API',
multiarray_funcs)
# Create dict name -> *Api instance
api_name = 'PyArray_API'
multiarray_api_dict = {}
for f in numpyapi_list:
name = f.name
index = multiarray_funcs[name][0]
annotations = multiarray_funcs[name][1:]
multiarray_api_dict[f.name] = FunctionApi(f.name, index, annotations,
f.return_type,
f.args, api_name)
for name, val in global_vars.items():
index, type = val
multiarray_api_dict[name] = GlobalVarApi(name, index, type, api_name)
for name, val in scalar_bool_values.items():
index = val[0]
multiarray_api_dict[name] = BoolValuesApi(name, index, api_name)
for name, val in types_api.items():
index = val[0]
internal_type = None if len(val) == 1 else val[1]
multiarray_api_dict[name] = TypeApi(
name, index, 'PyTypeObject', api_name, internal_type)
if len(multiarray_api_dict) != len(multiarray_api_index):
keys_dict = set(multiarray_api_dict.keys())
keys_index = set(multiarray_api_index.keys())
keys_index_dict = keys_index - keys_dict
keys_dict_index = keys_dict - keys_index
raise AssertionError(
f"Multiarray API size mismatch - index has extra keys {keys_index_dict}, "
f"dict has extra keys {keys_dict_index}"
)
extension_list = []
for name, index in genapi.order_dict(multiarray_api_index):
api_item = multiarray_api_dict[name]
# In NumPy 2.0 the API may have holes (which may be filled again)
# in that case, add `NULL` to fill it.
while len(init_list) < api_item.index:
init_list.append(" NULL")
extension_list.append(api_item.define_from_array_api_string())
init_list.append(api_item.array_api_define())
module_list.append(api_item.internal_define())
# In case we end with a "hole", append more NULLs
while len(init_list) <= unused_index_max:
init_list.append(" NULL")
# Write to header
s = h_template % ('\n'.join(module_list), '\n'.join(extension_list))
genapi.write_file(header_file, s)
# Write to c-code
s = c_template % ',\n'.join(init_list)
genapi.write_file(c_file, s)
return targets
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-o",
"--outdir",
type=str,
help="Path to the output directory"
)
parser.add_argument(
"-i",
"--ignore",
type=str,
help="An ignored input - may be useful to add a "
"dependency between custom targets"
)
args = parser.parse_args()
outdir_abs = os.path.join(os.getcwd(), args.outdir)
generate_api(outdir_abs)
if __name__ == "__main__":
main()