Skip to content

Commit d188432

Browse files
DinoVDinoV
committed
Makes symtable.symtable have parity for accepted datatypes
for source code as compile()
1 parent 2725cb0 commit d188432

6 files changed

Lines changed: 109 additions & 79 deletions

File tree

‎Include/pythonrun.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString(
119119
const char *filename, /* decoded from the filesystem encoding */
120120
int start);
121121
#ifndef Py_LIMITED_API
122+
PyAPI_FUNC(const char *) _Py_SourceAsString(
123+
PyObject *cmd,
124+
const char *funcname,
125+
const char *what,
126+
PyCompilerFlags *cf,
127+
PyObject **cmd_copy);
128+
122129
PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
123130
const char *str,
124131
PyObject *filename,
125132
int start);
133+
134+
PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
135+
const char *str,
136+
PyObject *filename,
137+
int start,
138+
PyCompilerFlags *flags);
126139
#endif
127140

128141
PyAPI_FUNC(void) PyErr_Print(void);

‎Lib/test/test_symtable.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,15 @@ def test_single(self):
215215
def test_exec(self):
216216
symbols = symtable.symtable("def f(x): return x", "?", "exec")
217217

218+
def test_bytes(self):
219+
top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
220+
self.assertIsNotNone(find_block(top, "Mine"))
221+
222+
code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
223+
224+
top = symtable.symtable(code, "?", "exec")
225+
self.assertIsNotNone(find_block(top, "\u017d"))
226+
218227

219228
if __name__ == '__main__':
220229
unittest.main()

‎Modules/clinic/symtablemodule.c.h‎

Lines changed: 5 additions & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Modules/symtablemodule.c‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ module _symtable
1414
/*[clinic input]
1515
_symtable.symtable
1616
17-
str: str
17+
str: object
1818
filename: object(converter='PyUnicode_FSDecoder')
1919
startstr: str
2020
/
@@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler.
2323
[clinic start generated code]*/
2424

2525
static PyObject *
26-
_symtable_symtable_impl(PyObject *module, const char *str,
27-
PyObject *filename, const char *startstr)
28-
/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/
26+
_symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename,
27+
const char *startstr)
28+
/*[clinic end generated code: output=da9d42d2103ea619 input=a8b56a8eee46c26b]*/
2929
{
3030
struct symtable *st;
3131
PyObject *t;
3232
int start;
33+
PyCompilerFlags cf;
34+
PyObject *source_copy = NULL;
35+
36+
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
37+
cf.cf_feature_version = PY_MINOR_VERSION;
38+
39+
const char *source = _Py_SourceAsString(str, "symtable", "string or bytes", &cf, &source_copy);
40+
if (source == NULL) {
41+
return NULL;
42+
}
3343

3444
if (strcmp(startstr, "exec") == 0)
3545
start = Py_file_input;
@@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str,
4151
PyErr_SetString(PyExc_ValueError,
4252
"symtable() arg 3 must be 'exec' or 'eval' or 'single'");
4353
Py_DECREF(filename);
54+
Py_XDECREF(source_copy);
4455
return NULL;
4556
}
46-
st = Py_SymtableStringObject(str, filename, start);
57+
st = _Py_SymtableStringObjectFlags(source, filename, start, &cf);
4758
Py_DECREF(filename);
48-
if (st == NULL)
59+
if (st == NULL) {
60+
Py_XDECREF(source_copy);
4961
return NULL;
62+
}
5063
t = (PyObject *)st->st_top;
5164
Py_INCREF(t);
5265
PyMem_Free((void *)st->st_future);

‎Python/bltinmodule.c‎

Lines changed: 3 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i)
687687
}
688688

689689

690-
static const char *
691-
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
692-
{
693-
const char *str;
694-
Py_ssize_t size;
695-
Py_buffer view;
696-
697-
*cmd_copy = NULL;
698-
if (PyUnicode_Check(cmd)) {
699-
cf->cf_flags |= PyCF_IGNORE_COOKIE;
700-
str = PyUnicode_AsUTF8AndSize(cmd, &size);
701-
if (str == NULL)
702-
return NULL;
703-
}
704-
else if (PyBytes_Check(cmd)) {
705-
str = PyBytes_AS_STRING(cmd);
706-
size = PyBytes_GET_SIZE(cmd);
707-
}
708-
else if (PyByteArray_Check(cmd)) {
709-
str = PyByteArray_AS_STRING(cmd);
710-
size = PyByteArray_GET_SIZE(cmd);
711-
}
712-
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
713-
/* Copy to NUL-terminated buffer. */
714-
*cmd_copy = PyBytes_FromStringAndSize(
715-
(const char *)view.buf, view.len);
716-
PyBuffer_Release(&view);
717-
if (*cmd_copy == NULL) {
718-
return NULL;
719-
}
720-
str = PyBytes_AS_STRING(*cmd_copy);
721-
size = PyBytes_GET_SIZE(*cmd_copy);
722-
}
723-
else {
724-
PyErr_Format(PyExc_TypeError,
725-
"%s() arg 1 must be a %s object",
726-
funcname, what);
727-
return NULL;
728-
}
729-
730-
if (strlen(str) != (size_t)size) {
731-
PyErr_SetString(PyExc_ValueError,
732-
"source code string cannot contain null bytes");
733-
Py_CLEAR(*cmd_copy);
734-
return NULL;
735-
}
736-
return str;
737-
}
738-
739690
/*[clinic input]
740691
compile as builtin_compile
741692
@@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
855806
goto finally;
856807
}
857808

858-
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
809+
str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy);
859810
if (str == NULL)
860811
goto error;
861812

@@ -987,7 +938,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
987938

988939
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
989940
cf.cf_feature_version = PY_MINOR_VERSION;
990-
str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
941+
str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy);
991942
if (str == NULL)
992943
return NULL;
993944

@@ -1075,7 +1026,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
10751026
PyCompilerFlags cf;
10761027
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
10771028
cf.cf_feature_version = PY_MINOR_VERSION;
1078-
str = source_as_string(source, "exec",
1029+
str = _Py_SourceAsString(source, "exec",
10791030
"string, bytes or code", &cf,
10801031
&source_copy);
10811032
if (str == NULL)

‎Python/pythonrun.c‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,21 +1184,77 @@ PyCompileString(const char *str, const char *filename, int start)
11841184
return Py_CompileStringFlags(str, filename, start, NULL);
11851185
}
11861186

1187+
const char *
1188+
_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
1189+
{
1190+
const char *str;
1191+
Py_ssize_t size;
1192+
Py_buffer view;
1193+
1194+
*cmd_copy = NULL;
1195+
if (PyUnicode_Check(cmd)) {
1196+
cf->cf_flags |= PyCF_IGNORE_COOKIE;
1197+
str = PyUnicode_AsUTF8AndSize(cmd, &size);
1198+
if (str == NULL)
1199+
return NULL;
1200+
}
1201+
else if (PyBytes_Check(cmd)) {
1202+
str = PyBytes_AS_STRING(cmd);
1203+
size = PyBytes_GET_SIZE(cmd);
1204+
}
1205+
else if (PyByteArray_Check(cmd)) {
1206+
str = PyByteArray_AS_STRING(cmd);
1207+
size = PyByteArray_GET_SIZE(cmd);
1208+
}
1209+
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
1210+
/* Copy to NUL-terminated buffer. */
1211+
*cmd_copy = PyBytes_FromStringAndSize(
1212+
(const char *)view.buf, view.len);
1213+
PyBuffer_Release(&view);
1214+
if (*cmd_copy == NULL) {
1215+
return NULL;
1216+
}
1217+
str = PyBytes_AS_STRING(*cmd_copy);
1218+
size = PyBytes_GET_SIZE(*cmd_copy);
1219+
}
1220+
else {
1221+
PyErr_Format(PyExc_TypeError,
1222+
"%s() arg 1 must be a %s object",
1223+
funcname, what);
1224+
return NULL;
1225+
}
1226+
1227+
if (strlen(str) != (size_t)size) {
1228+
PyErr_SetString(PyExc_ValueError,
1229+
"source code string cannot contain null bytes");
1230+
Py_CLEAR(*cmd_copy);
1231+
return NULL;
1232+
}
1233+
return str;
1234+
}
1235+
11871236
struct symtable *
11881237
Py_SymtableStringObject(const char *str, PyObject *filename, int start)
1238+
{
1239+
PyCompilerFlags flags;
1240+
1241+
flags.cf_flags = 0;
1242+
flags.cf_feature_version = PY_MINOR_VERSION;
1243+
return _Py_SymtableStringObjectFlags(str, filename, start, &flags);
1244+
}
1245+
1246+
struct symtable *
1247+
_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags)
11891248
{
11901249
struct symtable *st;
11911250
mod_ty mod;
1192-
PyCompilerFlags flags;
11931251
PyArena *arena;
11941252

11951253
arena = PyArena_New();
11961254
if (arena == NULL)
11971255
return NULL;
11981256

1199-
flags.cf_flags = 0;
1200-
flags.cf_feature_version = PY_MINOR_VERSION;
1201-
mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena);
1257+
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
12021258
if (mod == NULL) {
12031259
PyArena_Free(arena);
12041260
return NULL;

0 commit comments

Comments
 (0)