Skip to content

Commit 1831034

Browse files
dpgeorgedpgeorge
committed
py: Allow lexer to raise exceptions during construction.
This patch refactors the error handling in the lexer, to simplify it (ie reduce code size). A long time ago, when the lexer/parser/compiler were first written, the lexer and parser were designed so they didn't use exceptions (ie nlr) to report errors but rather returned an error code. Over time that has gradually changed, the parser in particular has more and more ways of raising exceptions. Also, the lexer never really handled all errors without raising, eg there were some memory errors which could raise an exception (and in these rare cases one would get a fatal nlr-not-handled fault). This patch accepts the fact that the lexer can raise exceptions in some cases and allows it to raise exceptions to handle all its errors, which are for the most part just out-of-memory errors during construction of the lexer. This makes the lexer a bit simpler, and also the persistent code stuff is simplified. What this means for users of the lexer is that calls to it must be wrapped in a nlr handler. But all uses of the lexer already have such an nlr handler for the parser (and compiler) so that doesn't put any extra burden on the callers.
1 parent 9773506 commit 1831034

7 files changed

Lines changed: 31 additions & 92 deletions

File tree

‎extmod/vfs_reader.c‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#include <stdio.h>
2828
#include <string.h>
2929

30-
#include "py/nlr.h"
30+
#include "py/runtime.h"
3131
#include "py/stream.h"
3232
#include "py/reader.h"
3333
#include "extmod/vfs.h"
@@ -69,30 +69,19 @@ STATIC void mp_reader_vfs_close(void *data) {
6969
m_del_obj(mp_reader_vfs_t, reader);
7070
}
7171

72-
int mp_reader_new_file(mp_reader_t *reader, const char *filename) {
73-
mp_reader_vfs_t *rf = m_new_obj_maybe(mp_reader_vfs_t);
74-
if (rf == NULL) {
75-
return MP_ENOMEM;
76-
}
77-
// TODO we really should just let this function raise a uPy exception
78-
nlr_buf_t nlr;
79-
if (nlr_push(&nlr) == 0) {
80-
mp_obj_t arg = mp_obj_new_str(filename, strlen(filename), false);
81-
rf->file = mp_vfs_open(1, &arg, (mp_map_t*)&mp_const_empty_map);
82-
int errcode;
83-
rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
84-
nlr_pop();
85-
if (errcode != 0) {
86-
return errcode;
87-
}
88-
} else {
89-
return MP_ENOENT; // assume error was "file not found"
72+
void mp_reader_new_file(mp_reader_t *reader, const char *filename) {
73+
mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
74+
mp_obj_t arg = mp_obj_new_str(filename, strlen(filename), false);
75+
rf->file = mp_vfs_open(1, &arg, (mp_map_t*)&mp_const_empty_map);
76+
int errcode;
77+
rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
78+
if (errcode != 0) {
79+
mp_raise_OSError(errcode);
9080
}
9181
rf->pos = 0;
9282
reader->data = rf;
9383
reader->readbyte = mp_reader_vfs_readbyte;
9484
reader->close = mp_reader_vfs_close;
95-
return 0; // success
9685
}
9786

9887
#endif // MICROPY_READER_VFS

‎py/builtinevex.c‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,6 @@ STATIC mp_obj_t eval_exec_helper(size_t n_args, const mp_obj_t *args, mp_parse_i
136136
mp_lexer_t *lex;
137137
if (MICROPY_PY_BUILTINS_EXECFILE && parse_input_kind == MP_PARSE_SINGLE_INPUT) {
138138
lex = mp_lexer_new_from_file(str);
139-
if (lex == NULL) {
140-
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_OSError, "could not open file '%s'", str));
141-
}
142139
parse_input_kind = MP_PARSE_FILE_INPUT;
143140
} else {
144141
lex = mp_lexer_new_from_str_len(MP_QSTR__lt_string_gt_, str, str_len, 0);

‎py/builtinimport.c‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -131,18 +131,7 @@ STATIC mp_import_stat_t find_file(const char *file_str, uint file_len, vstr_t *d
131131
}
132132

133133
#if MICROPY_ENABLE_COMPILER
134-
STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char *fname) {
135-
136-
if (lex == NULL) {
137-
// we verified the file exists using stat, but lexer could still fail
138-
if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) {
139-
mp_raise_msg(&mp_type_ImportError, "module not found");
140-
} else {
141-
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError,
142-
"no module named '%s'", fname));
143-
}
144-
}
145-
134+
STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) {
146135
#if MICROPY_PY___FILE__
147136
qstr source_name = lex->source_name;
148137
mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
@@ -207,7 +196,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
207196
// found the filename in the list of frozen files, then load and execute it.
208197
#if MICROPY_MODULE_FROZEN_STR
209198
if (frozen_type == MP_FROZEN_STR) {
210-
do_load_from_lexer(module_obj, modref, file_str);
199+
do_load_from_lexer(module_obj, modref);
211200
return;
212201
}
213202
#endif
@@ -235,7 +224,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
235224
#if MICROPY_ENABLE_COMPILER
236225
{
237226
mp_lexer_t *lex = mp_lexer_new_from_file(file_str);
238-
do_load_from_lexer(module_obj, lex, file_str);
227+
do_load_from_lexer(module_obj, lex);
239228
return;
240229
}
241230
#endif

‎py/lexer.c‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -699,13 +699,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
699699
}
700700

701701
mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
702-
mp_lexer_t *lex = m_new_obj_maybe(mp_lexer_t);
703-
704-
// check for memory allocation error
705-
if (lex == NULL) {
706-
reader.close(reader.data);
707-
return NULL;
708-
}
702+
mp_lexer_t *lex = m_new_obj(mp_lexer_t);
709703

710704
lex->source_name = src_name;
711705
lex->reader = reader;
@@ -715,16 +709,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
715709
lex->nested_bracket_level = 0;
716710
lex->alloc_indent_level = MICROPY_ALLOC_LEXER_INDENT_INIT;
717711
lex->num_indent_level = 1;
718-
lex->indent_level = m_new_maybe(uint16_t, lex->alloc_indent_level);
712+
lex->indent_level = m_new(uint16_t, lex->alloc_indent_level);
719713
vstr_init(&lex->vstr, 32);
720714

721-
// check for memory allocation error
722-
// note: vstr_init above may fail on malloc, but so may mp_lexer_to_next below
723-
if (lex->indent_level == NULL) {
724-
mp_lexer_free(lex);
725-
return NULL;
726-
}
727-
728715
// store sentinel for first indentation level
729716
lex->indent_level[0] = 0;
730717

@@ -764,31 +751,23 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
764751

765752
mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len, size_t free_len) {
766753
mp_reader_t reader;
767-
if (!mp_reader_new_mem(&reader, (const byte*)str, len, free_len)) {
768-
return NULL;
769-
}
754+
mp_reader_new_mem(&reader, (const byte*)str, len, free_len);
770755
return mp_lexer_new(src_name, reader);
771756
}
772757

773758
#if MICROPY_READER_POSIX || MICROPY_READER_VFS
774759

775760
mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
776761
mp_reader_t reader;
777-
int ret = mp_reader_new_file(&reader, filename);
778-
if (ret != 0) {
779-
return NULL;
780-
}
762+
mp_reader_new_file(&reader, filename);
781763
return mp_lexer_new(qstr_from_str(filename), reader);
782764
}
783765

784766
#if MICROPY_HELPER_LEXER_UNIX
785767

786768
mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
787769
mp_reader_t reader;
788-
int ret = mp_reader_new_file_from_fd(&reader, fd, close_fd);
789-
if (ret != 0) {
790-
return NULL;
791-
}
770+
mp_reader_new_file_from_fd(&reader, fd, close_fd);
792771
return mp_lexer_new(filename, reader);
793772
}
794773

‎py/persistentcode.c‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -225,18 +225,13 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) {
225225

226226
mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len) {
227227
mp_reader_t reader;
228-
if (!mp_reader_new_mem(&reader, buf, len, 0)) {
229-
m_malloc_fail(BYTES_PER_WORD); // we need to raise a MemoryError
230-
}
228+
mp_reader_new_mem(&reader, buf, len, 0);
231229
return mp_raw_code_load(&reader);
232230
}
233231

234232
mp_raw_code_t *mp_raw_code_load_file(const char *filename) {
235233
mp_reader_t reader;
236-
int ret = mp_reader_new_file(&reader, filename);
237-
if (ret != 0) {
238-
mp_raise_OSError(ret);
239-
}
234+
mp_reader_new_file(&reader, filename);
240235
return mp_raw_code_load(&reader);
241236
}
242237

‎py/reader.c‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <stdio.h>
2828
#include <assert.h>
2929

30+
#include "py/runtime.h"
3031
#include "py/mperrno.h"
3132
#include "py/reader.h"
3233

@@ -54,19 +55,15 @@ STATIC void mp_reader_mem_close(void *data) {
5455
m_del_obj(mp_reader_mem_t, reader);
5556
}
5657

57-
bool mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) {
58-
mp_reader_mem_t *rm = m_new_obj_maybe(mp_reader_mem_t);
59-
if (rm == NULL) {
60-
return false;
61-
}
58+
void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) {
59+
mp_reader_mem_t *rm = m_new_obj(mp_reader_mem_t);
6260
rm->free_len = free_len;
6361
rm->beg = buf;
6462
rm->cur = buf;
6563
rm->end = buf + len;
6664
reader->data = rm;
6765
reader->readbyte = mp_reader_mem_readbyte;
6866
reader->close = mp_reader_mem_close;
69-
return true;
7067
}
7168

7269
#if MICROPY_READER_POSIX
@@ -110,37 +107,30 @@ STATIC void mp_reader_posix_close(void *data) {
110107
m_del_obj(mp_reader_posix_t, reader);
111108
}
112109

113-
int mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
114-
mp_reader_posix_t *rp = m_new_obj_maybe(mp_reader_posix_t);
115-
if (rp == NULL) {
116-
if (close_fd) {
117-
close(fd);
118-
}
119-
return MP_ENOMEM;
120-
}
110+
void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd) {
111+
mp_reader_posix_t *rp = m_new_obj(mp_reader_posix_t);
121112
rp->close_fd = close_fd;
122113
rp->fd = fd;
123114
int n = read(rp->fd, rp->buf, sizeof(rp->buf));
124115
if (n == -1) {
125116
if (close_fd) {
126117
close(fd);
127118
}
128-
return errno;
119+
mp_raise_OSError(errno);
129120
}
130121
rp->len = n;
131122
rp->pos = 0;
132123
reader->data = rp;
133124
reader->readbyte = mp_reader_posix_readbyte;
134125
reader->close = mp_reader_posix_close;
135-
return 0; // success
136126
}
137127

138-
int mp_reader_new_file(mp_reader_t *reader, const char *filename) {
128+
void mp_reader_new_file(mp_reader_t *reader, const char *filename) {
139129
int fd = open(filename, O_RDONLY, 0644);
140130
if (fd < 0) {
141-
return errno;
131+
mp_raise_OSError(errno);
142132
}
143-
return mp_reader_new_file_from_fd(reader, fd, true);
133+
mp_reader_new_file_from_fd(reader, fd, true);
144134
}
145135

146136
#endif

‎py/reader.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ typedef struct _mp_reader_t {
3939
void (*close)(void *data);
4040
} mp_reader_t;
4141

42-
bool mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len);
43-
int mp_reader_new_file(mp_reader_t *reader, const char *filename);
44-
int mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd);
42+
void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len);
43+
void mp_reader_new_file(mp_reader_t *reader, const char *filename);
44+
void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd);
4545

4646
#endif // MICROPY_INCLUDED_PY_READER_H

0 commit comments

Comments
 (0)