Skip to content

Commit 5bdf165

Browse files
dpgeorgedpgeorge
committed
py/lexer: Make lexer use an mp_reader as its source.
1 parent 66d955c commit 5bdf165

3 files changed

Lines changed: 23 additions & 34 deletions

File tree

‎esp8266/lexerstr32.c‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,11 @@ typedef struct _mp_lexer_str32_buf_t {
3535
uint8_t byte_off;
3636
} mp_lexer_str32_buf_t;
3737

38-
STATIC mp_uint_t str32_buf_next_byte(mp_lexer_str32_buf_t *sb) {
38+
STATIC mp_uint_t str32_buf_next_byte(void *sb_in) {
39+
mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t*)sb_in;
3940
byte c = sb->val & 0xff;
4041
if (c == 0) {
41-
return MP_LEXER_EOF;
42+
return MP_READER_EOF;
4243
}
4344

4445
if (++sb->byte_off > 3) {
@@ -51,7 +52,8 @@ STATIC mp_uint_t str32_buf_next_byte(mp_lexer_str32_buf_t *sb) {
5152
return c;
5253
}
5354

54-
STATIC void str32_buf_free(mp_lexer_str32_buf_t *sb) {
55+
STATIC void str32_buf_free(void *sb_in) {
56+
mp_lexer_str32_buf_t *sb = (mp_lexer_str32_buf_t*)sb_in;
5557
m_del_obj(mp_lexer_str32_buf_t, sb);
5658
}
5759

@@ -63,7 +65,8 @@ mp_lexer_t *mp_lexer_new_from_str32(qstr src_name, const char *str, mp_uint_t le
6365
sb->byte_off = (uint32_t)str & 3;
6466
sb->src_cur = (uint32_t*)(str - sb->byte_off);
6567
sb->val = *sb->src_cur++ >> sb->byte_off * 8;
66-
return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_byte_t)str32_buf_next_byte, (mp_lexer_stream_close_t)str32_buf_free);
68+
mp_reader_t reader = {sb, str32_buf_next_byte, str32_buf_free};
69+
return mp_lexer_new(src_name, reader);
6770
}
6871

6972
#endif // MICROPY_ENABLE_COMPILER

‎py/lexer.c‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ STATIC bool str_strn_equal(const char *str, const char *strn, mp_uint_t len) {
5252
return i == len && *str == 0;
5353
}
5454

55+
#define MP_LEXER_EOF ((unichar)MP_READER_EOF)
5556
#define CUR_CHAR(lex) ((lex)->chr0)
5657

5758
STATIC bool is_end(mp_lexer_t *lex) {
@@ -145,15 +146,15 @@ STATIC void next_char(mp_lexer_t *lex) {
145146

146147
lex->chr0 = lex->chr1;
147148
lex->chr1 = lex->chr2;
148-
lex->chr2 = lex->stream_next_byte(lex->stream_data);
149+
lex->chr2 = lex->reader.readbyte(lex->reader.data);
149150

150151
if (lex->chr0 == '\r') {
151152
// CR is a new line, converted to LF
152153
lex->chr0 = '\n';
153154
if (lex->chr1 == '\n') {
154155
// CR LF is a single new line
155156
lex->chr1 = lex->chr2;
156-
lex->chr2 = lex->stream_next_byte(lex->stream_data);
157+
lex->chr2 = lex->reader.readbyte(lex->reader.data);
157158
}
158159
}
159160

@@ -689,21 +690,17 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
689690
}
690691
}
691692

692-
mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_byte_t stream_next_byte, mp_lexer_stream_close_t stream_close) {
693+
mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
693694
mp_lexer_t *lex = m_new_obj_maybe(mp_lexer_t);
694695

695696
// check for memory allocation error
696697
if (lex == NULL) {
697-
if (stream_close) {
698-
stream_close(stream_data);
699-
}
698+
reader.close(reader.data);
700699
return NULL;
701700
}
702701

703702
lex->source_name = src_name;
704-
lex->stream_data = stream_data;
705-
lex->stream_next_byte = stream_next_byte;
706-
lex->stream_close = stream_close;
703+
lex->reader = reader;
707704
lex->line = 1;
708705
lex->column = 1;
709706
lex->emit_dent = 0;
@@ -724,9 +721,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_
724721
lex->indent_level[0] = 0;
725722

726723
// preload characters
727-
lex->chr0 = stream_next_byte(stream_data);
728-
lex->chr1 = stream_next_byte(stream_data);
729-
lex->chr2 = stream_next_byte(stream_data);
724+
lex->chr0 = reader.readbyte(reader.data);
725+
lex->chr1 = reader.readbyte(reader.data);
726+
lex->chr2 = reader.readbyte(reader.data);
730727

731728
// if input stream is 0, 1 or 2 characters long and doesn't end in a newline, then insert a newline at the end
732729
if (lex->chr0 == MP_LEXER_EOF) {
@@ -756,7 +753,7 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t
756753
if (!mp_reader_new_mem(&reader, (const byte*)str, len, free_len)) {
757754
return NULL;
758755
}
759-
return mp_lexer_new(src_name, reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
756+
return mp_lexer_new(src_name, reader);
760757
}
761758

762759
#if MICROPY_READER_POSIX || MICROPY_READER_FATFS
@@ -767,7 +764,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
767764
if (ret != 0) {
768765
return NULL;
769766
}
770-
return mp_lexer_new(qstr_from_str(filename), reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
767+
return mp_lexer_new(qstr_from_str(filename), reader);
771768
}
772769

773770
#if MICROPY_HELPER_LEXER_UNIX
@@ -778,7 +775,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
778775
if (ret != 0) {
779776
return NULL;
780777
}
781-
return mp_lexer_new(filename, reader.data, (mp_lexer_stream_next_byte_t)reader.readbyte, (mp_lexer_stream_close_t)reader.close);
778+
return mp_lexer_new(filename, reader);
782779
}
783780

784781
#endif
@@ -787,9 +784,7 @@ mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd) {
787784

788785
void mp_lexer_free(mp_lexer_t *lex) {
789786
if (lex) {
790-
if (lex->stream_close) {
791-
lex->stream_close(lex->stream_data);
792-
}
787+
lex->reader.close(lex->reader.data);
793788
vstr_clear(&lex->vstr);
794789
m_del(uint16_t, lex->indent_level, lex->alloc_indent_level);
795790
m_del_obj(mp_lexer_t, lex);

‎py/lexer.h‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
#include "py/mpconfig.h"
3232
#include "py/qstr.h"
33+
#include "py/reader.h"
3334

3435
/* lexer.h -- simple tokeniser for Micro Python
3536
*
@@ -142,21 +143,11 @@ typedef enum _mp_token_kind_t {
142143
MP_TOKEN_DEL_MINUS_MORE,
143144
} mp_token_kind_t;
144145

145-
// the next-byte function must return the next byte in the stream
146-
// it must return MP_LEXER_EOF if end of stream
147-
// it can be called again after returning MP_LEXER_EOF, and in that case must return MP_LEXER_EOF
148-
#define MP_LEXER_EOF ((unichar)(-1))
149-
150-
typedef mp_uint_t (*mp_lexer_stream_next_byte_t)(void*);
151-
typedef void (*mp_lexer_stream_close_t)(void*);
152-
153146
// this data structure is exposed for efficiency
154147
// public members are: source_name, tok_line, tok_column, tok_kind, vstr
155148
typedef struct _mp_lexer_t {
156149
qstr source_name; // name of source
157-
void *stream_data; // data for stream
158-
mp_lexer_stream_next_byte_t stream_next_byte; // stream callback to get next byte
159-
mp_lexer_stream_close_t stream_close; // stream callback to free
150+
mp_reader_t reader; // stream source
160151

161152
unichar chr0, chr1, chr2; // current cached characters from source
162153

@@ -176,7 +167,7 @@ typedef struct _mp_lexer_t {
176167
vstr_t vstr; // token data
177168
} mp_lexer_t;
178169

179-
mp_lexer_t *mp_lexer_new(qstr src_name, void *stream_data, mp_lexer_stream_next_byte_t stream_next_byte, mp_lexer_stream_close_t stream_close);
170+
mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader);
180171
mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len);
181172

182173
void mp_lexer_free(mp_lexer_t *lex);

0 commit comments

Comments
 (0)