Skip to content

Commit 9193f89

Browse files
dpgeorgedpgeorge
committed
Move lexerstr to main py directory (everyone uses it).
1 parent dc141db commit 9193f89

File tree

12 files changed

+44
-80
lines changed

12 files changed

+44
-80
lines changed

‎py/lexer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ void mp_token_show_error_prefix(const mp_token_t *tok);
127127
bool mp_token_show_error(const mp_token_t *tok, const char *msg);
128128

129129
mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close);
130+
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len);
131+
130132
void mp_lexer_free(mp_lexer_t *lex);
131133
void mp_lexer_to_next(mp_lexer_t *lex);
132134
const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex);

‎py/lexerstr.c‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#include <stdint.h>
2+
#include <stdio.h>
3+
4+
#include "misc.h"
5+
#include "lexer.h"
6+
7+
typedef struct _mp_lexer_str_buf_t {
8+
uint free_len; // if > 0, src_beg will be freed when done by: m_free(src_beg, free_len)
9+
const char *src_beg; // beginning of source
10+
const char *src_cur; // current location in source
11+
const char *src_end; // end (exclusive) of source
12+
} mp_lexer_str_buf_t;
13+
14+
static unichar str_buf_next_char(mp_lexer_str_buf_t *sb) {
15+
if (sb->src_cur < sb->src_end) {
16+
return *sb->src_cur++;
17+
} else {
18+
return MP_LEXER_CHAR_EOF;
19+
}
20+
}
21+
22+
static void str_buf_free(mp_lexer_str_buf_t *sb) {
23+
if (sb->free_len > 0) {
24+
m_free((char*)sb->src_beg, sb->free_len);
25+
}
26+
}
27+
28+
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, uint free_len) {
29+
mp_lexer_str_buf_t *sb = m_new_obj(mp_lexer_str_buf_t);
30+
sb->free_len = free_len;
31+
sb->src_beg = str;
32+
sb->src_cur = str;
33+
sb->src_end = str + len;
34+
return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free);
35+
}

‎py/lexerunix.c‎

Lines changed: 2 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,10 @@
66
#include "misc.h"
77
#include "mpconfig.h"
88
#include "lexer.h"
9+
#include "lexerunix.h"
910

1011
#if MICROPY_ENABLE_LEXER_UNIX
1112

12-
typedef struct _str_buf_t {
13-
bool free; // free src_beg when done
14-
const char *src_beg; // beginning of source
15-
const char *src_cur; // current location in source
16-
const char *src_end; // end (exclusive) of source
17-
} str_buf_t;
18-
19-
unichar str_buf_next_char(str_buf_t *sb) {
20-
if (sb->src_cur < sb->src_end) {
21-
return *sb->src_cur++;
22-
} else {
23-
return MP_LEXER_CHAR_EOF;
24-
}
25-
}
26-
27-
void str_buf_free(str_buf_t *sb) {
28-
if (sb) {
29-
if (sb->free) {
30-
m_del(char, (char*)sb->src_beg, 0 /* unknown size of src_beg */);
31-
}
32-
m_del_obj(str_buf_t, sb);
33-
}
34-
}
35-
36-
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str) {
37-
str_buf_t *sb = m_new(str_buf_t, 1);
38-
sb->free = free_str;
39-
sb->src_beg = str;
40-
sb->src_cur = str;
41-
sb->src_end = str + len;
42-
return mp_lexer_new(src_name, sb, (mp_lexer_stream_next_char_t)str_buf_next_char, (mp_lexer_stream_close_t)str_buf_free);
43-
}
44-
4513
mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
4614
int fd = open(filename, O_RDONLY);
4715
if (fd < 0) {
@@ -59,7 +27,7 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
5927
return NULL;
6028
}
6129

62-
return mp_lexer_new_from_str_len(filename, data, size, true);
30+
return mp_lexer_new_from_str_len(filename, data, size, size);
6331
}
6432

6533
/******************************************************************************/

‎py/lexerunix.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str);
21
mp_lexer_t *mp_lexer_new_from_file(const char *filename);
32

43
void mp_import_set_directory(const char *dir);

‎py/py.mk‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ PY_O_BASENAME = \
2929
vstr.o \
3030
unicode.o \
3131
lexer.o \
32+
lexerstr.o \
3233
lexerunix.o \
3334
parse.o \
3435
scope.o \

‎stm/Makefile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ SRC_C = \
3030
string0.c \
3131
malloc0.c \
3232
systick.c \
33-
lexerstr.c \
3433
lexerfatfs.c \
3534
led.c \
3635
lcd.c \

‎stm/lexerfatfs.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
typedef struct _py_lexer_file_buf_t {
1+
typedef struct _mp_lexer_file_buf_t {
22
FIL fp;
33
char buf[20];
44
uint16_t len;

‎stm/lexerstr.c‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎stm/lexerstr.h‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎stm/main.c‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
#include "nlr.h"
2020
#include "misc.h"
2121
#include "lexer.h"
22-
#include "lexerstr.h"
2322
#include "lexerfatfs.h"
2423
#include "parse.h"
2524
#include "obj.h"
@@ -432,8 +431,7 @@ void do_repl(void) {
432431
}
433432
}
434433

435-
mp_lexer_str_buf_t sb;
436-
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), false, &sb);
434+
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", vstr_str(&line), vstr_len(&line), 0);
437435
mp_parse_node_t pn = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
438436
mp_lexer_free(lex);
439437

@@ -1059,8 +1057,7 @@ int main(void) {
10591057
" pass\n"
10601058
"f()\n";
10611059

1062-
mp_lexer_str_buf_t mp_lexer_str_buf;
1063-
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &mp_lexer_str_buf);
1060+
mp_lexer_t *lex = mp_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), 0);
10641061

10651062
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
10661063
printf("lex; al=%u\n", m_get_total_bytes_allocated());

0 commit comments

Comments
 (0)