Skip to content
This repository was archived by the owner on Aug 31, 2021. It is now read-only.

Commit df72c33

Browse files
peter-bpeter-b
authored
Merge pull request #4494 from runrevmark/lcb-hex_binary_literals
[[ LCB ]] Improve tokenisation of numbers and add hex and bin literals
2 parents 8101482 + b608944 commit df72c33

File tree

11 files changed

+564
-28
lines changed

11 files changed

+564
-28
lines changed

‎docs/guides/LiveCode Builder Language Reference.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ follows:
4040

4141
- **Identifier**: [A-Za-z_][A-Za-z0-9_.]*
4242
- **Integer**: [0-9]+
43+
- **Binary Integer**: 0b[01]+
44+
- **Hexadecimal Integer**: 0x[0-9a-fA-F]+
4345
- **Real**: [0-9]+"."[0-9]+([eE][-+]?[0-9]+)?
4446
- **String**: "[^\n\r"]*"
4547
- **Separator**: Any whitespace containing at least one newline

‎docs/lcb/notes/feature-hex_bin_literals.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# LiveCode Builder Language
2+
## Numeric literal improvements
3+
4+
* Base 2 (binary) integer literals can now be specified by using a "0b" prefix.
5+
e.g.
6+
7+
0b0000
8+
0b1010
9+
10+
* Base 16 (hexadecimal) integer literals can now be specified by using a "0x" prefix.
11+
e.g.
12+
13+
0xdeadbeef
14+
0x0123fedc
15+
16+
* Parsing of numeric literals, in general, has been tightened up. In particular,
17+
the compiler will detect invalid suffixes on numeric literals meaning you cannot
18+
accidentally elide a number with an identifier.
19+
e.g.
20+
21+
1.344foo -- ERROR
22+
0xabcdefgh -- ERROR
23+
0b010432 -- ERROR

‎tests/_compilertestrunner.livecodescript‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ private command runCompilerTest pInfo, pScriptFile, pTest
210210
put tempName() into tTestInterfaceFile
211211
put textEncode(tTestInfo["code"], "utf8") into url ("binfile:" & tTestFile)
212212
reportCompilerTestDiag format("output test source file to '%s'", tTestFile)
213-
reportCompilerTestDiag tTestInfo["code"]
213+
reportCompilerTestDiagWithLineNumbers tTestInfo["code"]
214214

215215
-- Build the command line
216216
local tCompilerCmdLine
@@ -643,6 +643,17 @@ private command reportCompilerTestDiag pMessage
643643
end if
644644
end reportCompilerTestDiag
645645

646+
private command reportCompilerTestDiagWithLineNumbers pMessage
647+
if $LCC_VERBOSE is not empty then
648+
local tLineNumber
649+
put 1 into tLineNumber
650+
repeat for each line tLine in pMessage
651+
write "DIAG:" && format("%4d", tLineNumber) && ":" && tLine & return to stderr
652+
add 1 to tLineNumber
653+
end repeat
654+
end if
655+
end reportCompilerTestDiagWithLineNumbers
656+
646657
----------------------------------------------------------------
647658
-- Logging helpers
648659
----------------------------------------------------------------

‎tests/lcb/compiler/frontend/numeric-literals.compilertest‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
%% Copyright (C) 2016 LiveCode Ltd.
2+
%%
3+
%% This file is part of LiveCode.
4+
%%
5+
%% LiveCode is free software; you can redistribute it and/or modify it under
6+
%% the terms of the GNU General Public License v3 as published by the Free
7+
%% Software Foundation.
8+
%%
9+
%% LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
10+
%% WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
%% FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
%% for more details.
13+
%%
14+
%% You should have received a copy of the GNU General Public License
15+
%% along with LiveCode. If not see <http://www.gnu.org/licenses/>.
16+
17+
%TEST DecimalIntegerLiterals
18+
module compiler_test
19+
constant Valid is [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
20+
constant InvalidPrefix is %{BEFORE_PREFIX}00
21+
constant InvalidSuffix is %{BEFORE_SUFFIX}100foo
22+
handler Compute()
23+
variable tVar
24+
put 100+200 into tVar
25+
end handler
26+
end module
27+
%EXPECT PASS
28+
%ERROR "Malformed integer literal" AT BEFORE_PREFIX
29+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX
30+
%ENDTEST
31+
32+
%% Note that decimal integer literals are always unsigned
33+
%TEST DecimalIntegerLiteralRange
34+
module compiler_test
35+
constant BiggerThanU8 is %{U8}256
36+
constant SmallerThanU8 is %{U8S}255
37+
constant BiggerThanU16 is %{U16}65536
38+
constant SmallerThanU16 is %{U16S}65535
39+
constant BiggerThanU32 is %{U32}4294967296
40+
constant SmallerThanU32 is %{U32S}4294967295
41+
constant BiggerThanU64 is %{U64}9223372036854775808
42+
constant SmallerThanU64 is %{U64S}9223372036854775807
43+
end module
44+
%EXPECT PASS
45+
%ERROR "Integer literal too big" at U32
46+
%ERROR "Integer literal too big" at U64
47+
%ERROR "Integer literal too big" at U64S
48+
%ENDTEST
49+
50+
%TEST BinaryIntegerLiterals
51+
module compiler_test
52+
constant ValidLower is [ 0b0, 0b1, 0b01, 0b10, 0b010, 0b101 ]
53+
constant ValidUpper is [ 0B0, 0B1, 0B01, 0B10, 0B010, 0B101 ]
54+
constant InvalidSuffixLower is %{BEFORE_SUFFIX_LOWER}0b01foo
55+
constant InvalidSuffixUpper is %{BEFORE_SUFFIX_UPPER}0B01foo
56+
handler Compute()
57+
variable tVar
58+
put 0b100+0b001 into tVar
59+
end handler
60+
end module
61+
%EXPECT PASS
62+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_LOWER
63+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_UPPER
64+
%ENDTEST
65+
66+
%TEST BinaryIntegerLiteralRange
67+
module compiler_test
68+
constant BiggerThanU8 is %{U8}0b111111111
69+
constant SmallerThanU8 is %{U8S}0b011111111
70+
constant BiggerThanU16 is %{U16}0b11111111111111111
71+
constant SmallerThanU16 is %{U16S}0b01111111111111111
72+
constant BiggerThanU32 is %{U32}0b111111111111111111111111111111111
73+
constant SmallerThanU32 is %{U32S}0b011111111111111111111111111111111
74+
constant BiggerThanU64 is %{U64}0b11111111111111111111111111111111111111111111111111111111111111111
75+
constant SmallerThanU64 is %{U64S}0b01111111111111111111111111111111111111111111111111111111111111111
76+
end module
77+
%EXPECT PASS
78+
%ERROR "Integer literal too big" at U32
79+
%ERROR "Integer literal too big" at U64
80+
%ERROR "Integer literal too big" at U64S
81+
%ENDTEST
82+
83+
%TEST HexadecimalIntegerLiterals
84+
module compiler_test
85+
constant ValidLower is [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,\
86+
0xb, 0xc, 0xd, 0xe, 0xf, 0x01, 0x10, 0x01, 0x0F, 0xF0 ]
87+
constant ValidLower is [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa,\
88+
0xB, 0xC, 0xD, 0xE, 0xF, 0x01, 0x10, 0x01, 0x0F, 0xF0 ]
89+
constant InvalidSuffixLower is %{BEFORE_SUFFIX_LOWER}0x01foo
90+
constant InvalidSuffixUpper is %{BEFORE_SUFFIX_UPPER}0X01foo
91+
handler Compute()
92+
variable tVar
93+
put 0x100f+0xf001 into tVar
94+
end handler
95+
end module
96+
%EXPECT PASS
97+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_LOWER
98+
%ERROR "Malformed integer literal" AT BEFORE_SUFFIX_UPPER
99+
%ENDTEST
100+
101+
%TEST HexadecimalIntegerLiteralRange
102+
module compiler_test
103+
constant BiggerThanU8 is %{U8}0x1FF
104+
constant SmallerThanU8 is %{U8S}0x0FF
105+
constant BiggerThanU16 is %{U16}0x1FFFF
106+
constant SmallerThanU16 is %{U16S}0x0FFFF
107+
constant BiggerThanU32 is %{U32}0x1FFFFFFFF
108+
constant SmallerThanU16 is %{U32S}0x0FFFFFFFF
109+
constant BiggerThanU64 is %{U64}0x1FFFFFFFFFFFFFFFF
110+
constant SmallerThanU16 is %{U64S}0x0FFFFFFFFFFFFFFFF
111+
end module
112+
%EXPECT PASS
113+
%ERROR "Integer literal too big" at U32
114+
%ERROR "Integer literal too big" at U64
115+
%ERROR "Integer literal too big" at U64S
116+
%ENDTEST
117+
118+
%% The general pattern for real literals is:
119+
%% I:Int P:Period F:ZeroesInt E:(Exp PlusMinus Int)
120+
%% Assuming all are optional then we have the following combinations
121+
%% I, IP, IPF, IPFE, IF, IFE, IE, IPE
122+
%% P, PF, PFE, PE
123+
%% F, FE
124+
%% EE
125+
%% Due to the underlying regex patterns there are the folllowing
126+
%% combinations which are the same:
127+
%% IF == I, IFE == IE
128+
%% Which leaves the following classification:
129+
%% I -- matches as integer
130+
%% IP - e.g. 1. -- VALID
131+
%% IPF - e.g. 1.1 -- VALID
132+
%% IPFE - e.g. 1.1e1 -- VALID
133+
%% IE - e.g. 1e1 -- VALID
134+
%% IPE - e.g. 1.e1 -- VALID
135+
%% P - . -- INVALID
136+
%% PF - e.g. .1 -- VALID
137+
%% PFE - e.g. .1e1 -- VALID
138+
%% PE - .e1 -- INVALID
139+
%% F - e.g. 01 -- INVALID
140+
%% FE - e.g. 01e1 -- INVALID
141+
%% E - e.g. e1 -- matches as identifier
142+
%%
143+
%TEST RealLiterals
144+
module compiler_test
145+
constant ValidInt is [ 0. , 1. , 10. ]
146+
constant ValidFrac is [ .0, .1, .00, .10, .01 ]
147+
constant ValidIntFrac is [ 0.0, 0.1, 0.00, 0.10, 0.01, \
148+
1.0, 1.1, 1.00, 1.10, 1.01, \
149+
10.0, 10.1, 10.00, 10.10, 10.01 ]
150+
constant ValidExp is [ 0e10, 1e1 , 10e10, 1e1 ]
151+
constant ValidDecExp is [ 0.e10, 0.e1 , .0e10, .0e1 ]
152+
constant ValidPosExp is [ 0.e+10, 0.e+1 , .0e+10, .0e+1 ]
153+
constant ValidNegExp is [ 0.e-10, 0.e-1 , .0e-10, .0e-1 ]
154+
155+
constant Invalid_PE is %{INVALID_PE}.e1
156+
constant Invalid_F is %{INVALID_F}01
157+
constant Invalid_FE is %{INVALID_FE}01e1
158+
159+
constant InvalidInt is [ %{INVALID_INT}01. ]
160+
constant InvalidIntFrac is [ %{INVALID_INTFRAC}01.0 ]
161+
constant InvalidExp is [ %{INVALID_EXP1}00e10, %{INVALID_EXP2}0e010 ]
162+
163+
constant IntSuffix is %{INT_SUFFIX}0.foo
164+
constant FracSuffix is %{FRAC_SUFFIX}.0foo
165+
constant IntFracSuffix is %{INTFRAC_SUFFIX}0.1foo
166+
constant ExpSuffix is %{EXP_SUFFIX}0e10foo
167+
constant DecExpSuffix is %{DECEXP_SUFFIX}.0e1foo
168+
constant PosExpSuffix is %{POSEXP_SUFFIX}.0e+1foo
169+
constant NegExpSuffix is %{NEGEXP_SUFFIX}.0e-1foo
170+
171+
constant IntDotSuffix is %{INTDOT_SUFFIX}0..
172+
constant FracDotSuffix is %{FRACDOT_SUFFIX}.0.
173+
constant IntFracDotSuffix is %{INTFRACDOT_SUFFIX}0.1.
174+
constant ExpDotSuffix is %{EXPDOT_SUFFIX}0e10.
175+
constant DecExpDotSuffix is %{DECEXPDOT_SUFFIX}.0e1.
176+
constant PosExpDotSuffix is %{POSEXPDOT_SUFFIX}.0e+1.
177+
constant NegExpDotSuffix is %{NEGEXPDOT_SUFFIX}.0e-1.
178+
179+
end module
180+
%EXPECT PASS
181+
%ERROR "Malformed real literal" AT INVALID_PE
182+
%ERROR "Malformed integer literal" AT INVALID_F
183+
%ERROR "Malformed real literal" AT INVALID_FE
184+
185+
%ERROR "Malformed real literal" AT INVALID_INT
186+
%ERROR "Malformed real literal" AT INVALID_INTFRAC
187+
%ERROR "Malformed real literal" AT INVALID_EXP1
188+
%ERROR "Malformed real literal" AT INVALID_EXP2
189+
190+
%ERROR "Malformed real literal" AT INT_SUFFIX
191+
%ERROR "Malformed real literal" AT FRAC_SUFFIX
192+
%ERROR "Malformed real literal" AT INTFRAC_SUFFIX
193+
%ERROR "Malformed real literal" AT EXP_SUFFIX
194+
%ERROR "Malformed real literal" AT DECEXP_SUFFIX
195+
%ERROR "Malformed real literal" AT POSEXP_SUFFIX
196+
%ERROR "Malformed real literal" AT NEGEXP_SUFFIX
197+
198+
%ERROR "Malformed real literal" AT INTDOT_SUFFIX
199+
%ERROR "Malformed real literal" AT FRACDOT_SUFFIX
200+
%ERROR "Malformed real literal" AT INTFRACDOT_SUFFIX
201+
%ERROR "Malformed real literal" AT EXPDOT_SUFFIX
202+
%ERROR "Malformed real literal" AT DECEXPDOT_SUFFIX
203+
%ERROR "Malformed real literal" AT POSEXPDOT_SUFFIX
204+
%ERROR "Malformed real literal" AT NEGEXPDOT_SUFFIX
205+
%ENDTEST
206+
207+
%TEST RealLiteralRange
208+
module compiler_test
209+
constant BiggerThanFloat is %{BIGF}3.40282347E+39
210+
constant SmallerThanFloat is %{SMALLF}3.40282347E+38
211+
constant BiggerThanDouble is %{BIGD}1.7976931348623157E+309
212+
constant SmallerThanDouble is %{SMALLD}1.7976931348623157E+308
213+
end module
214+
%EXPECT PASS
215+
%ERROR "Real literal too big" at BIGD
216+
%ENDTEST

‎toolchain/lc-compile/src/DOUBLE_LITERAL.t‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,51 @@
1-
[0-9]+"."[0-9]+([eE][-+]?[0-9]+)? {
2-
MakeDoubleLiteral(yytext, &yylval.attr[1]);
3-
yysetpos();
4-
return DOUBLE_LITERAL;
1+
/* This contains both INTEGER_LITERAL and DOUBLE_LITERAL rules */
2+
/* This ensures that they appear in this order, to ensure they match */
3+
/* correctly. */
4+
5+
[0-9]+"."[0-9]*([eE][-+]?[0-9]*)?([0-9a-zA-Z_\.]*) |
6+
[0-9]*"."[0-9]+([eE][-+]?[0-9]*)?([0-9a-zA-Z_\.]*) |
7+
[0-9]*"."[0-9]*([eE][-+]?[0-9]*)([0-9a-zA-Z_\.]*) |
8+
[0-9]+([eE][-+]?[0-9]*)([0-9a-zA-Z_\.]*) {
9+
yysetpos();
10+
if (IsDoubleLiteral(yytext))
11+
{
12+
if (MakeDoubleLiteral(yytext, &yylval.attr[1]) == 0)
13+
{
14+
long t_position;
15+
GetCurrentPosition(&t_position);
16+
Error_DoubleLiteralOutOfRange(t_position);
17+
yylval.attr[1] = 0;
18+
}
19+
}
20+
else
21+
{
22+
long t_position;
23+
GetCurrentPosition(&t_position);
24+
Error_InvalidDoubleLiteral(t_position);
25+
yylval.attr[1] = 0;
26+
}
27+
return DOUBLE_LITERAL;
28+
}
29+
30+
([0-9]+|0b[01]*|0x[0-9A-Fa-f]*)[0-9a-zA-Z_]* {
31+
yysetpos();
32+
if (IsIntegerLiteral(yytext))
33+
{
34+
if (MakeIntegerLiteral(yytext, &yylval.attr[1]) == 0)
35+
{
36+
long t_position;
37+
GetCurrentPosition(&t_position);
38+
Error_IntegerLiteralOutOfRange(t_position);
39+
yylval.attr[1] = 0;
40+
}
41+
}
42+
else
43+
{
44+
long t_position;
45+
GetCurrentPosition(&t_position);
46+
Error_InvalidIntegerLiteral(t_position);
47+
yylval.attr[1] = 0;
48+
}
49+
return INTEGER_LITERAL;
550
}
651

‎toolchain/lc-compile/src/INTEGER_LITERAL.t‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1 @@
1-
0|([1-9][0-9]*) {
2-
yysetpos();
3-
if (MakeIntegerLiteral(yytext, &yylval.attr[1]) == 0)
4-
{
5-
long t_position;
6-
GetCurrentPosition(&t_position);
7-
Error_IntegerLiteralOutOfRange(t_position);
8-
yylval.attr[1] = 0;
9-
}
10-
return INTEGER_LITERAL;
11-
}
12-
1+
/* See DOUBLE_LITERAL.t */

0 commit comments

Comments
 (0)