diff --git a/README.md b/README.md index 319ce98..6366df1 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,30 @@ yarn yarn start ``` -### Then open `http://localhost:8886` in a browser +### Then open `http://localhost:3000` in a browser + + + +# how to debug regex generator grammer parser +``` +yarn global add nearley +nearleyc grammer.ne -o src/lib/grammer.js +nearley-railroad grammer.ne -o grammar.html +python -m http.server 80 +``` +### open grammar.html in your browser to view railroad diagram of grammer +### http://localhost:80/grammar.html + +## nearley test with cli +```bash +nearley-test -i "#00ff00" src/lib/grammer.js +nearley-test -i "a\na" src/lib/grammer.js +``` +## nearely generate examples +```bash +nearley-unparse -n 3 src/lib/grammer.js +``` + ## Locally run integration tests with crypress @@ -39,16 +62,6 @@ yarn yarn run cy:ci ``` -# how to debug regex generator grammer parser -``` -yarn global add nearley -nearleyc grammer.ne -o src/lib/grammer.js -nearley-railroad grammer.ne -o grammar.html -python -m http.server 80 -``` -### open grammar.html in your browser to view railroad diagram of grammer -### http://localhost:80/grammar.html - ### Github actions how to setup DASHBOARDRECORDKEY ### [cypress setup api key](https://docs.cypress.io/guides/dashboard/projects#Set-up-a-project-to-record) diff --git a/grammar.html b/grammar.html index fc44317..805e3f9 100644 --- a/grammar.html +++ b/grammar.html @@ -32,143 +32,284 @@

main

- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +terminals + + + + + + + + + + + + + + + + + + + + + +
+

terminals

+
+ - + - - - - - - - + + + - - - + + + +NUMBERS + + + + + - - - + + + - - - -NUMBERS + + + +LETTERS - + - - + + - - - + + + - - - -LETTERS - - + + + +SYMBOLS - + - - + + - + + + + + + + +WHITESPACE - + - + + + + + + + + + + +NEWLINE - + - + + +

LETTERS

- + - + - + - - - - - - -/[a-zA-Z]/ + + +word - - - - + + + - + + + + +
+

SYMBOLS

+
+ + + + + + + + + + + + + + + +symbols - + - + - +

NUMBERS

- + - + - + - - - - - - -int + + +number + + - - + + + + + + +
+

WHITESPACE

+
+ + + + + + + + + + + - + + + +whitespace - + + + + + + + + +
+

NEWLINE

+
+ + + + + + + + + + + + + + + +newline - + - + - + diff --git a/grammer.ne b/grammer.ne index f82cbd5..38e4415 100644 --- a/grammer.ne +++ b/grammer.ne @@ -1,17 +1,19 @@ @builtin "whitespace.ne" # `_` means arbitrary amount of whitespace @builtin "number.ne" # `int`, `decimal`, and `percentage` number primitives - +# to playaround with this grammer output, i recommend copy pasting this whole file +# into this website. https://omrelli.ug/nearley-playground/ it makes playing with it much easier. :) @{% // Moo lexer documention is here: // https://github.com/no-context/moo const moo = require("moo") const lexer = moo.compile({ - whitespace: { match: /[\s\t\n]+/, lineBreaks: true }, + newline: { match: /\n+/, lineBreaks: true }, + whitespace: /[ \t]+/, number: /[0-9]+/, word: /[a-zA-Z]+/, - symbols: /[^a-zA-Z0-9\s\t\n\r]+/ + symbols: /[^a-zA-Z0-9 \t\n\r]+/ }); @@ -28,9 +30,10 @@ const un_nest = (d) =>{ main -> ( terminals ):* {% d=> un_nest(d[0]) %} -terminals -> NUMBERS | LETTERS | SYMBOLS | WHITESPACE {% d=> d[0] %} +terminals -> NUMBERS | LETTERS | SYMBOLS | WHITESPACE | NEWLINE {% d=> d[0] %} LETTERS -> %word {%([first])=>( {"type": first.type,"text":first.text, token: 'LETTERS'}) %} SYMBOLS -> %symbols {%([first])=>( {"type": first.type,"text":first.text, token: 'SYMBOLS'}) %} NUMBERS -> %number {%([first])=>( {"type": first.type,"text":first.text, token: 'NUMBERS'}) %} WHITESPACE -> %whitespace {%([first])=>( {"type": first.type,"text":first.text, token: 'WHITESPACE'}) %} +NEWLINE -> %newline {%([first])=>( {"type": first.type,"text":first.text, token: 'NEWLINE'}) %} diff --git a/src/PatternSelector.js b/src/PatternSelector.js index a374df8..fd0898b 100644 --- a/src/PatternSelector.js +++ b/src/PatternSelector.js @@ -45,6 +45,7 @@ const names = [ { name: "1234", label: 'NUMBERS' }, { name: "WHITESPACE", label: 'WHITESPACE' }, { name: "(){},?#...", label: 'SYMBOLS' }, + { name: "↩️", label: 'NEWLINE' }, ] const MAPPING = @@ -52,7 +53,8 @@ const MAPPING = LETTERS: "A-Z", NUMBERS: "1234", WHITESPACE: "WHITESPACE", - SYMBOLS: "(){},?#..." + SYMBOLS: "(){},?#...", + NEWLINE: "↩️" } @@ -105,10 +107,11 @@ const PatternSelector = ({ editor }) => { useEffect(() => { let tokens = { - WHITESPACE: `([\\t\\n\\r]+)`, + WHITESPACE: `([\\t ]+)`, NUMBERS: `([0-9]+)`, LETTERS: `([a-zA-Z]+)`, - SYMBOLS: `([^\\s\\t\\r\\-a-zA-Z0-9]+)` + SYMBOLS: `([^ \\t\\r\\-a-zA-Z0-9]+)`, + NEWLINE: `([\n]+)` } let parsed = chipData.map(x => x.label) let string_regex = parsed.reduce((accumulator, value, index, array) => { diff --git a/src/lib/grammer.js b/src/lib/grammer.js index 6e182dd..6bc1594 100644 --- a/src/lib/grammer.js +++ b/src/lib/grammer.js @@ -8,10 +8,11 @@ function id(x) { return x[0]; } const moo = require("moo") const lexer = moo.compile({ - whitespace: { match: /[\s\t\n]+/, lineBreaks: true }, + newline: { match: /\n+/, lineBreaks: true }, + whitespace: /[ \t]+/, number: /[0-9]+/, word: /[a-zA-Z]+/, - symbols: /[^a-zA-Z0-9\s\t\n\r]+/ + symbols: /[^a-zA-Z0-9 \t\n\r]+/ }); @@ -123,11 +124,13 @@ var grammar = { {"name": "terminals", "symbols": ["NUMBERS"]}, {"name": "terminals", "symbols": ["LETTERS"]}, {"name": "terminals", "symbols": ["SYMBOLS"]}, - {"name": "terminals", "symbols": ["WHITESPACE"], "postprocess": d=> d[0]}, + {"name": "terminals", "symbols": ["WHITESPACE"]}, + {"name": "terminals", "symbols": ["NEWLINE"], "postprocess": d=> d[0]}, {"name": "LETTERS", "symbols": [(lexer.has("word") ? {type: "word"} : word)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'LETTERS'})}, {"name": "SYMBOLS", "symbols": [(lexer.has("symbols") ? {type: "symbols"} : symbols)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'SYMBOLS'})}, {"name": "NUMBERS", "symbols": [(lexer.has("number") ? {type: "number"} : number)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'NUMBERS'})}, - {"name": "WHITESPACE", "symbols": [(lexer.has("whitespace") ? {type: "whitespace"} : whitespace)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'WHITESPACE'})} + {"name": "WHITESPACE", "symbols": [(lexer.has("whitespace") ? {type: "whitespace"} : whitespace)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'WHITESPACE'})}, + {"name": "NEWLINE", "symbols": [(lexer.has("newline") ? {type: "newline"} : newline)], "postprocess": ([first])=>( {"type": first.type,"text":first.text, token: 'NEWLINE'})} ] , ParserStart: "main" }