Skip to content

Commit d45eb93

Browse files
dcodeIOdcodeIO
committed
Allow intercepting comments when tokenizing
It appears that this isn't necessary for the compiler at this point, but might be good to have for future tooling.
1 parent 162096b commit d45eb93

File tree

7 files changed

+69
-18
lines changed

7 files changed

+69
-18
lines changed

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
node_modules/
22
npm-debug.*
3-
dist/
43
out/
54
raw/

‎dist/asc.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/asc.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/assemblyscript.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/assemblyscript.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/ast.ts‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ export enum NodeKind {
8787
// special
8888
DECORATOR,
8989
EXPORTMEMBER,
90-
SWITCHCASE
90+
SWITCHCASE,
91+
COMMENT
9192
}
9293

9394
/** Base class of all nodes. */
@@ -220,6 +221,18 @@ export abstract class Node {
220221
return stmt;
221222
}
222223

224+
static createComment(
225+
text: string,
226+
kind: CommentKind,
227+
range: Range
228+
): CommentNode {
229+
var node = new CommentNode();
230+
node.range = range;
231+
node.commentKind = kind;
232+
node.text = text;
233+
return node;
234+
}
235+
223236
// expressions
224237

225238
static createIdentifierExpression(
@@ -1042,15 +1055,15 @@ export class SignatureNode extends CommonTypeNode {
10421055
// special
10431056

10441057
/** Built-in decorator kinds. */
1045-
export const enum DecoratorKind {
1058+
export enum DecoratorKind {
10461059
CUSTOM,
10471060
GLOBAL,
10481061
OPERATOR,
10491062
UNMANAGED,
10501063
OFFSET
10511064
}
10521065

1053-
/** Depresents a decorator. */
1066+
/** Represents a decorator. */
10541067
export class DecoratorNode extends Node {
10551068
kind = NodeKind.DECORATOR;
10561069

@@ -1062,6 +1075,26 @@ export class DecoratorNode extends Node {
10621075
arguments: Expression[] | null;
10631076
}
10641077

1078+
/** Comment kinds. */
1079+
export enum CommentKind {
1080+
/** Line comment. */
1081+
LINE,
1082+
/** Triple-slash comment. */
1083+
TRIPLE,
1084+
/** Block comment. */
1085+
BLOCK
1086+
}
1087+
1088+
/** Represents a comment. */
1089+
export class CommentNode extends Node {
1090+
kind = NodeKind.COMMENT;
1091+
1092+
/** Comment kind. */
1093+
commentKind: CommentKind;
1094+
/** Comment text. */
1095+
text: string;
1096+
}
1097+
10651098
// expressions
10661099

10671100
/** Base class of all expression nodes. */
@@ -1076,7 +1109,7 @@ export class IdentifierExpression extends Expression {
10761109
}
10771110

10781111
/** Indicates the kind of a literal. */
1079-
export const enum LiteralKind {
1112+
export enum LiteralKind {
10801113
FLOAT,
10811114
INTEGER,
10821115
STRING,
@@ -1102,7 +1135,7 @@ export class ArrayLiteralExpression extends LiteralExpression {
11021135
}
11031136

11041137
/** Indicates the kind of an assertion. */
1105-
export const enum AssertionKind {
1138+
export enum AssertionKind {
11061139
PREFIX,
11071140
AS
11081141
}

‎src/tokenizer.ts‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ import {
2525
} from "./diagnostics";
2626

2727
import {
28-
Source
28+
Source,
29+
CommentKind
2930
} from "./ast";
3031

3132
import {
@@ -392,6 +393,8 @@ export class Tokenizer extends DiagnosticEmitter {
392393
nextTokenPos: i32 = 0;
393394
nextTokenOnNewLine: bool = false;
394395

396+
onComment: ((kind: CommentKind, text: string, range: Range) => void) | null = null;
397+
395398
constructor(source: Source, diagnostics: DiagnosticMessage[] | null = null) {
396399
super(diagnostics);
397400
this.source = source;
@@ -582,21 +585,31 @@ export class Tokenizer extends DiagnosticEmitter {
582585
return Token.DOT;
583586
}
584587
case CharCode.SLASH: {
588+
let commentStartPos = this.pos;
585589
++this.pos;
586590
if (maxTokenLength > 1 && this.pos < this.end) {
587591
if (text.charCodeAt(this.pos) == CharCode.SLASH) { // single-line
588-
// TODO: triple-slash?
589-
// if (
590-
// this.pos + 1 < this.end &&
591-
// text.charCodeAt(this.pos + 1) == CharCode.SLASH
592-
// ) {
593-
// }
592+
let commentKind = CommentKind.LINE;
593+
if (
594+
this.pos + 1 < this.end &&
595+
text.charCodeAt(this.pos + 1) == CharCode.SLASH
596+
) {
597+
++this.pos;
598+
commentKind = CommentKind.TRIPLE;
599+
}
594600
while (++this.pos < this.end) {
595-
if (isLineBreak(text.charCodeAt(this.pos))) {
601+
if (text.charCodeAt(this.pos) == CharCode.LINEFEED) {
596602
++this.pos;
597603
break;
598604
}
599605
}
606+
if (this.onComment) {
607+
this.onComment(
608+
commentKind,
609+
text.substring(commentStartPos, this.pos),
610+
this.range(commentStartPos, this.pos)
611+
);
612+
}
600613
break;
601614
}
602615
if (text.charCodeAt(this.pos) == CharCode.ASTERISK) { // multi-line
@@ -618,6 +631,12 @@ export class Tokenizer extends DiagnosticEmitter {
618631
DiagnosticCode._0_expected,
619632
this.range(this.pos), "*/"
620633
);
634+
} else if (this.onComment) {
635+
this.onComment(
636+
CommentKind.BLOCK,
637+
text.substring(commentStartPos, this.pos),
638+
this.range(commentStartPos, this.pos)
639+
);
621640
}
622641
break;
623642
}

0 commit comments

Comments
 (0)