Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 38 additions & 25 deletions packages/compiler/src/ml_parser/lexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,8 @@ class _Tokenizer {
}
} else if (this._attemptCharCode(chars.$SLASH)) {
this._consumeTagClose(start);
} else if (this._attemptCharCode(chars.$QUESTION)) {
this._consumeProcessingInstruction(start);
} else {
this._consumeTagOpen(start);
}
Expand Down Expand Up @@ -470,30 +472,7 @@ class _Tokenizer {
private _consumeLetDeclarationValue(): void {
const start = this._cursor.clone();
this._beginToken(TokenType.LET_VALUE, start);

while (this._cursor.peek() !== chars.$EOF) {
const char = this._cursor.peek();

// `@let` declarations terminate with a semicolon.
if (char === chars.$SEMICOLON) {
break;
}

// If we hit a quote, skip over its content since we don't care what's inside.
if (chars.isQuote(char)) {
this._cursor.advance();
this._attemptCharCodeUntilFn((inner) => {
if (inner === chars.$BACKSLASH) {
this._cursor.advance();
return false;
}
return inner === char;
});
}

this._cursor.advance();
}

this._attemptUntilCharIgnoreQuotes(chars.$SEMICOLON); // `@let` declarations terminate with a semicolon.
this._endToken([this._cursor.getChars(start)]);
}

Expand Down Expand Up @@ -665,6 +644,29 @@ class _Tokenizer {
}
}

private _attemptUntilCharIgnoreQuotes(endChar: number) {
while (this._cursor.peek() !== chars.$EOF) {
const char = this._cursor.peek();

if (char === endChar) {
break;
}

if (chars.isQuote(char)) {
this._cursor.advance();
this._attemptCharCodeUntilFn((inner) => {
if (inner === chars.$BACKSLASH) {
this._cursor.advance();
return false;
}
return inner === char;
});
}

this._cursor.advance();
}
}

private _readChar(): string {
// Don't rely upon reading directly from `_input` as the actual char value
// may have been generated from an escape sequence.
Expand Down Expand Up @@ -800,6 +802,16 @@ class _Tokenizer {
this._endToken([content]);
}

private _consumeProcessingInstruction(start: CharacterCursor) {
this._beginToken(TokenType.PROCESSING_INSTRUCTION, start);
const contentStart = this._cursor.clone();
this._attemptUntilCharIgnoreQuotes(chars.$QUESTION);
const content = this._cursor.getChars(contentStart);
this._cursor.advance();
this._requireCharCode(chars.$GT);
this._endToken([content]);
}

private _consumePrefixAndName(endPredicate: (code: number) => boolean): string[] {
const nameOrPrefixStart = this._cursor.clone();
let prefix: string = '';
Expand Down Expand Up @@ -1426,7 +1438,8 @@ class _Tokenizer {
(chars.$a <= code && code <= chars.$z) ||
(chars.$A <= code && code <= chars.$Z) ||
code === chars.$SLASH ||
code === chars.$BANG
code === chars.$BANG ||
code === chars.$QUESTION
) {
return true;
}
Expand Down
9 changes: 8 additions & 1 deletion packages/compiler/src/ml_parser/tokens.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export const enum TokenType {
DIRECTIVE_NAME,
DIRECTIVE_OPEN,
DIRECTIVE_CLOSE,
PROCESSING_INSTRUCTION,
EOF,
}

Expand Down Expand Up @@ -95,7 +96,8 @@ export type Token =
| IncompleteComponentOpenToken
| DirectiveNameToken
| DirectiveOpenToken
| DirectiveCloseToken;
| DirectiveCloseToken
| ProcessingInstructionToken;

export type InterpolatedTextToken = TextToken | InterpolationToken | EncodedEntityToken;

Expand Down Expand Up @@ -318,3 +320,8 @@ export interface DirectiveCloseToken extends TokenBase {
type: TokenType.DIRECTIVE_CLOSE;
parts: [];
}

export interface ProcessingInstructionToken extends TokenBase {
type: TokenType.PROCESSING_INSTRUCTION;
parts: [content: string];
}
14 changes: 14 additions & 0 deletions packages/compiler/test/ml_parser/html_parser_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1629,6 +1629,20 @@ describe('HtmlParser', () => {
});
});

describe('ignored syntax', () => {
it('should ignore doctype declaration', () => {
expect(humanizeDom(parser.parse(`<!DOCTYPE html>hello`, 'TestComp'))).toEqual([
[html.Text, 'hello', 0, ['hello']],
]);
});

it('should ignore processing instruction', () => {
expect(
humanizeDom(parser.parse(`<?xml version="1.0" encoding="UTF-8"?>hello`, 'TestComp')),
).toEqual([[html.Text, 'hello', 0, ['hello']]]);
});
});

describe('source spans', () => {
it('should store the location', () => {
expect(
Expand Down
39 changes: 39 additions & 0 deletions packages/compiler/test/ml_parser/lexer_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,45 @@ describe('HtmlLexer', () => {
});
});

describe('processing instructions', () => {
it('should parse a processing instruction', () => {
expect(tokenizeAndHumanizeParts('<?xml version="1.0" encoding="UTF-8"?>')).toEqual([
[TokenType.PROCESSING_INSTRUCTION, 'xml version="1.0" encoding="UTF-8"'],
[TokenType.EOF],
]);
});

it('should parse a processing instruction surrounded by other content', () => {
expect(
tokenizeAndHumanizeParts('<!DOCTYPE html>\n<?xml version="1.0" encoding="UTF-8"?> hello'),
).toEqual([
[TokenType.DOC_TYPE, 'DOCTYPE html'],
[TokenType.TEXT, '\n'],
[TokenType.PROCESSING_INSTRUCTION, 'xml version="1.0" encoding="UTF-8"'],
[TokenType.TEXT, ' hello'],
[TokenType.EOF],
]);
});

it('should parse a processing instruction that contains question marks inside quoted content', () => {
expect(tokenizeAndHumanizeParts('<?xml version="?" encoding="UTF?>-8"?>')).toEqual([
[TokenType.PROCESSING_INSTRUCTION, 'xml version="?" encoding="UTF?>-8"'],
[TokenType.EOF],
]);
});

it('should store the location of a processing instruction', () => {
expect(tokenizeAndHumanizeSourceSpans('<?xml version="1.0" encoding="UTF-8"?>')).toEqual([
[TokenType.PROCESSING_INSTRUCTION, '<?xml version="1.0" encoding="UTF-8"?>'],
[TokenType.EOF, ''],
]);
});

it('should report missing end of a processing instruction', () => {
expect(tokenizeAndHumanizeErrors('<?')).toEqual([['Unexpected character "EOF"', '0:2']]);
});
});

describe('CDATA', () => {
it('should parse CDATA', () => {
expect(tokenizeAndHumanizeParts('<![CDATA[t\ne\rs\r\nt]]>')).toEqual([
Expand Down
Loading