Skip to content

Commit a74f109

Browse files
rbucktonRyanCavanaugh
rbuckton
authored and
RyanCavanaugh
committed
Do not escape actual template literals on emit (microsoft#32844)
1 parent 370a596 commit a74f109

10 files changed

Lines changed: 209 additions & 62 deletions

File tree

‎src/compiler/factory.ts‎

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,30 +1329,100 @@ namespace ts {
13291329
: node;
13301330
}
13311331

1332-
export function createTemplateHead(text: string) {
1333-
const node = <TemplateHead>createSynthesizedNode(SyntaxKind.TemplateHead);
1332+
let rawTextScanner: Scanner | undefined;
1333+
const invalidValueSentinel: object = {};
1334+
1335+
function getCookedText(kind: TemplateLiteralToken["kind"], rawText: string) {
1336+
if (!rawTextScanner) {
1337+
rawTextScanner = createScanner(ScriptTarget.Latest, /*skipTrivia*/ false, LanguageVariant.Standard);
1338+
}
1339+
switch (kind) {
1340+
case SyntaxKind.NoSubstitutionTemplateLiteral:
1341+
rawTextScanner.setText("`" + rawText + "`");
1342+
break;
1343+
case SyntaxKind.TemplateHead:
1344+
// tslint:disable-next-line no-invalid-template-strings
1345+
rawTextScanner.setText("`" + rawText + "${");
1346+
break;
1347+
case SyntaxKind.TemplateMiddle:
1348+
// tslint:disable-next-line no-invalid-template-strings
1349+
rawTextScanner.setText("}" + rawText + "${");
1350+
break;
1351+
case SyntaxKind.TemplateTail:
1352+
rawTextScanner.setText("}" + rawText + "`");
1353+
break;
1354+
}
1355+
1356+
let token = rawTextScanner.scan();
1357+
if (token === SyntaxKind.CloseBracketToken) {
1358+
token = rawTextScanner.reScanTemplateToken();
1359+
}
1360+
1361+
if (rawTextScanner.isUnterminated()) {
1362+
rawTextScanner.setText(undefined);
1363+
return invalidValueSentinel;
1364+
}
1365+
1366+
let tokenValue: string | undefined;
1367+
switch (token) {
1368+
case SyntaxKind.NoSubstitutionTemplateLiteral:
1369+
case SyntaxKind.TemplateHead:
1370+
case SyntaxKind.TemplateMiddle:
1371+
case SyntaxKind.TemplateTail:
1372+
tokenValue = rawTextScanner.getTokenValue();
1373+
break;
1374+
}
1375+
1376+
if (rawTextScanner.scan() !== SyntaxKind.EndOfFileToken) {
1377+
rawTextScanner.setText(undefined);
1378+
return invalidValueSentinel;
1379+
}
1380+
1381+
rawTextScanner.setText(undefined);
1382+
return tokenValue;
1383+
}
1384+
1385+
function createTemplateLiteralLikeNode(kind: TemplateLiteralToken["kind"], text: string, rawText: string | undefined) {
1386+
const node = <TemplateLiteralLikeNode>createSynthesizedNode(kind);
13341387
node.text = text;
1388+
if (rawText === undefined || text === rawText) {
1389+
node.rawText = rawText;
1390+
}
1391+
else {
1392+
const cooked = getCookedText(kind, rawText);
1393+
if (typeof cooked === "object") {
1394+
return Debug.fail("Invalid raw text");
1395+
}
1396+
1397+
Debug.assert(text === cooked, "Expected argument 'text' to be the normalized (i.e. 'cooked') version of argument 'rawText'.");
1398+
node.rawText = rawText;
1399+
}
13351400
return node;
13361401
}
13371402

1338-
export function createTemplateMiddle(text: string) {
1339-
const node = <TemplateMiddle>createSynthesizedNode(SyntaxKind.TemplateMiddle);
1403+
export function createTemplateHead(text: string, rawText?: string) {
1404+
const node = <TemplateHead>createTemplateLiteralLikeNode(SyntaxKind.TemplateHead, text, rawText);
13401405
node.text = text;
13411406
return node;
13421407
}
13431408

1344-
export function createTemplateTail(text: string) {
1345-
const node = <TemplateTail>createSynthesizedNode(SyntaxKind.TemplateTail);
1409+
export function createTemplateMiddle(text: string, rawText?: string) {
1410+
const node = <TemplateMiddle>createTemplateLiteralLikeNode(SyntaxKind.TemplateMiddle, text, rawText);
13461411
node.text = text;
13471412
return node;
13481413
}
13491414

1350-
export function createNoSubstitutionTemplateLiteral(text: string) {
1351-
const node = <NoSubstitutionTemplateLiteral>createSynthesizedNode(SyntaxKind.NoSubstitutionTemplateLiteral);
1415+
export function createTemplateTail(text: string, rawText?: string) {
1416+
const node = <TemplateTail>createTemplateLiteralLikeNode(SyntaxKind.TemplateTail, text, rawText);
13521417
node.text = text;
13531418
return node;
13541419
}
13551420

1421+
export function createNoSubstitutionTemplateLiteral(text: string, rawText?: string) {
1422+
const node = <NoSubstitutionTemplateLiteral>createTemplateLiteralLikeNode(SyntaxKind.NoSubstitutionTemplateLiteral, text, rawText);
1423+
return node;
1424+
}
1425+
13561426
export function createYield(expression?: Expression): YieldExpression;
13571427
export function createYield(asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression;
13581428
export function createYield(asteriskTokenOrExpression?: AsteriskToken | undefined | Expression, expression?: Expression) {

‎src/compiler/parser.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2300,9 +2300,19 @@ namespace ts {
23002300
return <TemplateMiddle | TemplateTail>fragment;
23012301
}
23022302

2303-
function parseLiteralLikeNode(kind: SyntaxKind): LiteralExpression | LiteralLikeNode {
2304-
const node = <LiteralExpression>createNode(kind);
2303+
function parseLiteralLikeNode(kind: SyntaxKind): LiteralLikeNode {
2304+
const node = <LiteralLikeNode>createNode(kind);
23052305
node.text = scanner.getTokenValue();
2306+
switch (kind) {
2307+
case SyntaxKind.NoSubstitutionTemplateLiteral:
2308+
case SyntaxKind.TemplateHead:
2309+
case SyntaxKind.TemplateMiddle:
2310+
case SyntaxKind.TemplateTail:
2311+
const isLast = kind === SyntaxKind.NoSubstitutionTemplateLiteral || kind === SyntaxKind.TemplateTail;
2312+
const tokenText = scanner.getTokenText();
2313+
(<TemplateLiteralLikeNode>node).rawText = tokenText.substring(1, tokenText.length - (scanner.isUnterminated() ? 0 : isLast ? 1 : 2));
2314+
break;
2315+
}
23062316

23072317
if (scanner.hasExtendedUnicodeEscape()) {
23082318
node.hasExtendedUnicodeEscape = true;

‎src/compiler/transformers/es2015.ts‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3993,18 +3993,21 @@ namespace ts {
39933993
*
39943994
* @param node The ES6 template literal.
39953995
*/
3996-
function getRawLiteral(node: LiteralLikeNode) {
3996+
function getRawLiteral(node: TemplateLiteralLikeNode) {
39973997
// Find original source text, since we need to emit the raw strings of the tagged template.
39983998
// The raw strings contain the (escaped) strings of what the user wrote.
39993999
// Examples: `\n` is converted to "\\n", a template string with a newline to "\n".
4000-
let text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node);
4001-
4002-
// text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"),
4003-
// thus we need to remove those characters.
4004-
// First template piece starts with "`", others with "}"
4005-
// Last template piece ends with "`", others with "${"
4006-
const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail;
4007-
text = text.substring(1, text.length - (isLast ? 1 : 2));
4000+
let text = node.rawText;
4001+
if (text === undefined) {
4002+
text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node);
4003+
4004+
// text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"),
4005+
// thus we need to remove those characters.
4006+
// First template piece starts with "`", others with "}"
4007+
// Last template piece ends with "`", others with "${"
4008+
const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail;
4009+
text = text.substring(1, text.length - (isLast ? 1 : 2));
4010+
}
40084011

40094012
// Newline normalization:
40104013
// ES6 Spec 11.8.6.1 - Static Semantics of TV's and TRV's

‎src/compiler/types.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,6 +1646,10 @@ namespace ts {
16461646
hasExtendedUnicodeEscape?: boolean;
16471647
}
16481648

1649+
export interface TemplateLiteralLikeNode extends LiteralLikeNode {
1650+
rawText?: string;
1651+
}
1652+
16491653
// The text property of a LiteralExpression stores the interpreted value of the literal in text form. For a StringLiteral,
16501654
// or any literal of a template, this means quotes have been removed and escapes have been converted to actual characters.
16511655
// For a NumericLiteral, the stored value is the toString() representation of the number. For example 1, 1.00, and 1e0 are all stored as just "1".
@@ -1657,7 +1661,7 @@ namespace ts {
16571661
kind: SyntaxKind.RegularExpressionLiteral;
16581662
}
16591663

1660-
export interface NoSubstitutionTemplateLiteral extends LiteralExpression {
1664+
export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode {
16611665
kind: SyntaxKind.NoSubstitutionTemplateLiteral;
16621666
}
16631667

@@ -1696,17 +1700,17 @@ namespace ts {
16961700
kind: SyntaxKind.BigIntLiteral;
16971701
}
16981702

1699-
export interface TemplateHead extends LiteralLikeNode {
1703+
export interface TemplateHead extends TemplateLiteralLikeNode {
17001704
kind: SyntaxKind.TemplateHead;
17011705
parent: TemplateExpression;
17021706
}
17031707

1704-
export interface TemplateMiddle extends LiteralLikeNode {
1708+
export interface TemplateMiddle extends TemplateLiteralLikeNode {
17051709
kind: SyntaxKind.TemplateMiddle;
17061710
parent: TemplateSpan;
17071711
}
17081712

1709-
export interface TemplateTail extends LiteralLikeNode {
1713+
export interface TemplateTail extends TemplateLiteralLikeNode {
17101714
kind: SyntaxKind.TemplateTail;
17111715
parent: TemplateSpan;
17121716
}

‎src/compiler/utilities.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -566,8 +566,6 @@ namespace ts {
566566
return emitNode && emitNode.flags || 0;
567567
}
568568

569-
const escapeNoSubstitutionTemplateLiteralText = compose(escapeString, escapeTemplateSubstitution);
570-
const escapeNonAsciiNoSubstitutionTemplateLiteralText = compose(escapeNonAsciiString, escapeTemplateSubstitution);
571569
export function getLiteralText(node: LiteralLikeNode, sourceFile: SourceFile, neverAsciiEscape: boolean | undefined) {
572570
// If we don't need to downlevel and we can reach the original source text using
573571
// the node's parent reference, then simply get the text as it was originally written.
@@ -580,9 +578,7 @@ namespace ts {
580578

581579
// If a NoSubstitutionTemplateLiteral appears to have a substitution in it, the original text
582580
// had to include a backslash: `not \${a} substitution`.
583-
const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ?
584-
node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNoSubstitutionTemplateLiteralText : escapeString :
585-
node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNonAsciiNoSubstitutionTemplateLiteralText : escapeNonAsciiString;
581+
const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ? escapeString : escapeNonAsciiString;
586582

587583
// If we can't reach the original source text, use the canonical form if it's a number,
588584
// or a (possibly escaped) quoted form of the original text if it's string-like.
@@ -595,15 +591,23 @@ namespace ts {
595591
return '"' + escapeText(node.text, CharacterCodes.doubleQuote) + '"';
596592
}
597593
case SyntaxKind.NoSubstitutionTemplateLiteral:
598-
return "`" + escapeText(node.text, CharacterCodes.backtick) + "`";
599594
case SyntaxKind.TemplateHead:
600-
// tslint:disable-next-line no-invalid-template-strings
601-
return "`" + escapeText(node.text, CharacterCodes.backtick) + "${";
602595
case SyntaxKind.TemplateMiddle:
603-
// tslint:disable-next-line no-invalid-template-strings
604-
return "}" + escapeText(node.text, CharacterCodes.backtick) + "${";
605596
case SyntaxKind.TemplateTail:
606-
return "}" + escapeText(node.text, CharacterCodes.backtick) + "`";
597+
const rawText = (<TemplateLiteralLikeNode>node).rawText || escapeTemplateSubstitution(escapeText(node.text, CharacterCodes.backtick));
598+
switch (node.kind) {
599+
case SyntaxKind.NoSubstitutionTemplateLiteral:
600+
return "`" + rawText + "`";
601+
case SyntaxKind.TemplateHead:
602+
// tslint:disable-next-line no-invalid-template-strings
603+
return "`" + rawText + "${";
604+
case SyntaxKind.TemplateMiddle:
605+
// tslint:disable-next-line no-invalid-template-strings
606+
return "}" + rawText + "${";
607+
case SyntaxKind.TemplateTail:
608+
return "}" + rawText + "`";
609+
}
610+
break;
607611
case SyntaxKind.NumericLiteral:
608612
case SyntaxKind.BigIntLiteral:
609613
case SyntaxKind.RegularExpressionLiteral:
@@ -3178,7 +3182,8 @@ namespace ts {
31783182
// There is no reason for this other than that JSON.stringify does not handle it either.
31793183
const doubleQuoteEscapedCharsRegExp = /[\\\"\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
31803184
const singleQuoteEscapedCharsRegExp = /[\\\'\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
3181-
const backtickQuoteEscapedCharsRegExp = /[\\\`\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
3185+
// Template strings should be preserved as much as possible
3186+
const backtickQuoteEscapedCharsRegExp = /[\\\`]/g;
31823187
const escapedCharsMap = createMapFromTemplate({
31833188
"\t": "\\t",
31843189
"\v": "\\v",

‎src/harness/evaluator.ts‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ namespace evaluator {
22
declare var Symbol: SymbolConstructor;
33

44
const sourceFile = vpath.combine(vfs.srcFolder, "source.ts");
5+
const sourceFileJs = vpath.combine(vfs.srcFolder, "source.js");
56

67
function compile(sourceText: string, options?: ts.CompilerOptions) {
78
const fs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false);
@@ -32,9 +33,8 @@ namespace evaluator {
3233
// Add "asyncIterator" if missing
3334
if (!ts.hasProperty(FakeSymbol, "asyncIterator")) Object.defineProperty(FakeSymbol, "asyncIterator", { value: Symbol.for("Symbol.asyncIterator"), configurable: true });
3435

35-
function evaluate(result: compiler.CompilationResult, globals?: Record<string, any>) {
36-
globals = { Symbol: FakeSymbol, ...globals };
37-
36+
export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record<string, any>) {
37+
const result = compile(sourceText, options);
3838
if (ts.some(result.diagnostics)) {
3939
assert.ok(/*value*/ false, "Syntax error in evaluation source text:\n" + ts.formatDiagnostics(result.diagnostics, {
4040
getCanonicalFileName: file => file,
@@ -46,6 +46,12 @@ namespace evaluator {
4646
const output = result.getOutput(sourceFile, "js")!;
4747
assert.isDefined(output);
4848

49+
return evaluateJavaScript(output.text, globals, output.file);
50+
}
51+
52+
export function evaluateJavaScript(sourceText: string, globals?: Record<string, any>, sourceFile = sourceFileJs) {
53+
globals = { Symbol: FakeSymbol, ...globals };
54+
4955
const globalNames: string[] = [];
5056
const globalArgs: any[] = [];
5157
for (const name in globals) {
@@ -55,15 +61,11 @@ namespace evaluator {
5561
}
5662
}
5763

58-
const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${output.text} })`;
59-
// tslint:disable-next-line:no-eval
60-
const evaluateThunk = eval(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void;
64+
const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${sourceText} })`;
65+
// tslint:disable-next-line:no-eval no-unused-expression
66+
const evaluateThunk = (void 0, eval)(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void;
6167
const module: { exports: any; } = { exports: {} };
62-
evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(output.file), output.file, FakeSymbol, ...globalArgs);
68+
evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(sourceFile), sourceFile, FakeSymbol, ...globalArgs);
6369
return module.exports;
6470
}
65-
66-
export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record<string, any>) {
67-
return evaluate(compile(sourceText, options), globals);
68-
}
6971
}

‎src/testRunner/unittests/transform.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,24 @@ namespace ts {
6767
});
6868
}
6969

70+
function testBaselineAndEvaluate(testName: string, test: () => string, onEvaluate: (exports: any) => void) {
71+
describe(testName, () => {
72+
let sourceText!: string;
73+
before(() => {
74+
sourceText = test();
75+
});
76+
after(() => {
77+
sourceText = undefined!;
78+
});
79+
it("compare baselines", () => {
80+
Harness.Baseline.runBaseline(`transformApi/transformsCorrectly.${testName}.js`, sourceText);
81+
});
82+
it("evaluate", () => {
83+
onEvaluate(evaluator.evaluateJavaScript(sourceText));
84+
});
85+
});
86+
}
87+
7088
testBaseline("substitution", () => {
7189
return transformSourceFile(`var a = undefined;`, [replaceUndefinedWithVoid0]);
7290
});
@@ -440,6 +458,31 @@ namespace Foo {
440458

441459
});
442460

461+
testBaselineAndEvaluate("templateSpans", () => {
462+
return transpileModule("const x = String.raw`\n\nhello`; exports.stringLength = x.trim().length;", {
463+
compilerOptions: {
464+
target: ScriptTarget.ESNext,
465+
newLine: NewLineKind.CarriageReturnLineFeed,
466+
},
467+
transformers: {
468+
before: [transformSourceFile]
469+
}
470+
}).outputText;
471+
472+
function transformSourceFile(context: TransformationContext): Transformer<SourceFile> {
473+
function visitor(node: Node): VisitResult<Node> {
474+
if (isNoSubstitutionTemplateLiteral(node)) {
475+
return createNoSubstitutionTemplateLiteral(node.text, node.rawText);
476+
}
477+
else {
478+
return visitEachChild(node, visitor, context);
479+
}
480+
}
481+
return sourceFile => visitNode(sourceFile, visitor, isSourceFile);
482+
}
483+
}, exports => {
484+
assert.equal(exports.stringLength, 5);
485+
});
443486
});
444487
}
445488

0 commit comments

Comments
 (0)