Skip to content

Commit c92c491

Browse files
JLHwungJLHwung
authored
Improve Unicode handling in code-frame tokenizer (#17589)
* chore: enable prefer-string-starts-ends-with * fix: respect Unicode uppercase letter * skip test for Node.js 6 * perf: add fast path for ASCII identifiers
1 parent d3c7653 commit c92c491

25 files changed

Lines changed: 98 additions & 45 deletions

File tree

‎babel.config.js‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -198,10 +198,7 @@ module.exports = function (api) {
198198
].filter(Boolean),
199199
overrides: [
200200
{
201-
test: [
202-
"packages/babel-parser",
203-
"packages/babel-helper-validator-identifier",
204-
].map(normalize),
201+
test: ["packages/babel-parser"].map(normalize),
205202
plugins: [
206203
"babel-plugin-transform-charcodes",
207204
pluginBabelParserTokenType,
@@ -210,9 +207,11 @@ module.exports = function (api) {
210207
},
211208
{
212209
test: [
210+
"packages/babel-code-frame",
213211
"packages/babel-generator",
214212
"packages/babel-helper-create-class-features-plugin",
215213
"packages/babel-helper-string-parser",
214+
"packages/babel-helper-validator-identifier",
216215
].map(normalize),
217216
plugins: ["babel-plugin-transform-charcodes"],
218217
},

‎eslint.config.mts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,6 @@ export default defineConfig([
216216
"@typescript-eslint/no-unsafe-member-access": "off",
217217
"@typescript-eslint/prefer-for-of": "off",
218218
"@typescript-eslint/prefer-nullish-coalescing": "off",
219-
"@typescript-eslint/prefer-string-starts-ends-with": "off",
220219
"@typescript-eslint/restrict-template-expressions": "off",
221220
"@typescript-eslint/sort-type-constituents": "off",
222221
"@typescript-eslint/unbound-method": "off",

‎packages/babel-cli/src/babel/util.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export function readdir(
3535
if (dirent.isDirectory()) return false;
3636
const filename = dirent.name;
3737
return (
38-
(includeDotfiles || filename[0] !== ".") &&
38+
(includeDotfiles || !filename.startsWith(".")) &&
3939
(!filter || filter(filename))
4040
);
4141
})
@@ -54,7 +54,7 @@ export function readdir(
5454
if (stat.isDirectory()) return true;
5555

5656
return (
57-
(includeDotfiles || filename[0] !== ".") &&
57+
(includeDotfiles || !filename.startsWith(".")) &&
5858
(!filter || filter(filename))
5959
);
6060
},

‎packages/babel-code-frame/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"picocolors": "^1.1.1"
2222
},
2323
"devDependencies": {
24+
"charcodes": "^0.2.0",
2425
"import-meta-resolve": "^4.1.0",
2526
"strip-ansi": "^4.0.0"
2627
},

‎packages/babel-code-frame/src/highlight.ts‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import type { Token as JSToken, JSXToken } from "js-tokens";
22
import jsTokens from "js-tokens";
3+
// We inline this package
4+
// eslint-disable-next-line import/no-extraneous-dependencies
5+
import * as charCodes from "charcodes";
36

47
import {
58
isStrictReservedWord,
@@ -46,16 +49,29 @@ if (process.env.BABEL_8_BREAKING) {
4649
token: JSToken | JSXToken,
4750
): InternalTokenType | "uncolored" {
4851
if (token.type === "IdentifierName") {
52+
const tokenValue = token.value;
4953
if (
50-
isKeyword(token.value) ||
51-
isStrictReservedWord(token.value, true) ||
52-
sometimesKeywords.has(token.value)
54+
isKeyword(tokenValue) ||
55+
isStrictReservedWord(tokenValue, true) ||
56+
sometimesKeywords.has(tokenValue)
5357
) {
5458
return "keyword";
5559
}
5660

57-
if (token.value[0] !== token.value[0].toLowerCase()) {
58-
return "capitalized";
61+
const firstChar = tokenValue.charCodeAt(0);
62+
if (firstChar < 128) {
63+
// ASCII characters
64+
if (
65+
firstChar >= charCodes.uppercaseA &&
66+
firstChar <= charCodes.uppercaseZ
67+
) {
68+
return "capitalized";
69+
}
70+
} else {
71+
const firstChar = String.fromCodePoint(tokenValue.codePointAt(0));
72+
if (firstChar !== firstChar.toLowerCase()) {
73+
return "capitalized";
74+
}
5975
}
6076
}
6177

@@ -139,22 +155,24 @@ if (process.env.BABEL_8_BREAKING) {
139155
// typing it since the whole block will be removed in Babel 8
140156
const getTokenType = function (token: any, offset: number, text: string) {
141157
if (token.type === "name") {
158+
const tokenValue = token.value;
142159
if (
143-
isKeyword(token.value) ||
144-
isStrictReservedWord(token.value, true) ||
145-
sometimesKeywords.has(token.value)
160+
isKeyword(tokenValue) ||
161+
isStrictReservedWord(tokenValue, true) ||
162+
sometimesKeywords.has(tokenValue)
146163
) {
147164
return "keyword";
148165
}
149166

150167
if (
151-
JSX_TAG.test(token.value) &&
168+
JSX_TAG.test(tokenValue) &&
152169
(text[offset - 1] === "<" || text.slice(offset - 2, offset) === "</")
153170
) {
154171
return "jsxIdentifier";
155172
}
156173

157-
if (token.value[0] !== token.value[0].toLowerCase()) {
174+
const firstChar = String.fromCodePoint(tokenValue.codePointAt(0));
175+
if (firstChar !== firstChar.toLowerCase()) {
158176
return "capitalized";
159177
}
160178
}

‎packages/babel-code-frame/test/color-detection.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import stripAnsi from "strip-ansi";
22
import colors from "picocolors";
33

44
import _codeFrame, { codeFrameColumns } from "../lib/index.js";
5+
import { itGte } from "$repo-utils";
6+
const nodeGte8 = itGte("8.0.0");
57
const codeFrame = _codeFrame.default || _codeFrame;
68

79
const compose = (f, g) => v => f(g(v));
@@ -127,6 +129,40 @@ describe("highlight", function () {
127129
),
128130
);
129131
});
132+
133+
// Node.js 6 does not map upper case letter U+10400 to U+10428
134+
nodeGte8("unicode capitalized", function () {
135+
const gutter = colors.gray;
136+
const yellow = colors.yellow;
137+
const cyan = colors.cyan;
138+
139+
const rawLines = ["var 𐐔𐐯𐑅𐐨𐑉𐐯𐐻, deseret;"].join("\n");
140+
141+
expect(
142+
JSON.stringify(
143+
codeFrame(rawLines, 0, null, {
144+
linesAbove: 1,
145+
linesBelow: 1,
146+
forceColor: true,
147+
}),
148+
),
149+
).toEqual(
150+
JSON.stringify(
151+
colors.reset(
152+
" " +
153+
gutter(" 1 |") +
154+
" " +
155+
cyan("var") +
156+
" " +
157+
yellow("𐐔𐐯𐑅𐐨𐑉𐐯𐐻") +
158+
yellow(",") +
159+
" " +
160+
"deseret" +
161+
yellow(";"),
162+
),
163+
),
164+
);
165+
});
130166
});
131167

132168
describe("when colors are not supported", () => {

‎packages/babel-core/src/config/files/configuration.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ const readIgnoreConfig = makeStaticFileCache((filepath, content) => {
199199
.filter(Boolean);
200200

201201
for (const pattern of ignorePatterns) {
202-
if (pattern[0] === "!") {
202+
if (pattern.startsWith("!")) {
203203
throw new ConfigError(
204204
`Negation of file paths is not supported.`,
205205
filepath,

‎packages/babel-core/src/config/full.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ function enhanceError<T extends Function>(context: ConfigContext, fn: T): T {
227227
} catch (e) {
228228
// There are a few case where thrown errors will try to annotate themselves multiple times, so
229229
// to keep things simple we just bail out if re-wrapping the message.
230-
if (!/^\[BABEL\]/.test(e.message)) {
230+
if (!e.message.startsWith("[BABEL]")) {
231231
e.message = `[BABEL] ${context.filename ?? "unknown file"}: ${
232232
e.message
233233
}`;

‎packages/babel-core/src/config/pattern-to-regex.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export default function pathToPattern(
3939
if (part === "*") return last ? starPatLast : starPat;
4040

4141
// *.ext matches a wildcard with an extension.
42-
if (part.indexOf("*.") === 0) {
42+
if (part.startsWith("*.")) {
4343
return (
4444
substitution + escapeRegExp(part.slice(1)) + (last ? endSep : sep)
4545
);

‎packages/babel-helper-check-duplicate-nodes/src/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ export default function checkDuplicateNodes(ast: t.Node) {
1515
const hidePrivateProperties = (key: string, val: unknown) => {
1616
// Hides properties like _shadowedFunctionLiteral,
1717
// which makes the AST circular
18-
if (key[0] === "_") return "[Private]";
18+
if (key.startsWith("_")) return "[Private]";
1919
return val;
2020
};
2121

0 commit comments

Comments
 (0)