Skip to content

Commit b45aae3

Browse files
author
Mikhail Arkhipov
authored
Formatting on Enter (microsoft#649)
* Basic tokenizer * Fixed property names * Tests, round I * Tests, round II * tokenizer test * Remove temorary change * Fix merge issue * Merge conflict * Merge conflict * Completion test * Fix last line * Fix javascript math * Make test await for results * Add license headers * Rename definitions to types * License headers * Fix typo in completion details (typo) * Fix hover test * Russian translations * Update to better translation * Fix typo * microsoft#70 How to get all parameter info when filling in a function param list * Fix microsoft#70 How to get all parameter info when filling in a function param list * Clean up * Clean imports * CR feedback * Trim whitespace for test stability * More tests * Better handle no-parameters documentation * Better handle ellipsis and Python3 * microsoft#385 Auto-Indentation doesn't work after comment * microsoft#141 Auto indentation broken when return keyword involved * Undo changes * On type formatting * Fix warnings * Round I * Round 2 * Round 3 * Round 4 * Round 5 * no message * Round 6 * Round 7 * Clean up targets and messages * Settings propagation * Tests * Test warning * Fix installer tests * Tests * Test fixes * Fix terminal service and tests async/await * Fix mock setup * Test fix * Test async/await fix * Test fix + activate tslint on awaits * Use command manager * Work around updateSettings * Multiroot fixes, partial * More workarounds * Multiroot tests * Fix installer test * Test fixes * Disable prospector * Enable dispose in all cases * Fix event firing * Min pylint options * Min checkers & pylintrc discovery * Fix Windows path in tests for Travis * Fix Mac test * Test fix * Work around VSC issue with formatting on save microsoft#624 * Workaround test * Unused * Old file * Brace and colon handling * More tests * Don't format inside strings and comments * Provider tests * Remove duplicate code
1 parent d001142 commit b45aae3

21 files changed

Lines changed: 928 additions & 88 deletions

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1594,6 +1594,7 @@
15941594
"tree-kill": "^1.1.0",
15951595
"typescript-char": "^0.0.0",
15961596
"uint64be": "^1.0.1",
1597+
"unicode": "^10.0.0",
15971598
"untildify": "^3.0.2",
15981599
"vscode-debugadapter": "^1.0.1",
15991600
"vscode-debugprotocol": "^1.0.1",

‎src/client/extension.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import { StopWatch } from './telemetry/stopWatch';
5757
import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry';
5858
import { ICodeExecutionManager } from './terminals/types';
5959
import { BlockFormatProviders } from './typeFormatters/blockFormatProvider';
60+
import { OnEnterFormatter } from './typeFormatters/onEnterFormatter';
6061
import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants';
6162
import * as tests from './unittests/main';
6263
import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry';
@@ -190,6 +191,8 @@ export async function activate(context: vscode.ExtensionContext) {
190191
context.subscriptions.push(new WorkspaceSymbols(serviceContainer));
191192

192193
context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
194+
context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
195+
193196
// In case we have CR LF
194197
const triggerCharacters: string[] = os.EOL.split('');
195198
triggerCharacters.shift();

‎src/client/formatters/lineFormatter.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
// tslint:disable-next-line:import-name
5+
import Char from 'typescript-char';
6+
import { BraceCounter } from '../language/braceCounter';
7+
import { TextBuilder } from '../language/textBuilder';
8+
import { Tokenizer } from '../language/tokenizer';
9+
import { ITextRangeCollection, IToken, TokenType } from '../language/types';
10+
11+
export class LineFormatter {
12+
private builder: TextBuilder;
13+
private tokens: ITextRangeCollection<IToken>;
14+
private braceCounter: BraceCounter;
15+
private text: string;
16+
17+
// tslint:disable-next-line:cyclomatic-complexity
18+
public formatLine(text: string): string {
19+
this.tokens = new Tokenizer().tokenize(text);
20+
this.text = text;
21+
this.builder = new TextBuilder();
22+
this.braceCounter = new BraceCounter();
23+
24+
if (this.tokens.count === 0) {
25+
return this.text;
26+
}
27+
28+
const ws = this.text.substr(0, this.tokens.getItemAt(0).start);
29+
if (ws.length > 0) {
30+
this.builder.append(ws); // Preserve leading indentation
31+
}
32+
33+
for (let i = 0; i < this.tokens.count; i += 1) {
34+
const t = this.tokens.getItemAt(i);
35+
const prev = i > 0 ? this.tokens.getItemAt(i - 1) : undefined;
36+
const next = i < this.tokens.count - 1 ? this.tokens.getItemAt(i + 1) : undefined;
37+
38+
switch (t.type) {
39+
case TokenType.Operator:
40+
this.handleOperator(i);
41+
break;
42+
43+
case TokenType.Comma:
44+
this.builder.append(',');
45+
if (next && !this.isCloseBraceType(next.type)) {
46+
this.builder.softAppendSpace();
47+
}
48+
break;
49+
50+
case TokenType.Identifier:
51+
if (!prev || (!this.isOpenBraceType(prev.type) && prev.type !== TokenType.Colon)) {
52+
this.builder.softAppendSpace();
53+
}
54+
this.builder.append(this.text.substring(t.start, t.end));
55+
break;
56+
57+
case TokenType.Colon:
58+
// x: 1 if not in slice, x[1:y] if inside the slice
59+
this.builder.append(':');
60+
if (!this.braceCounter.isOpened(TokenType.OpenBracket) && (next && next.type !== TokenType.Colon)) {
61+
// Not inside opened [[ ... ] sequence
62+
this.builder.softAppendSpace();
63+
}
64+
break;
65+
66+
case TokenType.Comment:
67+
// add space before in-line comment
68+
if (prev) {
69+
this.builder.softAppendSpace();
70+
}
71+
this.builder.append(this.text.substring(t.start, t.end));
72+
break;
73+
74+
default:
75+
this.handleOther(t);
76+
break;
77+
}
78+
}
79+
return this.builder.getText();
80+
}
81+
82+
private handleOperator(index: number): void {
83+
const t = this.tokens.getItemAt(index);
84+
if (index >= 2 && t.length === 1 && this.text.charCodeAt(t.start) === Char.Equal) {
85+
if (this.braceCounter.isOpened(TokenType.OpenBrace)) {
86+
// Check if this is = in function arguments. If so, do not
87+
// add spaces around it.
88+
const prev = this.tokens.getItemAt(index - 1);
89+
const prevPrev = this.tokens.getItemAt(index - 2);
90+
if (prev.type === TokenType.Identifier &&
91+
(prevPrev.type === TokenType.Comma || prevPrev.type === TokenType.OpenBrace)) {
92+
this.builder.append('=');
93+
return;
94+
}
95+
}
96+
}
97+
this.builder.softAppendSpace();
98+
this.builder.append(this.text.substring(t.start, t.end));
99+
this.builder.softAppendSpace();
100+
}
101+
102+
private handleOther(t: IToken): void {
103+
if (this.isBraceType(t.type)) {
104+
this.braceCounter.countBrace(t);
105+
}
106+
this.builder.append(this.text.substring(t.start, t.end));
107+
}
108+
109+
private isOpenBraceType(type: TokenType): boolean {
110+
return type === TokenType.OpenBrace || type === TokenType.OpenBracket || type === TokenType.OpenCurly;
111+
}
112+
private isCloseBraceType(type: TokenType): boolean {
113+
return type === TokenType.CloseBrace || type === TokenType.CloseBracket || type === TokenType.CloseCurly;
114+
}
115+
private isBraceType(type: TokenType): boolean {
116+
return this.isOpenBraceType(type) || this.isCloseBraceType(type);
117+
}
118+
}

‎src/client/language/braceCounter.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import { IToken, TokenType } from './types';
5+
6+
class BracePair {
7+
public readonly openBrace: TokenType;
8+
public readonly closeBrace: TokenType;
9+
10+
constructor(openBrace: TokenType, closeBrace: TokenType) {
11+
this.openBrace = openBrace;
12+
this.closeBrace = closeBrace;
13+
}
14+
}
15+
16+
class Stack {
17+
private store: IToken[] = [];
18+
public push(val: IToken) {
19+
this.store.push(val);
20+
}
21+
public pop(): IToken | undefined {
22+
return this.store.pop();
23+
}
24+
public get length(): number {
25+
return this.store.length;
26+
}
27+
}
28+
29+
export class BraceCounter {
30+
private readonly bracePairs: BracePair[] = [
31+
new BracePair(TokenType.OpenBrace, TokenType.CloseBrace),
32+
new BracePair(TokenType.OpenBracket, TokenType.CloseBracket),
33+
new BracePair(TokenType.OpenCurly, TokenType.CloseCurly)
34+
];
35+
private braceStacks: Stack[] = [new Stack(), new Stack(), new Stack()];
36+
37+
public get count(): number {
38+
let c = 0;
39+
for (const s of this.braceStacks) {
40+
c += s.length;
41+
}
42+
return c;
43+
}
44+
45+
public isOpened(type: TokenType): boolean {
46+
for (let i = 0; i < this.bracePairs.length; i += 1) {
47+
const pair = this.bracePairs[i];
48+
if (pair.openBrace === type || pair.closeBrace === type) {
49+
return this.braceStacks[i].length > 0;
50+
}
51+
}
52+
return false;
53+
}
54+
55+
public countBrace(brace: IToken): boolean {
56+
for (let i = 0; i < this.bracePairs.length; i += 1) {
57+
const pair = this.bracePairs[i];
58+
if (pair.openBrace === brace.type) {
59+
this.braceStacks[i].push(brace);
60+
return true;
61+
}
62+
if (pair.closeBrace === brace.type) {
63+
if (this.braceStacks[i].length > 0) {
64+
this.braceStacks[i].pop();
65+
}
66+
return true;
67+
}
68+
}
69+
return false;
70+
}
71+
}

‎src/client/language/characterStream.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
// tslint:disable-next-line:import-name
66
import Char from 'typescript-char';
7+
import { isLineBreak, isWhiteSpace } from './characters';
78
import { TextIterator } from './textIterator';
89
import { ICharacterStream, ITextIterator } from './types';
910

@@ -70,11 +71,11 @@ export class CharacterStream implements ICharacterStream {
7071
}
7172

7273
public isAtWhiteSpace(): boolean {
73-
return this.currentChar <= Char.Space || this.currentChar === 0x200B; // Unicode whitespace
74+
return isWhiteSpace(this.currentChar);
7475
}
7576

7677
public isAtLineBreak(): boolean {
77-
return this.currentChar === Char.CarriageReturn || this.currentChar === Char.LineFeed;
78+
return isLineBreak(this.currentChar);
7879
}
7980

8081
public skipLineBreak(): void {

‎src/client/language/characters.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
// tslint:disable-next-line:import-name
5+
import Char from 'typescript-char';
6+
import { getUnicodeCategory, UnicodeCategory } from './unicode';
7+
8+
export function isIdentifierStartChar(ch: number) {
9+
switch (ch) {
10+
// Underscore is explicitly allowed to start an identifier
11+
case Char.Underscore:
12+
return true;
13+
// Characters with the Other_ID_Start property
14+
case 0x1885:
15+
case 0x1886:
16+
case 0x2118:
17+
case 0x212E:
18+
case 0x309B:
19+
case 0x309C:
20+
return true;
21+
default:
22+
break;
23+
}
24+
25+
const cat = getUnicodeCategory(ch);
26+
switch (cat) {
27+
// Supported categories for starting an identifier
28+
case UnicodeCategory.UppercaseLetter:
29+
case UnicodeCategory.LowercaseLetter:
30+
case UnicodeCategory.TitlecaseLetter:
31+
case UnicodeCategory.ModifierLetter:
32+
case UnicodeCategory.OtherLetter:
33+
case UnicodeCategory.LetterNumber:
34+
return true;
35+
default:
36+
break;
37+
}
38+
return false;
39+
}
40+
41+
export function isIdentifierChar(ch: number) {
42+
if (isIdentifierStartChar(ch)) {
43+
return true;
44+
}
45+
46+
switch (ch) {
47+
// Characters with the Other_ID_Continue property
48+
case 0x00B7:
49+
case 0x0387:
50+
case 0x1369:
51+
case 0x136A:
52+
case 0x136B:
53+
case 0x136C:
54+
case 0x136D:
55+
case 0x136E:
56+
case 0x136F:
57+
case 0x1370:
58+
case 0x1371:
59+
case 0x19DA:
60+
return true;
61+
default:
62+
break;
63+
}
64+
65+
switch (getUnicodeCategory(ch)) {
66+
// Supported categories for continuing an identifier
67+
case UnicodeCategory.NonSpacingMark:
68+
case UnicodeCategory.SpacingCombiningMark:
69+
case UnicodeCategory.DecimalDigitNumber:
70+
case UnicodeCategory.ConnectorPunctuation:
71+
return true;
72+
default:
73+
break;
74+
}
75+
return false;
76+
}
77+
78+
export function isWhiteSpace(ch: number): boolean {
79+
return ch <= Char.Space || ch === 0x200B; // Unicode whitespace
80+
}
81+
82+
export function isLineBreak(ch: number): boolean {
83+
return ch === Char.CarriageReturn || ch === Char.LineFeed;
84+
}
85+
86+
export function isDecimal(ch: number): boolean {
87+
return ch >= Char._0 && ch <= Char._9;
88+
}
89+
90+
export function isHex(ch: number): boolean {
91+
return isDecimal(ch) || (ch >= Char.a && ch <= Char.f) || (ch >= Char.A && ch <= Char.F);
92+
}
93+
94+
export function isOctal(ch: number): boolean {
95+
return ch >= Char._0 && ch <= Char._7;
96+
}
97+
98+
export function isBinary(ch: number): boolean {
99+
return ch === Char._0 || ch === Char._1;
100+
}

‎src/client/language/textBuilder.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import { isWhiteSpace } from './characters';
5+
6+
// Copyright (c) Microsoft Corporation. All rights reserved.
7+
// Licensed under the MIT License.
8+
9+
export class TextBuilder {
10+
private segments: string[] = [];
11+
12+
public getText(): string {
13+
if (this.isLastWhiteSpace()) {
14+
this.segments.pop();
15+
}
16+
return this.segments.join('');
17+
}
18+
19+
public softAppendSpace(): void {
20+
if (!this.isLastWhiteSpace() && this.segments.length > 0) {
21+
this.segments.push(' ');
22+
}
23+
}
24+
25+
public append(text: string): void {
26+
this.segments.push(text);
27+
}
28+
29+
private isLastWhiteSpace(): boolean {
30+
return this.segments.length > 0 && this.isWhitespace(this.segments[this.segments.length - 1]);
31+
}
32+
33+
private isWhitespace(s: string): boolean {
34+
for (let i = 0; i < s.length; i += 1) {
35+
if (!isWhiteSpace(s.charCodeAt(i))) {
36+
return false;
37+
}
38+
}
39+
return true;
40+
}
41+
}

0 commit comments

Comments
 (0)