Skip to content

Commit 556b576

Browse files
joke1196ghislainpiot
joke1196
authored and
ghislainpiot
committed
SONARPY-2036: Enrich EOF tokens (SonarSource#1891)
1 parent 05e7ffe commit 556b576

5 files changed

Lines changed: 88 additions & 15 deletions

File tree

‎python-frontend/src/main/java/org/sonar/python/tree/TokenEnricher.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,15 @@
1919
*/
2020
package org.sonar.python.tree;
2121

22-
import com.sonar.sslr.api.GenericTokenType;
2322
import com.sonar.sslr.api.Token;
24-
import com.sonar.sslr.api.TokenType;
2523
import java.util.List;
2624
import java.util.Map;
2725
import java.util.Set;
2826
import org.sonar.python.IPythonLocation;
29-
import org.sonar.python.api.PythonTokenType;
3027

3128
public class TokenEnricher {
3229

3330
private static final Set<Character> ESCAPED_CHARS = Set.of('"', '\'', '\\', '\b', '\f', '\n', '\r', '\t');
34-
private static final Set<TokenType> TOKEN_TYPES_TO_IGNORE = Set.of(PythonTokenType.DEDENT, GenericTokenType.EOF, PythonTokenType.INDENT);
3531

3632
private TokenEnricher() {
3733
}
@@ -41,7 +37,7 @@ public static List<TokenImpl> enrichTokens(List<Token> tokens, Map<Integer, IPyt
4137
}
4238

4339
public static TokenImpl enrichToken(Token token, Map<Integer, IPythonLocation> offsetMap) {
44-
if (!offsetMap.isEmpty() && !TOKEN_TYPES_TO_IGNORE.contains(token.getType())) {
40+
if (!offsetMap.isEmpty()) {
4541
IPythonLocation location = offsetMap.get(token.getLine());
4642
if (location == null) {
4743
throw new IllegalStateException(String.format("No IPythonLocation found for line %s", token.getLine()));

‎python-frontend/src/test/java/org/sonar/python/tree/TokenEnricherTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,8 @@ void shouldComputeColCorrectly() {
116116
assertThat(stringToken.includedEscapeChars()).isEqualTo(2);
117117

118118
var eofToken = tokens.get(tokens.size() - 1);
119-
assertThat(eofToken.line()).isEqualTo(1);
120-
assertThat(eofToken.column()).isEqualTo(24);
119+
assertThat(eofToken.line()).isEqualTo(100);
120+
assertThat(eofToken.column()).isEqualTo(329);
121121
assertThat(eofToken.includedEscapeChars()).isZero();
122122
}
123123
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/IpynbNotebookParser.java‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131

3232
public class IpynbNotebookParser {
3333

34-
public static final String SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER = "#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER\n";
34+
public static final String SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER = "#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER";
3535

3636
public static GeneratedIPythonFile parseNotebook(PythonInputFile inputFile) {
3737
try {
@@ -50,7 +50,9 @@ private IpynbNotebookParser(PythonInputFile inputFile) {
5050

5151
// Keys are the aggregated source line number
5252
private final Map<Integer, IPythonLocation> locationMap = new HashMap<>();
53-
private int aggregatedSourceLine = 1;
53+
private int aggregatedSourceLine = 0;
54+
private int lastPythonLine = 0;
55+
private boolean isFirstCell = true;
5456

5557
public GeneratedIPythonFile parseNotebook() throws IOException {
5658
String content = inputFile.wrappedFile().contents();
@@ -68,12 +70,15 @@ public GeneratedIPythonFile parseNotebook() throws IOException {
6870
}
6971
}
7072
}
73+
// Account for EOF token
74+
addDefaultLocation(lastPythonLine, jParser.currentTokenLocation());
7175
}
7276

7377
return new GeneratedIPythonFile(inputFile.wrappedFile(), aggregatedSource.toString(), locationMap);
7478
}
7579

7680
private void processCodeCell(JsonParser jParser) throws IOException {
81+
7782
while (!jParser.isClosed()) {
7883
JsonToken jsonToken = jParser.nextToken();
7984
if (JsonToken.FIELD_NAME.equals(jsonToken) && "source".equals(jParser.currentName())) {
@@ -87,26 +92,38 @@ private void processCodeCell(JsonParser jParser) throws IOException {
8792
}
8893
}
8994

95+
private void appendNewLineAfterPreviousCellDelimiter() {
96+
if (!isFirstCell) {
97+
aggregatedSource.append("\n");
98+
} else {
99+
isFirstCell = false;
100+
}
101+
}
102+
90103
private boolean parseSourceArray(JsonParser jParser, JsonToken jsonToken) throws IOException {
91104
if (jsonToken != JsonToken.START_ARRAY) {
92105
return false;
93106
}
107+
appendNewLineAfterPreviousCellDelimiter();
108+
JsonLocation tokenLocation = jParser.currentTokenLocation();
94109
while (jParser.nextToken() != JsonToken.END_ARRAY) {
95110
String sourceLine = jParser.getValueAsString();
96-
var tokenLocation = jParser.currentTokenLocation();
111+
tokenLocation = jParser.currentTokenLocation();
97112
var countEscapedChar = countEscapeCharacters(sourceLine, new LinkedHashMap<>(), tokenLocation.getColumnNr());
98113
addLineToSource(sourceLine, tokenLocation, countEscapedChar);
99114
}
100115
aggregatedSource.append("\n");
101116
// Account for the last cell delimiter
102-
addDelimiterToSource();
117+
addDelimiterToSource(tokenLocation);
118+
lastPythonLine = aggregatedSourceLine;
103119
return true;
104120
}
105121

106122
private boolean parseSourceMultilineString(JsonParser jParser, JsonToken jsonToken) throws IOException {
107123
if (jsonToken != JsonToken.VALUE_STRING) {
108124
return false;
109125
}
126+
appendNewLineAfterPreviousCellDelimiter();
110127
String sourceLine = jParser.getValueAsString();
111128
JsonLocation tokenLocation = jParser.currentTokenLocation();
112129
var previousLen = 0;
@@ -122,7 +139,8 @@ private boolean parseSourceMultilineString(JsonParser jParser, JsonToken jsonTok
122139
previousExtraChars = currentCount;
123140
}
124141
// Account for the last cell delimiter
125-
addDelimiterToSource();
142+
addDelimiterToSource(tokenLocation);
143+
lastPythonLine = aggregatedSourceLine;
126144
return true;
127145
}
128146

@@ -132,13 +150,18 @@ private void addLineToSource(String sourceLine, JsonLocation tokenLocation, Map<
132150

133151
private void addLineToSource(String sourceLine, IPythonLocation location) {
134152
aggregatedSource.append(sourceLine);
135-
locationMap.put(aggregatedSourceLine, location);
136153
aggregatedSourceLine++;
154+
locationMap.put(aggregatedSourceLine, location);
137155
}
138156

139-
private void addDelimiterToSource() {
157+
private void addDelimiterToSource(JsonLocation tokenLocation) {
140158
aggregatedSource.append(SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER);
141159
aggregatedSourceLine++;
160+
addDefaultLocation(aggregatedSourceLine, tokenLocation);
161+
}
162+
163+
private void addDefaultLocation(int line, JsonLocation tokenLocation) {
164+
locationMap.putIfAbsent(line, new IPythonLocation(tokenLocation.getLineNr(), tokenLocation.getColumnNr(), Map.of(-1, 0)));
142165
}
143166

144167
private static Map<Integer, Integer> countEscapeCharacters(String sourceLine, Map<Integer, Integer> colMap, int colOffSet) {

‎sonar-python-plugin/src/test/java/org/sonar/plugins/python/IpynbNotebookParserTest.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ void testParseNotebook() throws IOException {
4040

4141
var result = IpynbNotebookParser.parseNotebook(inputFile);
4242

43-
assertThat(result.locationMap().keySet()).hasSize(20);
43+
assertThat(result.locationMap().keySet()).hasSize(27);
4444
assertThat(result.contents()).hasLineCount(27);
4545
assertThat(StringUtils.countMatches(result.contents(), IpynbNotebookParser.SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER))
4646
.isEqualTo(7);
@@ -52,6 +52,24 @@ void testParseNotebook() throws IOException {
5252
assertThat(result.locationMap()).extracting(map -> map.get(25))
5353
.isEqualTo(new IPythonLocation(90, 15, Map.of(4, 19, 39, 62, 41, 64, 42, 65, 46, 71, -1, 7)));
5454
assertThat(result.locationMap()).extracting(map -> map.get(26)).isEqualTo(new IPythonLocation(90, 71, Map.of(-1, 0)));
55+
// last line with the cell delimiter which contains the EOF token
56+
assertThat(result.locationMap()).extracting(map -> map.get(27)).isEqualTo(new IPythonLocation(90, 14, Map.of(-1, 0)));
57+
}
58+
59+
@Test
60+
void testParseNotebookWithEmptyLines() throws IOException {
61+
var inputFile = createInputFile(baseDir, "notebook_with_empty_lines.ipynb", InputFile.Status.CHANGED, InputFile.Type.MAIN);
62+
63+
var result = IpynbNotebookParser.parseNotebook(inputFile);
64+
65+
assertThat(result.locationMap().keySet()).hasSize(4);
66+
assertThat(result.contents()).hasLineCount(5);
67+
assertThat(StringUtils.countMatches(result.contents(), IpynbNotebookParser.SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER))
68+
.isEqualTo(1);
69+
assertThat(result.locationMap()).extracting(map -> map.get(3)).isEqualTo(new IPythonLocation(11, 5, Map.of(-1, 0)));
70+
71+
// last line with the cell delimiter which contains the EOF token
72+
assertThat(result.locationMap()).extracting(map -> map.get(4)).isEqualTo(new IPythonLocation(11, 5, Map.of(-1, 0)));
5573
}
5674

5775
@Test

‎sonar-python-plugin/src/test/resources/org/sonar/plugins/python/notebook_with_empty_lines.ipynb‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"x = None\n",
10+
"\n",
11+
"\n"
12+
]
13+
}
14+
],
15+
"metadata": {
16+
"kernelspec": {
17+
"display_name": "jupyter-experiment_venv",
18+
"language": "python",
19+
"name": "python3"
20+
},
21+
"language_info": {
22+
"codemirror_mode": {
23+
"name": "ipython",
24+
"version": 3
25+
},
26+
"file_extension": ".py",
27+
"mimetype": "text/x-python",
28+
"name": "python",
29+
"nbconvert_exporter": "python",
30+
"pygments_lexer": "ipython3",
31+
"version": "3.12.2"
32+
}
33+
},
34+
"nbformat": 4,
35+
"nbformat_minor": 2
36+
}

0 commit comments

Comments
 (0)