Merge pull request microsoft#1367 from Microsoft/tokenRanges

CyrusNajmabadi · CyrusNajmabadi · commit 51804ee91ed6 · 2014-12-04T12:42:57.000-08:00
Simplify scanner by removing need for a 'onComment' callback.
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -945,10 +945,8 @@ module ts {
     }
 
     export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, version: string, isOpen: boolean = false): SourceFile {
-        var scanner: Scanner;
         var token: SyntaxKind;
         var parsingContext: ParsingContext;
-        var commentRanges: TextRange[];
         var identifiers: Map<string> = {};
         var identifierCount = 0;
         var nodeCount = 0;
@@ -1128,10 +1126,6 @@ module ts {
             parseErrorAtPosition(pos, 0, message);
         }
 
-        function onComment(pos: number, end: number) {
-            if (commentRanges) commentRanges.push({ pos: pos, end: end });
-        }
-
         function getNodePos(): number {
             return scanner.getStartPos();
         }
@@ -4201,14 +4195,25 @@ module ts {
         }
 
         function processReferenceComments(): ReferenceComments {
+            var triviaScanner = createScanner(languageVersion, /*skipTrivia*/false, sourceText);
             var referencedFiles: FileReference[] = [];
             var amdDependencies: string[] = [];
             var amdModuleName: string;
-            commentRanges = [];
-            token = scanner.scan();
 
-            for (var i = 0; i < commentRanges.length; i++) {
-                var range = commentRanges[i];
+            // Keep scanning all the leading trivia in the file until we get to something that 
+            // isn't trivia.  Any single line comment will be analyzed to see if it is a 
+            // reference comment.
+            while (true) {
+                var kind = triviaScanner.scan();
+                if (kind === SyntaxKind.WhitespaceTrivia || kind === SyntaxKind.NewLineTrivia || kind === SyntaxKind.MultiLineCommentTrivia) {
+                    continue;
+                }
+                if (kind !== SyntaxKind.SingleLineCommentTrivia) {
+                    break;
+                }
+
+                var range = { pos: triviaScanner.getTokenPos(), end: triviaScanner.getTextPos() };
+
                 var comment = sourceText.substring(range.pos, range.end);
                 var referencePathMatchResult = getFileReferenceFromReferencePath(comment, range);
                 if (referencePathMatchResult) {
@@ -4239,7 +4244,7 @@ module ts {
                     }
                 }
             }
-            commentRanges = undefined;
+
             return {
                 referencedFiles,
                 amdDependencies,
@@ -4275,7 +4280,6 @@ module ts {
             return syntacticDiagnostics;
         }
 
-        scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError, onComment);
         var rootNodeFlags: NodeFlags = 0;
         if (fileExtensionIs(filename, ".d.ts")) {
             rootNodeFlags = NodeFlags.DeclarationFile;
@@ -4301,6 +4305,10 @@ module ts {
         sourceFile.amdDependencies = referenceComments.amdDependencies;
         sourceFile.amdModuleName = referenceComments.amdModuleName;
 
+        // Create and prime the scanner before parsing the source elements.
+        var scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError);
+        nextToken();
+
         sourceFile.statements = parseList(ParsingContext.SourceElements, /*checkForStrictMode*/ true, parseSourceElement);
         Debug.assert(token === SyntaxKind.EndOfFileToken);
         sourceFile.endOfFileToken = parseTokenNode();
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -472,7 +472,7 @@ module ts {
             ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);
     }
 
-    export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner {
+    export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {
         var pos: number;       // Current position (end position of text of current token)
         var len: number;       // Length of text
         var startPos: number;  // Start position of whitespace before current token
@@ -908,9 +908,6 @@ module ts {
                                 pos++;
 
                             }
-                            if (onComment) {
-                                onComment(tokenPos, pos);
-                            }
 
                             if (skipTrivia) {
                                 continue;
@@ -943,10 +940,6 @@ module ts {
                                 error(Diagnostics.Asterisk_Slash_expected);
                             }
 
-                            if (onComment) {
-                                onComment(tokenPos, pos);
-                            }
-
                             if (skipTrivia) {
                                 continue;
                             }

Original file line number	Diff line number	Diff line change
`@@ -472,7 +472,7 @@ module ts {`
`472`	`472`	`ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);`
`473`	`473`	`}`
`474`	`474`
`475`		`- export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner {`
	`475`	`+ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {`
`476`	`476`	`var pos: number; // Current position (end position of text of current token)`
`477`	`477`	`var len: number; // Length of text`
`478`	`478`	`var startPos: number; // Start position of whitespace before current token`
`@@ -908,9 +908,6 @@ module ts {`
`908`	`908`	`pos++;`
`909`	`909`
`910`	`910`	`}`
`911`		`- if (onComment) {`
`912`		`- onComment(tokenPos, pos);`
`913`		`- }`
`914`	`911`
`915`	`912`	`if (skipTrivia) {`
`916`	`913`	`continue;`
`@@ -943,10 +940,6 @@ module ts {`
`943`	`940`	`error(Diagnostics.Asterisk_Slash_expected);`
`944`	`941`	`}`
`945`	`942`
`946`		`- if (onComment) {`
`947`		`- onComment(tokenPos, pos);`
`948`		`- }`
`949`		`-`
`950`	`943`	`if (skipTrivia) {`
`951`	`944`	`continue;`
`952`	`945`	`}`