Simplify scanner by removing need for a 'onComment' callback.

CyrusNajmabadi · CyrusNajmabadi · commit 2510a5f9075d · 2014-12-04T06:40:36.000-08:00
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -951,10 +951,8 @@ module ts {
     }
 
     export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, version: string, isOpen: boolean = false): SourceFile {
-        var scanner: Scanner;
         var token: SyntaxKind;
         var parsingContext: ParsingContext;
-        var commentRanges: TextRange[];
         var identifiers: Map<string> = {};
         var identifierCount = 0;
         var nodeCount = 0;
@@ -1136,10 +1134,6 @@ module ts {
             parseErrorAtPosition(pos, 0, message);
         }
 
-        function onComment(pos: number, end: number) {
-            if (commentRanges) commentRanges.push({ pos: pos, end: end });
-        }
-
         function getNodePos(): number {
             return scanner.getStartPos();
         }
@@ -4173,14 +4167,25 @@ module ts {
         }
 
         function processReferenceComments(): ReferenceComments {
+            var triviaScanner = createScanner(languageVersion, /*skipTrivia*/false, sourceText);
             var referencedFiles: FileReference[] = [];
             var amdDependencies: string[] = [];
             var amdModuleName: string;
-            commentRanges = [];
-            token = scanner.scan();
 
-            for (var i = 0; i < commentRanges.length; i++) {
-                var range = commentRanges[i];
+            // Keep scanning all the leading trivia in the file until we get to something that 
+            // isn't trivia.  Any single line comment will be analyzed to see if it is a 
+            // reference comment.
+            while (true) {
+                var kind = triviaScanner.scan();
+                if (kind === SyntaxKind.WhitespaceTrivia || kind === SyntaxKind.NewLineTrivia || kind === SyntaxKind.MultiLineCommentTrivia) {
+                    continue;
+                }
+                if (kind !== SyntaxKind.SingleLineCommentTrivia) {
+                    break;
+                }
+
+                var range = { pos: triviaScanner.getTokenPos(), end: triviaScanner.getTextPos() };
+
                 var comment = sourceText.substring(range.pos, range.end);
                 var referencePathMatchResult = getFileReferenceFromReferencePath(comment, range);
                 if (referencePathMatchResult) {
@@ -4211,7 +4216,7 @@ module ts {
                     }
                 }
             }
-            commentRanges = undefined;
+
             return {
                 referencedFiles,
                 amdDependencies,
@@ -4247,7 +4252,6 @@ module ts {
             return syntacticDiagnostics;
         }
 
-        scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError, onComment);
         var rootNodeFlags: NodeFlags = 0;
         if (fileExtensionIs(filename, ".d.ts")) {
             rootNodeFlags = NodeFlags.DeclarationFile;
@@ -4273,6 +4277,10 @@ module ts {
         sourceFile.amdDependencies = referenceComments.amdDependencies;
         sourceFile.amdModuleName = referenceComments.amdModuleName;
 
+        // Create and prime the scanner before parsing the source elements.
+        var scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError);
+        nextToken();
+
         sourceFile.statements = parseList(ParsingContext.SourceElements, /*checkForStrictMode*/ true, parseSourceElement);
         Debug.assert(token === SyntaxKind.EndOfFileToken);
         sourceFile.endOfFileToken = parseTokenNode();
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -463,7 +463,7 @@ module ts {
             ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);
     }
 
-    export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner {
+    export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {
         var pos: number;       // Current position (end position of text of current token)
         var len: number;       // Length of text
         var startPos: number;  // Start position of whitespace before current token
@@ -899,9 +899,6 @@ module ts {
                                 pos++;
 
                             }
-                            if (onComment) {
-                                onComment(tokenPos, pos);
-                            }
 
                             if (skipTrivia) {
                                 continue;
@@ -934,10 +931,6 @@ module ts {
                                 error(Diagnostics.Asterisk_Slash_expected);
                             }
 
-                            if (onComment) {
-                                onComment(tokenPos, pos);
-                            }
-
                             if (skipTrivia) {
                                 continue;
                             }

Original file line number	Diff line number	Diff line change
`@@ -463,7 +463,7 @@ module ts {`
`463`	`463`	`ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);`
`464`	`464`	`}`
`465`	`465`
`466`		`- export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner {`
	`466`	`+ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {`
`467`	`467`	`var pos: number; // Current position (end position of text of current token)`
`468`	`468`	`var len: number; // Length of text`
`469`	`469`	`var startPos: number; // Start position of whitespace before current token`
`@@ -899,9 +899,6 @@ module ts {`
`899`	`899`	`pos++;`
`900`	`900`
`901`	`901`	`}`
`902`		`- if (onComment) {`
`903`		`- onComment(tokenPos, pos);`
`904`		`- }`
`905`	`902`
`906`	`903`	`if (skipTrivia) {`
`907`	`904`	`continue;`
`@@ -934,10 +931,6 @@ module ts {`
`934`	`931`	`error(Diagnostics.Asterisk_Slash_expected);`
`935`	`932`	`}`
`936`	`933`
`937`		`- if (onComment) {`
`938`		`- onComment(tokenPos, pos);`
`939`		`- }`
`940`		`-`
`941`	`934`	`if (skipTrivia) {`
`942`	`935`	`continue;`
`943`	`936`	`}`