+ add a custom legacy override: useWhitespaceAfterStartToken

fjn · fjn · commit 92eca3748f1c · 2021-11-09T09:44:33.000+08:00
diff --git a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java
@@ -9,11 +9,13 @@ public class LegacyOverrides {
   private final boolean evaluateMapKeys;
   private final boolean iterateOverMapKeys;
   private final boolean usePyishObjectMapper;
+  private final boolean useWhitespaceAfterStartToken;
 
   private LegacyOverrides(Builder builder) {
     evaluateMapKeys = builder.evaluateMapKeys;
     iterateOverMapKeys = builder.iterateOverMapKeys;
     usePyishObjectMapper = builder.usePyishObjectMapper;
+    useWhitespaceAfterStartToken = builder.useWhitespaceAfterStartToken;
   }
 
   public static Builder newBuilder() {
@@ -32,10 +34,15 @@ public boolean isUsePyishObjectMapper() {
     return usePyishObjectMapper;
   }
 
+  public boolean isUseWhitespaceAfterStartToken(){
+    return useWhitespaceAfterStartToken;
+  }
+
   public static class Builder {
     private boolean evaluateMapKeys = false;
     private boolean iterateOverMapKeys = false;
     private boolean usePyishObjectMapper = false;
+    private boolean useWhitespaceAfterStartToken = false;
 
     private Builder() {}
 
@@ -47,7 +54,8 @@ public static Builder from(LegacyOverrides legacyOverrides) {
       return new Builder()
         .withEvaluateMapKeys(legacyOverrides.evaluateMapKeys)
         .withIterateOverMapKeys(legacyOverrides.iterateOverMapKeys)
-        .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper);
+        .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper)
+        .withUseWhitespaceAfterStartToken(legacyOverrides.useWhitespaceAfterStartToken);
     }
 
     public Builder withEvaluateMapKeys(boolean evaluateMapKeys) {
@@ -64,5 +72,10 @@ public Builder withUsePyishObjectMapper(boolean usePyishObjectMapper) {
       this.usePyishObjectMapper = usePyishObjectMapper;
       return this;
     }
+
+    public Builder withUseWhitespaceAfterStartToken(boolean useWhitespace){
+      this.useWhitespaceAfterStartToken = useWhitespace;
+      return this;
+    }
   }
 }
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
@@ -88,9 +88,13 @@ private Token getNextToken() {
       if (c == symbols.getPrefix()) {
         if (currPost < length) {
           c = is[currPost];
-          boolean hasNextChar = (currPost + 1) < length;
-          boolean nextCharIsWhitespace = hasNextChar && (' ' == is[currPost + 1]);
-          if (nextCharIsWhitespace) {
+          boolean startTokenFound = true;
+          if(config.getLegacyOverrides().isUseWhitespaceAfterStartToken()){
+            boolean hasNextChar = (currPost + 1) < length;
+            boolean nextCharIsWhitespace = hasNextChar && (' ' == is[currPost + 1]);
+            startTokenFound = nextCharIsWhitespace;
+          }
+          if (startTokenFound) {
             if (c == symbols.getNote()) {
               if (inComment == 1 || inRaw == 1) {
                 continue;
diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java
@@ -10,6 +10,8 @@
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.stream.Collectors;
+
+import com.hubspot.jinjava.LegacyOverrides;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -260,8 +262,18 @@ public void testEscapedQuoteWithinAttrValue() {
   }
 
   @Test
-  public void testCommentWith$Char(){
+  public void testCommentWithWhitespaceChar(){
     List<Token> tokens = tokens("comment-without-whitespace");
+    assertThat(tokens.get(0).content.trim()).isEqualTo("$");
+
+    LegacyOverrides legacyOverrides = LegacyOverrides.newBuilder()
+            .withUseWhitespaceAfterStartToken(true)
+            .build();
+    JinjavaConfig config = JinjavaConfig.newBuilder()
+            .withLegacyOverrides(legacyOverrides)
+            .build();
+    TokenScanner scanner = fixture("comment-without-whitespace", config);
+    tokens = Lists.newArrayList(scanner);
     assertThat(tokens.get(0).content.trim()).isEqualTo("${#array[@]}");
   }
 
@@ -298,13 +310,17 @@ private List<Token> tokens(String fixture) {
   }
 
   private TokenScanner fixture(String fixture) {
+    return fixture(fixture, config);
+  }
+
+  private TokenScanner fixture(String fixture, JinjavaConfig config) {
     try {
       return new TokenScanner(
-        Resources.toString(
-          Resources.getResource(String.format("parse/tokenizer/%s.jinja", fixture)),
-          StandardCharsets.UTF_8
-        ),
-        config
+              Resources.toString(
+                      Resources.getResource(String.format("parse/tokenizer/%s.jinja", fixture)),
+                      StandardCharsets.UTF_8
+              ),
+              config
       );
     } catch (Exception e) {
       throw new RuntimeException(e);
