new approach for snippet injection (googleapis#2838)

pongad · web-flow · commit aa23ce4917e4 · 2018-07-09T14:06:12.000-07:00
The current snippet injector does not work properly with google-java-format, because GJF formats short javadoc comments on one line, eg "/** comment */". However, the injector script looks for "/**" on a line by itself. The script will also not work if/when we move to Java 8, due to lack of parser support. This PR takes a different approach of not caring about Java syntax and copy-paste everything in the SNIPPET block. While less powerful, it is more robust. As written, the script is also easier to use. There's no need to tell it what file contains snippets and where to copy the snippets to. The script recursively scan given directories. Updates googleapis#2413. * license * Add test case for getSnip * Add support for cloud region tags to snippet.go
diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java
@@ -753,13 +753,19 @@ public int hashCode() {
    * Updates dataset information.
    *
    * <p>Example of updating a dataset by changing its description.
-   * <pre> {@code
-   * String datasetName = "my_dataset_name";
-   * String newDescription = "some_new_description";
-   * Dataset oldDataset = bigquery.getDataset(datasetName);
-   * DatasetInfo datasetInfo = oldDataset.toBuilder().setDescription(newDescription).build();
-   * Dataset newDataset = bigquery.update(datasetInfo);
-   * }</pre>
+   * <!--SNIPPET bigquery_update_table_description-->
+   * <pre>{@code
+   *    // String datasetName = "my_dataset_name";
+   *    // String tableName = "my_table_name";
+   *    // String newDescription = "new_description";
+   *
+   *    Table beforeTable = bigquery.getTable(datasetName, tableName);
+   *    TableInfo tableInfo = beforeTable.toBuilder()
+   *        .setDescription(newDescription)
+   *        .build();
+   *    Table afterTable = bigquery.update(tableInfo);
+   *     }</pre>
+   * <!--SNIPPET bigquery_update_table_description-->
    *
    * @throws BigQueryException upon failure
    */
@@ -785,7 +791,7 @@ public int hashCode() {
    * String datasetName = "my_dataset_name";
    * String tableName = "my_table_name";
    * Table beforeTable = bigquery.getTable(datasetName, tableName);
-   * 
+   *
    * // Set table to expire 5 days from now.
    * long expirationMillis = DateTime.now().plusDays(5).getMillis();
    * TableInfo tableInfo = beforeTable.toBuilder()
@@ -1104,7 +1110,7 @@ TableResult listTableData(
    * // BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
    * String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
    * QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query).build();
-   * 
+   *
    * // Print the results.
    * for (FieldValueList row : bigquery.query(queryConfig).iterateAll()) {
    *   for (FieldValue val : row) {
diff --git a/google-cloud-clients/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/MessageReceiver.java b/google-cloud-clients/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/MessageReceiver.java
@@ -22,27 +22,31 @@
 public interface MessageReceiver {
   /**
    * Called when a message is received by the subscriber. The implementation must arrange for {@link
-   * AckReplyConsumer#ack()} or {@link
-   * AckReplyConsumer#nack()} to be called after processing the {@code message}.
+   * AckReplyConsumer#ack()} or {@link AckReplyConsumer#nack()} to be called after processing the
+   * {@code message}.
+   * <!--SNIPPET receiveMessage-->
    *
-   * <p>This {@code MessageReceiver} passes all messages to a {@code BlockingQueue}.
-   * This method can be called concurrently from multiple threads,
-   * so it is important that the queue be thread-safe.
+   * <pre>{@code
+   * // This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}. This method can
+   * // be called concurrently from multiple threads, so it is important that the queue be
+   * // thread-safe.
+   * //
+   * // This example is for illustration. Implementations may directly process messages instead of
+   * // sending them to queues.
+   * MessageReceiver receiver =
+   *     new MessageReceiver() {
+   *       public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
+   *         if (blockingQueue.offer(message)) {
+   *           consumer.ack();
+   *         } else {
+   *           consumer.nack();
+   *         }
+   *       }
+   *     };
    *
-   * This example is for illustration. Implementations may directly process messages
-   * instead of sending them to queues.
-   * <pre> {@code
-   * MessageReceiver receiver = new MessageReceiver() {
-   *   public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
-   *     if (blockingQueue.offer(message)) {
-   *       consumer.ack();
-   *     } else {
-   *       consumer.nack();
-   *     }
-   *   }
-   * };
    * }</pre>
    *
+   * <!--SNIPPET receiveMessage-->
    */
   void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer);
 }
diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java
@@ -121,12 +121,12 @@ public Dataset updateDataset(String datasetName, String newDescription) {
   /**
    * Example of updating a table by changing its description.
    */
-  // [TARGET update(TableInfo, TableOption...)]
-  // [VARIABLE "my_dataset_name"]
-  // [VARIABLE "my_table_name"]
-  // [VARIABLE "new_description"]
   public Table updateTableDescription(String datasetName, String tableName, String newDescription) {
     // [START bigquery_update_table_description]
+    // String datasetName = "my_dataset_name";
+    // String tableName = "my_table_name";
+    // String newDescription = "new_description";
+
     Table beforeTable = bigquery.getTable(datasetName, tableName);
     TableInfo tableInfo = beforeTable.toBuilder()
         .setDescription(newDescription)
diff --git a/google-cloud-examples/src/main/java/com/google/cloud/examples/pubsub/snippets/MessageReceiverSnippets.java b/google-cloud-examples/src/main/java/com/google/cloud/examples/pubsub/snippets/MessageReceiverSnippets.java
@@ -36,25 +36,25 @@ public MessageReceiverSnippets(BlockingQueue<PubsubMessage> blockingQueue) {
     this.blockingQueue = blockingQueue;
   }
 
-  /**
-   * This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}.
-   * This method can be called concurrently from multiple threads,
-   * so it is important that the queue be thread-safe.
-   *
-   * This example is for illustration. Implementations may directly process messages
-   * instead of sending them to queues.
-   */
-  // [TARGET receiveMessage(PubsubMessage, AckReplyConsumer)]
   public MessageReceiver messageReceiver() {
-    MessageReceiver receiver = new MessageReceiver() {
-      public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
-        if (blockingQueue.offer(message)) {
-          consumer.ack();
-        } else {
-          consumer.nack();
-        }
-      }
-    };
+    // SNIPPET receiveMessage
+    // This {@code MessageReceiver} passes all messages to a {@link BlockingQueue}. This method can
+    // be called concurrently from multiple threads, so it is important that the queue be
+    // thread-safe.
+    //
+    // This example is for illustration. Implementations may directly process messages instead of
+    // sending them to queues.
+    MessageReceiver receiver =
+        new MessageReceiver() {
+          public void receiveMessage(final PubsubMessage message, final AckReplyConsumer consumer) {
+            if (blockingQueue.offer(message)) {
+              consumer.ack();
+            } else {
+              consumer.nack();
+            }
+          }
+        };
+    // SNIPPET receiveMessage
     return receiver;
   }
 }
diff --git a/utilities/snippets.go b/utilities/snippets.go
@@ -0,0 +1,230 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime/pprof"
+	"strings"
+)
+
+func init() {
+	log.SetFlags(0)
+	log.SetPrefix("snippet: ")
+}
+
+func main() {
+	cpuprof := flag.String("cpuprofile", "", "write CPU profile to this file")
+	flag.Parse()
+
+	if cp := *cpuprof; cp != "" {
+		f, err := os.Create(cp)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	files := map[string]string{}
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.Mode().IsRegular() || filepath.Ext(path) != ".java" {
+			return nil
+		}
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		files[path] = string(b)
+		return nil
+	}
+	for _, dir := range flag.Args() {
+		if err := filepath.Walk(dir, walkFn); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	snip := map[string]string{}
+	for file, txt := range files {
+		if err := getSnip(file, txt, snip); err != nil {
+			log.Fatal(err)
+		}
+		if err := getCloud(file, txt, snip); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	rd := rewriteData{
+		rewrite: map[string]string{},
+		used:    map[string]bool{},
+	}
+	for file, txt := range files {
+		if err := writeSnip(file, txt, snip, rd); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for file, txt := range rd.rewrite {
+		if err := ioutil.WriteFile(file, []byte(txt), 0644); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for key := range snip {
+		if !rd.used[key] {
+			log.Printf("unused snippet: %q", key)
+		}
+	}
+}
+
+func getCloud(file, txt string, snip map[string]string) error {
+	const cloudPrefix = "// [START "
+	const cloudSuffix = "// [END %s]"
+
+	ftxt := txt
+	for {
+		if p := strings.Index(txt, cloudPrefix); p >= 0 {
+			txt = txt[p:]
+		} else {
+			return nil
+		}
+
+		var tag string
+		if p := strings.Index(txt, "]\n"); p >= 0 {
+			// "// [START foo]" -> "foo"
+			tag = txt[10:p]
+			txt = txt[p+1:]
+		} else {
+			tag = txt
+			txt = ""
+		}
+
+		endTag := fmt.Sprintf(cloudSuffix, tag)
+		if p := strings.Index(txt, endTag); p >= 0 {
+			key := fmt.Sprintf("<!--SNIPPET %s-->", tag)
+			snipTxt := strings.Trim(txt[:p], "\n\r")
+			if _, exist := snip[key]; exist {
+				snip[key] = strings.Join([]string{snip[key], snipTxt}, "")
+			}
+
+			snip[key] = snipTxt
+			txt = txt[p+len(endTag):]
+		} else {
+			return fmt.Errorf("[START %s]:%d snippet %q not closed", file, lineNum(ftxt, txt), tag)
+		}
+	}
+}
+
+func getSnip(file, txt string, snip map[string]string) error {
+	const snipPrefix = "// SNIPPET "
+
+	ftxt := txt
+	for {
+		if p := strings.Index(txt, snipPrefix); p >= 0 {
+			txt = txt[p:]
+		} else {
+			return nil
+		}
+
+		var key string
+		if p := strings.IndexByte(txt, '\n'); p >= 0 {
+			key = txt[:p]
+			txt = txt[p:]
+		} else {
+			key = txt
+			txt = ""
+		}
+
+		if p := strings.Index(txt, key); p >= 0 {
+			// "// SNIPPET foo" -> "<!--SNIPPET foo-->"
+			key = fmt.Sprintf("<!--%s-->", strings.TrimSpace(key[3:]))
+
+			if _, exist := snip[key]; exist {
+				return fmt.Errorf("%s:%d snippet %q has already been defined", file, lineNum(ftxt, txt), key)
+			}
+
+			snip[key] = strings.Trim(txt[:p], "\n\r")
+			txt = txt[p+len(snipPrefix):]
+		} else {
+			return fmt.Errorf("%s:%d snippet %q not closed", file, lineNum(ftxt, txt), key)
+		}
+	}
+}
+
+type rewriteData struct {
+	rewrite map[string]string
+	used    map[string]bool
+}
+
+func writeSnip(file, txt string, snip map[string]string, rd rewriteData) error {
+	const snipPrefix = "<!--SNIPPET "
+
+	ftxt := txt
+	var buf bytes.Buffer
+	for {
+		if p := strings.Index(txt, snipPrefix); p >= 0 {
+			buf.WriteString(txt[:p])
+			txt = txt[p:]
+		} else if buf.Len() == 0 {
+			return nil
+		} else {
+			buf.WriteString(txt)
+			rd.rewrite[file] = buf.String()
+			return nil
+		}
+
+		var key string
+		if p := strings.Index(txt, "-->"); p >= 0 {
+			key = txt[:p+3]
+			txt = txt[p+3:]
+		}
+
+		rep, ok := snip[key]
+		if ok {
+			rd.used[key] = true
+		} else {
+			return fmt.Errorf("%s:%d snippet target %q undefined", file, lineNum(ftxt, txt), key)
+		}
+
+		if p := strings.Index(txt, key); p >= 0 {
+			buf.WriteString(key)
+			buf.WriteString("\n   * <pre>{@code\n   *")
+			buf.WriteString(strings.Replace(rep, "\n", "\n   *", -1))
+			buf.WriteString(" }</pre>\n   * ")
+			buf.WriteString(key)
+			txt = txt[p+len(key):]
+		} else {
+			return fmt.Errorf("%s:%d snippet target %q not closed", file, lineNum(ftxt, txt), key)
+		}
+	}
+}
+
+// Give strings s and suf where suf is a suffix of s, lineNum reports
+// the line number on which suf starts.
+func lineNum(s, suf string) int {
+	pre := s[:len(s)-len(suf)]
+	return strings.Count(pre, "\n") + 1
+}
diff --git a/utilities/snippets_test.go b/utilities/snippets_test.go
