Fix problems with the format tasks (#1390)

mindhog · web-flow · commit 7344c424d128 · 2021-10-18T08:10:09.000-04:00
* Fix problems with the format tasks

The format check is using python2, and if "python" doesn't exist on the path
(or isn't python 2, or there is any other error in the python code or in the
shell script...) the format check just succeeds.

This change:
- Refactors out the gradle code that finds a python3 executable and use it
  to get the python executable to be used for the format check.
- Upgrades google-java-format-diff.py to python3 and removes #! line.
- Fixes shell script to ensure that failures are propagated.
- Suppresses error output when checking for python commands.

Tested:
- verified that python errors cause the build to fail
- verified that introducing a bad format diff causes check to fail
- verified that javaIncrementalFormatDryRun shows the diffs that would be
  introduced.
- verified that javaIncrementalFormatApply reformats a file.
- verified that well formatted code passes the format check.
- verified that an invalid or missing PYTHON env var causes
  google-java-format-git-diff.sh to fail with the appropriate error.

* Fix presubmit issues

Omit the format presubmit when not in a git repo and remove unused "string"
import.
diff --git a/build.gradle b/build.gradle
@@ -200,33 +200,39 @@ rootProject.ext {
   pyver = { exe ->
     try {
       ext.execInBash(
-          exe + " -c 'import sys; print(sys.hexversion)'", "/") as Integer
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
     } catch (org.gradle.process.internal.ExecException e) {
       return -1;
     }
   }
-}
-
-task runPresubmits(type: Exec) {
-
-  args('config/presubmits.py')
 
-  doFirst {
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
     // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
     // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
     // Debian so it seems like that should be available anywhere).
     def MIN_PY_VER = 0x3070300
     if (pyver('python') >= MIN_PY_VER) {
-      executable 'python'
+      return 'python'
     } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
-      executable '/usr/bin/python3'
+      return '/usr/bin/python3'
     } else {
       throw new GradleException("No usable Python version found (build " +
                                 "requires at least python 3.7.3)");
     }
   }
 }
 
+task runPresubmits(type: Exec) {
+
+  args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
+}
+
 def javadocSource = []
 def javadocClasspath = []
 def javadocDependentTasks = []
@@ -438,28 +444,34 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
 // Checks if modified lines in Java source files need reformatting.
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
diff --git a/java-format/google-java-format-diff.py b/java-format/google-java-format-diff.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python2.7
 #
 #===- google-java-format-diff.py - google-java-format Diff Reformatter -----===#
 #
@@ -24,7 +23,6 @@
 import argparse
 import difflib
 import re
-import string
 import subprocess
 import io
 import sys
@@ -99,7 +97,7 @@ def main():
     base_command = [binary]
 
   # Reformat files containing changes in place.
-  for filename, lines in lines_by_file.iteritems():
+  for filename, lines in lines_by_file.items():
     if args.i and args.verbose:
       print('Formatting ' + filename)
     command = base_command[:]
@@ -120,11 +118,11 @@ def main():
     if not args.i:
       with open(filename) as f:
         code = f.readlines()
-      formatted_code = io.BytesIO(stdout).readlines()
+      formatted_code = io.StringIO(stdout.decode()).readlines()
       diff = difflib.unified_diff(code, formatted_code,
                                   filename, filename,
                                   '(before formatting)', '(after formatting)')
-      diff_string = string.join(diff, '')
+      diff_string = ''.join(diff)
       if len(diff_string) > 0:
         sys.stdout.write(diff_string)
 
diff --git a/java-format/google-java-format-git-diff.sh b/java-format/google-java-format-git-diff.sh
@@ -42,6 +42,16 @@ where:
 SCRIPT_DIR="$(realpath $(dirname $0))"
 JAR_NAME="google-java-format-1.8-all-deps.jar"
 
+# Make sure we have a valid python interpreter.
+if [ -z "$PYTHON" ]; then
+  echo "You must specify the name of a python3 interpreter in the PYTHON" \
+       "environment variable."
+  exit 1
+elif ! "$PYTHON" -c ''; then
+  echo "Invalid python interpreter: $PYTHON"
+  exit 1
+fi
+
 # Locate the java binary.
 if [ -n "$JAVA_HOME" ]; then
   JAVA_BIN="$JAVA_HOME/bin/java"
@@ -69,10 +79,14 @@ function runGoogleJavaFormatAgainstDiffs() {
   shift
 
   git diff -U0 "$forkPoint" | \
-      ${SCRIPT_DIR}/google-java-format-diff.py \
-      --java-binary "$JAVA_BIN" \
-      --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \
-      -p1 "$@" | tee gjf.out
+      "${PYTHON}" "${SCRIPT_DIR}/google-java-format-diff.py" \
+          --java-binary "$JAVA_BIN" \
+          --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \
+          -p1 "$@" | \
+      tee gjf.out
+
+  # If any of the commands in the last pipe failed, return false.
+  [[ ! "${PIPESTATUS[@]}" =~ [^0\ ] ]]
 }
 
 # Show the file names in a diff preceeded by a message.
@@ -96,7 +110,11 @@ function callGoogleJavaFormatDiff() {
   local callResult
   case "$1" in
     "check")
-      local output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint")
+      # We need to do explicit checks for an error and "exit 1" if there was
+      # one here (though not elsewhere), "set -e" doesn't catch this case,
+      # it's not clear why.
+      local output
+      output=$(runGoogleJavaFormatAgainstDiffs "$forkPoint") || exit 1
       echo "$output" | showFileNames "\033[1mNeeds formatting: "
       callResult=$(echo -n "$output" | wc -l)
       ;;
