Move to Python 3. Python3.6 causing multiple issues with newer Ubuntu versions >= 18.04 (EFForg#19987)

zoracon · web-flow · commit 32a5b370158b · 2021-03-27T13:45:58.000-07:00
Fixes EFForg#19023, EFForg#18951
diff --git a/browser-dist/edge.sh b/browser-dist/edge.sh
@@ -2,7 +2,7 @@
 # Written for transparency and reproducibility on Edge upload
 # See browser-dist.md for more info
 
-VERSION=`python3.6 -c "import json ; print(json.loads(open('../chromium/manifest.json').read())['version'])"`
+VERSION=`python3 -c "import json ; print(json.loads(open('../chromium/manifest.json').read())['version'])"`
 crx_cws="../pkg/https-everywhere-$VERSION-cws.crx"
 crx_eff="../pkg/https-everywhere-$VERSION-eff.crx"
 
diff --git a/browser-dist/opera.sh b/browser-dist/opera.sh
@@ -84,7 +84,7 @@ if [ -n "$1" ]; then
   git submodule update --recursive -f
 fi
 
-VERSION=`python3.6 -c "import json ; print(json.loads(open('../chromium/manifest.json').read())['version'])"`
+VERSION=`python3 -c "import json ; print(json.loads(open('../chromium/manifest.json').read())['version'])"`
 
 echo "Building version" $VERSION
 
@@ -100,8 +100,8 @@ cp -a ../../chromium/* ./
 # Turn the Firefox translations into the appropriate Chrome format:
 rm -rf _locales/
 mkdir _locales/
-python3.6 ../../utils/chromium-translations.py ../../translations/ _locales/
-python3.6 ../../utils/chromium-translations.py ../../src/chrome/locale/ _locales/
+python3 ../../utils/chromium-translations.py ../../translations/ _locales/
+python3 ../../utils/chromium-translations.py ../../src/chrome/locale/ _locales/
 do_not_ship="*.py *.xml"
 rm -f $do_not_ship
 
@@ -111,7 +111,7 @@ cp ../../lib-wasm/pkg/*.js wasm
 
 cd ../..
 
-python3.6 ./utils/merge-rulesets.py || exit 5
+python3 ./utils/merge-rulesets.py || exit 5
 
 cp src/chrome/content/rules/default.rulesets.json pkg/crx-opera/rules/default.rulesets.json
 
@@ -128,7 +128,7 @@ cd ../..
 
 # Remove the 'applications' manifest key from the crx version of the extension, change the 'author' string to a hash, and add the "update_url" manifest key
 # "update_url" needs to be present to avoid problems reported in https://bugs.chromium.org/p/chromium/issues/detail?id=805755
-python3.6 -c "import json; m=json.loads(open('pkg/crx-opera/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; open('pkg/crx-opera/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+python3 -c "import json; m=json.loads(open('pkg/crx-opera/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; open('pkg/crx-opera/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
 
 # If the --remove-update-channels flag is set, remove all out-of-band update channels
 if $REMOVE_UPDATE_CHANNELS; then
diff --git a/docs/en_US/ruleset-update-channels.md b/docs/en_US/ruleset-update-channels.md
@@ -123,7 +123,7 @@ You will need python 3.6 on your system or available via docker for the next ste
 
 Next, run
 
-    python3.6 utils/merge-rulesets.py
+    python3 utils/merge-rulesets.py
 
 You should see the following output:
 
diff --git a/hooks/precommit b/hooks/precommit
@@ -39,7 +39,7 @@ if [ "$CHANGED_RULESETS" ]; then
             continue
         fi
 
-        python3.6 test/validations/special/run.py --quiet $FILE
+        python3 test/validations/special/run.py --quiet $FILE
         RESULT=$?
 
         if [ $RESULT -ne 0 ]; then
diff --git a/install-dev-dependencies.sh b/install-dev-dependencies.sh
@@ -41,7 +41,7 @@ if type apt-get>/dev/null 2>&1;  then
     CHROMEDRIVER="chromium-driver"
   fi
   $SUDO_SHIM apt-get install -y libxml2-dev libxml2-utils libxslt1-dev \
-    python3.6-dev $BROWSERS zip sqlite3 python3-pip libcurl4-openssl-dev xvfb \
+    python3-dev $BROWSERS zip sqlite3 python3-pip libcurl4-openssl-dev xvfb \
     nodejs \
     npm \
     libssl-dev git curl $CHROMEDRIVER
diff --git a/make.sh b/make.sh
@@ -85,7 +85,7 @@ if [ -n "$1" ]; then
   git submodule update --recursive -f
 fi
 
-VERSION=`python3.6 -c "import json ; print(json.loads(open('chromium/manifest.json').read())['version'])"`
+VERSION=`python3 -c "import json ; print(json.loads(open('chromium/manifest.json').read())['version'])"`
 
 echo "Building version" $VERSION
 
@@ -104,8 +104,8 @@ cp -a ../../chromium/* ./
 # Turn the Firefox translations into the appropriate Chrome format:
 rm -rf _locales/
 mkdir _locales/
-python3.6 ../../utils/chromium-translations.py ../../translations/ _locales/
-python3.6 ../../utils/chromium-translations.py ../../src/chrome/locale/ _locales/
+python3 ../../utils/chromium-translations.py ../../translations/ _locales/
+python3 ../../utils/chromium-translations.py ../../src/chrome/locale/ _locales/
 do_not_ship="*.py *.xml"
 rm -f $do_not_ship
 
@@ -115,7 +115,7 @@ cp ../../lib-wasm/pkg/*.js wasm
 
 cd ../..
 
-python3.6 ./utils/merge-rulesets.py || exit 5
+python3 ./utils/merge-rulesets.py || exit 5
 
 cp src/chrome/content/rules/default.rulesets pkg/crx-cws/rules/default.rulesets
 
@@ -134,16 +134,16 @@ cp -a src/META-INF pkg/xpi-eff
 
 # Remove the 'applications' manifest key from the crx version of the extension, change the 'author' string to a hash, and add the "update_url" manifest key
 # "update_url" needs to be present to avoid problems reported in https://bugs.chromium.org/p/chromium/issues/detail?id=805755
-python3.6 -c "import json; m=json.loads(open('pkg/crx-cws/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; m['update_url'] = 'https://clients2.google.com/service/update2/crx'; open('pkg/crx-cws/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
-python3.6 -c "import json; m=json.loads(open('pkg/crx-eff/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; open('pkg/crx-eff/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+python3 -c "import json; m=json.loads(open('pkg/crx-cws/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; m['update_url'] = 'https://clients2.google.com/service/update2/crx'; open('pkg/crx-cws/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+python3 -c "import json; m=json.loads(open('pkg/crx-eff/manifest.json').read()); m['author']={'email': 'eff.software.projects@gmail.com'}; del m['applications']; open('pkg/crx-eff/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
 # Remove the 'update_url' manifest key from the xpi version of the extension delivered to AMO
-python3.6 -c "import json; m=json.loads(open('pkg/xpi-amo/manifest.json').read()); del m['applications']['gecko']['update_url']; m['applications']['gecko']['id'] = 'https-everywhere@eff.org'; open('pkg/xpi-amo/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+python3 -c "import json; m=json.loads(open('pkg/xpi-amo/manifest.json').read()); del m['applications']['gecko']['update_url']; m['applications']['gecko']['id'] = 'https-everywhere@eff.org'; open('pkg/xpi-amo/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
 
 # If the --remove-extension-update flag is set, ensure the extension is unable to update
 if $REMOVE_EXTENSION_UPDATE; then
   echo "Flag --remove-extension-update specified.  Removing the XPI extensions' ability to update."
-  python3.6 -c "import json; m=json.loads(open('pkg/xpi-amo/manifest.json').read()); m['applications']['gecko']['update_url'] = 'https://127.0.0.1'; open('pkg/xpi-amo/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
-  python3.6 -c "import json; m=json.loads(open('pkg/xpi-eff/manifest.json').read()); m['applications']['gecko']['update_url'] = 'https://127.0.0.1'; open('pkg/xpi-eff/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+  python3 -c "import json; m=json.loads(open('pkg/xpi-amo/manifest.json').read()); m['applications']['gecko']['update_url'] = 'https://127.0.0.1'; open('pkg/xpi-amo/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
+  python3 -c "import json; m=json.loads(open('pkg/xpi-eff/manifest.json').read()); m['applications']['gecko']['update_url'] = 'https://127.0.0.1'; open('pkg/xpi-eff/manifest.json','w').write(json.dumps(m,indent=4,sort_keys=True))"
 fi
 
 # If the --remove-update-channels flag is set, remove all out-of-band update channels
diff --git a/test/chromium.sh b/test/chromium.sh
@@ -37,5 +37,5 @@ else
 	./make.sh
 	echo "running tests"
 	CRX_NAME="`ls -tr pkg/*.crx | tail -1`"
-	$XVFB_RUN python3.6 test/script.py Chrome $CRX_NAME
+	$XVFB_RUN python3 test/script.py Chrome $CRX_NAME
 fi
diff --git a/test/fetch.sh b/test/fetch.sh
@@ -19,7 +19,7 @@ if [ "$TO_BE_TESTED" ]; then
   # Do the actual test, using https-everywhere-checker.
   OUTPUT_FILE=`mktemp`
   trap 'rm "$OUTPUT_FILE"' EXIT
-  python3.6 $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE
+  python3 $RULETESTFOLDER/src/https_everywhere_checker/check_rules.py $RULETESTFOLDER/http.checker.config $TO_BE_TESTED 2>&1 | tee $OUTPUT_FILE
   # Unfortunately, no specific exit codes are available for connection
   # failures, so we catch those with grep.
   if [[ `cat $OUTPUT_FILE | grep ERROR | wc -l` -ge 1 ]]; then
diff --git a/test/firefox.sh b/test/firefox.sh
@@ -58,8 +58,8 @@ else
 
   PATH=/home/user/geckodriver:$PATH
   if [ -n "$FIREFOX" ]; then
-    $XVFB_RUN python3.6 test/script.py Firefox "$PROFILE_DIRECTORY" $FIREFOX
+    $XVFB_RUN python3 test/script.py Firefox "$PROFILE_DIRECTORY" $FIREFOX
   else
-    $XVFB_RUN python3.6 test/script.py Firefox "$PROFILE_DIRECTORY"
+    $XVFB_RUN python3 test/script.py Firefox "$PROFILE_DIRECTORY"
   fi
 fi
diff --git a/test/manual.sh b/test/manual.sh
@@ -3,4 +3,4 @@
 # coverage for their rules, and performs a network "fetch" test to alert the
 # contributor of potential problems.
 
-exec python3.6 test/rules/src/https_everywhere_checker/check_rules.py test/rules/manual.checker.config "$@"
+exec python3 test/rules/src/https_everywhere_checker/check_rules.py test/rules/manual.checker.config "$@"
diff --git a/test/rules/src/https_everywhere_checker/check_rules.py b/test/rules/src/https_everywhere_checker/check_rules.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 import binascii
 import argparse
diff --git a/test/script.py b/test/script.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 #
 # Run Selenium tests for HTTPS Everywhere
 #
-# This script may be executed as `python3.6 script.py [directory of CRX]`
+# This script may be executed as `python3 script.py [directory of CRX]`
 #
 # The script is compatible with Python 3.6.
 # Selenium, WebDriver and Google Chrome (or Chromium) must be installed
@@ -41,7 +41,7 @@ class bcolors:
 
     try:
         # First argument is optional, if not specified will search path.
-        driver = webdriver.Chrome(chromedriver_path, chrome_options=chromeOps)
+        driver = webdriver.Chrome(chromedriver_path, options=chromeOps)
     except WebDriverException as e:
         error = e.__str__()
 
diff --git a/test/validations.sh b/test/validations.sh
@@ -4,7 +4,7 @@
 utils/remove-obsolete-references.sh
 test/validations/path/run.sh
 test/validations/test-coverage/run.sh
-python3.6 test/validations/securecookie/run.py
-python3.6 test/validations/filename/run.py
-python3.6 test/validations/relaxng/run.py
-python3.6 test/validations/special/run.py --quiet
+python3 test/validations/securecookie/run.py
+python3 test/validations/filename/run.py
+python3 test/validations/relaxng/run.py
+python3 test/validations/special/run.py --quiet
diff --git a/test/validations/filename/run.py b/test/validations/filename/run.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 #
 # Validates and provides a generator for ruleset filenames
 #
diff --git a/test/validations/relaxng/run.py b/test/validations/relaxng/run.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 # -*- encoding: utf-8 -*-
 
 import argparse
diff --git a/test/validations/securecookie/run.py b/test/validations/securecookie/run.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 # This python utility check for wildcard securecookies which
 # can be normalized, warn and exit with non-zero when such
diff --git a/test/validations/special/duplicate-allowlist-cleanup.sh b/test/validations/special/duplicate-allowlist-cleanup.sh
@@ -4,7 +4,7 @@ TMPFILE=`mktemp /tmp/buffer.XXXXXXXX`
 trap 'rm "$TMPFILE"' EXIT
 
 for host in `cat test/validations/special/duplicate-allowlist.txt`; do
-    REGEX_ESCAPED_HOST=`python3.6 -c "import re; print(re.escape('$host'))"`
+    REGEX_ESCAPED_HOST=`python3 -c "import re; print(re.escape('$host'))"`
     REPEATS=`egrep -l "<target\s+host=\s*\"$REGEX_ESCAPED_HOST\"\s*/>" src/chrome/content/rules/*.xml | wc -l`
     if [ $REPEATS -gt 1 ]; then
         echo $host
diff --git a/test/validations/special/run.py b/test/validations/special/run.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 import glob
 import argparse
diff --git a/test/validations/test-coverage/run.sh b/test/validations/test-coverage/run.sh
@@ -24,10 +24,10 @@ if ! [ -d test/rules ] ; then
   exit 1
 fi
 if [ $# -gt 0 ] ; then
-  exec python3.6 test/rules/src/https_everywhere_checker/check_rules.py \
+  exec python3 test/rules/src/https_everywhere_checker/check_rules.py \
     test/rules/coverage.checker.config "$@"
 fi
-if ! python3.6 test/rules/src/https_everywhere_checker/check_rules.py \
+if ! python3 test/rules/src/https_everywhere_checker/check_rules.py \
       test/rules/coverage.checker.config; then
   echo '
 Ruleset test coverage was insufficient.
diff --git a/utils/chromium-translations.py b/utils/chromium-translations.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 """
 Given two directories, copy Firefox-style https-everywhere.dtd translations from
 the first directory into appropriately-named Chromium-style messages.json
diff --git a/utils/create-platform-certs/create_platform_certs.sh b/utils/create-platform-certs/create_platform_certs.sh
@@ -17,7 +17,7 @@ git rm -r -f -q "${SPLIT_CERT_DIR}"
 
 perl "${MK_CA_BUNDLE_PL_EXEC}" -n "${COMBINED_CERT_FILE}"
 
-python3.6 "${SPLIT_COMBINED_CERT_FILE_EXEC}" "${COMBINED_CERT_FILE}" "${SPLIT_CERT_DIR}"
+python3 "${SPLIT_COMBINED_CERT_FILE_EXEC}" "${COMBINED_CERT_FILE}" "${SPLIT_CERT_DIR}"
 
 c_rehash "${SPLIT_CERT_DIR}" > /dev/null
 
diff --git a/utils/create-platform-certs/split_combined_cert_file.py b/utils/create-platform-certs/split_combined_cert_file.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 import argparse
 import codecs
 import os
diff --git a/utils/create_zip.py b/utils/create_zip.py
@@ -1,10 +1,10 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 # Uses the Python zip implementation to create deterministic zip's
 # Author: Yan Zhu, yan@mit.edu
 
 """
-Usage: python3.6 create_zip.py -x <exclusions> -n <name of zipped file> <directory>
+Usage: python3 create_zip.py -x <exclusions> -n <name of zipped file> <directory>
 """
 
 import argparse
diff --git a/utils/merge-rulesets.py b/utils/merge-rulesets.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 # Merge all the .xml rulesets into a single "default.rulesets" file -- this
 # prevents inodes from wasting disk space, but more importantly, this works
diff --git a/utils/setversion.py b/utils/setversion.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 from datetime import date
 import json
 
diff --git a/utils/standalone-translations.py b/utils/standalone-translations.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 """
 Generate a messages.json translations file for HTTPS Everywhere Standalone
 """
diff --git a/utils/zipfile_deterministic.py b/utils/zipfile_deterministic.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 """
 A fork of the Python 3.6 zipfile module, modified to remove sources of entropy
 and ensure bit-by-bit determinism for the output file.
