EFForg · Hainish · Feb 9, 2018 · Feb 8, 2018 · Feb 8, 2018 · Feb 8, 2018
diff --git a/.travis.yml b/.travis.yml
@@ -9,31 +9,31 @@ services:
 matrix:
   fast_finish: true
   include:
-    - env: INFO="lint"
+    - env: TEST="lint"
       language: node_js
       node_js:
         - "node"
-    - env: INFO="unittests"
+    - env: TEST="unittests"
       language: node_js
       node_js: node
         - "node"
-    - env: INFO="rules"
-    - env: INFO="fetch"
-    - env: INFO="preloaded"
+    - env: TEST="validations"
+    - env: TEST="fetch"
+    - env: TEST="preloaded"
     - addons:
         chrome: beta
-      env: INFO="chrome beta" BROWSER=google-chrome-beta
+      env: TEST="chrome beta" BROWSER=google-chrome-beta
     - addons:
         chrome: stable
-      env: INFO="chrome stable" BROWSER=google-chrome-stable
+      env: TEST="chrome stable" BROWSER=google-chrome-stable
     - addons:
         firefox: latest
-      env: INFO="firefox" BROWSER=firefox
+      env: TEST="firefox" BROWSER=firefox
     - addons:
         firefox: latest-esr
-      env: INFO="firefox esr" BROWSER=firefox
+      env: TEST="firefox esr" BROWSER=firefox
 before_install:
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
-before_script: travis_retry ./setup_travis.sh
-script: . ./run_travis.sh
+before_script: travis_retry test/setup_travis.sh
+script: . test/run_travis.sh
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -148,7 +148,7 @@ Avoid using the left-wildcard (`<target host="*.example.com" />`) unless you int
 
 Instead, prefer listing explicit target hosts and a single rewrite from `"^http:"` to `"^https:"`. This saves you time as a ruleset author because each explicit target host automatically creates an implicit test URL, reducing the need to add your own test URLs. These also make it easier for someone reading the ruleset to figure out which subdomains are covered.
 
-If you know all subdomains of a given domain support HTTPS, go ahead and use a left-wildcard, along with a plain rewrite from `"^http:"` to `"^https:"`. Make sure to add a bunch of test URLs for the more important subdomains. 
+If you know all subdomains of a given domain support HTTPS, go ahead and use a left-wildcard, along with a plain rewrite from `"^http:"` to `"^https:"`. Make sure to add a bunch of test URLs for the more important subdomains.
 
 #### Edge-Case: Right-Wildcards
 
@@ -334,7 +334,7 @@ In `utils` we have a tool called `hsts-prune` which removes `targets` from rules
 Every new pull request automatically has the `hsts-prune` utility applied to it as part of the continual integration process.  If a new PR introduces a `target` which is preloaded, it will fail the CI test suite.  See:
 
 - `.travis.yml`
-- `test/travis.sh`
+- `test/run_travis.sh`
 
 * * *
 

diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@ Get the packages you need and install a git hook to run tests before push:
 
     bash install-dev-dependencies.sh
 
-Run all the tests:
+Run the ruleset validations and browser tests:
 
     bash test.sh
 
@@ -50,7 +50,7 @@ This is the source tree for HTTPS Everywhere for Firefox and Chrome.
 
 Important directories you might want to know about
 
-    
+
     chromium/                 WebExtension source code (for Firefox & Chromium/chrome)
     chromium/external         External dependencies
     chromium/test             Unit tests
@@ -60,7 +60,7 @@ Important directories you might want to know about
     src/chrome/content/rules  Ruleset files live here
 
     test/                     Travis unit test source code live here
-    
+
     utils/                    Various utilities (includes some Travis test source)
 
 Hacking on the Source Code

diff --git a/fetch-test.sh b/fetch-test.sh
diff --git a/ruleset-testing.md b/ruleset-testing.md
@@ -53,11 +53,10 @@ Submitting changes to any ruleset that does not meet the coverage requirements
 will break the build. This means that even fixes of existing rules may require
 additional work to bring them up to snuff.
 
-To run the tests locally, you'll need the https-everywhere-checker, which is now
-a submodule of https-everywhere. To set it up, run:
+To run the tests locally, first install the development dependencies:
 
     ./install-dev-dependencies.sh
 
 To test a specific ruleset:
 
-    ./fetch-test.sh rules/Example.xml
+    test/manual.sh rules/Example.xml
diff --git a/run_travis.sh b/run_travis.sh
diff --git a/test.sh b/test.sh
@@ -14,6 +14,6 @@ else
     git rev-parse && cd "$(git rev-parse --show-toplevel)"
 fi
 
-./test/rules.sh
+./test/validations.sh
 ./test/firefox.sh $@
 ./test/chromium.sh $@
diff --git a/test/manual.sh b/test/manual.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+# This script assists ruleset creators in ensuring that there is proper test
+# coverage for their rules, and performs a network "fetch" test to alert the
+# contributor of potential problems.
+
+exec python2.7 test/rules/src/https_everywhere_checker/check_rules.py test/rules/manual.checker.config "$@"
diff --git a/test/run_travis.sh b/test/run_travis.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+set -x
+toplevel="$(git rev-parse --show-toplevel)"
+testdir="${toplevel}/test/selenium"
+srcdir="${toplevel}/chromium"
+linter="${toplevel}/utils/eslint/node_modules/.bin/eslint --ignore-path ${srcdir}/.eslintignore"
+
+
+function run_lint {
+  $linter $srcdir
+  if [ $? != 0 ]; then
+    echo "Linting errors"
+    exit 1
+  fi
+}
+
+function run_unittests {
+  pushd ${srcdir}
+    npm run cover # run with coverage
+    if [ $? != 0 ]; then
+        echo "unittest errors"
+        exit 1
+    fi
+    npm run report
+  popd
+}
+
+function run_selenium {
+  ENABLE_XVFB=1 py.test -v --capture=no ${testdir} # autodiscover and run the tests
+}
+
+if [ "$TEST" == "lint" ]; then
+    echo "running lint tests"
+    run_lint
+elif [ "$TEST" == "unittests" ]; then
+    echo "Running unittests"
+    run_unittests
+elif [ "$TEST" == "validations" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then
+
+    # Folder paths, relative to parent
+    RULESETFOLDER="${toplevel}/src/chrome/content/rules"
+
+    # Go to git repo root; taken from ../test.sh. Note that
+    # $GIT_DIR is .git in this case.
+    if [ -n "$GIT_DIR" ]
+    then
+      # $GIT_DIR is set, so we're running as a hook.
+      cd $GIT_DIR
+      cd ..
+    else
+      # Let's CD to the right place.
+      cd $toplevel
+    fi
+
+    # Fetch the current GitHub version of HTTPS-E to compare to its master
+    git remote add upstream-for-travis https://github.com/EFForg/https-everywhere.git
+    trap 'git remote remove upstream-for-travis' EXIT
+
+    # Only do a shallow fetch if we're in Travis.  No need otherwise.
+    if [ $TRAVIS ]; then
+      git fetch --depth=50 upstream-for-travis master
+    else
+      git fetch upstream-for-travis master
+    fi
+
+    COMMON_BASE_COMMIT=$(git merge-base upstream-for-travis/master HEAD)
+    RULESETS_CHANGED=$(git diff --name-only $COMMON_BASE_COMMIT | grep $RULESETFOLDER | grep '.xml')
+    if [ "$(git diff --name-only $COMMON_BASE_COMMIT)" != "$RULESETS_CHANGED" ]; then
+      ONLY_RULESETS_CHANGED=false
+    fi
+
+    # At this point, if anything fails, the test should fail
+    set -e
+
+    if [ "$TEST" == "validations" ]; then
+      echo >&2 "Performing validations on rulesets."
+      docker run --rm -ti -v $(pwd):/opt httpse bash -c "test/validations.sh"
+    fi
+
+    if [ "$TEST" == "fetch" ]; then
+      echo >&2 "Testing test URLs in all changed rulesets."
+      # --privileged is required here for miredo to create a network tunnel
+      docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" --privileged httpse bash -c "service miredo start && service tor start && test/fetch.sh"
+    fi
+
+    if [ "$TEST" == "preloaded" ]; then
+      echo >&2 "Ensuring rulesets do not introduce targets which are already HSTS preloaded."
+      docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" node bash -c "cd /opt/utils/hsts-prune && npm install && node index.js"
+      [ `git diff --name-only $RULESETFOLDER | wc -l` -eq 0 ]
+    fi
+
+    exit 0
+
+else
+    case $BROWSER in
+      *chrome*)
+        echo "running tests on chrome"
+        run_selenium
+        ;;
+      *firefox*)
+        echo "running tests on firefox"
+        run_selenium
+        ;;
+      *)
+        echo "bad TEST variable, got $TEST"
+        exit 1
+        ;;
+    esac
+fi
diff --git a/setup_travis.sh → test/setup_travis.sh b/setup_travis.sh → test/setup_travis.sh
@@ -46,7 +46,7 @@ function setup_docker {
   docker build -t httpse .
 }
 
-case $INFO in
+case $TEST in
   *chrome*)
     setup_chrome
     browser_setup
@@ -61,13 +61,13 @@ case $INFO in
   *unittests*)
     setup_unittests
     ;;
-  *rules*|*fetch*)
+  *validations*|*fetch*)
     setup_docker
     ;;
   *preloaded*)
     ;;
   *)
-    echo "bad INFO variable, got $INFO"
+    echo "bad TEST variable, got $TEST"
     exit 1
     ;;
 esac
diff --git a/test/travis.sh b/test/travis.sh
diff --git a/test/validations.sh b/test/validations.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+# Perform validations on rulesets.
+
+utils/remove-obsolete-references.sh
+test/validations/path/run.sh
+test/validations/test-coverage/run.sh
+python2.7 test/validations/securecookie/run.py
+python2.7 test/validations/filename/run.py
+python2.7 test/validations/relaxng/run.py
+python2.7 test/validations/special/run.py --quiet
diff --git a/utils/ruleset_filenames_validate.py → test/validations/filename/run.py b/utils/ruleset_filenames_validate.py → test/validations/filename/run.py