From 5ac2149b9e5793024de667acf22874f755fbfeee Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 13:39:48 -0800 Subject: [PATCH 1/6] s/INFO/test --- .travis.yml | 18 +++++++++--------- run_travis.sh | 9 ++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29d23029aa2e..f2ad846157e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,29 +9,29 @@ services: matrix: fast_finish: true include: - - env: INFO="lint" + - env: TEST="lint" language: node_js node_js: - "node" - - env: INFO="unittests" + - env: TEST="unittests" language: node_js node_js: node - "node" - - env: INFO="rules" - - env: INFO="fetch" - - env: INFO="preloaded" + - env: TEST="rules" + - env: TEST="fetch" + - env: TEST="preloaded" - addons: chrome: beta - env: INFO="chrome beta" BROWSER=google-chrome-beta + env: TEST="chrome beta" BROWSER=google-chrome-beta - addons: chrome: stable - env: INFO="chrome stable" BROWSER=google-chrome-stable + env: TEST="chrome stable" BROWSER=google-chrome-stable - addons: firefox: latest - env: INFO="firefox" BROWSER=firefox + env: TEST="firefox" BROWSER=firefox - addons: firefox: latest-esr - env: INFO="firefox esr" BROWSER=firefox + env: TEST="firefox esr" BROWSER=firefox before_install: - export DISPLAY=:99.0 - sh -e /etc/init.d/xvfb start diff --git a/run_travis.sh b/run_travis.sh index 7680cbbbe9c6..50d561f268b5 100755 --- a/run_travis.sh +++ b/run_travis.sh @@ -29,14 +29,13 @@ function run_selenium { ENABLE_XVFB=1 py.test -v --capture=no ${testdir} # autodiscover and run the tests } -if [ "$INFO" == "lint" ]; then +if [ "$TEST" == "lint" ]; then echo "running lint tests" run_lint -elif [ "$INFO" == "unittests" ]; then +elif [ "$TEST" == "unittests" ]; then echo "Running unittests" run_unittests -elif [ "$INFO" == "rules" ] || [ "$INFO" == "fetch" ] || [ "$INFO" == "preloaded" ]; then - export TEST=${INFO} +elif [ "$TEST" == "rules" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then ${toplevel}/test/travis.sh # run old travis tests else case $BROWSER in @@ -49,7 +48,7 @@ else run_selenium ;; *) - echo "bad INFO variable, got $INFO" + echo "bad TEST variable, got $TEST" exit 1 ;; esac From 017a5fe38000e7fd3642c50f550517ad07499781 Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 13:52:53 -0800 Subject: [PATCH 2/6] Naming should be more sensible --- README.md | 6 ++--- test.sh | 2 +- test/rules.sh | 54 ++++++++----------------------------------- test/test-coverage.sh | 45 ++++++++++++++++++++++++++++++++++++ test/travis.sh | 10 ++------ 5 files changed, 60 insertions(+), 57 deletions(-) mode change 100755 => 100644 test/rules.sh create mode 100755 test/test-coverage.sh diff --git a/README.md b/README.md index 330fc8e5d746..978b72d22de7 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Get the packages you need and install a git hook to run tests before push: bash install-dev-dependencies.sh -Run all the tests: +Run the test coverage and browser tests: bash test.sh @@ -50,7 +50,7 @@ This is the source tree for HTTPS Everywhere for Firefox and Chrome. Important directories you might want to know about - + chromium/ WebExtension source code (for Firefox & Chromium/chrome) chromium/external External dependencies chromium/test Unit tests @@ -60,7 +60,7 @@ Important directories you might want to know about src/chrome/content/rules Ruleset files live here test/ Travis unit test source code live here - + utils/ Various utilities (includes some Travis test source) Hacking on the Source Code diff --git a/test.sh b/test.sh index 441bd3557157..d2ee948a900e 100755 --- a/test.sh +++ b/test.sh @@ -14,6 +14,6 @@ else git rev-parse && cd "$(git rev-parse --show-toplevel)" fi -./test/rules.sh +./test/test-coverage.sh ./test/firefox.sh $@ ./test/chromium.sh $@ diff --git a/test/rules.sh b/test/rules.sh old mode 100755 new mode 100644 index 7bef2f2d5d84..a99f15c23243 --- a/test/rules.sh +++ b/test/rules.sh @@ -1,45 +1,9 @@ -#!/bin/bash -ex -# -# Test that all rulesets modified after a certain date have sufficient test -# coverage, according to the ruleset checker. -# - -# Get to the repo root directory, even when we're symlinked as a hook. -if [ -n "$GIT_DIR" ] -then - # $GIT_DIR is set, so we're running as a hook. - cd $GIT_DIR -else - # Git command exists? Cool, let's CD to the right place. - git rev-parse && cd "$(git rev-parse --show-toplevel)" -fi - -source utils/mktemp.sh - -TMP="$(mktemp)" -trap 'rm "$TMP"' EXIT -if ! [ -d test/rules ] ; then - echo "Submodule https-everywhere-checker is missing. Run" - echo "./install-dev-dependencies.sh" - exit 1 -fi -if [ $# -gt 0 ] ; then - exec python2.7 test/rules/src/https_everywhere_checker/check_rules.py \ - test/rules/coverage.checker.config "$@" -fi -if ! python2.7 test/rules/src/https_everywhere_checker/check_rules.py \ - test/rules/coverage.checker.config; then - echo ' -Ruleset test coverage was insufficient. - -Under the new ruleset testing rules (February 2015), any modified ruleset -must have sufficient test coverage. You can often improve test coverage by -adding tags, or by restructuring the rule to avoid -wildcard tags. See these documents: -https://github.com/EFForg/https-everywhere/blob/master/ruleset-testing.md -https://github.com/EFForg/https-everywhere/blob/master/CONTRIBUTING.md#ruleset-style-guide -' - exit 1 -else - exit 0 -fi +#!/bin/bash + +python2.7 utils/ruleset_filenames_validate.py +RULESETFOLDER="$RULESETFOLDER" COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" utils/check-ruleset-path.sh +utils/remove-obsolete-references.sh +python2.7 utils/validate.py +python2.7 utils/trivial-validate.py --quiet +test/test-coverage.sh +python2.7 utils/normalize-securecookie.py diff --git a/test/test-coverage.sh b/test/test-coverage.sh new file mode 100755 index 000000000000..7bef2f2d5d84 --- /dev/null +++ b/test/test-coverage.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex +# +# Test that all rulesets modified after a certain date have sufficient test +# coverage, according to the ruleset checker. +# + +# Get to the repo root directory, even when we're symlinked as a hook. +if [ -n "$GIT_DIR" ] +then + # $GIT_DIR is set, so we're running as a hook. + cd $GIT_DIR +else + # Git command exists? Cool, let's CD to the right place. + git rev-parse && cd "$(git rev-parse --show-toplevel)" +fi + +source utils/mktemp.sh + +TMP="$(mktemp)" +trap 'rm "$TMP"' EXIT +if ! [ -d test/rules ] ; then + echo "Submodule https-everywhere-checker is missing. Run" + echo "./install-dev-dependencies.sh" + exit 1 +fi +if [ $# -gt 0 ] ; then + exec python2.7 test/rules/src/https_everywhere_checker/check_rules.py \ + test/rules/coverage.checker.config "$@" +fi +if ! python2.7 test/rules/src/https_everywhere_checker/check_rules.py \ + test/rules/coverage.checker.config; then + echo ' +Ruleset test coverage was insufficient. + +Under the new ruleset testing rules (February 2015), any modified ruleset +must have sufficient test coverage. You can often improve test coverage by +adding tags, or by restructuring the rule to avoid +wildcard tags. See these documents: +https://github.com/EFForg/https-everywhere/blob/master/ruleset-testing.md +https://github.com/EFForg/https-everywhere/blob/master/CONTRIBUTING.md#ruleset-style-guide +' + exit 1 +else + exit 0 +fi diff --git a/test/travis.sh b/test/travis.sh index 8086f2177a73..a338f0dd2497 100755 --- a/test/travis.sh +++ b/test/travis.sh @@ -37,14 +37,8 @@ fi set -e if [ "$TEST" == "rules" ]; then - echo >&2 "Performing comprehensive coverage test." - docker run --rm -ti -v $(pwd):/opt httpse python utils/ruleset_filenames_validate.py - docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "utils/check-ruleset-path.sh" - docker run --rm -ti -v $(pwd):/opt httpse bash -c "utils/remove-obsolete-references.sh" - docker run --rm -ti -v $(pwd):/opt httpse python utils/validate.py - docker run --rm -ti -v $(pwd):/opt httpse python utils/trivial-validate.py --quiet - docker run --rm -ti -v $(pwd):/opt httpse bash -c "test/rules.sh" - docker run --rm -ti -v $(pwd):/opt httpse python utils/normalize-securecookie.py + echo >&2 "Performing validations on rules." + docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/rules.sh" fi if [ "$TEST" == "fetch" ]; then From df20586c1f2b825da2c17d4a6f8c63d02307f16a Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 14:11:37 -0800 Subject: [PATCH 3/6] Rearrange travis files --- .travis.yml | 4 +- CONTRIBUTING.md | 4 +- run_travis.sh | 55 ------------ test/run_travis.sh | 109 ++++++++++++++++++++++++ setup_travis.sh => test/setup_travis.sh | 4 +- test/travis.sh | 56 ------------ 6 files changed, 115 insertions(+), 117 deletions(-) delete mode 100755 run_travis.sh create mode 100755 test/run_travis.sh rename setup_travis.sh => test/setup_travis.sh (97%) delete mode 100755 test/travis.sh diff --git a/.travis.yml b/.travis.yml index f2ad846157e9..e1eca9e00614 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,5 +35,5 @@ matrix: before_install: - export DISPLAY=:99.0 - sh -e /etc/init.d/xvfb start -before_script: travis_retry ./setup_travis.sh -script: . ./run_travis.sh +before_script: travis_retry test/setup_travis.sh +script: . test/run_travis.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 843d7ec08bce..041595491a77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,7 +148,7 @@ Avoid using the left-wildcard (``) unless you int Instead, prefer listing explicit target hosts and a single rewrite from `"^http:"` to `"^https:"`. This saves you time as a ruleset author because each explicit target host automatically creates an implicit test URL, reducing the need to add your own test URLs. These also make it easier for someone reading the ruleset to figure out which subdomains are covered. -If you know all subdomains of a given domain support HTTPS, go ahead and use a left-wildcard, along with a plain rewrite from `"^http:"` to `"^https:"`. Make sure to add a bunch of test URLs for the more important subdomains. +If you know all subdomains of a given domain support HTTPS, go ahead and use a left-wildcard, along with a plain rewrite from `"^http:"` to `"^https:"`. Make sure to add a bunch of test URLs for the more important subdomains. #### Edge-Case: Right-Wildcards @@ -334,7 +334,7 @@ In `utils` we have a tool called `hsts-prune` which removes `targets` from rules Every new pull request automatically has the `hsts-prune` utility applied to it as part of the continual integration process. If a new PR introduces a `target` which is preloaded, it will fail the CI test suite. See: - `.travis.yml` -- `test/travis.sh` +- `test/run_travis.sh` * * * diff --git a/run_travis.sh b/run_travis.sh deleted file mode 100755 index 50d561f268b5..000000000000 --- a/run_travis.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -x -toplevel="$(git rev-parse --show-toplevel)" -testdir="${toplevel}/test/selenium" -srcdir="${toplevel}/chromium" -linter="${toplevel}/utils/eslint/node_modules/.bin/eslint --ignore-path ${srcdir}/.eslintignore" - - -function run_lint { - $linter $srcdir - if [ $? != 0 ]; then - echo "Linting errors" - exit 1 - fi -} - -function run_unittests { - pushd ${srcdir} - npm run cover # run with coverage - if [ $? != 0 ]; then - echo "unittest errors" - exit 1 - fi - npm run report - popd -} - -function run_selenium { - ENABLE_XVFB=1 py.test -v --capture=no ${testdir} # autodiscover and run the tests -} - -if [ "$TEST" == "lint" ]; then - echo "running lint tests" - run_lint -elif [ "$TEST" == "unittests" ]; then - echo "Running unittests" - run_unittests -elif [ "$TEST" == "rules" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then - ${toplevel}/test/travis.sh # run old travis tests -else - case $BROWSER in - *chrome*) - echo "running tests on chrome" - run_selenium - ;; - *firefox*) - echo "running tests on firefox" - run_selenium - ;; - *) - echo "bad TEST variable, got $TEST" - exit 1 - ;; - esac -fi diff --git a/test/run_travis.sh b/test/run_travis.sh new file mode 100755 index 000000000000..2c533bd6f85a --- /dev/null +++ b/test/run_travis.sh @@ -0,0 +1,109 @@ +#!/bin/bash +set -x +toplevel="$(git rev-parse --show-toplevel)" +testdir="${toplevel}/test/selenium" +srcdir="${toplevel}/chromium" +linter="${toplevel}/utils/eslint/node_modules/.bin/eslint --ignore-path ${srcdir}/.eslintignore" + + +function run_lint { + $linter $srcdir + if [ $? != 0 ]; then + echo "Linting errors" + exit 1 + fi +} + +function run_unittests { + pushd ${srcdir} + npm run cover # run with coverage + if [ $? != 0 ]; then + echo "unittest errors" + exit 1 + fi + npm run report + popd +} + +function run_selenium { + ENABLE_XVFB=1 py.test -v --capture=no ${testdir} # autodiscover and run the tests +} + +if [ "$TEST" == "lint" ]; then + echo "running lint tests" + run_lint +elif [ "$TEST" == "unittests" ]; then + echo "Running unittests" + run_unittests +elif [ "$TEST" == "rules" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then + + # Folder paths, relative to parent + RULESETFOLDER="${toplevel}/src/chrome/content/rules" + + # Go to git repo root; taken from ../test.sh. Note that + # $GIT_DIR is .git in this case. + if [ -n "$GIT_DIR" ] + then + # $GIT_DIR is set, so we're running as a hook. + cd $GIT_DIR + cd .. + else + # Let's CD to the right place. + cd $toplevel + fi + + # Fetch the current GitHub version of HTTPS-E to compare to its master + git remote add upstream-for-travis https://github.com/EFForg/https-everywhere.git + trap 'git remote remove upstream-for-travis' EXIT + + # Only do a shallow fetch if we're in Travis. No need otherwise. + if [ $TRAVIS ]; then + git fetch --depth=50 upstream-for-travis master + else + git fetch upstream-for-travis master + fi + + COMMON_BASE_COMMIT=$(git merge-base upstream-for-travis/master HEAD) + RULESETS_CHANGED=$(git diff --name-only $COMMON_BASE_COMMIT | grep $RULESETFOLDER | grep '.xml') + if [ "$(git diff --name-only $COMMON_BASE_COMMIT)" != "$RULESETS_CHANGED" ]; then + ONLY_RULESETS_CHANGED=false + fi + + # At this point, if anything fails, the test should fail + set -e + + if [ "$TEST" == "rules" ]; then + echo >&2 "Performing validations on rules." + docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/rules.sh" + fi + + if [ "$TEST" == "fetch" ]; then + echo >&2 "Testing test URLs in all changed rulesets." + # --privileged is required here for miredo to create a network tunnel + docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" --privileged httpse bash -c "service miredo start && service tor start && test/fetch.sh" + fi + + if [ "$TEST" == "preloaded" ]; then + echo >&2 "Ensuring rulesets do not introduce targets which are already HSTS preloaded." + docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" node bash -c "cd /opt/utils/hsts-prune && npm install && node index.js" + [ `git diff --name-only $RULESETFOLDER | wc -l` -eq 0 ] + fi + + exit 0 + +else + case $BROWSER in + *chrome*) + echo "running tests on chrome" + run_selenium + ;; + *firefox*) + echo "running tests on firefox" + run_selenium + ;; + *) + echo "bad TEST variable, got $TEST" + exit 1 + ;; + esac +fi diff --git a/setup_travis.sh b/test/setup_travis.sh similarity index 97% rename from setup_travis.sh rename to test/setup_travis.sh index b371515419f8..92c2544730e4 100755 --- a/setup_travis.sh +++ b/test/setup_travis.sh @@ -46,7 +46,7 @@ function setup_docker { docker build -t httpse . } -case $INFO in +case $TEST in *chrome*) setup_chrome browser_setup @@ -67,7 +67,7 @@ case $INFO in *preloaded*) ;; *) - echo "bad INFO variable, got $INFO" + echo "bad TEST variable, got $TEST" exit 1 ;; esac diff --git a/test/travis.sh b/test/travis.sh deleted file mode 100755 index a338f0dd2497..000000000000 --- a/test/travis.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Wrapper for travis tests - -# Folder paths, relative to parent -RULESETFOLDER="src/chrome/content/rules" - -# Go to git repo root; taken from ../test.sh. Note that -# $GIT_DIR is .git in this case. -if [ -n "$GIT_DIR" ] -then - # $GIT_DIR is set, so we're running as a hook. - cd $GIT_DIR - cd .. -else - # Git command exists? Cool, let's CD to the right place. - git rev-parse && cd "$(git rev-parse --show-toplevel)" -fi - -# Fetch the current GitHub version of HTTPS-E to compare to its master -git remote add upstream-for-travis https://github.com/EFForg/https-everywhere.git -trap 'git remote remove upstream-for-travis' EXIT - -# Only do a shallow fetch if we're in Travis. No need otherwise. -if [ $TRAVIS ]; then - git fetch --depth=50 upstream-for-travis master -else - git fetch upstream-for-travis master -fi - -COMMON_BASE_COMMIT=$(git merge-base upstream-for-travis/master HEAD) -RULESETS_CHANGED=$(git diff --name-only $COMMON_BASE_COMMIT | grep $RULESETFOLDER | grep '.xml') -if [ "$(git diff --name-only $COMMON_BASE_COMMIT)" != "$RULESETS_CHANGED" ]; then - ONLY_RULESETS_CHANGED=false -fi - -# At this point, if anything fails, the test should fail -set -e - -if [ "$TEST" == "rules" ]; then - echo >&2 "Performing validations on rules." - docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/rules.sh" -fi - -if [ "$TEST" == "fetch" ]; then - echo >&2 "Testing test URLs in all changed rulesets." - # --privileged is required here for miredo to create a network tunnel - docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" --privileged httpse bash -c "service miredo start && service tor start && test/fetch.sh" -fi - -if [ "$TEST" == "preloaded" ]; then - echo >&2 "Ensuring rulesets do not introduce targets which are already HSTS preloaded." - docker run --rm -ti -v $(pwd):/opt -e RULESETS_CHANGED="$RULESETS_CHANGED" node bash -c "cd /opt/utils/hsts-prune && npm install && node index.js" - [ `git diff --name-only $RULESETFOLDER | wc -l` -eq 0 ] -fi - -exit 0 From 950f5d620bc10d4e3e533d4b8527fb93f94e2615 Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 14:18:00 -0800 Subject: [PATCH 4/6] Fix rules --- test/rules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 test/rules.sh diff --git a/test/rules.sh b/test/rules.sh old mode 100644 new mode 100755 index a99f15c23243..c3487c46d549 --- a/test/rules.sh +++ b/test/rules.sh @@ -1,7 +1,7 @@ #!/bin/bash python2.7 utils/ruleset_filenames_validate.py -RULESETFOLDER="$RULESETFOLDER" COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" utils/check-ruleset-path.sh +utils/check-ruleset-path.sh utils/remove-obsolete-references.sh python2.7 utils/validate.py python2.7 utils/trivial-validate.py --quiet From 8d46bf8aa55e483df11717a095160492abb9ab8a Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 15:17:18 -0800 Subject: [PATCH 5/6] Move validation tests into test/validations/* --- .travis.yml | 2 +- ruleset-testing.md | 3 +-- test.sh | 2 +- test/rules.sh | 9 --------- test/run_travis.sh | 8 ++++---- test/validations.sh | 10 ++++++++++ .../validations/filename/run.py | 0 .../validations/path/run.sh | 0 utils/validate.py => test/validations/relaxng/run.py | 10 +++++----- .../relaxng.xml => test/validations/relaxng/schema.xml | 0 .../validations/securecookie/run.py | 0 .../validations/special/run.py | 8 ++++---- .../test-coverage/run.sh} | 0 13 files changed, 26 insertions(+), 26 deletions(-) delete mode 100755 test/rules.sh create mode 100755 test/validations.sh rename utils/ruleset_filenames_validate.py => test/validations/filename/run.py (100%) rename utils/check-ruleset-path.sh => test/validations/path/run.sh (100%) rename utils/validate.py => test/validations/relaxng/run.py (79%) rename utils/relaxng.xml => test/validations/relaxng/schema.xml (100%) rename utils/normalize-securecookie.py => test/validations/securecookie/run.py (100%) rename utils/trivial-validate.py => test/validations/special/run.py (94%) rename test/{test-coverage.sh => validations/test-coverage/run.sh} (100%) diff --git a/.travis.yml b/.travis.yml index e1eca9e00614..3fda2a067fb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: language: node_js node_js: node - "node" - - env: TEST="rules" + - env: TEST="validations" - env: TEST="fetch" - env: TEST="preloaded" - addons: diff --git a/ruleset-testing.md b/ruleset-testing.md index 7e8a26fa0b0f..579a7488c4fb 100644 --- a/ruleset-testing.md +++ b/ruleset-testing.md @@ -53,8 +53,7 @@ Submitting changes to any ruleset that does not meet the coverage requirements will break the build. This means that even fixes of existing rules may require additional work to bring them up to snuff. -To run the tests locally, you'll need the https-everywhere-checker, which is now -a submodule of https-everywhere. To set it up, run: +To run the tests locally, first install the development dependencies: ./install-dev-dependencies.sh diff --git a/test.sh b/test.sh index d2ee948a900e..ede71befaa3d 100755 --- a/test.sh +++ b/test.sh @@ -14,6 +14,6 @@ else git rev-parse && cd "$(git rev-parse --show-toplevel)" fi -./test/test-coverage.sh +./test/validations/test-coverage/run.sh ./test/firefox.sh $@ ./test/chromium.sh $@ diff --git a/test/rules.sh b/test/rules.sh deleted file mode 100755 index c3487c46d549..000000000000 --- a/test/rules.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -python2.7 utils/ruleset_filenames_validate.py -utils/check-ruleset-path.sh -utils/remove-obsolete-references.sh -python2.7 utils/validate.py -python2.7 utils/trivial-validate.py --quiet -test/test-coverage.sh -python2.7 utils/normalize-securecookie.py diff --git a/test/run_travis.sh b/test/run_travis.sh index 2c533bd6f85a..692755be7bbe 100755 --- a/test/run_travis.sh +++ b/test/run_travis.sh @@ -35,7 +35,7 @@ if [ "$TEST" == "lint" ]; then elif [ "$TEST" == "unittests" ]; then echo "Running unittests" run_unittests -elif [ "$TEST" == "rules" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then +elif [ "$TEST" == "validations" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded" ]; then # Folder paths, relative to parent RULESETFOLDER="${toplevel}/src/chrome/content/rules" @@ -72,9 +72,9 @@ elif [ "$TEST" == "rules" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "preloaded # At this point, if anything fails, the test should fail set -e - if [ "$TEST" == "rules" ]; then - echo >&2 "Performing validations on rules." - docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/rules.sh" + if [ "$TEST" == "validations" ]; then + echo >&2 "Performing validations on rulesets." + docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/validations.sh" fi if [ "$TEST" == "fetch" ]; then diff --git a/test/validations.sh b/test/validations.sh new file mode 100755 index 000000000000..ef5bec0aeab0 --- /dev/null +++ b/test/validations.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Perform validations on rulesets. + +utils/remove-obsolete-references.sh +test/validations/path/run.sh +test/validations/test-coverage/run.sh +python2.7 test/validations/securecookie/run.py +python2.7 test/validations/filename/run.py +python2.7 test/validations/relaxng/run.py +python2.7 test/validations/special/run.py --quiet diff --git a/utils/ruleset_filenames_validate.py b/test/validations/filename/run.py similarity index 100% rename from utils/ruleset_filenames_validate.py rename to test/validations/filename/run.py diff --git a/utils/check-ruleset-path.sh b/test/validations/path/run.sh similarity index 100% rename from utils/check-ruleset-path.sh rename to test/validations/path/run.sh diff --git a/utils/validate.py b/test/validations/relaxng/run.py similarity index 79% rename from utils/validate.py rename to test/validations/relaxng/run.py index f7ad6dd2cc44..32503a58c22f 100644 --- a/utils/validate.py +++ b/test/validations/relaxng/run.py @@ -9,7 +9,7 @@ from lxml import etree # commandline arguments parsing (nobody use it, though) -parser = argparse.ArgumentParser(description="Validate rulesets against relaxng.xml") +parser = argparse.ArgumentParser(description="Validate rulesets against relaxng schema.xml") parser.add_argument("--source_dir", default="src/chrome/content/rules") args = parser.parse_args() @@ -18,12 +18,12 @@ files = glob.glob(os.path.join(args.source_dir, "*.xml")) # read the schema file -relaxng_doc = etree.parse('utils/relaxng.xml') +relaxng_doc = etree.parse('test/validations/relaxng/schema.xml') relaxng = etree.RelaxNG(relaxng_doc) exit_code = 0 -print("Validation of rulesets against utils/relaxng.xml begins...") +print("Validation of rulesets against relaxng schema.xml begins...") for filename in sorted(files): tree = etree.parse(filename) @@ -34,13 +34,13 @@ print(("%s %s:%s:%s: %s" % (e.level_name, e.filename, e.line, e.column, e.message))) if exit_code == 0: - message = "Validation of rulesets against utils/relaxng.xml succeeded." + message = "Validation of rulesets against relaxng schema.xml succeeded." else: message = "\nTwo very common reasons for this are the following:\n" \ " - missing caret (^) in 'from' attribute: it should be \"^http:\" and not \"http:\"\n" \ " - missing trailing slashes in 'from' or 'to' when specifying full hostnames: \n" \ " it should be \"https://eff.org/\" and not \"https://eff.org\"\n\n" \ - "Validation of rulesets against utils/relaxng.xml failed." + "Validation of rulesets against relaxng schema.xml failed." print(message) exit(exit_code) diff --git a/utils/relaxng.xml b/test/validations/relaxng/schema.xml similarity index 100% rename from utils/relaxng.xml rename to test/validations/relaxng/schema.xml diff --git a/utils/normalize-securecookie.py b/test/validations/securecookie/run.py similarity index 100% rename from utils/normalize-securecookie.py rename to test/validations/securecookie/run.py diff --git a/utils/trivial-validate.py b/test/validations/special/run.py similarity index 94% rename from utils/trivial-validate.py rename to test/validations/special/run.py index 1b5ef5103663..b9733b42a13d 100755 --- a/utils/trivial-validate.py +++ b/test/validations/special/run.py @@ -37,7 +37,7 @@ def fail(s): with open(thispath + '/duplicate-whitelist.txt') as duplicate_fh: duplicate_allowed_list = [x.rstrip('\n') for x in duplicate_fh.readlines()] -filenames = glob.glob(thispath + '/../src/chrome/content/rules/*') +filenames = glob.glob(thispath + '/../../../src/chrome/content/rules/*') def test_bad_regexp(tree, rulename, from_attrib, to): # Rules with invalid regular expressions. @@ -150,7 +150,7 @@ def nomes_all(where=sys.argv[1:]): xpath_from = etree.XPath("/ruleset/rule/@from") xpath_to = etree.XPath("/ruleset/rule/@to") -print("Complex validations & assertions of rulesets using utils/trivial-validate.py begins...") +print("Complex validations & assertions of rulesets using test/validations/special/run.py begins...") host_counter = Counter() for filename in filenames: @@ -203,9 +203,9 @@ def nomes_all(where=sys.argv[1:]): fail("%s failed: %s" % (host, is_valid_target_host.__doc__)) if failure > 0: - print("Complex validations & assertions of rulesets using utils/trivial-validate.py failed.") + print("Complex validations & assertions of rulesets using test/validations/special/run.py failed.") else: - print("Complex validations & assertions of rulesets using utils/trivial-validate.py succeeded.") + print("Complex validations & assertions of rulesets using test/validations/special/run.py succeeded.") sys.exit(failure) diff --git a/test/test-coverage.sh b/test/validations/test-coverage/run.sh similarity index 100% rename from test/test-coverage.sh rename to test/validations/test-coverage/run.sh From 9fa6c8630dfee0a841962d92fd394cf01db1b5ac Mon Sep 17 00:00:00 2001 From: William Budington Date: Thu, 8 Feb 2018 16:11:41 -0800 Subject: [PATCH 6/6] moar --- README.md | 2 +- fetch-test.sh | 2 -- ruleset-testing.md | 2 +- test.sh | 2 +- test/manual.sh | 6 ++++++ test/run_travis.sh | 2 +- test/setup_travis.sh | 2 +- test/validations.sh | 2 +- test/validations/path/run.sh | 8 +++++--- .../validations/special}/duplicate-whitelist-cleanup.sh | 0 .../validations/special}/duplicate-whitelist.txt | 0 test/validations/test-coverage/run.sh | 2 +- 12 files changed, 18 insertions(+), 12 deletions(-) delete mode 100755 fetch-test.sh create mode 100755 test/manual.sh rename {utils => test/validations/special}/duplicate-whitelist-cleanup.sh (100%) rename {utils => test/validations/special}/duplicate-whitelist.txt (100%) diff --git a/README.md b/README.md index 978b72d22de7..9e10be63cde6 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Get the packages you need and install a git hook to run tests before push: bash install-dev-dependencies.sh -Run the test coverage and browser tests: +Run the ruleset validations and browser tests: bash test.sh diff --git a/fetch-test.sh b/fetch-test.sh deleted file mode 100755 index b6457444c00e..000000000000 --- a/fetch-test.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec python2.7 test/rules/src/https_everywhere_checker/check_rules.py test/rules/manual.checker.config "$@" diff --git a/ruleset-testing.md b/ruleset-testing.md index 579a7488c4fb..9d3a6fb82304 100644 --- a/ruleset-testing.md +++ b/ruleset-testing.md @@ -59,4 +59,4 @@ To run the tests locally, first install the development dependencies: To test a specific ruleset: - ./fetch-test.sh rules/Example.xml + test/manual.sh rules/Example.xml diff --git a/test.sh b/test.sh index ede71befaa3d..64c9b27c68b8 100755 --- a/test.sh +++ b/test.sh @@ -14,6 +14,6 @@ else git rev-parse && cd "$(git rev-parse --show-toplevel)" fi -./test/validations/test-coverage/run.sh +./test/validations.sh ./test/firefox.sh $@ ./test/chromium.sh $@ diff --git a/test/manual.sh b/test/manual.sh new file mode 100755 index 000000000000..938c82400e64 --- /dev/null +++ b/test/manual.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# This script assists ruleset creators in ensuring that there is proper test +# coverage for their rules, and performs a network "fetch" test to alert the +# contributor of potential problems. + +exec python2.7 test/rules/src/https_everywhere_checker/check_rules.py test/rules/manual.checker.config "$@" diff --git a/test/run_travis.sh b/test/run_travis.sh index 692755be7bbe..12557f5e0219 100755 --- a/test/run_travis.sh +++ b/test/run_travis.sh @@ -74,7 +74,7 @@ elif [ "$TEST" == "validations" ] || [ "$TEST" == "fetch" ] || [ "$TEST" == "pre if [ "$TEST" == "validations" ]; then echo >&2 "Performing validations on rulesets." - docker run --rm -ti -v $(pwd):/opt -e RULESETFOLDER="$RULESETFOLDER" -e COMMON_BASE_COMMIT="$COMMON_BASE_COMMIT" httpse bash -c "test/validations.sh" + docker run --rm -ti -v $(pwd):/opt httpse bash -c "test/validations.sh" fi if [ "$TEST" == "fetch" ]; then diff --git a/test/setup_travis.sh b/test/setup_travis.sh index 92c2544730e4..31f0e5a9c4f9 100755 --- a/test/setup_travis.sh +++ b/test/setup_travis.sh @@ -61,7 +61,7 @@ case $TEST in *unittests*) setup_unittests ;; - *rules*|*fetch*) + *validations*|*fetch*) setup_docker ;; *preloaded*) diff --git a/test/validations.sh b/test/validations.sh index ef5bec0aeab0..c13a127c5e9f 100755 --- a/test/validations.sh +++ b/test/validations.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -ex # Perform validations on rulesets. utils/remove-obsolete-references.sh diff --git a/test/validations/path/run.sh b/test/validations/path/run.sh index b1250c9ee369..b8b8ff1fc189 100755 --- a/test/validations/path/run.sh +++ b/test/validations/path/run.sh @@ -1,10 +1,12 @@ #!/bin/bash -CHANGED_FILES=`git diff --name-only $COMMON_BASE_COMMIT | grep -v $RULESETFOLDER | grep '.xml'` +RULESETFOLDER=src/chrome/content/rules + +FILES=`find . | grep -v $RULESETFOLDER | grep '\.xml$'` EXIT_CODE=0 -if [ "$CHANGED_FILES" != "" ]; then +if [ "$FILES" != "" ]; then while read FILE; do # check if changed file is actually a ruleset egrep -q "^]+>" "$FILE" @@ -14,7 +16,7 @@ if [ "$CHANGED_FILES" != "" ]; then echo >&2 "ERROR: $FILE Inclusion of ruleset outside of $RULESETFOLDER" EXIT_CODE=1 fi - done <<< "$CHANGED_FILES" + done <<< "$FILES" fi exit $EXIT_CODE diff --git a/utils/duplicate-whitelist-cleanup.sh b/test/validations/special/duplicate-whitelist-cleanup.sh similarity index 100% rename from utils/duplicate-whitelist-cleanup.sh rename to test/validations/special/duplicate-whitelist-cleanup.sh diff --git a/utils/duplicate-whitelist.txt b/test/validations/special/duplicate-whitelist.txt similarity index 100% rename from utils/duplicate-whitelist.txt rename to test/validations/special/duplicate-whitelist.txt diff --git a/test/validations/test-coverage/run.sh b/test/validations/test-coverage/run.sh index 7bef2f2d5d84..268a705f8cbd 100755 --- a/test/validations/test-coverage/run.sh +++ b/test/validations/test-coverage/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # # Test that all rulesets modified after a certain date have sufficient test # coverage, according to the ruleset checker.