Skip to content

Commit f067a45

Browse files
kripkenkripken
authored
Skip tests that fail on windows and enable all the rest (WebAssembly#3035)
This lets us run most tests at least on that platform. Add a new function for skipping those tests, skip_if_on_windows, so that it's easy to find which tests are disabled on windows for later fixing efforts. This fixes a few minor issues for windows, like comparisons should ignore \r in some cases. Rename all passes tests that use --dwarfdump to contain "dwarf" in their name, which makes it easy to skip those (and is clearer anyhow).
1 parent 454a1cb commit f067a45

32 files changed

Lines changed: 47 additions & 14 deletions

‎.github/workflows/ci.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,6 @@ jobs:
9292

9393
- name: test
9494
run: python check.py --binaryen-bin=out/install/bin
95-
# Currently disabled on windows due to a single test failure.
96-
# https://github.com/WebAssembly/binaryen/issues/2781
97-
if: matrix.os != 'windows-latest'
9895

9996
build-clang:
10097
name: clang

‎check.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,11 @@ def run_spec_tests():
190190
print('\n[ checking wasm-shell spec testcases... ]\n')
191191

192192
for wast in shared.options.spec_tests:
193-
print('..', os.path.basename(wast))
193+
base = os.path.basename(wast)
194+
print('..', base)
195+
# windows has some failures that need to be investigated
196+
if base == 'names.wast' and shared.skip_if_on_windows('spec: ' + base):
197+
continue
194198

195199
def run_spec_test(wast):
196200
cmd = shared.WASM_SHELL + [wast]
@@ -214,13 +218,13 @@ def check_expected(actual, expected):
214218
if actual != expected:
215219
shared.fail(actual, expected)
216220

217-
expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log')
221+
expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log')
218222

219223
# some spec tests should fail (actual process failure, not just assert_invalid)
220224
try:
221225
actual = run_spec_test(wast)
222226
except Exception as e:
223-
if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in os.path.basename(wast):
227+
if ('wasm-validator error' in str(e) or 'parse exception' in str(e)) and '.fail.' in base:
224228
print('<< test failed as expected >>')
225229
continue # don't try all the binary format stuff TODO
226230
else:
@@ -229,7 +233,7 @@ def check_expected(actual, expected):
229233
check_expected(actual, expected)
230234

231235
# skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
232-
if 'exports.wast' in os.path.basename(wast): # FIXME
236+
if 'exports.wast' in base: # FIXME
233237
continue
234238

235239
# check binary format. here we can verify execution of the final
@@ -240,7 +244,7 @@ def check_expected(actual, expected):
240244

241245
# FIXME Remove reference type tests from this list after nullref is
242246
# implemented in V8
243-
if os.path.basename(wast) not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
247+
if base not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
244248
split_num = 0
245249
actual = ''
246250
for module, asserts in support.split_wast(wast):
@@ -254,7 +258,7 @@ def check_expected(actual, expected):
254258
open(result_wast, 'a').write('\n' + '\n'.join(asserts))
255259
actual += run_spec_test(result_wast)
256260
# compare all the outputs to the expected output
257-
check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(wast) + '.log'))
261+
check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))
258262
else:
259263
# handle unsplittable wast files
260264
run_spec_test(wast)
@@ -282,6 +286,9 @@ def run_gcc_tests():
282286
if not shared.NATIVECC or not shared.NATIVEXX:
283287
shared.fail_with_error('Native compiler (e.g. gcc/g++) was not found in PATH!')
284288
return
289+
# windows + gcc will need some work
290+
if shared.skip_if_on_windows('gcc'):
291+
return
285292

286293
for t in sorted(os.listdir(shared.get_test_dir('example'))):
287294
output_file = 'example'
@@ -332,6 +339,10 @@ def run_gcc_tests():
332339
def run_unittest():
333340
print('\n[ checking unit tests...]\n')
334341

342+
# windows has some failures that need to be investigated
343+
if shared.skip_if_on_windows('unit'):
344+
return
345+
335346
# equivalent to `python -m unittest discover -s ./test -v`
336347
suite = unittest.defaultTestLoader.discover(os.path.dirname(shared.options.binaryen_test))
337348
result = unittest.TextTestRunner(verbosity=2, failfast=shared.options.abort_on_first_failure).run(suite)

‎scripts/test/shared.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -376,13 +376,13 @@ def get_tests(test_dir, extensions=[]):
376376

377377
# 11/27/2019: We updated the spec test suite to upstream spec repo. For some
378378
# files that started failing after this update, we added the new files to this
379-
# blacklist and preserved old ones by renaming them to 'old_[FILENAME].wast'
379+
# skip-list and preserved old ones by renaming them to 'old_[FILENAME].wast'
380380
# not to lose coverage. When the cause of the error is fixed or the unsupported
381381
# construct gets support so the new test passes, we can delete the
382382
# corresponding 'old_[FILENAME].wast' file. When you fix the new file and
383383
# delete the old file, make sure you rename the corresponding .wast.log file in
384384
# expected-output/ if any.
385-
SPEC_TEST_BLACKLIST = [
385+
SPEC_TESTS_TO_SKIP = [
386386
# Stacky code / notation
387387
'block.wast',
388388
'call.wast',
@@ -437,7 +437,7 @@ def get_tests(test_dir, extensions=[]):
437437
'unreached-invalid.wast' # 'assert_invalid' failure
438438
]
439439
options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
440-
in SPEC_TEST_BLACKLIST]
440+
in SPEC_TESTS_TO_SKIP]
441441

442442

443443
# check utilities
@@ -507,3 +507,13 @@ def with_pass_debug(check):
507507
else:
508508
if 'BINARYEN_PASS_DEBUG' in os.environ:
509509
del os.environ['BINARYEN_PASS_DEBUG']
510+
511+
512+
# checks if we are on windows, and if so logs out that a test is being skipped,
513+
# and returns True. This is a central location for all test skipping on
514+
# windows, so that we can easily find which tests are skipped.
515+
def skip_if_on_windows(name):
516+
if get_platform() == 'windows':
517+
print('skipping test "%s" on windows' % name)
518+
return True
519+
return False

‎scripts/test/wasm2js.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ def test_asserts_output():
121121

122122
def test_wasm2js():
123123
print('\n[ checking wasm2js testcases... ]\n')
124+
if shared.skip_if_on_windows('wasm2js'):
125+
return
124126
test_wasm2js_output()
125127
test_asserts_output()
126128

‎scripts/test/wasm_opt.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ def test_wasm_opt():
4545

4646
for t in shared.get_tests(shared.get_test_dir('passes'), ['.wast', '.wasm']):
4747
print('..', os.path.basename(t))
48+
# windows has some failures that need to be investigated:
49+
# * ttf tests have different outputs - order of execution of params?
50+
# * dwarf tests print windows slashes instead of unix
51+
if ('translate-to-fuzz' in t or 'dwarf' in t) and \
52+
shared.skip_if_on_windows('fuzz translation tests'):
53+
continue
4854
binary = '.wasm' in t
4955
base = os.path.basename(t).replace('.wast', '').replace('.wasm', '')
5056
passname = base

test/passes/fannkuch0.bin.txt renamed to test/passes/fannkuch0_dwarf.bin.txt

File renamed without changes.

test/passes/fannkuch0.passes renamed to test/passes/fannkuch0_dwarf.passes

File renamed without changes.

test/passes/fannkuch0.wasm renamed to test/passes/fannkuch0_dwarf.wasm

File renamed without changes.

test/passes/fannkuch3.bin.txt renamed to test/passes/fannkuch3_dwarf.bin.txt

File renamed without changes.

test/passes/fannkuch3.passes renamed to test/passes/fannkuch3_dwarf.passes

File renamed without changes.

0 commit comments

Comments
 (0)