diff --git a/check.py b/check.py index 5eddc89b407..4349ce95149 100755 --- a/check.py +++ b/check.py @@ -196,16 +196,6 @@ def run_opt_test(wast, stdout=None, stderr=None): support.run_command(cmd, stdout=stdout) -def check_expected(actual, expected, stdout=None): - if expected and os.path.exists(expected): - expected = open(expected).read() - print(' (using expected output)', file=stdout) - actual = actual.strip() - expected = expected.strip() - if actual != expected: - shared.fail(actual, expected) - - def run_one_spec_test(wast: Path, stdout=None, stderr=None): test_name = wast.name @@ -217,11 +207,9 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None): if test_name == 'names.wast' and shared.skip_if_on_windows('spec: ' + test_name): return - expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log') - # some spec tests should fail (actual process failure, not just assert_invalid) try: - actual = run_spec_test(str(wast), stdout=stdout, stderr=stderr) + run_spec_test(str(wast), stdout=stdout, stderr=stderr) except Exception as e: if ('wasm-validator error' in str(e) or 'error: ' in str(e)) and '.fail.' in test_name: print('<< test failed as expected >>', file=stdout) @@ -229,11 +217,8 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None): else: shared.fail_with_error(str(e)) - check_expected(actual, expected, stdout=stdout) - # check binary format. here we can verify execution of the final # result, no need for an output verification - actual = '' transformed_path = base_name + ".transformed" with open(transformed_path, 'w') as transformed_spec_file: for i, (module, asserts) in enumerate(support.split_wast(str(wast))): @@ -252,8 +237,7 @@ def run_one_spec_test(wast: Path, stdout=None, stderr=None): transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts)) # compare all the outputs to the expected output - actual = run_spec_test(transformed_path, stdout=stdout, stderr=stderr) - check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', test_name + '.log'), stdout=stdout) + run_spec_test(transformed_path, stdout=stdout, stderr=stderr) def run_spec_test_with_wrapped_stdout(output_queue, wast: Path): diff --git a/scripts/auto_update_tests.py b/scripts/auto_update_tests.py index 527b1280fb0..5d9548b15fd 100755 --- a/scripts/auto_update_tests.py +++ b/scripts/auto_update_tests.py @@ -124,24 +124,6 @@ def update_reduce_tests(): support.run_command(shared.WASM_DIS + ['c.wasm', '-o', expected]) -def update_spec_tests(): - print('\n[ updating wasm-shell spec testcases... ]\n') - - for t in shared.options.spec_tests: - print('..', os.path.basename(t)) - - cmd = shared.WASM_SHELL + [t] - expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(t) + '.log') - if os.path.isfile(expected): - stdout = support.run_command(cmd, stderr=subprocess.PIPE) - # filter out binaryen interpreter logging that the spec suite - # doesn't expect - filtered = [line for line in stdout.splitlines() if not line.startswith('[trap')] - stdout = '\n'.join(filtered) + '\n' - with open(expected, 'w') as o: - o.write(stdout) - - def update_lit_tests(): print('\n[ updating lit testcases... ]\n') script = os.path.join(shared.options.binaryen_root, @@ -170,7 +152,6 @@ def update_lit_tests(): ('ctor-eval', update_ctor_eval_tests), ('wasm-metadce', update_metadce_tests), ('wasm-reduce', update_reduce_tests), - ('spec', update_spec_tests), ('lld', lld.update_lld_tests), ('wasm2js', wasm2js.update_wasm2js_tests), ('binaryenjs', binaryenjs.update_binaryen_js_tests), diff --git a/scripts/test/shared.py b/scripts/test/shared.py index 30c3593d6d2..cb687f4ed54 100644 --- a/scripts/test/shared.py +++ b/scripts/test/shared.py @@ -400,9 +400,7 @@ def get_tests(test_dir, extensions=[], recursive=False): # skip-list and preserved old ones by renaming them to 'old_[FILENAME].wast' # not to lose coverage. When the cause of the error is fixed or the unsupported # construct gets support so the new test passes, we can delete the -# corresponding 'old_[FILENAME].wast' file. When you fix the new file and -# delete the old file, make sure you rename the corresponding .wast.log file in -# expected-output/ if any. +# corresponding 'old_[FILENAME].wast' file. # Paths are relative to the test/spec directory SPEC_TESTS_TO_SKIP = [ # Requires us to write our own floating point parser diff --git a/scripts/test/wasm2js.py b/scripts/test/wasm2js.py index 3c70c061609..146b69d3db4 100644 --- a/scripts/test/wasm2js.py +++ b/scripts/test/wasm2js.py @@ -126,12 +126,6 @@ def test_wasm2js_output(): all_out += out shared.fail_if_not_identical_to_file(''.join(all_js), expected_file) - expected_out = os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(t) + '.log') - if os.path.exists(expected_out): - expected_out = open(expected_out).read() - else: - expected_out = '' - shared.fail_if_not_identical(all_out, expected_out) def test_asserts_output(): diff --git a/test/spec/expected-output/func_ptrs.wast.log b/test/spec/expected-output/func_ptrs.wast.log deleted file mode 100644 index 8f757d3c048..00000000000 --- a/test/spec/expected-output/func_ptrs.wast.log +++ /dev/null @@ -1 +0,0 @@ -83 : i32 diff --git a/test/spec/expected-output/names.wast.log b/test/spec/expected-output/names.wast.log deleted file mode 100644 index b694f30336a..00000000000 --- a/test/spec/expected-output/names.wast.log +++ /dev/null @@ -1,2 +0,0 @@ -42 : i32 -123 : i32 diff --git a/test/spec/expected-output/old_imports.wast.log b/test/spec/expected-output/old_imports.wast.log deleted file mode 100644 index 59c70fea97a..00000000000 --- a/test/spec/expected-output/old_imports.wast.log +++ /dev/null @@ -1,10 +0,0 @@ -13 : i32 -14 : i32 -42 : f32 -13 : i32 -13 : i32 -24 : i64 -25 : i64 -53 : f64 -24 : i64 -24 : i64 diff --git a/test/spec/expected-output/old_start.wast.log b/test/spec/expected-output/old_start.wast.log deleted file mode 100644 index f74aa5848ac..00000000000 --- a/test/spec/expected-output/old_start.wast.log +++ /dev/null @@ -1,2 +0,0 @@ -1 : i32 -2 : i32