You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ek...@apache.org on 2024/02/26 15:58:24 UTC
(tvm) branch main updated: [AOT][Testing] Print output values on test failure (#16611)
This is an automated email from the ASF dual-hosted git repository.
ekalda pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new b3fa6cb873 [AOT][Testing] Print output values on test failure (#16611)
b3fa6cb873 is described below
commit b3fa6cb873c71bfc15054bc9abbcc111c8413c9b
Author: Luke Hutton <lu...@arm.com>
AuthorDate: Mon Feb 26 15:58:16 2024 +0000
[AOT][Testing] Print output values on test failure (#16611)
This commit enhances the AOT test harness to print the "actual" and
"reference" values when there is a mismatch. This helps when
debugging a failing test. Sample output:
```
Actual, Reference
8.502946, 8.887751
9.810405, 9.108611
8.563767, 9.041000
10.019511, 9.190888
....
```
---
python/tvm/testing/aot.py | 76 ++++++++++++++++++++-----
tests/python/relay/aot/test_aot_test_harness.py | 61 ++++++++++++++++++++
tests/python/relay/aot/test_crt_aot.py | 1 +
3 files changed, 123 insertions(+), 15 deletions(-)
diff --git a/python/tvm/testing/aot.py b/python/tvm/testing/aot.py
index 9ee3a84c8a..8d74f545a3 100644
--- a/python/tvm/testing/aot.py
+++ b/python/tvm/testing/aot.py
@@ -425,7 +425,14 @@ def _emit_main_packed_call(main_file, input_map, output_list, mod_name):
main_file.write("\n")
-def _emit_main_compare(main_file, outputs, output_tolerance, mod_name, use_interface_c=False):
+def _emit_main_compare(
+ main_file,
+ outputs,
+ output_tolerance,
+ mod_name,
+ use_interface_c=False,
+ print_output_on_mismatch=False,
+):
for key in outputs:
sanitized_tensor_name = re.sub(r"\W", "_", key)
expected_data_name = _mangle_name(mod_name, f"expected_output_data_{sanitized_tensor_name}")
@@ -433,9 +440,11 @@ def _emit_main_compare(main_file, outputs, output_tolerance, mod_name, use_inter
comparison_function = "abs"
tolerance = output_tolerance or 0
+ value_format_specifier = "%d"
if is_float_dtype:
comparison_function = "fabs"
tolerance = output_tolerance or 0.001
+ value_format_specifier = "%f"
data_length_var_name = (
_mangle_name(mod_name, f"output_data_{sanitized_tensor_name}") + "_len"
@@ -447,15 +456,34 @@ def _emit_main_compare(main_file, outputs, output_tolerance, mod_name, use_inter
)
else:
actual_data_name = _mangle_name(mod_name, f"output_data_{sanitized_tensor_name}")
- main_file.write(
- f"for (int i = 0; i<{data_length_var_name}; i++) {{\n"
- f"\tif ({comparison_function}({actual_data_name}[i]-"
- f"{expected_data_name}[i]) > {tolerance}) {{\n"
- f'\t\tprintf("{AOT_FAILURE_TOKEN}\\n");\n'
- f"\t\treturn -1;\n"
- f"\t}}\n"
- f"}}"
- )
+
+ if print_output_on_mismatch:
+ main_file.write(
+ f"int mismatch = 0;"
+ f'printf("Actual, Reference\\n");\n'
+ f"for (int i = 0; i<{data_length_var_name}; i++) {{\n"
+ f"\tif ({comparison_function}({actual_data_name}[i]-"
+ f"{expected_data_name}[i]) > {tolerance}) {{\n"
+ f'\t\tprintf("{value_format_specifier}, {value_format_specifier}\\n"'
+ f", {actual_data_name}[i], {expected_data_name}[i]);\n"
+ f"\t\tmismatch = 1;\n"
+ f"\t}}\n"
+ f"}}"
+ f"if (mismatch == 1) {{\n"
+ f'\tprintf("{AOT_FAILURE_TOKEN}\\n");\n'
+ f"\treturn -1;\n"
+ f"}}"
+ )
+ else:
+ main_file.write(
+ f"for (int i = 0; i<{data_length_var_name}; i++) {{\n"
+ f"\tif ({comparison_function}({actual_data_name}[i]-"
+ f"{expected_data_name}[i]) > {tolerance}) {{\n"
+ f'\t\tprintf("{AOT_FAILURE_TOKEN}\\n");\n'
+ f"\t\treturn -1;\n"
+ f"\t}}\n"
+ f"}}"
+ )
def _emit_main_init_memory_manager(main_file):
@@ -500,6 +528,7 @@ def _create_main(
use_stack_allocator=True,
use_workspace_io=False,
debug_last_error=False,
+ print_output_on_mismatch=False,
):
file_path = pathlib.Path(f"{output_path}/" + test_name).resolve()
# create header file
@@ -568,7 +597,12 @@ def _create_main(
for compiled_model in compiled_models:
model = compiled_model.model
_emit_main_compare(
- main_file, model.outputs, model.output_tolerance, model.name, interface_api == "c"
+ main_file,
+ model.outputs,
+ model.output_tolerance,
+ model.name,
+ interface_api == "c",
+ print_output_on_mismatch,
)
_emit_main_epilogue(main_file, custom_epilogue)
@@ -709,6 +743,7 @@ def run_and_check(
use_workspace_io: bool = False,
debug_last_error: bool = False,
checker: Optional[Callable[[str], bool]] = None,
+ print_output_on_mismatch: bool = False,
):
"""
This method uses the original test data and compiled runtime.Modules
@@ -789,6 +824,7 @@ def run_and_check(
use_stack_allocator,
use_workspace_io,
debug_last_error,
+ print_output_on_mismatch,
)
if checker and (not checker(base_path)):
@@ -832,7 +868,10 @@ def run_and_check(
_subprocess_check_log_output(run_command, build_path, run_log_path)
with open(run_log_path) as run_log:
- assert AOT_SUCCESS_TOKEN in run_log.read()
+ run_log_out = run_log.read()
+ if print_output_on_mismatch and AOT_FAILURE_TOKEN in run_log_out:
+ print(run_log_out)
+ assert AOT_SUCCESS_TOKEN in run_log_out
return True
@@ -861,15 +900,21 @@ def compile_and_run(
schedule_name: str = None,
debug_last_error: bool = False,
checker: Optional[Callable[[str], bool]] = None,
+ print_output_on_mismatch: bool = False,
) -> bool:
"""This is a wrapper API to compile and run models as test for AoT
Parameters
----------
test_dir : str
- This path will contain build, codegen, include directories
- verbose: bool
- Prints commands to build and run AOT test runner
+ This path will contain build, codegen, include directories.
+
+ verbose : bool
+ Prints commands to build and run AOT test runner.
+
+ print_output_on_mismatch : bool
+ Print both the output and reference values side-by-side
+ when there is a mismatch.
"""
if target_opts:
@@ -904,6 +949,7 @@ def compile_and_run(
verbose=verbose,
debug_last_error=debug_last_error,
checker=checker,
+ print_output_on_mismatch=print_output_on_mismatch,
)
diff --git a/tests/python/relay/aot/test_aot_test_harness.py b/tests/python/relay/aot/test_aot_test_harness.py
new file mode 100644
index 0000000000..8ec9506f9f
--- /dev/null
+++ b/tests/python/relay/aot/test_aot_test_harness.py
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Tests for the AOT test harness.
+"""
+
+import pytest
+import numpy as np
+
+import tvm
+from tvm import relay
+from tvm.testing.aot import AOTTestRunner, compile_and_run, AOTTestModel
+
+
+def test_output_on_mismatch_option():
+ """
+ Test the print_output_on_mismatch option when there is a mismatch.
+ """
+ interface_api = "packed"
+ use_unpacked_api = True
+ test_runner = AOTTestRunner()
+ dtype = "float32"
+
+ two = relay.add(relay.const(1, dtype=dtype), relay.const(1, dtype=dtype))
+ func = relay.Function([], two)
+ outputs = {
+ "output": np.array(
+ [
+ 0,
+ ]
+ ).astype(dtype)
+ }
+
+ msg = ".*Actual, Reference\n2.000000, 0.000000\nAOT_TEST_FAILURE.*"
+ with pytest.raises(RuntimeError, match=msg):
+ compile_and_run(
+ AOTTestModel(module=tvm.IRModule.from_expr(func), inputs={}, outputs=outputs),
+ test_runner,
+ interface_api,
+ use_unpacked_api,
+ print_output_on_mismatch=True,
+ )
+
+
+if __name__ == "__main__":
+ tvm.testing.main()
diff --git a/tests/python/relay/aot/test_crt_aot.py b/tests/python/relay/aot/test_crt_aot.py
index f7e5af18d2..1c0f354d31 100644
--- a/tests/python/relay/aot/test_crt_aot.py
+++ b/tests/python/relay/aot/test_crt_aot.py
@@ -93,6 +93,7 @@ def test_conv_with_params(interface_api, use_unpacked_api, test_runner):
test_runner,
interface_api,
use_unpacked_api,
+ print_output_on_mismatch=True,
)