You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by me...@apache.org on 2022/06/17 16:15:49 UTC
[tvm] branch main updated: [hexagon][testing] add test-skip logic; fixes (#11737)
This is an automated email from the ASF dual-hosted git repository.
mehrdadh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 2ffd955779 [hexagon][testing] add test-skip logic; fixes (#11737)
2ffd955779 is described below
commit 2ffd9557794ab34b20deed1d0ed5e38cb2f81e20
Author: Christian Convey <cc...@octoml.ai>
AuthorDate: Fri Jun 17 12:15:43 2022 -0400
[hexagon][testing] add test-skip logic; fixes (#11737)
- Skip Hexagon benchmarks whenever the env. var `ANDROID_SERIAL_NUMBER`
has the value `simulator`.
This is a temporary hack to prevent the CI pre-commit hook from
running benchmarks, due to the extra time required.
- Fix a bug where the elementwise-add benchmark code was broken by
an earlier change to the `HexagonLauncherRPC` class.
- Rename `benchmark_elemwise_add.py` to `test_benchmark_elemwise_add.py`
so that it's noticed by the CI test infrastructure.
(CI tests are sometimes run in contexts _other than_ the pre-commit
hook.)
- Miscellaneous small changes to
`tests/python/contrib/test_hexagon/benchmark_util.py`.
---
.../python/contrib/test_hexagon/benchmark_util.py | 34 ++++++++++++++++
...mwise_add.py => test_benchmark_elemwise_add.py} | 46 ++++++++++------------
2 files changed, 54 insertions(+), 26 deletions(-)
diff --git a/tests/python/contrib/test_hexagon/benchmark_util.py b/tests/python/contrib/test_hexagon/benchmark_util.py
index 113c7780c1..35fe6bad64 100644
--- a/tests/python/contrib/test_hexagon/benchmark_util.py
+++ b/tests/python/contrib/test_hexagon/benchmark_util.py
@@ -16,6 +16,40 @@
# under the License.
import csv
+import os
+
+
+def skip_bencharks_flag_and_reason():
+ """
+ Returns one of these tuples:
+ (False, '') or
+ (True, (a string describing why the test should be skipped))
+
+ NOTE: This function is a temporary measure to prevent the TVM CI system
+ running benchmark scripts every time the CI pre-commit hook executes.
+ This should go away when a better system is in place to govern when various
+ tests / benchmarks are executed.
+ """
+ asn = os.environ.get("ANDROID_SERIAL_NUMBER")
+
+ if asn == "simulator":
+ return (True, "Skipping benchmarks when ANDROID_SERIAL_NUMBER='simluator'")
+ else:
+ return (False, "")
+
+
+class UnsupportedException(Exception):
+ """
+ Indicates that the specified benchmarking configuration is known to
+ currently be unsupported. The Exception message may provide more detail.
+ """
+
+
+class NumericalAccuracyException(Exception):
+ """
+ Indicates that the benchmarking configuration appeared to run successfully,
+ but the output data didn't have the expected accuracy.
+ """
class BenchmarksTable:
diff --git a/tests/python/contrib/test_hexagon/benchmark_elemwise_add.py b/tests/python/contrib/test_hexagon/test_benchmark_elemwise_add.py
similarity index 93%
rename from tests/python/contrib/test_hexagon/benchmark_elemwise_add.py
rename to tests/python/contrib/test_hexagon/test_benchmark_elemwise_add.py
index 70266d7939..f7f5f3e176 100644
--- a/tests/python/contrib/test_hexagon/benchmark_elemwise_add.py
+++ b/tests/python/contrib/test_hexagon/test_benchmark_elemwise_add.py
@@ -28,7 +28,9 @@ import tvm.script
from tvm.script import tir as T
from tvm import te
from tvm.contrib.hexagon.build import HexagonLauncherRPC
-from . import benchmark_util
+from . import benchmark_util as bu
+
+_SHOULD_SKIP_BENCHMARKS, _SKIP_BENCHMARKS_REASON = bu.skip_bencharks_flag_and_reason()
# This is a fixed detail of the v68 architecture.
HVX_VECTOR_BYTES = 128
@@ -43,7 +45,7 @@ _SUPER_TARGET = tvm.target.Target(_HEXAGON_TARGET, host=_HEXAGON_TARGET)
# triggering TIME_WAIT state on the server socket. This prevents another
# server to bind to the same port until the wait time elapses.
-_BT = benchmark_util.BenchmarksTable()
+_BT = bu.BenchmarksTable()
_CSV_COLUMN_ORDER = [
# Identifies which TE-compute / TIRScript is used as the basis for the
@@ -88,21 +90,6 @@ print("OUTPUT DIRECTORY: {}".format(_HOST_OUTPUT_DIR))
print("-" * 80)
print()
-
-class UnsupportedException(Exception):
- """
- Indicates that the specified benchmarking configuration is known to
- currently be unsupported. The Exception message may provide more detail.
- """
-
-
-class NumericalAccuracyException(Exception):
- """
- Indicates that the benchmarking configuration appeared to run successfully,
- but the output data didn't have the expected accuracy.
- """
-
-
from typing import Tuple
@@ -129,7 +116,7 @@ def _get_irmod_elemwise_add(
dtype_str = str(dtype)
if mem_scope == "global.vtcm":
- raise UnsupportedException("This benchmark kernel does not yet support VTCM buffers.")
+ raise bu.UnsupportedException("This benchmark kernel does not yet support VTCM buffers.")
# This check is currently elided by the one above, but it should become relevant as soon
# as we add VTCM support to this kernel generator.
@@ -147,7 +134,7 @@ def _get_irmod_elemwise_add(
estimated_vtcm_needed_bytes = shape[0] * shape[1] * dtype_bytes * num_vtcm_tensors
if estimated_vtcm_needed_bytes > estimated_vtcm_budget_bytes:
- raise UnsupportedException("Expect to exceed VTCM budget.")
+ raise bu.UnsupportedException("Expect to exceed VTCM budget.")
@tvm.script.ir_module
class BenchmarkModule:
@@ -190,10 +177,10 @@ def _benchmark_hexagon_elementwise_add_kernel(
"mem_scope": mem_scope,
}
- desc = benchmark_util.get_benchmark_decription(keys_dict)
+ desc = bu.get_benchmark_decription(keys_dict)
# Create the host-side directory for this benchmark run's files / logs...
- host_files_dir_name = benchmark_util.get_benchmark_id(keys_dict)
+ host_files_dir_name = bu.get_benchmark_id(keys_dict)
host_files_dir_path = os.path.join(_HOST_OUTPUT_DIR, host_files_dir_name)
os.mkdir(host_files_dir_path)
@@ -238,7 +225,9 @@ def _benchmark_hexagon_elementwise_add_kernel(
# Upload the .so to the Android device's file system (or wherever is appropriate
# when using the Hexagon simulator)...
target_dso_binary_filename = "test_binary.so"
- hexagon_launcher.upload(host_dso_binary_path, target_dso_binary_filename)
+ target_dso_binary_pathname = hexagon_launcher.upload(
+ host_dso_binary_path, target_dso_binary_filename
+ )
# Generate our testing / validation data...
(
@@ -251,7 +240,7 @@ def _benchmark_hexagon_elementwise_add_kernel(
# On the target device / simulator, make our Hexagon-native shared object
# available for use...
loaded_hexagon_module: tvm.runtime.module.Module = hexagon_launcher.load_module(
- target_dso_binary_filename, sess
+ target_dso_binary_pathname, sess
)
# Create the target-side tensors to hold the primfunc's inputs and outputs...
@@ -296,11 +285,11 @@ def _benchmark_hexagon_elementwise_add_kernel(
result, host_numpy_C_data_expected, rel_tolerance, abs_tolerance
)
except AssertionError as e:
- raise NumericalAccuracyException(str(e))
+ raise bu.NumericalAccuracyException(str(e))
_BT.record_success(timing_result, **keys_dict)
- except NumericalAccuracyException as e:
+ except bu.NumericalAccuracyException as e:
print()
print(f"FAIL: Numerical accuracy error. See log file.")
@@ -309,7 +298,7 @@ def _benchmark_hexagon_elementwise_add_kernel(
_BT.record_fail(**keys_dict, comments=f"Numerical accuracy error. See log file.")
- except UnsupportedException as e:
+ except bu.UnsupportedException as e:
print()
print(f"SKIP: {e}")
@@ -381,6 +370,7 @@ def _get_elemwise_add_reference_value_tensors(shape: list, dtype: str):
]
+@pytest.mark.skipif(_SHOULD_SKIP_BENCHMARKS, reason=_SKIP_BENCHMARKS_REASON)
@tvm.testing.requires_hexagon
def test_elemwise_add(hexagon_launcher: HexagonLauncherRPC):
for dtype in [
@@ -432,3 +422,7 @@ def test_elemwise_add(hexagon_launcher: HexagonLauncherRPC):
if _BT.has_fail() > 0:
pytest.fail("At least one benchmark configuration failed", pytrace=False)
+
+
+if __name__ == "__main__":
+ tvm.testing.main()