You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by me...@apache.org on 2022/05/11 23:22:32 UTC
[tvm] branch main updated: [Hexagon] capture gtest output and return over FFI (#11239)
This is an automated email from the ASF dual-hosted git repository.
mehrdadh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 1a8c64bb91 [Hexagon] capture gtest output and return over FFI (#11239)
1a8c64bb91 is described below
commit 1a8c64bb915e9bcdeefc1933592ceaa08210abff
Author: Adam Straw <as...@octoml.ai>
AuthorDate: Wed May 11 16:22:27 2022 -0700
[Hexagon] capture gtest output and return over FFI (#11239)
* [Hexagon] capture gtest output and return over FFI
* rename to test_hexagon_unit_tests.py so it will run in CI
* rename to run_unit_tests.cc
* pass back gtest error code along with gtest output
* skip Hexagon unit tests if gtest not enabled
* pass gtest_args as pytest argument
* change env variable to HEXAGON_GTEST
* set HEXAGON_GTEST in the environment to enable Hexagon unit tests
* add back try / except around get_function
---
docker/Dockerfile.ci_hexagon | 1 +
tests/cpp-runtime/hexagon/run_unit_tests.cc | 122 +++++++++++++++++++++
tests/python/contrib/test_hexagon/conftest.py | 10 ++
.../contrib/test_hexagon/test_run_unit_tests.py | 47 ++++++++
tests/python/contrib/test_hexagon/unit_tests.py | 42 -------
tests/scripts/task_build_hexagon_api.sh | 5 +-
tests/scripts/task_python_hexagon.sh | 3 +
7 files changed, 187 insertions(+), 43 deletions(-)
diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon
index b98276dddd..54bdf5d316 100644
--- a/docker/Dockerfile.ci_hexagon
+++ b/docker/Dockerfile.ci_hexagon
@@ -63,6 +63,7 @@ ENV CLANG_LLVM_HOME /opt/clang-llvm
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/opt/clang-llvm/lib
ENV PATH /opt/clang-llvm/bin:$PATH
ENV HEXAGON_TOOLCHAIN "${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/8.5.08/Tools"
+ENV HEXAGON_GTEST "${HEXAGON_SDK_PATH}/utils/googletest/gtest"
# sccache
COPY install/ubuntu_install_sccache.sh /install/ubuntu_install_sccache.sh
diff --git a/tests/cpp-runtime/hexagon/run_unit_tests.cc b/tests/cpp-runtime/hexagon/run_unit_tests.cc
new file mode 100644
index 0000000000..6ad770da33
--- /dev/null
+++ b/tests/cpp-runtime/hexagon/run_unit_tests.cc
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+
+#include <string>
+#include <vector>
+
+#include "../src/support/utils.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+class GtestPrinter : public testing::EmptyTestEventListener {
+ void OnTestProgramStart(const testing::UnitTest& unit_test) override {
+ gtest_out_ << "[==========] Running " << unit_test.test_to_run_count() << " test(s) from "
+ << unit_test.test_suite_to_run_count() << " test suite(s).\n";
+ }
+
+ void OnTestProgramEnd(const testing::UnitTest& unit_test) override {
+ gtest_out_ << "[==========] " << unit_test.test_to_run_count() << " test(s) from "
+ << unit_test.test_suite_to_run_count() << " test suite(s) ran. ("
+ << unit_test.elapsed_time() << " ms total)\n";
+ gtest_out_ << "[ PASSED ] " << unit_test.successful_test_count() << " test(s)\n";
+
+ if (unit_test.failed_test_count()) {
+ gtest_out_ << "[ FAILED ] " << unit_test.failed_test_count() << " test(s)\n";
+ }
+ }
+
+ void OnTestSuiteStart(const testing::TestSuite& test_suite) override {
+ gtest_out_ << "[----------] " << test_suite.test_to_run_count() << " test(s) from "
+ << test_suite.name() << "\n";
+ }
+
+ void OnTestSuiteEnd(const testing::TestSuite& test_suite) override {
+ gtest_out_ << "[----------] " << test_suite.test_to_run_count() << " test(s) from "
+ << test_suite.name() << " (" << test_suite.elapsed_time() << " ms total)\n";
+ }
+
+ void OnTestStart(const testing::TestInfo& test_info) override {
+ gtest_out_ << "[ RUN ] " << test_info.test_suite_name() << "." << test_info.name() << "\n";
+ }
+
+ void OnTestEnd(const testing::TestInfo& test_info) override {
+ for (int i = 0; i < test_info.result()->total_part_count(); ++i) {
+ gtest_out_ << test_info.result()->GetTestPartResult(i).message() << "\n";
+ }
+ if (test_info.result()->Passed()) {
+ gtest_out_ << "[ OK ]";
+ } else {
+ gtest_out_ << "[ FAILED ]";
+ }
+ gtest_out_ << " " << test_info.test_suite_name() << "." << test_info.name() << " ("
+ << test_info.result()->elapsed_time() << " ms)\n";
+ }
+
+ std::stringstream gtest_out_;
+
+ public:
+ std::string GetOutput() { return gtest_out_.str(); }
+};
+
+TVM_REGISTER_GLOBAL("hexagon.run_unit_tests").set_body([](TVMArgs args, TVMRetValue* rv) {
+ // gtest args are passed into this packed func as a singular string
+ // split gtest args using <space> delimiter and build argument vector
+ std::vector<std::string> parsed_args = tvm::support::Split(args[0], ' ');
+ std::vector<char*> argv;
+
+ // add executable name
+ argv.push_back(const_cast<char*>("hexagon_run_unit_tests"));
+
+ // add parsed arguments
+ for (int i = 0; i < parsed_args.size(); ++i) {
+ argv.push_back(const_cast<char*>(parsed_args[i].data()));
+ }
+
+ // end of parsed arguments
+ argv.push_back(nullptr);
+
+ // set argument count
+ int argc = argv.size() - 1;
+
+ // initialize gtest with arguments and run
+ ::testing::InitGoogleTest(&argc, argv.data());
+
+ // add printer to capture gtest output in a string
+ GtestPrinter* gprinter = new GtestPrinter();
+ testing::TestEventListeners& listeners = testing::UnitTest::GetInstance()->listeners();
+ listeners.Append(gprinter);
+
+ int gtest_error_code = RUN_ALL_TESTS();
+ std::string gtest_output = gprinter->GetOutput();
+ std::stringstream gtest_error_code_and_output;
+ gtest_error_code_and_output << gtest_error_code << std::endl;
+ gtest_error_code_and_output << gtest_output;
+ *rv = gtest_error_code_and_output.str();
+ delete gprinter;
+});
+
+} // namespace hexagon
+} // namespace runtime
+} // namespace tvm
diff --git a/tests/python/contrib/test_hexagon/conftest.py b/tests/python/contrib/test_hexagon/conftest.py
index 7a90317d55..e09329b76b 100644
--- a/tests/python/contrib/test_hexagon/conftest.py
+++ b/tests/python/contrib/test_hexagon/conftest.py
@@ -218,3 +218,13 @@ def aot_target(aot_host_target):
yield aot_host_target
else:
assert False, "Incorrect AoT host target: {aot_host_target}. Options are [c, llvm]."
+
+
+def pytest_addoption(parser):
+ parser.addoption("--gtest_args", action="store", default="")
+
+
+def pytest_generate_tests(metafunc):
+ option_value = metafunc.config.option.gtest_args
+ if "gtest_args" in metafunc.fixturenames and option_value is not None:
+ metafunc.parametrize("gtest_args", [option_value])
diff --git a/tests/python/contrib/test_hexagon/test_run_unit_tests.py b/tests/python/contrib/test_hexagon/test_run_unit_tests.py
new file mode 100644
index 0000000000..3a383d30e5
--- /dev/null
+++ b/tests/python/contrib/test_hexagon/test_run_unit_tests.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import pytest
+import numpy as np
+from tvm.contrib.hexagon.build import HexagonLauncher
+from .conftest import requires_hexagon_toolchain
+
+
+# use pytest -sv to observe gtest output
+# use --gtest_args to pass arguments to gtest
+# for example to run all "foo" tests twice and observe gtest output run
+# pytest -sv <this file> --gtests_args="--gtest_filter=*foo* --gtest_repeat=2"
+@requires_hexagon_toolchain
+@pytest.mark.skipif(
+ os.environ.get("HEXAGON_GTEST") == None,
+ reason="Test requires environment variable HEXAGON_GTEST set with a path to a Hexagon gtest version normally located at /path/to/hexagon/sdk/utils/googletest/gtest",
+)
+def test_run_unit_tests(hexagon_session, gtest_args):
+ try:
+ func = hexagon_session._rpc.get_function("hexagon.run_unit_tests")
+ except:
+ print(
+ "Test requires TVM Runtime to be built with a Hexagon gtest version using Hexagon API cmake flag -DUSE_HEXAGON_GTEST=${HEXAGON_GTEST}"
+ )
+ raise
+
+ gtest_error_code_and_output = func(gtest_args)
+ gtest_error_code = int(gtest_error_code_and_output.splitlines()[0])
+ gtest_output = gtest_error_code_and_output.split("\n", 1)[-1]
+ print(gtest_output)
+ np.testing.assert_equal(gtest_error_code, 0)
diff --git a/tests/python/contrib/test_hexagon/unit_tests.py b/tests/python/contrib/test_hexagon/unit_tests.py
deleted file mode 100644
index d340cba5b1..0000000000
--- a/tests/python/contrib/test_hexagon/unit_tests.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import pytest
-import numpy as np
-from tvm.contrib.hexagon.build import HexagonLauncher
-from .conftest import requires_hexagon_toolchain
-
-
-@requires_hexagon_toolchain
-def test_cache_read_write_2d(hexagon_session):
- # arguments to pass to gtest
- # e.g.
- # 1) to run all tests use:
- # gtest_args = ""
- # 2) to run all tests with "foo" in their name twice use:
- # gtest_args = "--gtest_repeat=2 --gtest_filter=*foo*"
- gtest_args = ""
- try:
- func = hexagon_session._rpc.get_function("hexagon.run_all_tests")
- result = func(gtest_args)
- except:
- print(
- "This test requires the USE_HEXAGON_GTEST cmake flag to be specified with a path to a Hexagon gtest version normally located at /path/to/hexagon/sdk/utils/googletest/gtest"
- )
- result = 1
-
- np.testing.assert_equal(result, 0)
diff --git a/tests/scripts/task_build_hexagon_api.sh b/tests/scripts/task_build_hexagon_api.sh
index a3b501d9c5..c5d05eaad8 100755
--- a/tests/scripts/task_build_hexagon_api.sh
+++ b/tests/scripts/task_build_hexagon_api.sh
@@ -37,6 +37,9 @@ cd build
output_binary_directory=$(realpath ${PWD}/../../../build/hexagon_api_output)
rm -rf ${output_binary_directory}
+# should be removed after Hexagon Docker update
+export HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest"
+
cmake -DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-28 \
-DUSE_ANDROID_TOOLCHAIN="${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake" \
@@ -44,6 +47,6 @@ cmake -DANDROID_ABI=arm64-v8a \
-DUSE_HEXAGON_SDK="${HEXAGON_SDK_PATH}" \
-DUSE_HEXAGON_TOOLCHAIN="${HEXAGON_TOOLCHAIN}" \
-DUSE_OUTPUT_BINARY_DIR="${output_binary_directory}" \
- -DUSE_HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest" ..
+ -DUSE_HEXAGON_GTEST="${HEXAGON_GTEST}" ..
make -j$(nproc)
diff --git a/tests/scripts/task_python_hexagon.sh b/tests/scripts/task_python_hexagon.sh
index 274b348f09..b639ac02a6 100755
--- a/tests/scripts/task_python_hexagon.sh
+++ b/tests/scripts/task_python_hexagon.sh
@@ -43,6 +43,9 @@ if [[ "${device_serial}" == "simulator" ]]; then
export HEXAGON_SDK_ROOT=${HEXAGON_SDK_PATH}
fi
+# should be removed after Hexagon Docker update
+export HEXAGON_GTEST="${HEXAGON_SDK_PATH}/utils/googletest/gtest"
+
export ANDROID_SERIAL_NUMBER=${device_serial}
run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon