You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2025/02/15 02:00:17 UTC
(tvm) 01/01: Cleanup relay related python components
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch refactor
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 4e4ab3c47153678be0dc5563db7897146d197fa7
Author: tqchen <ti...@gmail.com>
AuthorDate: Fri Feb 14 17:27:05 2025 -0500
Cleanup relay related python components
---
apps/cpp_clml/CMakeLists.txt | 61 -
apps/cpp_clml/README.md | 145 --
apps/cpp_clml/clml_runner.cc | 822 ---------
apps/cpp_clml/clml_runner.h | 262 ---
apps/cpp_clml/main.cc | 243 ---
apps/cpp_clml/scripts/clml_codegen.py | 64 -
apps/cpp_clml/scripts/clml_codegen_json.py | 51 -
apps/cpp_clml/scripts/compare_npy.py | 41 -
ci/jenkins/generated/arm_jenkinsfile.groovy | 126 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 174 +-
ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 22 -
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 15 -
docs/conf.py | 1 -
docs/reference/api/python/auto_scheduler.rst | 23 -
docs/reference/api/python/autotvm.rst | 90 -
docs/reference/api/python/graph_executor.rst | 21 -
docs/reference/api/python/index.rst | 18 -
docs/reference/api/python/relay/analysis.rst | 25 -
docs/reference/api/python/relay/backend.rst | 33 -
.../api/python/relay/dataflow_pattern.rst | 25 -
docs/reference/api/python/relay/frontend.rst | 25 -
docs/reference/api/python/relay/image.rst | 26 -
docs/reference/api/python/relay/index.rst | 30 -
docs/reference/api/python/relay/nn.rst | 23 -
docs/reference/api/python/relay/testing.rst | 51 -
docs/reference/api/python/relay/transform.rst | 25 -
docs/reference/api/python/relay/vision.rst | 25 -
docs/reference/api/python/topi.rst | 8 -
jvm/core/src/test/scripts/test_graph_executor.py | 69 -
python/tvm/__init__.py | 3 -
python/tvm/contrib/cutlass/__init__.py | 2 +-
python/tvm/contrib/cutlass/build.py | 256 +--
python/tvm/contrib/graph_executor.py | 549 ------
python/tvm/contrib/pipeline_executor.py | 345 ----
python/tvm/contrib/pipeline_executor_build.py | 674 --------
python/tvm/contrib/target/coreml.py | 2 +-
python/tvm/contrib/target/onnx.py | 1110 ------------
python/tvm/contrib/torch/__init__.py | 78 -
python/tvm/contrib/torch/as_torch.py | 156 --
python/tvm/contrib/torch/module.py | 138 --
python/tvm/contrib/torch/optimize_torch.py | 177 --
python/tvm/contrib/torch/pytorch_tvm.py | 292 ----
python/tvm/contrib/xcode.py | 1 -
python/tvm/driver/build_module.py | 1 +
python/tvm/driver/tvmc/__init__.py | 39 -
python/tvm/driver/tvmc/__main__.py | 24 -
python/tvm/driver/tvmc/arguments.py | 52 -
python/tvm/driver/tvmc/autotuner.py | 864 ---------
python/tvm/driver/tvmc/compiler.py | 743 --------
python/tvm/driver/tvmc/composite_target.py | 109 --
python/tvm/driver/tvmc/config_options.py | 210 ---
python/tvm/driver/tvmc/fmtopt.py | 116 --
python/tvm/driver/tvmc/frontends.py | 478 -----
python/tvm/driver/tvmc/main.py | 122 --
python/tvm/driver/tvmc/model.py | 467 -----
python/tvm/driver/tvmc/pass_config.py | 194 ---
python/tvm/driver/tvmc/pass_list.py | 54 -
python/tvm/driver/tvmc/project.py | 233 ---
python/tvm/driver/tvmc/registry.py | 104 --
python/tvm/driver/tvmc/result_utils.py | 60 -
python/tvm/driver/tvmc/runner.py | 578 -------
python/tvm/driver/tvmc/shape_parser.py | 67 -
python/tvm/driver/tvmc/target.py | 431 -----
python/tvm/driver/tvmc/tracker.py | 57 -
python/tvm/driver/tvmc/transform.py | 274 ---
python/tvm/driver/tvmc/workspace_pools.py | 237 ---
python/tvm/exec/autotvm_log_editor.py | 59 -
python/tvm/ir/__init__.py | 1 -
python/tvm/ir/adt.py | 88 -
python/tvm/ir/expr.py | 9 +-
python/tvm/meta_schedule/__init__.py | 2 -
python/tvm/meta_schedule/relay_integration.py | 467 -----
python/tvm/meta_schedule/testing/relay_workload.py | 317 ----
python/tvm/meta_schedule/testing/te_workload.py | 41 -
python/tvm/meta_schedule/testing/tlcbench.py | 122 --
python/tvm/parser.py | 53 -
python/tvm/relax/testing/__init__.py | 1 -
python/tvm/relax/testing/relay_translator.py | 273 ---
python/tvm/relax/testing/transform.py | 104 +-
python/tvm/runtime/__init__.py | 1 -
python/tvm/runtime/executor/__init__.py | 26 -
python/tvm/runtime/executor/aot_executor.py | 201 ---
python/tvm/runtime/vm.py | 738 --------
python/tvm/testing/utils.py | 7 +-
python/tvm/topi/__init__.py | 2 -
python/tvm/topi/nn/batch_matmul.py | 12 +-
python/tvm/topi/nn/conv2d.py | 1 -
python/tvm/topi/nn/conv2d_transpose.py | 1 -
python/tvm/topi/nn/conv3d_transpose.py | 61 -
python/tvm/topi/nn/dense.py | 6 +-
python/tvm/topi/sparse/__init__.py | 24 -
python/tvm/topi/sparse/csrmm.py | 127 --
python/tvm/topi/sparse/csrmv.py | 117 --
python/tvm/topi/sparse/dense.py | 217 ---
python/tvm/topi/sparse/utils.py | 358 ----
python/tvm/topi/testing/__init__.py | 9 -
python/tvm/topi/testing/common.py | 92 -
rust/Cargo.toml | 11 +-
rust/compiler-ext/Cargo.toml | 32 -
rust/compiler-ext/src/lib.rs | 35 -
rust/tvm-graph-rt/Cargo.toml | 45 -
rust/tvm-graph-rt/README.md | 20 -
rust/tvm-graph-rt/src/allocator.rs | 73 -
rust/tvm-graph-rt/src/array.rs | 401 -----
rust/tvm-graph-rt/src/errors.rs | 55 -
rust/tvm-graph-rt/src/graph.rs | 523 ------
rust/tvm-graph-rt/src/lib.rs | 76 -
rust/tvm-graph-rt/src/module/dso.rs | 148 --
rust/tvm-graph-rt/src/module/mod.rs | 65 -
rust/tvm-graph-rt/src/module/syslib.rs | 73 -
rust/tvm-graph-rt/src/threading.rs | 263 ---
rust/tvm-graph-rt/src/workspace.rs | 139 --
rust/tvm-graph-rt/tests/.gitignore | 3 -
rust/tvm-graph-rt/tests/build_model.py | 54 -
rust/tvm-graph-rt/tests/test_graph_serde.rs | 72 -
rust/tvm-graph-rt/tests/test_nn/Cargo.toml | 33 -
rust/tvm-graph-rt/tests/test_nn/build.rs | 70 -
.../tests/test_nn/src/build_test_graph.py | 55 -
rust/tvm-graph-rt/tests/test_nn/src/main.rs | 99 --
rust/tvm-graph-rt/tests/test_tvm_basic/Cargo.toml | 32 -
rust/tvm-graph-rt/tests/test_tvm_basic/build.rs | 70 -
.../tests/test_tvm_basic/src/build_test_lib.py | 42 -
rust/tvm-graph-rt/tests/test_tvm_basic/src/main.rs | 47 -
rust/tvm-graph-rt/tests/test_tvm_dso/Cargo.toml | 31 -
rust/tvm-graph-rt/tests/test_tvm_dso/build.rs | 46 -
.../tests/test_tvm_dso/src/build_test_lib.py | 43 -
rust/tvm-graph-rt/tests/test_tvm_dso/src/main.rs | 39 -
rust/tvm-graph-rt/tests/test_wasm32/.cargo/config | 2 -
rust/tvm-graph-rt/tests/test_wasm32/Cargo.toml | 30 -
rust/tvm-graph-rt/tests/test_wasm32/build.rs | 77 -
.../tests/test_wasm32/src/build_test_lib.py | 44 -
rust/tvm-graph-rt/tests/test_wasm32/src/main.rs | 54 -
rust/tvm-rt/Cargo.toml | 3 +
rust/tvm/.gitignore | 7 -
rust/tvm/.travis.yml | 22 -
rust/tvm/Cargo.toml | 103 --
rust/tvm/README.md | 58 -
rust/tvm/examples/resnet/Cargo.toml | 34 -
rust/tvm/examples/resnet/README.md | 45 -
rust/tvm/examples/resnet/build.rs | 57 -
rust/tvm/examples/resnet/src/build_resnet.py | 160 --
rust/tvm/examples/resnet/src/main.rs | 139 --
rust/tvm/src/bin/tyck.rs | 49 -
rust/tvm/src/compiler/graph_rt.rs | 124 --
rust/tvm/src/compiler/mod.rs | 20 -
rust/tvm/src/ir/arith.rs | 46 -
rust/tvm/src/ir/attrs.rs | 29 -
rust/tvm/src/ir/diagnostics/codespan.rs | 216 ---
rust/tvm/src/ir/diagnostics/mod.rs | 246 ---
rust/tvm/src/ir/expr.rs | 104 --
rust/tvm/src/ir/function.rs | 46 -
rust/tvm/src/ir/mod.rs | 34 -
rust/tvm/src/ir/module.rs | 399 -----
rust/tvm/src/ir/op.rs | 43 -
rust/tvm/src/ir/relay/attrs/mod.rs | 22 -
rust/tvm/src/ir/relay/attrs/nn.rs | 258 ---
rust/tvm/src/ir/relay/attrs/reduce.rs | 48 -
rust/tvm/src/ir/relay/attrs/transform.rs | 140 --
rust/tvm/src/ir/relay/mod.rs | 589 -------
rust/tvm/src/ir/source_map.rs | 57 -
rust/tvm/src/ir/span.rs | 71 -
rust/tvm/src/ir/tir.rs | 100 --
rust/tvm/src/ir/ty.rs | 312 ----
rust/tvm/src/lib.rs | 76 -
rust/tvm/src/python.rs | 70 -
rust/tvm/src/runtime/mod.rs | 20 -
rust/tvm/src/transform.rs | 113 --
rust/tvm/tests/basics/.gitignore | 7 -
rust/tvm/tests/basics/Cargo.toml | 36 -
rust/tvm/tests/basics/build.rs | 52 -
rust/tvm/tests/basics/src/main.rs | 54 -
rust/tvm/tests/basics/src/tvm_add.py | 47 -
rust/tvm/tests/callback/Cargo.toml | 26 -
rust/tvm/tests/callback/src/bin/array.rs | 63 -
rust/tvm/tests/callback/src/bin/error.rs | 45 -
rust/tvm/tests/callback/src/bin/float.rs | 50 -
rust/tvm/tests/callback/src/bin/int.rs | 45 -
rust/tvm/tests/callback/src/bin/string.rs | 59 -
tests/cpp/relay/backend/aot/aot_lower_main_test.cc | 63 -
tests/cpp/relay/backend/executor_test.cc | 77 -
tests/cpp/relay/backend/graph_plan_token_alloc.cc | 540 ------
tests/cpp/relay/backend/runtime_test.cc | 77 -
.../cpp/relay/collage/candidate_partition_test.cc | 221 ---
tests/cpp/relay/collage/partition_rule_test.cc | 715 --------
tests/cpp/relay/df_pattern_rewrite_test.cc | 100 --
tests/cpp/relay/ir/indexed_graph_test.cc | 205 ---
tests/cpp/relay/op/memory/on_device_test.cc | 89 -
tests/cpp/relay/transforms/device_domains_test.cc | 76 -
tests/cpp/relay/with_fields_test.cc | 215 ---
tests/cpp/relay_build_module_test.cc | 193 ---
tests/cpp/relay_dismantler_test.cc | 164 --
tests/cpp/relay_pass_type_infer_test.cc | 43 -
tests/cpp/relay_text_printer_test.cc | 58 -
tests/cpp/relay_transform_sequential_test.cc | 135 --
tests/lint/pylint.sh | 25 -
tests/python/codegen/test_target_codegen_vulkan.py | 5 +-
tests/python/contrib/test_amx.py | 131 --
.../contrib/test_arm_compute_lib/__init__.py | 17 -
.../contrib/test_arm_compute_lib/infrastructure.py | 341 ----
.../contrib/test_arm_compute_lib/test_add.py | 168 --
.../test_arm_compute_lib/test_concatenate.py | 159 --
.../contrib/test_arm_compute_lib/test_config.json | 8 -
.../contrib/test_arm_compute_lib/test_conv2d.py | 672 -------
.../contrib/test_arm_compute_lib/test_dense.py | 430 -----
.../contrib/test_arm_compute_lib/test_maximum.py | 105 --
.../contrib/test_arm_compute_lib/test_network.py | 231 ---
.../contrib/test_arm_compute_lib/test_pooling.py | 419 -----
.../contrib/test_arm_compute_lib/test_reshape.py | 111 --
.../contrib/test_arm_compute_lib/test_runtime.py | 134 --
tests/python/contrib/test_bnns/__init__.py | 17 -
tests/python/contrib/test_bnns/infrastructure.py | 328 ----
tests/python/contrib/test_bnns/test_conv2d.py | 177 --
.../contrib/test_bnns/test_conv2d_patterns.py | 107 --
tests/python/contrib/test_bnns/test_dense.py | 190 --
tests/python/contrib/test_bnns/test_matmul.py | 113 --
.../python/contrib/test_bnns/test_normalization.py | 201 ---
.../contrib/test_bnns/test_onnx_topologies.py | 140 --
tests/python/contrib/test_bnns/test_pooling.py | 289 ----
tests/python/contrib/test_clml/__init__.py | 17 -
tests/python/contrib/test_clml/conftest.py | 39 -
tests/python/contrib/test_clml/infrastructure.py | 250 ---
.../test_clml/test_adreno_collage_targets.py | 354 ----
tests/python/contrib/test_clml/test_compiler.py | 42 -
tests/python/contrib/test_clml/test_network.py | 158 --
tests/python/contrib/test_clml/test_ops.py | 1362 ---------------
tests/python/contrib/test_coreml_codegen.py | 252 ---
tests/python/contrib/test_cublas.py | 385 ----
tests/python/contrib/test_cudnn.py | 627 -------
tests/python/contrib/test_cutlass.py | 1288 --------------
tests/python/contrib/test_dnnl.py | 1829 --------------------
.../test_hexagon/metaschedule_e2e/__init__.py | 18 -
.../test_hexagon/metaschedule_e2e/export_models.py | 79 -
.../metaschedule_e2e/test_resnet50_fp16.py | 135 --
.../metaschedule_e2e/test_resnet50_int8.py | 539 ------
tests/python/contrib/test_hexagon/test_autotvm.py | 175 --
.../test_hexagon/test_fixed_point_multiply.py | 310 ----
tests/python/contrib/test_hexagon/test_models.py | 87 -
.../test_hexagon/test_qnn_op_integration.py | 576 ------
.../test_hexagon/test_relay_simplify_conv_pat.py | 203 ---
.../test_hexagon/test_relay_simplify_qnn_concat.py | 99 --
.../contrib/test_hexagon/test_relay_transforms.py | 120 --
tests/python/contrib/test_hexagon/topi/__init__.py | 18 -
.../contrib/test_hexagon/topi/slice_op/__init__.py | 18 -
.../topi/slice_op/test_argmax_slice.py | 119 --
.../topi/slice_op/test_avg_pool2d_slice.py | 478 -----
.../test_hexagon/topi/slice_op/test_cast_slice.py | 203 ---
.../test_hexagon/topi/slice_op/test_clip_slice.py | 125 --
.../topi/slice_op/test_conv2d_slice.py | 338 ----
.../test_hexagon/topi/slice_op/test_dense_slice.py | 282 ---
.../topi/slice_op/test_depthwise_conv2d_slice.py | 340 ----
.../topi/slice_op/test_dequantize_slice.py | 124 --
.../topi/slice_op/test_global_avg_pool2d.py | 167 --
.../topi/slice_op/test_max_pool2d_slice.py | 405 -----
.../test_hexagon/topi/slice_op/test_relu_slice.py | 118 --
.../topi/slice_op/test_softmax_slice.py | 134 --
.../test_hexagon/topi/slice_op/test_tanh_slice.py | 108 --
.../test_hexagon/topi/test_adaptive_avg_pool1d.py | 185 --
.../topi/test_add_subtract_multiply.py | 411 -----
.../contrib/test_hexagon/topi/test_batch_matmul.py | 159 --
.../test_hexagon/topi/test_conv2d_fp16_intrin.py | 246 ---
.../contrib/test_hexagon/topi/test_conv2d_nchw.py | 247 ---
.../contrib/test_hexagon/topi/test_conv2d_nhwc.py | 128 --
.../test_hexagon/topi/test_conv2d_quant_intrin.py | 261 ---
.../test_hexagon/topi/test_conv2d_transpose.py | 130 --
.../python/contrib/test_hexagon/topi/test_dense.py | 129 --
.../test_hexagon/topi/test_depth_to_space.py | 135 --
.../test_hexagon/topi/test_depthwise_conv2d.py | 300 ----
tests/python/contrib/test_hexagon/topi/test_pad.py | 59 -
.../contrib/test_hexagon/topi/test_pooling.py | 751 --------
.../contrib/test_hexagon/topi/test_quantize.py | 128 --
.../contrib/test_hexagon/topi/test_reduce.py | 164 --
.../contrib/test_hexagon/topi/test_reshape.py | 188 --
.../contrib/test_hexagon/topi/test_resize2d.py | 203 ---
.../contrib/test_hexagon/topi/test_softmax.py | 106 --
tests/python/contrib/test_libtorch_ops.py | 93 -
tests/python/contrib/test_mrvl/__init__.py | 17 -
tests/python/contrib/test_mrvl/infrastructure.py | 153 --
tests/python/contrib/test_mrvl/test_mrvl.py | 295 ----
.../contrib/test_msc/test_translate_relay.py | 1143 ------------
tests/python/contrib/test_nnpack.py | 220 ---
tests/python/contrib/test_onnx.py | 754 --------
tests/python/contrib/test_onnx_model.py | 171 --
tests/python/contrib/test_rpc_server_device.py | 442 -----
tests/python/contrib/test_tedd.py | 214 ---
tests/python/contrib/test_tensorrt.py | 1410 ---------------
tests/python/contrib/test_tensorrt_int8_exp.py | 156 --
tests/python/contrib/test_thrust.py | 146 --
tests/python/contrib/test_uma/test_partition.py | 94 -
tests/python/contrib/test_uma/test_target.py | 104 --
.../test_uma/test_uma_lowering_with_umalower.py | 121 --
tests/python/contrib/test_uma/test_uma_utils.py | 87 -
.../test_uma/test_uma_vanilla_accelerator.py | 57 -
tests/python/contrib/test_vitis_ai/__init__.py | 18 -
.../python/contrib/test_vitis_ai/infrastructure.py | 165 --
.../contrib/test_vitis_ai/test_vitis_ai_codegen.py | 383 ----
.../test_vitis_ai_runtime_cpu_part.py | 87 -
.../test_meta_schedule_byoc_tensorrt.py | 172 --
.../test_meta_schedule_cpu_dot_product.py | 274 ---
.../meta_schedule/test_meta_schedule_database.py | 15 +-
...meta_schedule_mma_m16n8k8_auto_tensorization.py | 1239 -------------
.../test_meta_schedule_multi_anchor.py | 88 -
.../test_meta_schedule_relay_integration.py | 1002 -----------
.../test_meta_schedule_relay_tir_compute.py | 174 --
...meta_schedule_schedule_cuda_layout_transform.py | 471 -----
.../test_meta_schedule_space_cpu_winograd.py | 167 --
.../test_meta_schedule_space_cuda_winograd.py | 593 -------
.../quantization/test_quantization_accuracy.py | 229 ---
.../test_quantization_accuracy_for_vit.py | 153 --
tests/python/relax/test_blockbuilder_core.py | 7 +-
tests/python/relax/test_codegen_coreml.py | 2 +
tests/python/relax/test_dataflow_pattern.py | 4 +-
tests/python/relax/test_relay_translator.py | 346 ----
.../relax/test_transform_gradient_numeric.py | 5 +-
tests/python/relax/test_vm_build.py | 1 -
tests/python/runtime/test_runtime_container.py | 1 -
tests/python/runtime/test_runtime_module_export.py | 1 +
tests/python/runtime/test_runtime_module_load.py | 2 +-
tests/python/runtime/test_runtime_rpc.py | 28 +-
.../test_tir_analysis_device_constraint_utils.py | 68 -
.../tir-analysis/test_tir_analysis_stmt_finding.py | 54 -
.../test_tir_analysis_verify_gpu_code.py | 28 -
tests/python/tir-base/test_debug_info.py | 1 -
.../test_tir_transform_common_subexpr_elim.py | 47 +-
.../test_tir_transform_fp8_legalize.py | 4 +
.../tir-transform/test_tir_transform_hoist_if.py | 64 -
.../test_tir_transform_narrow_datatype.py | 72 +-
...sform_plan_update_buffer_allocation_location.py | 54 +-
.../test_tir_transform_storage_flatten.py | 3 +-
tests/python/tir-usmp/test_tir_usmp_algo.py | 45 -
.../tvmscript/test_tvmscript_printer_highlight.py | 24 -
.../test_tvmscript_printer_structural_equal.py | 1 -
tests/scripts/ci.py | 1 -
tests/scripts/task_java_unittest.sh | 1 -
tests/scripts/task_python_adreno.sh | 2 +
tests/scripts/task_python_arm_compute_library.sh | 2 -
tests/scripts/task_python_integration.sh | 9 -
tests/scripts/task_python_topi.sh | 8 -
tests/scripts/task_python_unittest.sh | 4 +-
web/tests/python/prepare_test_libs.py | 10 +-
339 files changed, 65 insertions(+), 57705 deletions(-)
diff --git a/apps/cpp_clml/CMakeLists.txt b/apps/cpp_clml/CMakeLists.txt
deleted file mode 100644
index 8c0fd53bf9..0000000000
--- a/apps/cpp_clml/CMakeLists.txt
+++ /dev/null
@@ -1,61 +0,0 @@
-cmake_minimum_required(VERSION 3.13)
-
-project(clml_run VERSION 2.0)
-
-if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
- message( FATAL_ERROR "CMAKE_TOOLCHAIN_FILE Not set, forcing exit. Suggested value: {ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake." )
-endif(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
-
-if(NOT DEFINED ANDROID_ABI)
- message( FATAL_ERROR "ANDROID_ABI Not set, forcing exit. Suggested value(s): arm64-v8a (64), armeabi-v7a (32)" )
-endif(NOT DEFINED ANDROID_ABI)
-
-if(NOT DEFINED CLML_SDK)
- message( FATAL_ERROR "CLML_SDK Not set, forcing exit." )
-endif(NOT DEFINED CLML_SDK)
-
-if (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY STREQUAL "ONLY")
- set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
-endif()
-
-find_library(CLML_LIBRARIES NAMES libOpenCL.so NO_DEFAULT_PATH PATHS ${CLML_SDK}/lib ${CLML_SDK}/lib64)
-
-# CMake/Android variables
-set( ANDROID_STL c++_static CACHE STRING "Target Android STL") # default
-
-# Source variables
-set( OPENCL_INCLUDE_DIRS ${CLML_SDK} CACHE PATH "filepath to OpenCL headers")
-
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED True)
-
-#we do not want to pass -fno-exceptions
-if(${CMAKE_CXX_FLAGS} MATCHES "-fno-exceptions")
- message ( WARNING "Disabling -fno-exceptions")
- string(REGEX REPLACE "-fno-exceptions" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-endif()
-
-#we do not want to pass -fno-rtti
-if(${CMAKE_CXX_FLAGS} MATCHES "-fno-rtti")
- message ( WARNING "Disabling -fno-rtti")
- string(REGEX REPLACE "-fno-rtti" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-endif()
-
-set(COMMON_SOURCE_FILES
- clml_models.cc
- clml_runner.cc
- clml_runner.h
- main.cc
- ../../3rdparty/cnpy/cnpy.cpp
- )
-
-include_directories(
- src
- ${OPENCL_INCLUDE_DIRS}
- "../../3rdparty/dmlc-core/include"
- "../../3rdparty/cnpy/"
- )
-
-add_executable(clml_run ${COMMON_SOURCE_FILES})
-target_link_options(clml_run PRIVATE -Wl,--unresolved-symbols=ignore-in-shared-libs)
-target_link_libraries(clml_run ${CLML_LIBRARIES} z)
diff --git a/apps/cpp_clml/README.md b/apps/cpp_clml/README.md
deleted file mode 100644
index 3200492122..0000000000
--- a/apps/cpp_clml/README.md
+++ /dev/null
@@ -1,145 +0,0 @@
-<!--- Licensed to the Apache Software Foundation (ASF) under one -->
-<!--- or more contributor license agreements. See the NOTICE file -->
-<!--- distributed with this work for additional information -->
-<!--- regarding copyright ownership. The ASF licenses this file -->
-<!--- to you under the Apache License, Version 2.0 (the -->
-<!--- "License"); you may not use this file except in compliance -->
-<!--- with the License. You may obtain a copy of the License at -->
-
-<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
-
-<!--- Unless required by applicable law or agreed to in writing, -->
-<!--- software distributed under the License is distributed on an -->
-<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
-<!--- KIND, either express or implied. See the License for the -->
-<!--- specific language governing permissions and limitations -->
-<!--- under the License. -->
-
-# OpenCLML Debug Tool
-
-Tool to generate OpenCLML source file given a model from any framework and compile it as a native application that runs on Android target.
-This tool helps to debug or triage OpenCLML offloaded sub graphs as a standalone application.
-
-### Codegen
-
-Models can be downloaded from well known frameworks like Tensorflow, PyTorch, TFLite, Onnx ..etc.
-Assuming ```resnet50.h5``` is a Keras ResNet50 model file, use the below command to generate a OpenCLML source for the model.
-
-```bash
-python3 scripts/clml_codegen.py resnet50.h5
-```
-
-Above command generates ```clml_models.cc``` and ```clml_params.npz```.
-```clml_models.cc``` contains cpp representation of all OpenCLML subgraphs offloaded by TVM compilation. This file will be used to build tool ```clml_run```.
-```clml_params.npz``` is a numpy dump of all params involved in all sub graphs of TVM module. This file to be copied to target.
-
-### Build Tool
-
-Copy the generated models source ```clml_models.cc``` under ```cpp_clml```.
-
-Below commands will compile the tool ```clml_run``` from generated source and other static dependents.
-
-```bash
-cmake -S . -B build_64 -D ANDROID_ABI=arm64-v8a -D CLML_SDK=<CLML SDK PATH> -D CMAKE_TOOLCHAIN_FILE=<ANDROID NDK PATH>/build/cmake/android.toolchain.cmake -D ANDROID_PLATFORM=latest
-cmake --build build_64
-```
-
-### Run the tool
-
-Copy ```clml_params.npz``` and ```clml_run``` to the target Android device
-
-```bash
-Android:/data/local/tmp $ ./clml_run --dump-meta
-Input =
-Output =
-Params =
-DumpMeta = 1
-.....
-Subgraph Name: tvmgen_default_clml_main_1
- Input Count : 1
- Output Count : 1
- Input MetaInfo
- Input: tvmgen_default_clml_main_1_input_0
- Dtype : float32
- Shape : [1, 1, 1, 2048]
- Output MetaInfo
- Output: tvmgen_default_clml_main_1_layer_out_5
- Dtype : float32
- Shape : [1, 1000]
-
-Subgraph Name: tvmgen_default_clml_main_0
- Input Count : 1
- Output Count : 1
- Input MetaInfo
- Input: tvmgen_default_clml_main_0_input_0
- Dtype : float32
- Shape : [1, 3, 230, 230]
- Output MetaInfo
- Output: tvmgen_default_clml_main_0_layer_out_406
- Dtype : float32
- Shape : [1, 2048, 1, 1]
-.....
-```
-
-The meta information above indicates that the ResNet50 model is partitioned such a way that there exists two OpenCLML subgraphs.
-
-Below command runs the models by setting the parameters from ```clml_params.npz```.
-
-```bash
-Android:/data/local/tmp $ ./clml_run --params=./clml_params.npz
-Input =
-Output =
-Params = ./clml_params.npz
-DumpMeta = 1
-......
-CLMLRunner Loading Params:./clml_params.npz
-CLMLRunner Loading Params:./clml_params.npz
-CLMLRunner::Run :tvmgen_default_clml_main_1
-CLMLRunner::Run :tvmgen_default_clml_main_0
-......
-```
-
-Below command can set the model inputs from ```input.npz``` and can output sub graph outputs to ```output.npz```.
-```input.npz``` should have numpy arrays for ```tvmgen_default_clml_main_1_input_0``` from sub graph ```tvmgen_default_clml_main_1``` and ```tvmgen_default_clml_main_0_input_0``` from sub graph ```tvmgen_default_clml_main_0```.
-
-```bash
-Android:/data/local/tmp $ ./clml_run --params=./clml_params.npz --input=./input.npz --output=./output.npz <
-Input = ./input.npz
-Output = ./output.npz
-Params = ./clml_params.npz
-DumpMeta = 0
-Call Build Modules
-CLMLRunner Constructor: Input:./input.npz Output:./output.npz Params:./clml_params.npz
-CLML Target version:3
-CLMLRunner Loading Params:./clml_params.npz
-CLMLRunner Loading Inputs:./input.npz
-Set Input For:tvmgen_default_clml_main_1_input_0
-
-CLMLRunner Constructor: Input:./input.npz Output:./output.npz Params:./clml_params.npz
-CLML Target version:3
-CLMLRunner Loading Params:./clml_params.npz
-CLMLRunner Loading Inputs:./input.npz
-Set Input For:tvmgen_default_clml_main_0_input_0
-
-Loop Through the Modules
-CLMLRunner::Run :tvmgen_default_clml_main_1
-Saving Output:tvmgen_default_clml_main_1_layer_out_5
-CLMLRunner::Run :tvmgen_default_clml_main_0
-Saving Output:tvmgen_default_clml_main_0_layer_out_406
-......
-```
-
-The generated output file ```output.npz``` contains all the output from all sub modules.
-In this case it contains ```tvmgen_default_clml_main_1_layer_out_5``` for sub graph ```tvmgen_default_clml_main_1``` and ```tvmgen_default_clml_main_0_layer_out_406``` for sub graph ```tvmgen_default_clml_main_0``` as shown below.
-
-
-```bash
-Android:/data/local/tmp $ unzip -l output.npz
-Archive: output.npz
- Length Date Time Name
---------- ---------- ----- ----
- 4080 1980-00-00 00:00 tvmgen_default_clml_main_1_layer_out_5.npy
- 8272 1980-00-00 00:00 tvmgen_default_clml_main_0_layer_out_406.npy
---------- -------
- 12352 2 files
-```
diff --git a/apps/cpp_clml/clml_runner.cc b/apps/cpp_clml/clml_runner.cc
deleted file mode 100644
index 0a5508635e..0000000000
--- a/apps/cpp_clml/clml_runner.cc
+++ /dev/null
@@ -1,822 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*!
- * \file clml_runner.cc
- * \brief CLML model runner implementation.
- */
-
-#include "clml_runner.h"
-
-#include <fstream>
-#include <iostream>
-#include <streambuf>
-#include <string>
-
-namespace tvm {
-namespace runtime {
-
-/*!
- * \brief Constructor for CLMLRunner.
- * \param name is unique name for the sub graph or this CLML Runner.
- * \param args tool or utility arguments.
- * \param arg_platform_id is the OpenCL platform.
- * \param arg_context is the OpenCL context.
- * \param arg_device_id is the OpenCL device_id.
- * \param arg_queue is the OpenCL queue.
- */
-CLMLRunner::CLMLRunner(std::string name, ToolArgs& args, cl_platform_id arg_platform_id,
- cl_context arg_context, cl_device_id arg_device_id,
- cl_command_queue arg_queue)
- : r_args(args),
- r_name(name),
- platform(arg_platform_id),
- context(arg_context),
- device_id(arg_device_id),
- queue(arg_queue) {
- LOG(INFO) << "CLMLRunner Constructor:" << name << " Input:" << r_args.input
- << " Output:" << r_args.output << " Params:" << r_args.params;
- cl_int result;
-
- // Query and Get CLML Interface
- static const cl_uint MAX_VERSIONS = 256;
- cl_int majorVersions[MAX_VERSIONS];
- cl_int minorVersions[MAX_VERSIONS];
- cl_uint numVersions = 0;
- result = clQueryMLInterfaceVersionsQCOM(nullptr, nullptr, 0, &numVersions);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
- CLML_SDK_TEST_AND_EXIT(numVersions > 0u);
- CLML_SDK_TEST_AND_EXIT(numVersions <= MAX_VERSIONS);
-
- result = clQueryMLInterfaceVersionsQCOM(majorVersions, minorVersions, numVersions, nullptr);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- for (cl_uint i = 0; i < numVersions; ++i) {
- if (majorVersions[i] == CL_QCOM_ML_OPS_H_MAJOR_VERSION) {
- this->h_ClmlIntf = GET_ML_INTERFACE(0);
- LOG(INFO) << "CLML Target version:" << majorVersions[i];
- break;
- }
- }
- CLML_SDK_TEST_AND_EXIT(this->h_ClmlIntf != nullptr);
-
- result = h_ClmlIntf->clCreateMLTuningCacheQCOM(&tuning_cache);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- if (!r_args.params.empty()) {
- LOG(INFO) << "CLMLRunner Loading Params:" << r_args.params;
- npz_params = cnpy::npz_load(r_args.params);
- } else {
- LOG(INFO) << "CLMLRunner : No parameters supplied";
- }
-
- if (!r_args.input.empty()) {
- LOG(INFO) << "CLMLRunner Loading Inputs:" << r_args.input;
- npz_input = cnpy::npz_load(r_args.input);
- } else {
- LOG(INFO) << "CLMLRunner : No Input's given. Asuming a dry-run.";
- }
-}
-
-/*!
- * \brief Call one cycle of execution for the model.
- * \return 0 on success else error code.
- */
-int CLMLRunner::Run(void) {
- LOG(INFO) << "CLMLRunner::Run :" << GetModName();
- cl_int result;
-
- for (size_t i = 0; i < this->function.size(); ++i) {
- result = h_ClmlIntf->clEnqueueMLOpQCOM(queue, this->function[i], this->descriptorSet, 0,
- nullptr, nullptr);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
- }
- if (!r_args.output.empty()) {
- for (auto it = this->outputs.begin(); it != this->outputs.end(); it++) {
- auto out_name = it->first;
- auto out_desc = it->second;
- auto dtype = outputs_dtypes[out_name];
- auto shape = outputs_shapes[out_name];
- size_t size = 1;
- for (auto si : shape) size *= si;
- if (dtype == "float32") {
- void* data = (void*)malloc(size * 4);
- CopyDataFromCLMLTensor(out_desc, data);
- LOG(INFO) << "Saving Output:" << out_name;
- cnpy::npz_save<float>(r_args.output, out_name, (float*)data, shape, "a");
- free(data);
- } else if (dtype == "int8") {
- void* data = (void*)malloc(size);
- CopyDataFromCLMLTensor(out_desc, data);
- LOG(INFO) << "Saving Output:" << out_name;
- cnpy::npz_save<int8_t>(r_args.output, out_name, (int8_t*)data, shape, "a");
- free(data);
- } else {
- LOG(WARNING) << "Unsupported dtype to dump :" << dtype;
- }
- }
- }
- return 0;
-}
-
-/*!
- * \brief Set meta information.
- * \param minfo is the meta information of the sub graph.
- */
-void CLMLRunner::SetMetaInfo(std::string minfo) { this->meta_info = minfo; }
-
-/*!
- * \brief Print the meta information.
- */
-void CLMLRunner::PrintMetaInfo(void) { LOG(INFO) << "\n" << this->meta_info; }
-
-/*!
- * \brief Copy the bytedata into tensor.
- * \param tensor is tensor descriptor to copy data.
- * \param data is pointer to bytedata.
- * \param layout is source data layout
- */
-void CLMLRunner::CopyDataToCLMLTensor(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> tensor,
- void* data, cl_ml_tensor_layout_qcom layout) {
- cl_int result = 0;
- cl_event evt = nullptr;
- result = h_ClmlIntf->clEnqueueWriteMLTensorDataQCOM(this->queue, data, layout, tensor->tensor,
- tensor->memory,
- 0, // n waitlist
- nullptr, // waitlist
- &evt); // event
- CLML_SDK_TEST_AND_EXIT((evt != nullptr) && result == CL_SUCCESS);
-}
-
-/*!
- * \brief Copy the bytedata into tensor.
- * \param tensor is tensor descriptor to copy data.
- * \param data is pointer to bytedata.
- * \param layout is source data layout
- */
-void CLMLRunner::CopyDataFromCLMLTensor(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> tensor,
- void* data, cl_ml_tensor_layout_qcom layout) {
- cl_int result = 0;
- cl_event readEvent = nullptr;
- // Read the output tensor
- result = h_ClmlIntf->clEnqueueReadMLTensorDataQCOM(this->queue, tensor->tensor, tensor->memory,
- data, layout,
- 0, // n waitlist
- nullptr, // waitlist
- &readEvent); // event
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
- result = clWaitForEvents(1, &readEvent);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-}
-
-/*!
- * \brief Allocate backing memory for tensor descriptor.
- * \param pTensorMemDesc is tensor descriptor.
- * \return memory alocation status (CL_SUCCESS or error code).
- */
-cl_int CLMLRunner::AllocateTensorMemory(
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> pTensorMemDesc) {
- uint32_t size = 0;
- cl_int result = CL_OUT_OF_HOST_MEMORY;
- cl_mem buffer = nullptr;
-
- result = h_ClmlIntf->clGetMLTensorMemorySizeQCOM(context, pTensorMemDesc->tensor, &size);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- buffer = clCreateBuffer(context, CL_MEM_READ_WRITE, size, nullptr, &result);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- pTensorMemDesc->memory = buffer;
-
- return result;
-}
-
-/*!
- * \brief Allocate memory for all tensor dectiptor in storage map.
- * Also set data for tensors given params and input numpy dumps
- */
-void CLMLRunner::AllocateMemAndPopulateParams(void) {
- cl_int result;
- for (auto it = this->storage_map.begin(); it != this->storage_map.end(); it++) {
- auto node_id = it->first;
- auto tensor_desc = it->second;
-
- AllocateTensorMemory(tensor_desc);
-
- if (npz_params.find(node_id) != npz_params.end()) {
- CopyDataToCLMLTensor(tensor_desc, npz_params[node_id].data<char>());
- }
-
- if (npz_input.find(node_id) != npz_input.end()) {
- LOG(INFO) << "Set Input For:" << node_id;
- CopyDataToCLMLTensor(tensor_desc, npz_input[node_id].data<char>());
- }
-
- this->tensorMemDescs.push_back(*tensor_desc);
- }
- if (!r_args.dump_meta) {
- // Cross check all params
- for (auto nid : consts) {
- if (npz_params.find(nid) == npz_params.end()) {
- LOG(WARNING) << "Param not found in npz:" << nid;
- }
- }
- }
- // Initialize Tensor Descriptors
- result = h_ClmlIntf->clCreateMLTensorMemoryDescriptorSetQCOM(&this->descriptorSet);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- result = h_ClmlIntf->clUpdateMLTensorMemoryDescriptorSetQCOM(
- this->descriptorSet, static_cast<uint32_t>(this->tensorMemDescs.size()),
- this->tensorMemDescs.data());
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-}
-
-/*!
- * \brief Initializes an unused tensor.
- * It is used across operators.
- */
-void CLMLRunner::MakeUnusedTensor(void) {
- cl_int result;
- cl_ml_tensor_desc_qcom desc = {};
- desc.num_dimensions = CL_TENSOR_UNUSED_QCOM;
- this->unusedTensor = std::make_shared<cl_ml_tensor_memory_desc_qcom>();
- result = this->h_ClmlIntf->clCreateMLTensorQCOM(this->context, nullptr, &desc,
- &(this->unusedTensor->tensor));
- CLML_SDK_TEST_AND_EXIT(this->unusedTensor && result == CL_SUCCESS);
-}
-
-/*!
- * \brief Convert string datatype to cl channel type.
- * \param dtype the datatype as string.
- * \return cl channel type corresponding to the datatype.
- */
-cl_channel_type MakeCLDataType(const std::string& dtype) {
- if (dtype == "float32") {
- return CL_FLOAT;
- } else if (dtype == "float16") {
- return CL_HALF_FLOAT;
- } else {
- LOG(FATAL) << "Datatype: " << dtype << " unsupported by CLML runtime";
- }
- return CL_FLOAT;
-}
-
-/*!
- * \brief Map operator arthemetic mode based on data type and accumulation type.
- * \param data_type is cl channel type for computation.
- * \param acc_tpe is cl channel type for accumulation.
- * \return the arthemetic mode.
- */
-cl_arithmetic_mode_qcom MakeCLArithMode(const cl_channel_type& data_type,
- const cl_channel_type& acc_type = CL_FLOAT) {
- if (data_type == CL_FLOAT && acc_type == CL_FLOAT) {
- return CL_ARITHMETIC_MODE_FP32_QCOM;
- } else if (data_type == CL_HALF_FLOAT && acc_type == CL_FLOAT) {
- return CL_ARITHMETIC_MODE_FP16_ACC32_QCOM;
- } else if (data_type == CL_HALF_FLOAT && acc_type == CL_HALF_FLOAT) {
- return CL_ARITHMETIC_MODE_FP16_QCOM;
- } else {
- LOG(FATAL) << "Datatype " << data_type << " unsupported by CLML runtime";
- }
-}
-
-/*!
- * \brief Creates a tensor descriptor.
- * \param shape is shape of tensor.
- * \param dtype tensor data type as string.
- * \param layout is the data layout to be used.
- * \return newly created tensor descriptor.
- */
-std::shared_ptr<cl_ml_tensor_memory_desc_qcom> CLMLRunner::MakeCLMLTensor(
- std::vector<size_t> shape, std::string dtype, cl_ml_tensor_layout_qcom layout) {
- cl_int result;
- tensor_dims_t dims;
- // Make sure the tensors with dimensions less than 4 are padded with 1.
- shape.push_back(1);
- shape.push_back(1);
- shape.push_back(1);
-
- dims.n = shape[0];
- dims.c = shape[1];
- dims.h = shape[2];
- dims.w = shape[3];
- cl_channel_type cl_dtype = MakeCLDataType(dtype);
- auto tensor_dsc = std::make_shared<cl_ml_tensor_memory_desc_qcom>();
- cl_ml_tensor_desc_qcom desc = {
- cl_dtype, layout, dims.n, dims.c, dims.h, dims.w, 0, CL_TENSOR_DIMENSIONS_4D_QCOM, {0}};
- result =
- this->h_ClmlIntf->clCreateMLTensorQCOM(this->context, nullptr, &desc, &tensor_dsc->tensor);
- CLML_SDK_TEST_AND_EXIT(tensor_dsc->tensor && result == CL_SUCCESS);
- return tensor_dsc;
-}
-
-/*!
- * \brief Convolution2D implementation.
- * \param input_desc is input tensor descriptor.
- * \param weight_desc is the kernel as tensor descriptor.
- * \param bias_desc is bias as tensor descriptor.
- * \param output_desc is the placeholder for convolution output.
- * \param padding padding to be applied on input tensor.
- * \param dilation is convolution dilation parameter.
- * \param strides is convolution strides parameter.
- * \param groups number of groups.
- * \param mode is it normal convolution of depthwise convolution.
- * \param activation activation to be applied on result.
- * \param has_bias is bias tensor valid.
- * \param has_activation is activation to be applied.
- * \param dtype operator data type.
- */
-void CLMLRunner::MakeConv2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bias_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> padding, std::vector<cl_uint> dilation,
- std::vector<cl_uint> strides, int groups, cl_convolution_mode_qcom mode,
- cl_activation_function_qcom activation, bool has_bias, bool has_act,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_int result;
- if (CL_CONVOLUTION_MODE_CONVOLUTION_QCOM == mode) {
- CLML_SDK_TEST_AND_EXIT(groups == 1); // CLML convolution only supports group size of 1
- } else {
- groups = 1; // Don't need to pass groups to depthwise
- }
- cl_ml_op_activation_desc_qcom act_desc = {activation, CL_PROPAGATE_NAN_QCOM, cl_arithmetic_mode};
- cl_uint clml_padding_b[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {padding[0], padding[1]};
- cl_uint clml_padding_a[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {padding[2], padding[3]};
- cl_uint clml_strides[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {strides[0], strides[1]};
- cl_uint clml_dilation[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {dilation[0], dilation[1]};
-
- cl_ml_op_convolution_desc_qcom conv_desc{mode,
- static_cast<cl_uint>(groups),
- 4,
- {clml_padding_b[0], clml_padding_b[1]},
- {clml_padding_a[0], clml_padding_a[1]},
- {clml_strides[0], clml_strides[1]},
- {clml_dilation[0], clml_dilation[1]},
- 0,
- cl_arithmetic_mode};
- cl_ml_op_qcom op = nullptr;
- if (!has_act) {
- result = h_ClmlIntf->clCreateMLOpConvolutionForwardQCOM(
- this->context, 0, &conv_desc, input_desc->tensor, weight_desc->tensor, bias_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- } else {
- result = h_ClmlIntf->clCreateMLOpFusedConvolutionActivationForwardQCOM(
- this->context, 0, &conv_desc, &act_desc, input_desc->tensor, weight_desc->tensor,
- bias_desc->tensor, nullptr, output_desc->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- }
- this->function.push_back(op);
-}
-
-/*!
- * \brief Fused Convolution2D+BatchNorm implementation.
- * \param input_desc is input tensor descriptor.
- * \param weight_desc is the kernel as tensor descriptor.
- * \param bias_desc is bias as tensor descriptor.
- * \param output_desc is the placeholder for convolution output.
- * \param bn_scale fused batchnorm scale tensor descriptor.
- * \param bn_bias fused batchnorm scale tensor descriptor.
- * \param bn_mean fused batchnorm mean tensor descriptor.
- * \param bn_var fused batchnorm variance tensor descriptor.
- * \param bn_attrs batchnorm other attributes.
- * \param padding padding to be applied on input tensor.
- * \param dilation is convolution dilation parameter.
- * \param strides is convolution strides parameter.
- * \param groups number of groups.
- * \param mode is it normal convolution of depthwise convolution.
- * \param activation activation to be applied on result.
- * \param has_bias is bias tensor valid.
- * \param has_activation is activation to be applied.
- * \param dtype operator data type.
- */
-void CLMLRunner::MakeConv2DWithBN(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bias_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_scale,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_bias,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_mean,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_var,
- std::vector<float> bn_attrs, std::vector<cl_uint> padding,
- std::vector<cl_uint> dilation, std::vector<cl_uint> strides,
- int groups, cl_convolution_mode_qcom mode,
- cl_activation_function_qcom activation, bool has_bias,
- bool has_act, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_int result;
- if (CL_CONVOLUTION_MODE_CONVOLUTION_QCOM == mode) {
- CLML_SDK_TEST_AND_EXIT(groups == 1); // CLML convolution only supports group size of 1
- } else {
- groups = 1; // Don't need to pass groups to depthwise
- }
- cl_ml_op_activation_desc_qcom act_desc = {activation, CL_PROPAGATE_NAN_QCOM, cl_arithmetic_mode};
- cl_uint clml_padding_b[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {padding[0], padding[1]};
- cl_uint clml_padding_a[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {padding[2], padding[3]};
- cl_uint clml_strides[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {strides[0], strides[1]};
- cl_uint clml_dilation[CL_ML_TENSOR_MAX_SPATIAL_DIMS_QCOM] = {dilation[0], dilation[1]};
-
- cl_ml_op_convolution_desc_qcom conv_desc{mode,
- static_cast<cl_uint>(groups),
- 4,
- {clml_padding_b[0], clml_padding_b[1]},
- {clml_padding_a[0], clml_padding_a[1]},
- {clml_strides[0], clml_strides[1]},
- {clml_dilation[0], clml_dilation[1]},
- 0,
- cl_arithmetic_mode};
- cl_ml_op_qcom op = nullptr;
- cl_ml_op_batchnorm_desc_qcom bn_desc = {CL_BATCHNORM_MODE_SPATIAL_QCOM, cl_arithmetic_mode};
- if (!has_act) {
- result = h_ClmlIntf->clCreateMLOpFusedConvolutionBatchNormForwardQCOM(
- this->context, 0, &conv_desc, &bn_desc, input_desc->tensor, weight_desc->tensor,
- bias_desc->tensor, output_desc->tensor, bn_mean->tensor, bn_var->tensor, bn_scale->tensor,
- bn_bias->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- } else {
- result = h_ClmlIntf->clCreateMLOpFusedConvolutionBatchNormActivationForwardQCOM(
- this->context, 0, &conv_desc, &bn_desc, &act_desc, input_desc->tensor, weight_desc->tensor,
- bias_desc->tensor, output_desc->tensor, nullptr, bn_mean->tensor, bn_var->tensor,
- bn_scale->tensor, bn_bias->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- }
- this->function.push_back(op);
-}
-
-/*!
- * \brief All types of ReLU(6) implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param relu_type the pf ReLU activation.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeRelu(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- cl_activation_function_qcom relu_type, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
- cl_ml_op_activation_desc_qcom act_desc = {relu_type, CL_PROPAGATE_NAN_QCOM, cl_arithmetic_mode};
-
- result = h_ClmlIntf->clCreateMLOpActivationForwardQCOM(
- this->context, 0, &act_desc, input_desc->tensor, this->unusedTensor->tensor,
- output_desc->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Batch Normalization operator implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param bn_scale fused batchnorm scale tensor descriptor.
- * \param bn_bias fused batchnorm scale tensor descriptor.
- * \param bn_mean fused batchnorm mean tensor descriptor.
- * \param bn_var fused batchnorm variance tensor descriptor.
- * \param bn_attrs batchnorm other attributes.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeBatchNorm(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_scale,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_bias,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_mean,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_var,
- std::vector<float> bn_attrs, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_ml_op_batchnorm_desc_qcom bn_desc = {CL_BATCHNORM_MODE_SPATIAL_QCOM, cl_arithmetic_mode};
-
- result = h_ClmlIntf->clCreateMLOpBatchNormForwardQCOM(
- this->context, 0, &bn_desc, input_desc->tensor, bn_mean->tensor, bn_var->tensor,
- bn_scale->tensor, bn_bias->tensor, output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief All types of Pool2D operator implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param pool_size pooling window size.
- * \param strides stride for pooling.
- * \param padding is the input padding.
- * \param pool_type is type of poling (max, avg ...etc).
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakePool2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> pool_size, std::vector<cl_uint> strides,
- std::vector<cl_uint> padding, std::string pool_type,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_ml_op_pooling_desc_qcom pool_desc = {
- pool_type == "nn.max_pool2d" ? CL_POOLING_MODE_MAX_QCOM
- : CL_POOLING_MODE_AVERAGE_EXCLUDE_PADDING_QCOM,
- 4, // reserved
- {padding[0], padding[1]},
- {padding[2], padding[3]},
- {strides[0], strides[1]},
- {pool_size[0], pool_size[1]},
- CL_PROPAGATE_NAN_QCOM,
- cl_arithmetic_mode,
- };
-
- result = h_ClmlIntf->clCreateMLOpPoolingForwardQCOM(
- this->context, 0, &pool_desc, input_desc->tensor, this->unusedTensor->tensor,
- output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief All types of Global Pooling 2D operator implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param in_shape is the input tensor shape.
- * \param pool_type is the pool type (max or avg).
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeGlobalPool2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> in_shape, std::string pool_type,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
- cl_ml_op_pooling_desc_qcom pool_desc = {
- pool_type == "nn.global_max_pool2d" ? CL_POOLING_MODE_MAX_QCOM
- : CL_POOLING_MODE_AVERAGE_EXCLUDE_PADDING_QCOM,
- 4, // reserved
- {0, 0},
- {0, 0},
- {1, 1},
- {in_shape[2], in_shape[3]},
- CL_PROPAGATE_NAN_QCOM,
- cl_arithmetic_mode,
- };
-
- result = h_ClmlIntf->clCreateMLOpPoolingForwardQCOM(
- this->context, 0, &pool_desc, input_desc->tensor, this->unusedTensor->tensor,
- output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Reshape Operator.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeReshape(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- result = h_ClmlIntf->clCreateMLOpReshapeQCOM(this->context, 0, input_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Concatenate operator implementation.
- * \param in_list list of input tensor descriptors to concatenate.
- * \param output_desc output tensor descriptor.
- * \param axis is the dimention on which we join the tensors.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeConcatenate(
- std::vector<std::shared_ptr<cl_ml_tensor_memory_desc_qcom>> in_list,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, int axis, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_ml_tensor_qcom* concatInputs = new cl_ml_tensor_qcom[in_list.size()];
- for (int i = 0; i < in_list.size(); i++) {
- concatInputs[i] = in_list[i]->tensor;
- }
- cl_ml_op_concat_desc_qcom concatDesc = {1, (cl_uint)in_list.size(), cl_arithmetic_mode};
- result = h_ClmlIntf->clCreateMLOpConcatQCOM(this->context, 0, &concatDesc, concatInputs,
- output_desc->tensor, &op, tuning_cache);
- delete[] concatInputs;
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Dense operator implementation.
- * \param input_desc input tensor descriptor.
- * \param weight_desc weight tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param bias_desc bias tensor descriptor.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeDense(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> in_shape, std::vector<cl_uint> wt_shape,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
- cl_gemm_transform_qcom b_transform = CL_GEMM_TRANSFORM_NONE_QCOM;
-
- if (in_shape[1] == wt_shape[1]) {
- b_transform = CL_GEMM_TRANSFORM_TRANSPOSE_QCOM;
- }
-
- cl_ml_op_gemm_desc_qcom gemmDesc = {in_shape[0], // m
- wt_shape[0], // n
- wt_shape[1], // k
- CL_GEMM_TRANSFORM_NONE_QCOM, // A transform
- b_transform, // B transform
- {{1.0}, CL_FLOAT}, // alpha
- {{0.0}, CL_FLOAT}, // beta
- cl_arithmetic_mode};
-
- result =
- h_ClmlIntf->clCreateMLOpGemmQCOM(this->context, 0, &gemmDesc, input_desc->tensor,
- weight_desc->tensor, output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief SoftMax operator implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeSoftMax(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_ml_op_softmax_desc_qcom softmax_desc = {CL_SOFTMAX_ALGORITHM_ACCURATE_QCOM,
- CL_SOFTMAX_MODE_INSTANCE_QCOM, cl_arithmetic_mode};
-
- result = h_ClmlIntf->clCreateMLOpSoftmaxQCOM(this->context, 0, &softmax_desc, input_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief .
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param pad_mode type of padding to be applied (constant, edge, reflect ...etc).
- * \param padding amount of padding.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakePad(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string pad_mode, std::vector<cl_uint> padding, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_pad_mode_qcom clml_pad_mode = CL_PAD_MODE_CONSTANT_QCOM;
- if (pad_mode == "constant")
- clml_pad_mode = CL_PAD_MODE_CONSTANT_QCOM;
- else if (pad_mode == "edge")
- clml_pad_mode = CL_PAD_MODE_SYMMETRIC_QCOM;
- else if (pad_mode == "reflect")
- clml_pad_mode = CL_PAD_MODE_REFLECT_QCOM;
- else
- LOG(FATAL) << "Padding mode not supported by CLML:" << pad_mode;
-
- cl_ml_op_pad_desc_qcom pad_desc{clml_pad_mode,
- {0, 0},
- {padding[0], padding[1], padding[2], padding[3], 0, 0, 0, 0},
- cl_arithmetic_mode};
-
- result = h_ClmlIntf->clCreateMLOpPadQCOM(this->context, 0, &pad_desc, input_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Batch Flatten operator implementation.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeBatchFlatten(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- result = h_ClmlIntf->clCreateMLOpReshapeQCOM(this->context, 0, input_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief Clip operator.
- * \param input_desc input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param a_max is the upper bound to clip.
- * \param a_min is the lower bound to clip.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeClip(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, float a_max,
- float a_min, std::string dtype) {
- LOG(INFO) << "MakeClip called";
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_ml_op_clip_desc_qcom clip_desc = {
- CL_CLIP_BY_VALUE_QCOM, {{a_max}, CL_FLOAT}, {{a_min}, CL_FLOAT}, cl_arithmetic_mode};
-
- result = h_ClmlIntf->clCreateMLOpClipQCOM(this->context, 0, &clip_desc, input_desc->tensor,
- output_desc->tensor, &op, tuning_cache);
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-/*!
- * \brief All types of Binary operators.
- * \param input_a first input tensor descriptor.
- * \param input_b second input tensor descriptor.
- * \param output_desc output tensor descriptor.
- * \param op_name is the binary operator.
- * \param dtype operator datatype.
- */
-void CLMLRunner::MakeBinaryOp(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_a,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_b,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string op_name, std::string dtype) {
- cl_arithmetic_mode_qcom cl_arithmetic_mode = MakeCLArithMode(MakeCLDataType(dtype));
- cl_ml_op_qcom op = nullptr;
- cl_int result;
-
- cl_binary_op_qcom binary_op = CL_TENSOR_OP_ADD_QCOM;
- if (op_name == "subtract")
- binary_op = CL_TENSOR_OP_SUB_QCOM;
- else if (op_name == "multiply")
- binary_op = CL_TENSOR_OP_MUL_QCOM;
- else if (op_name == "divide")
- binary_op = CL_TENSOR_OP_DIV_QCOM;
- else if (op_name == "minimum")
- binary_op = CL_TENSOR_OP_MIN_QCOM;
- else if (op_name == "maximum")
- binary_op = CL_TENSOR_OP_MAX_QCOM;
- cl_ml_op_binary_desc_qcom add_desc = {
- binary_op, {{1.0}, CL_FLOAT}, {{1.0}, CL_FLOAT}, {{0.0}, CL_FLOAT}, cl_arithmetic_mode};
-
- result =
- h_ClmlIntf->clCreateMLOpBinaryQCOM(this->context, 0, &add_desc, input_a->tensor,
- input_b->tensor, output_desc->tensor, &op, tuning_cache);
-
- CLML_SDK_TEST_AND_EXIT(op && result == CL_SUCCESS);
- this->function.push_back(op);
-}
-
-} // namespace runtime
-} // namespace tvm
diff --git a/apps/cpp_clml/clml_runner.h b/apps/cpp_clml/clml_runner.h
deleted file mode 100644
index a1e78fcb66..0000000000
--- a/apps/cpp_clml/clml_runner.h
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*!
- * \file clml_runner.h
- * \brief CLML model runner.
- */
-#ifndef CLML_APPS_CPP_RCLML_RUNNER_H_
-#define CLML_APPS_CPP_RCLML_RUNNER_H_
-
-#include <csignal>
-#include <cstdio>
-#include <cstdlib>
-#include <iostream>
-#include <string>
-#if defined(__linux__) || defined(__ANDROID__)
-#include <unistd.h>
-#endif
-
-#include <CL/cl_qcom_ml_ops.h>
-#include <cnpy.h>
-#include <dmlc/io.h>
-
-#include "CL/cl.h"
-
-#define CLML_SDK_TEST_AND_EXIT(expression) \
- { \
- { \
- int _n_ = !(expression); \
- if (_n_) { \
- fprintf(stderr, "Error on line %d of %s\nFailing expression: %s\n", __LINE__, __FILE__, \
- #expression); \
- exit(1); \
- } \
- } \
- }
-
-#define CAT_I(a, b) a##b
-#define CAT(a, b) CAT_I(a, b)
-#define GET_ML_INTERFACE CAT(CAT(clGetMLInterfaceV, CL_QCOM_ML_OPS_H_MAJOR_VERSION), QCOM)
-#define GET_ML_API_INTERFACE CAT(CAT(CLMLInterfaceV, CL_QCOM_ML_OPS_H_MAJOR_VERSION), QCOM)
-
-namespace tvm {
-namespace runtime {
-
-/**
- * \brief Tensor dimensions, batch, channel, height, width
- *
- */
-struct tensor_dims_t {
- uint32_t n, c, h, w;
-};
-
-/*!
- * \brief Tool Arguments.
- * \arg input Numpy file for the model input
- * \arg output Numpy file name to dump the model output as numpy
- * \arg parsms Numpy file holding the params for models
- */
-struct ToolArgs {
- std::string input;
- std::string output;
- std::string params;
- bool dump_meta = false;
-};
-
-/*!
- * \brief encapsulates CLML Runner functionality for the sub graph
- */
-class CLMLRunner {
- public:
- /*! \brief Constructor */
- CLMLRunner(std::string name, ToolArgs& args, cl_platform_id arg_platform_id,
- cl_context arg_context, cl_device_id arg_device_id, cl_command_queue arg_queue);
-
- /*! \brief Returns the name for this sub graph */
- std::string GetModName(void) { return r_name; }
- /*! \brief Executes one cycle all CLML ops */
- int Run(void);
- /*! \brief set meta information */
- void SetMetaInfo(std::string minfo);
- /*! \brief Print function to show all meta information */
- void PrintMetaInfo(void);
- /*! \brief initializes the unusedTensor */
- void MakeUnusedTensor(void);
- /*! \brief Copy given bytestream of data to the tensor */
- void CopyDataToCLMLTensor(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> tensor, void* data,
- cl_ml_tensor_layout_qcom layout = CL_TENSOR_LAYOUT_NCHW_QCOM);
- /*! \brief Copy tensor data to data in expected layout format */
- void CopyDataFromCLMLTensor(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> tensor, void* data,
- cl_ml_tensor_layout_qcom layout = CL_TENSOR_LAYOUT_NCHW_QCOM);
- /*! \brief Allocates memory for the tensor descriptor */
- cl_int AllocateTensorMemory(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> pTensorMemDesc);
- /*!
- * \brief Allocates memory for all tensor descriptor in storage map.
- * Also initializes the parameter nodes, inputs from given numpy dumps if provided.
- */
- void AllocateMemAndPopulateParams(void);
- /*! \brief Create a tensor descriptor given it's shape, dtype and layout */
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> MakeCLMLTensor(
- std::vector<size_t> shape, std::string dtype = "float32",
- cl_ml_tensor_layout_qcom layout = CL_TENSOR_LAYOUT_OPTIMAL_QCOM);
- /*! \brief Conv2D layer implementattion */
- void MakeConv2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bias_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> padding, std::vector<cl_uint> dilation,
- std::vector<cl_uint> strides, int groups, cl_convolution_mode_qcom mode,
- cl_activation_function_qcom activation, bool has_bias, bool has_act,
- std::string dtype);
-
- /*! \brief Conv2D with Fused BatchNorm layer implementattion */
- void MakeConv2DWithBN(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bias_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_scale,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_bias,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_mean,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_var,
- std::vector<float> bn_attrs, std::vector<cl_uint> padding,
- std::vector<cl_uint> dilation, std::vector<cl_uint> strides, int groups,
- cl_convolution_mode_qcom mode, cl_activation_function_qcom activation,
- bool has_bias, bool has_act, std::string dtype);
-
- /*! \brief ReLU layer implementattion */
- void MakeRelu(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- cl_activation_function_qcom relu_type, std::string dtype);
-
- /*! \brief Batch Normalization layer implementattion */
- void MakeBatchNorm(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_scale,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_bias,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_mean,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> bn_var,
- std::vector<float> bn_attrs, std::string dtype);
-
- /*! \brief Pool2D (with all variants) layer implementattion */
- void MakePool2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> pool_size, std::vector<cl_uint> strides,
- std::vector<cl_uint> padding, std::string pool_type, std::string dtype);
-
- /*! \brief GlobalPool2D (with all variants) layer implementattion */
- void MakeGlobalPool2D(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> in_shape, std::string pool_type, std::string dtype);
-
- /*! \brief Reshape layer implementattion */
- void MakeReshape(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, std::string dtype);
-
- /*! \brief Concatenate layer implementattion */
- void MakeConcatenate(std::vector<std::shared_ptr<cl_ml_tensor_memory_desc_qcom>> in_list,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, int axis,
- std::string dtype);
-
- /*! \brief Dense layer implementattion */
- void MakeDense(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> weight_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::vector<cl_uint> in_shape, std::vector<cl_uint> wt_shape, std::string dtype);
-
- /*! \brief SoftMax layer implementattion */
- void MakeSoftMax(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, std::string dtype);
-
- /*! \brief Pad layer implementattion */
- void MakePad(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, std::string pad_mode,
- std::vector<cl_uint> padding, std::string dtype);
-
- /*! \brief Batch Flatten layer implementattion */
- void MakeBatchFlatten(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc,
- std::string dtype);
-
- /*! \brief Clip layer implementattion */
- void MakeClip(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_desc,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, float a_max,
- float a_min, std::string dtype);
-
- /*! \brief Binary Operator (with all types) layer implementattion */
- void MakeBinaryOp(std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_a,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> input_b,
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> output_desc, std::string op_name,
- std::string dtype);
-
- /*! \brief Vector of created operators */
- std::vector<cl_ml_op_qcom> function;
- /*! \brief Vector of graph's input tensor descriptors */
- std::vector<std::shared_ptr<cl_ml_tensor_memory_desc_qcom>> inputs;
- /*! \brief Map of graph's output tensor descriptors with names */
- std::map<std::string, std::shared_ptr<cl_ml_tensor_memory_desc_qcom>> outputs;
- /*! \brief Map of graph's output tensor names and dtypes */
- std::map<std::string, std::string> outputs_dtypes;
- /*! \brief Map of graph's output tensor names and shapes */
- std::map<std::string, std::vector<size_t>> outputs_shapes;
- /*! \brief Overall storage map for all tensor descriptors involved */
- std::map<std::string, std::shared_ptr<cl_ml_tensor_memory_desc_qcom>> storage_map;
- /*! \brief List of const tensor of the graph */
- std::vector<std::string> consts;
- /*! \brief List of all memory descriptor in graph */
- std::vector<cl_ml_tensor_memory_desc_qcom> tensorMemDescs;
- /*! \brief Tensor memory descriptor set */
- cl_ml_tensor_mem_desc_set_qcom descriptorSet;
- /*! \brief Unused tensor used across various ops */
- std::shared_ptr<cl_ml_tensor_memory_desc_qcom> unusedTensor;
-
- /*! \brief ML API interface */
- GET_ML_API_INTERFACE* h_ClmlIntf = nullptr;
- /*! \brief Tuning cache object */
- cl_ml_tuningcache_qcom tuning_cache = nullptr;
- /*! \brief Flag to inticate a tuning run */
- bool is_tuning_run;
- /*! \brief The tuning file for loading or storing cache */
- char* tuning_file;
-
- /*! \brief OpenCL platform */
- cl_platform_id platform{nullptr};
- /*! \brief OpenCL context */
- cl_context context{nullptr};
- /*! \brief OpenCL device */
- cl_device_id device_id{nullptr};
- /*! \brief OpenCL Queue */
- cl_command_queue queue{nullptr};
- /*! \brief Numpy object for params */
- cnpy::npz_t npz_params;
- /*! \brief Numpy object for inputs */
- cnpy::npz_t npz_input;
-
- private:
- /*! \brief unique name for the runner */
- std::string r_name;
- /*! \brief arguments */
- ToolArgs r_args;
- /*! \brief Holds meta information from clml codegen */
- std::string meta_info;
-};
-
-} // namespace runtime
-} // namespace tvm
-#endif // CLML_APPS_CPP_RCLML_RUNNER_H_
diff --git a/apps/cpp_clml/main.cc b/apps/cpp_clml/main.cc
deleted file mode 100644
index b918618a17..0000000000
--- a/apps/cpp_clml/main.cc
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*!
- * \file main.cc
- * \brief CLML Model execution application.
- */
-
-#include "clml_runner.h"
-
-using namespace tvm::runtime;
-
-/*!
- * \brief Auto generated model file (clml_models.cc) entry function definition.
- * \param args The tool arguments to forward
- * \param arg_platform OpenCL platform
- * \param arg_context OpenCL context
- * \param arg_device_id OpenCL device id
- * \param queue OpenCL queue
- * \return List of CLMLRunner objects corresponding to all sub graphs of a TVM module.
- */
-std::vector<CLMLRunner> BuildModules(ToolArgs& args, cl_platform_id arg_platform,
- cl_context arg_context, cl_device_id arg_device_id,
- cl_command_queue queue);
-
-static const std::string kUsage =
- "Command line usage\n"
- "--input - Numpy file for the model input (optional and we use random of not given)\n"
- "--output - Numpy file name to dump the model output as numpy\n"
- "--params - Numpy file with params\n"
- "--dump-meta - Dump model meta information\n"
- "\n"
- " Example\n"
- " ./clml_run --dump-meta\n"
- " ./clml_run --params=clmlparams.npz\n"
- " ./clml_run --input=input.npz --output=output.npz --params=clml_params.npz\n"
- "\n";
-
-/*!
- * \brief PrintArgs print the contents of ToolArgs
- * \param args ToolArgs structure
- */
-void PrintArgs(const ToolArgs& args) {
- LOG(INFO) << "Input = " << args.input;
- LOG(INFO) << "Output = " << args.output;
- LOG(INFO) << "Params = " << args.params;
- LOG(INFO) << "DumpMeta = " << args.dump_meta;
-}
-
-#if defined(__linux__) || defined(__ANDROID__)
-/*!
- * \brief CtrlCHandler, exits if Ctrl+C is pressed
- * \param s signal
- */
-void CtrlCHandler(int s) {
- LOG(INFO) << "User pressed Ctrl+C, Exiting";
- exit(1);
-}
-
-/*!
- * \brief HandleCtrlC Register for handling Ctrl+C event.
- */
-void HandleCtrlC() {
- // Ctrl+C handler
- struct sigaction sigIntHandler;
- sigIntHandler.sa_handler = CtrlCHandler;
- sigemptyset(&sigIntHandler.sa_mask);
- sigIntHandler.sa_flags = 0;
- sigaction(SIGINT, &sigIntHandler, nullptr);
-}
-#endif
-/*!
- * \brief GetCmdOption Parse and find the command option.
- * \param argc arg counter
- * \param argv arg values
- * \param option command line option to search for.
- * \param key whether the option itself is key
- * \return value corresponding to option.
- */
-std::string GetCmdOption(int argc, char* argv[], std::string option, bool key = false) {
- std::string cmd;
- for (int i = 1; i < argc; ++i) {
- std::string arg = argv[i];
- if (arg.find(option) == 0) {
- if (key) {
- cmd = argv[i];
- return cmd;
- }
- // We assume "=" is the end of option.
- // ICHECK_EQ(*option.rbegin(), '=');
- cmd = arg.substr(arg.find('=') + 1);
- return cmd;
- }
- }
- return cmd;
-}
-
-/*!
- * \brief ParseCmdArgs parses the command line arguments.
- * \param argc arg counter
- * \param argv arg values
- * \param args the output structure which holds the parsed values
- */
-void ParseCmdArgs(int argc, char* argv[], struct ToolArgs& args) {
- const std::string input = GetCmdOption(argc, argv, "--input=");
- if (!input.empty()) {
- args.input = input;
- }
-
- const std::string output = GetCmdOption(argc, argv, "--output=");
- if (!output.empty()) {
- args.output = output;
- }
-
- const std::string params = GetCmdOption(argc, argv, "--params=");
- if (!params.empty()) {
- args.params = params;
- }
-
- const std::string pmeta = GetCmdOption(argc, argv, "--dump-meta", true);
- if (!pmeta.empty()) {
- args.dump_meta = true;
- }
-}
-
-/*!
- * \brief Check CLML extension availability in the CL device.
- * \param platform_id OpenCL platform
- * \param device_id OpenCL device id
- * \return true if extension present else false.
- */
-bool ExtensionStringPresent(cl_platform_id platform_id, cl_device_id device_id) {
- cl_int result = 0;
- size_t reqd_size = 0;
- result = clGetDeviceInfo(device_id, CL_DEVICE_EXTENSIONS, 0, nullptr, &reqd_size);
- CLML_SDK_TEST_AND_EXIT(reqd_size > 0u && result == CL_SUCCESS);
-
- std::vector<char> buf(reqd_size);
- result = clGetDeviceInfo(device_id, CL_DEVICE_EXTENSIONS, reqd_size, buf.data(), nullptr);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- std::string extensions(buf.data());
- LOG(WARNING) << "OpenCL Extensions:" << extensions;
- return (extensions.find("cl_qcom_ml_ops") != std::string::npos);
-}
-
-/*!
- * \brief Loads and Executes the model on given Target.
- * \param args tool arguments
- * \return result of operation.
- */
-int ExecuteModel(ToolArgs& args) {
-#if defined(__linux__) || defined(__ANDROID__)
- // Ctrl+C handler
- HandleCtrlC();
-#endif
-
- // Init OpenCL Environment
- cl_int result;
- cl_event readEvent = nullptr;
- cl_platform_id platform = nullptr;
- cl_context context = nullptr;
- cl_device_id device_id = nullptr;
- cl_command_queue queue = nullptr;
-
- // Initialize Context and Command Queue
- result = clGetPlatformIDs(1, &platform, nullptr);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- uint32_t num_devices = 0;
- result = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 0, nullptr, &num_devices);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS && num_devices == 1);
-
- result = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device_id, nullptr);
- CLML_SDK_TEST_AND_EXIT(device_id && result == CL_SUCCESS);
-
- CLML_SDK_TEST_AND_EXIT(ExtensionStringPresent(platform, device_id) == true);
-
- context = clCreateContext(0, 1, &device_id, nullptr, nullptr, &result);
- CLML_SDK_TEST_AND_EXIT(result == CL_SUCCESS);
-
- cl_command_queue_properties queue_props = 0;
-
- queue = clCreateCommandQueue(context, device_id, queue_props, &result);
- CLML_SDK_TEST_AND_EXIT(queue && result == CL_SUCCESS);
-
- // Populate the runner with model
- LOG(INFO) << "Call Build Modules\n";
- auto runners = BuildModules(args, platform, context, device_id, queue);
-
- LOG(INFO) << "Loop Through the Modules";
- for (auto runner : runners) {
- if (args.dump_meta) {
- // Print Meta Information
- runner.PrintMetaInfo();
- }
-
- // Run the model
- runner.Run();
- }
-
- return 0;
-}
-
-/*!
- * \brief main The main function.
- * \param argc arg counter
- * \param argv arg values
- * \return result of operation.
- */
-int main(int argc, char* argv[]) {
- if (argc <= 1) {
- LOG(INFO) << kUsage;
- return 0;
- }
-
- ToolArgs args;
- ParseCmdArgs(argc, argv, args);
- PrintArgs(args);
-
- if (ExecuteModel(args)) {
- PrintArgs(args);
- LOG(INFO) << kUsage;
- return -1;
- }
- return 0;
-}
diff --git a/apps/cpp_clml/scripts/clml_codegen.py b/apps/cpp_clml/scripts/clml_codegen.py
deleted file mode 100644
index 7540812ed5..0000000000
--- a/apps/cpp_clml/scripts/clml_codegen.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import sys
-import numpy as np
-
-import tvm
-from tvm import relay
-from tvm.driver import tvmc
-from tvm.relay.op.contrib import clml
-from tvm.contrib import utils
-from string import Template
-
-
-def main():
- print("CLML Codegen")
- if len(sys.argv) != 2:
- print("Usage: python clml_codegen.py <model_path>")
- return
-
- tvmc_model = tvmc.load(sys.argv[1])
- mod = tvmc_model.mod
- params = tvmc_model.params
- with tvm.transform.PassContext(opt_level=3):
- mod = tvmc.transform.convert_graph_layout(mod, "NCHW")
- with tvm.transform.PassContext(opt_level=3, disabled_pass=["AlterOpLayout"]):
- clml_mod = clml.partition_for_clml(mod, params)
- libm = relay.build(
- clml_mod,
- target="opencl",
- target_host="llvm -mtriple=aarch64-linux-gnu",
- params=params,
- )
-
- # Extract CLML related params
- (clml_params_save, gen_src) = clml.CLMLGenSrc(libm).get_artifacts()
- np.savez("clml_params.npz", **clml_params_save)
-
- f_src = open("../clml_models.cc", "w")
- f_src.write("\n".join(gen_src))
- f_src.close()
- os.popen("clang-format-15 -i ../clml_models.cc")
-
-
-if __name__ == "__main__":
- main()
diff --git a/apps/cpp_clml/scripts/clml_codegen_json.py b/apps/cpp_clml/scripts/clml_codegen_json.py
deleted file mode 100644
index c3fbf835d8..0000000000
--- a/apps/cpp_clml/scripts/clml_codegen_json.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import sys
-import json
-import numpy as np
-
-import tvm
-from tvm import relay
-from tvm.driver import tvmc
-from tvm.relay.op.contrib import clml
-from tvm.contrib import utils
-from string import Template
-
-
-def main():
- print("CLML Codegen From JSON")
- if len(sys.argv) != 3:
- print("Usage: python clml_codegen_json.py <json path> <outfile path>")
- return
-
- with open(sys.argv[1], "r") as file:
- codegen = json.load(file)
- (_, gen_src) = clml.CLMLGenSrc(codegen).get_artifacts()
-
- f_src = open(sys.argv[2], "w")
- f_src.write("\n".join(gen_src))
- f_src.close()
- os.popen("clang-format-15 -i " + sys.argv[2])
-
-
-if __name__ == "__main__":
- main()
diff --git a/apps/cpp_clml/scripts/compare_npy.py b/apps/cpp_clml/scripts/compare_npy.py
deleted file mode 100644
index 8e3c3a8b63..0000000000
--- a/apps/cpp_clml/scripts/compare_npy.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import sys
-import numpy as np
-
-
-def main():
- print("Compare given numpy array in npz files")
- if len(sys.argv) != 4:
- print("Usage: python compare_npy.py <npz file 1> <npz file 2> <np array to cpmpare>")
- return
-
- in1 = np.load(sys.argv[1])
- in2 = np.load(sys.argv[2])
-
- print(sys.argv[1] + "->" + sys.argv[3] + ":", in1[sys.argv[3]].shape)
- print(sys.argv[2] + "->" + sys.argv[3] + ":", in1[sys.argv[3]].shape)
-
- np.testing.assert_allclose(in1[sys.argv[3]], in2[sys.argv[3]], rtol=1e-5, atol=1e-5)
-
-
-if __name__ == "__main__":
- main()
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
index 0eab7ac6cb..a8949af697 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2025-02-09T12:21:01.773823
+// Generated at 2025-02-14T17:21:20.745906
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -725,108 +725,6 @@ def shard_run_integration_aarch64_4_of_4(node_type) {
-def shard_run_topi_aarch64_1_of_2(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=topi: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_arm)
- cpp_unittest(ci_arm)
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh",
- label: 'Run test_arm_compute_lib test',
- )
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('topi: aarch64 1 of 2')
- }
-}
-
-def shard_run_topi_aarch64_2_of_2(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=topi: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_arm)
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh",
- label: 'Run test_arm_compute_lib test',
- )
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('topi: aarch64 2 of 2')
- }
-}
-
-
def test() {
stage('Test') {
environment {
@@ -877,28 +775,6 @@ def test() {
shard_run_integration_aarch64_4_of_4('ARM-GRAVITON3')
}
},
- 'topi: aarch64 1 of 2': {
- try {
- shard_run_topi_aarch64_1_of_2('ARM-GRAVITON3-SPOT')
- } catch (Throwable ex) {
- // mark the current stage as success
- // and try again via on demand node
- echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
- currentBuild.result = 'SUCCESS'
- shard_run_topi_aarch64_1_of_2('ARM-GRAVITON3')
- }
- },
- 'topi: aarch64 2 of 2': {
- try {
- shard_run_topi_aarch64_2_of_2('ARM-GRAVITON3-SPOT')
- } catch (Throwable ex) {
- // mark the current stage as success
- // and try again via on demand node
- echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
- currentBuild.result = 'SUCCESS'
- shard_run_topi_aarch64_2_of_2('ARM-GRAVITON3')
- }
- },
)
}
}
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 8c169ffd36..89d7784705 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2025-02-09T12:21:01.812601
+// Generated at 2025-02-14T17:21:20.702568
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -719,145 +719,6 @@ def shard_run_unittest_GPU_3_of_3(node_type) {
-def shard_run_topi_GPU_1_of_3(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_gpu)
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('topi: GPU 1 of 3')
- }
-}
-
-def shard_run_topi_GPU_2_of_3(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_gpu)
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('topi: GPU 2 of 3')
- }
-}
-
-def shard_run_topi_GPU_3_of_3(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_gpu)
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('topi: GPU 3 of 3')
- }
-}
-
-
def shard_run_docs_GPU_1_of_1(node_type) {
echo 'Begin running on node_type ' + node_type
@@ -955,39 +816,6 @@ def test() {
shard_run_unittest_GPU_3_of_3('GPU')
}
},
- 'topi: GPU 1 of 3': {
- try {
- shard_run_topi_GPU_1_of_3('GPU-SPOT')
- } catch (Throwable ex) {
- // mark the current stage as success
- // and try again via on demand node
- echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
- currentBuild.result = 'SUCCESS'
- shard_run_topi_GPU_1_of_3('GPU')
- }
- },
- 'topi: GPU 2 of 3': {
- try {
- shard_run_topi_GPU_2_of_3('GPU-SPOT')
- } catch (Throwable ex) {
- // mark the current stage as success
- // and try again via on demand node
- echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
- currentBuild.result = 'SUCCESS'
- shard_run_topi_GPU_2_of_3('GPU')
- }
- },
- 'topi: GPU 3 of 3': {
- try {
- shard_run_topi_GPU_3_of_3('GPU-SPOT')
- } catch (Throwable ex) {
- // mark the current stage as success
- // and try again via on demand node
- echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
- currentBuild.result = 'SUCCESS'
- shard_run_topi_GPU_3_of_3('GPU')
- }
- },
'docs: GPU 1 of 1': {
try {
shard_run_docs_GPU_1_of_1('GPU-SPOT')
diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
index f869509105..aa999408a7 100644
--- a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
@@ -53,27 +53,5 @@
)
{% endcall %}
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="topi: aarch64",
- ws="tvm/ut-python-arm",
- platform="arm",
- docker_image="ci_arm",
- num_shards=2,
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='arm') }}
- ci_setup(ci_arm)
- {% if shard_index == 1 %}
- cpp_unittest(ci_arm)
- {% endif %}
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh",
- label: 'Run test_arm_compute_lib test',
- )
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
-{% endcall %}
{{ m.invoke_tests(node="ARM-GRAVITON3", test_method_names=test_method_names) -}}
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index e57e8c0433..0adb492b98 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -87,21 +87,6 @@
)
{% endcall %}
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="topi: GPU",
- num_shards=3,
- ws="tvm/topi-python-gpu",
- platform="gpu",
- docker_image="ci_gpu",
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='gpu') }}
- ci_setup(ci_gpu)
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
- label: 'Run TOPI tests',
- )
-{% endcall %}
{% call(shard_index, num_shards) m.sharded_test_step(
name="docs: GPU",
diff --git a/docs/conf.py b/docs/conf.py
index cffb616f3b..6f9567c9b1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -487,7 +487,6 @@ autodoc_default_options = {
tvm_alias_check_map = {
"tvm.te": ["tvm.tir"],
"tvm.tir": ["tvm.ir", "tvm.runtime"],
- "tvm.relay": ["tvm.ir", "tvm.tir"],
}
## Setup header and other configs
diff --git a/docs/reference/api/python/auto_scheduler.rst b/docs/reference/api/python/auto_scheduler.rst
deleted file mode 100644
index 8fa1823073..0000000000
--- a/docs/reference/api/python/auto_scheduler.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.auto_scheduler
-------------------
-.. automodule:: tvm.auto_scheduler
- :members:
- :imported-members:
- :autosummary:
diff --git a/docs/reference/api/python/autotvm.rst b/docs/reference/api/python/autotvm.rst
deleted file mode 100644
index 5bde9ac479..0000000000
--- a/docs/reference/api/python/autotvm.rst
+++ /dev/null
@@ -1,90 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.autotvm
------------
-.. automodule:: tvm.autotvm
-.. autofunction:: tvm.autotvm.apply_history_best
-
-tvm.autotvm.measure
-~~~~~~~~~~~~~~~~~~~
-.. automodule:: tvm.autotvm.measure.measure
-
-.. autoclass:: tvm.autotvm.measure.MeasureInput
- :members:
-
-.. autoclass:: tvm.autotvm.measure.MeasureResult
- :members:
-
-.. autofunction:: tvm.autotvm.measure.measure_option
-
-.. autofunction:: tvm.autotvm.measure.create_measure_batch
-
-.. autoclass:: tvm.autotvm.measure.measure_methods.LocalBuilder
-
-.. autoclass:: tvm.autotvm.measure.measure_methods.RPCRunner
-
-.. autoclass:: tvm.autotvm.measure.measure_methods.LocalRunner
-
-tvm.autotvm.tuner
-~~~~~~~~~~~~~~~~~
-.. automodule:: tvm.autotvm.tuner
- :members:
-
-.. autoclass:: tvm.autotvm.tuner.Tuner
- :members:
-
-.. autoclass:: tvm.autotvm.tuner.RandomTuner
- :members:
- :inherited-members:
-
-.. autoclass:: tvm.autotvm.tuner.GridSearchTuner
- :members:
- :inherited-members:
-
-.. autoclass:: tvm.autotvm.tuner.GATuner
- :members:
- :inherited-members:
-
-.. autoclass:: tvm.autotvm.tuner.XGBTuner
- :members:
- :inherited-members:
-
-.. automodule:: tvm.autotvm.tuner.callback
- :members:
-
-tvm.autotvm.task
-~~~~~~~~~~~~~~~~
-.. automodule:: tvm.autotvm.task
- :members:
-
-.. automodule:: tvm.autotvm.task.task
- :members:
-
-.. automodule:: tvm.autotvm.task.space
- :members:
-
-.. automodule:: tvm.autotvm.task.dispatcher
- :members:
-
-.. automodule:: tvm.autotvm.task.topi_integration
- :members:
-
-tvm.autotvm.record
-~~~~~~~~~~~~~~~~~~
-.. automodule:: tvm.autotvm.record
- :members:
diff --git a/docs/reference/api/python/graph_executor.rst b/docs/reference/api/python/graph_executor.rst
deleted file mode 100644
index 1af93e8845..0000000000
--- a/docs/reference/api/python/graph_executor.rst
+++ /dev/null
@@ -1,21 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.contrib.graph_executor
---------------------------
-.. automodule:: tvm.contrib.graph_executor
- :members:
diff --git a/docs/reference/api/python/index.rst b/docs/reference/api/python/index.rst
index c4082354a8..2e0a9c7f2f 100644
--- a/docs/reference/api/python/index.rst
+++ b/docs/reference/api/python/index.rst
@@ -85,21 +85,3 @@ Python API
rpc
contrib
-
-.. toctree::
- :maxdepth: 1
- :caption: Legacy
-
- relay/index
- relay/frontend
- relay/nn
- relay/vision
- relay/image
- relay/transform
- relay/analysis
- relay/backend
- relay/dataflow_pattern
- relay/testing
- autotvm
- auto_scheduler
- graph_executor
diff --git a/docs/reference/api/python/relay/analysis.rst b/docs/reference/api/python/relay/analysis.rst
deleted file mode 100644
index ee2ae2fe1e..0000000000
--- a/docs/reference/api/python/relay/analysis.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.analysis
-----------------------
-
-.. automodule:: tvm.relay.analysis
- :members:
- :imported-members:
- :exclude-members: Object, RelayExpr, IRModule, Type
- :autosummary:
diff --git a/docs/reference/api/python/relay/backend.rst b/docs/reference/api/python/relay/backend.rst
deleted file mode 100644
index e717ee10ff..0000000000
--- a/docs/reference/api/python/relay/backend.rst
+++ /dev/null
@@ -1,33 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.backend
------------------
-
-.. automodule:: tvm.relay.backend
-
-.. automodule:: tvm.relay.backend.interpreter
- :members:
-
-.. automodule:: tvm.relay.backend.te_compiler
- :members:
-
-.. automodule:: tvm.relay.backend.graph_executor_codegen
- :members:
-
-.. automodule:: tvm.relay.backend.vm
- :members:
diff --git a/docs/reference/api/python/relay/dataflow_pattern.rst b/docs/reference/api/python/relay/dataflow_pattern.rst
deleted file mode 100644
index fe1d4e95e5..0000000000
--- a/docs/reference/api/python/relay/dataflow_pattern.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.dataflow_pattern
---------------------------
-
-.. automodule:: tvm.relay.dataflow_pattern
- :members:
- :imported-members:
- :exclude-members: Object, Node
- :autosummary:
diff --git a/docs/reference/api/python/relay/frontend.rst b/docs/reference/api/python/relay/frontend.rst
deleted file mode 100644
index a62e36e7b7..0000000000
--- a/docs/reference/api/python/relay/frontend.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-tvm.relay.frontend
-------------------
-
-.. automodule:: tvm.relay.frontend
- :members:
- :imported-members:
- :autosummary:
diff --git a/docs/reference/api/python/relay/image.rst b/docs/reference/api/python/relay/image.rst
deleted file mode 100644
index 3e51627184..0000000000
--- a/docs/reference/api/python/relay/image.rst
+++ /dev/null
@@ -1,26 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-tvm.relay.image
----------------
-
-.. automodule:: tvm.relay.image
- :members:
- :imported-members:
- :exclude-members: Expr, Constant
- :autosummary:
diff --git a/docs/reference/api/python/relay/index.rst b/docs/reference/api/python/relay/index.rst
deleted file mode 100644
index 399bba3e5b..0000000000
--- a/docs/reference/api/python/relay/index.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay
----------
-
-.. automodule:: tvm.relay
- :members:
- :imported-members:
- :exclude-members: RelayExpr,
- Type, TypeKind,
- TypeVar, GlobalTypeVar, TypeConstraint, FuncType, TupleType, IncompleteType,
- TypeCall, TypeRelation, TensorType, RelayRefType, GlobalVar, SourceName,
- Span, Var, Op, Constructor
- :noindex: TypeData
- :autosummary:
diff --git a/docs/reference/api/python/relay/nn.rst b/docs/reference/api/python/relay/nn.rst
deleted file mode 100644
index b54b752ba1..0000000000
--- a/docs/reference/api/python/relay/nn.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.nn
-------------
-.. automodule:: tvm.relay.nn
- :members:
- :imported-members:
- :autosummary:
diff --git a/docs/reference/api/python/relay/testing.rst b/docs/reference/api/python/relay/testing.rst
deleted file mode 100644
index 1d0bfe08e1..0000000000
--- a/docs/reference/api/python/relay/testing.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.testing
------------------
-
-.. automodule:: tvm.relay.testing
- :members:
- :imported-members:
- :autosummary:
-
-.. automodule:: tvm.relay.testing.mlp
- :members:
-
-.. automodule:: tvm.relay.testing.resnet
- :members:
-
-.. automodule:: tvm.relay.testing.dcgan
- :members:
-
-.. automodule:: tvm.relay.testing.mobilenet
- :members:
-
-.. automodule:: tvm.relay.testing.lstm
- :members:
-
-.. automodule:: tvm.relay.testing.inception_v3
- :members:
-
-.. automodule:: tvm.relay.testing.squeezenet
- :members:
-
-.. automodule:: tvm.relay.testing.vgg
- :members:
-
-.. automodule:: tvm.relay.testing.densenet
- :members:
diff --git a/docs/reference/api/python/relay/transform.rst b/docs/reference/api/python/relay/transform.rst
deleted file mode 100644
index 4a8747606e..0000000000
--- a/docs/reference/api/python/relay/transform.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-tvm.relay.transform
-----------------------
-
-.. automodule:: tvm.relay.transform
- :members:
- :imported-members:
- :autosummary:
- :exclude-members: FunctionPass
diff --git a/docs/reference/api/python/relay/vision.rst b/docs/reference/api/python/relay/vision.rst
deleted file mode 100644
index f6bedee8ce..0000000000
--- a/docs/reference/api/python/relay/vision.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-tvm.relay.vision
-----------------
-
-.. automodule:: tvm.relay.vision
- :members:
- :imported-members:
- :autosummary:
diff --git a/docs/reference/api/python/topi.rst b/docs/reference/api/python/topi.rst
index 0528844d68..ce44e07ded 100644
--- a/docs/reference/api/python/topi.rst
+++ b/docs/reference/api/python/topi.rst
@@ -37,11 +37,3 @@ tvm.topi.image
:members:
:imported-members:
:autosummary:
-
-
-tvm.topi.sparse
-~~~~~~~~~~~~~~~
-.. automodule:: tvm.topi.sparse
- :members:
- :imported-members:
- :autosummary:
diff --git a/jvm/core/src/test/scripts/test_graph_executor.py b/jvm/core/src/test/scripts/test_graph_executor.py
deleted file mode 100644
index 676b008205..0000000000
--- a/jvm/core/src/test/scripts/test_graph_executor.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import os
-
-import tvm
-from tvm import te
-import json
-from tvm.contrib import graph_executor
-
-
-def dump_graph_lib(target_dir):
- dim = 4
- A = te.placeholder((dim,), name="A")
- B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
- sched = te.create_schedule(B.op)
-
- node0 = {"op": "null", "name": "x", "inputs": []}
- node1 = {
- "op": "tvm_op",
- "name": "add",
- "inputs": [[0, 0, 0]],
- "attrs": {"func_name": "myadd", "flatten_data": "1", "num_inputs": "1", "num_outputs": "1"},
- }
- nodes = [node0, node1]
- arg_nodes = [0]
- node_row_ptr = [0, 1, 2]
- outputs = [[1, 0, 0]]
- shape = (4,)
- attrs = {
- "shape": ["list_shape", [shape, shape]],
- "dltype": ["list_str", ["float32", "float32"]],
- "storage_id": ["list_int", [0, 1]],
- }
- graph = {
- "nodes": nodes,
- "arg_nodes": arg_nodes,
- "node_row_ptr": node_row_ptr,
- "heads": outputs,
- "attrs": attrs,
- }
-
- graph = json.dumps(graph)
- mlib = tvm.build(sched, [A, B], "llvm", name="myadd")
-
- mlib.export_library(os.path.join(target_dir, "graph_addone_lib.so"))
- with open(os.path.join(target_dir, "graph_addone.json"), "w") as fo:
- fo.write(graph)
-
-
-if __name__ == "__main__":
- import sys
-
- if len(sys.argv) != 2:
- sys.exit(-1)
- dump_graph_lib(sys.argv[1])
diff --git a/python/tvm/__init__.py b/python/tvm/__init__.py
index 32ec9b5d3f..ab11f33cc0 100644
--- a/python/tvm/__init__.py
+++ b/python/tvm/__init__.py
@@ -63,9 +63,6 @@ from . import te
# tvm.driver
from .driver import build, lower
-# tvm.parser
-from . import parser
-
# others
from . import arith
diff --git a/python/tvm/contrib/cutlass/__init__.py b/python/tvm/contrib/cutlass/__init__.py
index 4b56ac4e16..5c3ee3ae25 100644
--- a/python/tvm/contrib/cutlass/__init__.py
+++ b/python/tvm/contrib/cutlass/__init__.py
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
"""BYOC support for CUTLASS."""
-from .build import has_cutlass, num_cutlass_partitions, finalize_modules, finalize_modules_vm
+from .build import has_cutlass, num_cutlass_partitions, finalize_modules
diff --git a/python/tvm/contrib/cutlass/build.py b/python/tvm/contrib/cutlass/build.py
index 5c09c79bd9..ba598e9b22 100644
--- a/python/tvm/contrib/cutlass/build.py
+++ b/python/tvm/contrib/cutlass/build.py
@@ -25,7 +25,7 @@ from functools import reduce
from typing import Optional, Sequence
import tvm
-from tvm import relax, relay, runtime
+from tvm import relax, runtime
from tvm._ffi.registry import register_func
from tvm.contrib.nvcc import get_cuda_version
from tvm.topi.utils import get_const_tuple
@@ -89,35 +89,6 @@ def _get_cutlass_compile_options(sm, threads, use_fast_math=False):
return kwargs
-class OpAnnotator(tvm.relay.ExprVisitor):
- """Annotates partitioned functions with shape and dtype information."""
-
- def __init__(self):
- super().__init__()
- self.signature = {}
-
- def visit_call(self, call):
- op = call.op
- if isinstance(op, relay.Function) and "Composite" in op.attrs:
- self.signature["op_type"] = op.attrs["Composite"]
- for i, arg in enumerate(op.params):
- self.signature[f"arg{i}_shape"] = arg.checked_type.shape
- self.signature[f"arg{i}_dtype"] = arg.checked_type.dtype
- self.signature["ret_shape"] = op.ret_type.shape
- self.signature["ret_dtype"] = op.ret_type.dtype
- self.visit(op.body)
-
- elif isinstance(op, tvm.ir.Op) and op.name in [
- "nn.conv2d",
- "nn.conv2d_transpose",
- "nn.conv2d_backward_weight",
- ]:
- self.op_attrs = call.attrs
-
- for arg in call.args:
- self.visit(arg)
-
-
def select_gemm_kernel(
cutlass_profiler,
op_type,
@@ -389,148 +360,6 @@ def tune_cutlass_kernels(
return mod, num_cutlass_partition
-def tune_cutlass_function(
- func,
- use_3xtf32,
- split_k_slices,
- profile_all_alignments,
- find_first_valid,
- use_multiprocessing,
- gemm_profiler,
- conv2d_profiler,
-):
- """Given a function intended to be offloaded to CUTLASS, profile each workload to select which
- kernels to emit.
-
- Parameters
- ----------
- func : IRModule
- The Relay Function to tune for.
-
- use_3xtf32 : bool
- Wheter or not use slower but very accurate (compared to tf32) 3xtf32 mode for
- fp32 inputs on tensorcore.
-
- split_k_slices : list of int
- Split factor candidates for split-K GEMM. If split-K > 1, the GEMM K-loop is computed in
- parallel accross split-K blocks, and a seperate global reduction kernel is launched to
- accumulate partial reductions. The profiler will pick the best split-k factor from the
- given candidate list. Note that the larger split-K factor requires a larger workspace.
- Currently, parallel split-k has been tested only for wgrad. For GEMM and other conv2d
- kinds, split_k_slices is ignored.
-
- profile_all_alignments : bool
- When True, profile all kernal variants with smaller alignments than the largest possible.
-
- find_first_valid : bool
- Whether or not profile all candidate kernels, or stop profiling after
- the first applicable kernel is found.
-
- use_multiprocessing : bool
- Whether or not compile profiler executables for different kernels in parallel.
-
- gemm_profiler : CutlassGemmProfiler
- Profiler for dense operators. May cache results between tuned functions.
-
- conv2d_profiler : CutlassConv2DProfiler
- Profiler for conv2d operators. May cach results between tuned functions.
-
- Returns
- -------
- annot_func : Function
- The input function with attributes capturing the best CUTLASS kernel found by tuning.
- """
- annotator = OpAnnotator()
- annotator.visit(func)
- out_shape = annotator.signature["ret_shape"]
- out_dtype = annotator.signature["ret_dtype"]
- op_type = annotator.signature["op_type"]
-
- new_attrs = {"op_type": op_type}
- new_attrs.update(annotator.signature)
- new_attrs.update(func.attrs)
- arg0_shape = new_attrs["arg0_shape"]
- arg1_shape = new_attrs["arg1_shape"]
- arg0_dtype = new_attrs["arg0_dtype"]
- arg1_dtype = new_attrs["arg1_dtype"]
-
- if "conv2d" in op_type:
- new_attrs["padding"] = annotator.op_attrs.padding
- new_attrs["strides"] = annotator.op_attrs.strides
- new_attrs["dilation"] = annotator.op_attrs.dilation
-
- if "conv2d_transpose" in op_type:
- d_shape = out_shape
- w_shape = arg1_shape
- elif "conv2d_backward_weight" in op_type:
- d_shape = arg1_shape
- w_shape = out_shape
- else:
- d_shape = arg0_shape
- w_shape = arg1_shape
-
- new_attrs.update(
- handle_conv2d(
- conv2d_profiler,
- op_type,
- d_shape,
- w_shape,
- annotator.op_attrs.padding,
- annotator.op_attrs.strides,
- annotator.op_attrs.dilation,
- out_dtype,
- arg0_dtype,
- arg1_dtype,
- use_3xtf32,
- split_k_slices,
- profile_all_alignments,
- find_first_valid,
- use_multiprocessing,
- )
- )
- elif "batch_matmul" in op_type:
- new_attrs.update(
- handle_batch_matmul(
- gemm_profiler,
- op_type,
- arg0_shape,
- arg1_shape,
- out_dtype,
- arg0_dtype,
- arg1_dtype,
- use_3xtf32,
- find_first_valid,
- use_multiprocessing,
- )
- )
- elif "dense" in op_type:
- new_attrs.update(
- handle_dense(
- gemm_profiler,
- op_type,
- arg0_shape,
- arg1_shape,
- out_dtype,
- arg0_dtype,
- arg1_dtype,
- use_3xtf32,
- find_first_valid,
- use_multiprocessing,
- )
- )
- else:
- raise ValueError(f"{op_type} unsupported composite")
-
- new_attrs = tvm.ir.make_node("DictAttrs", **new_attrs)
- return relay.Function(
- func.params,
- func.body,
- ret_type=func.ret_type,
- type_params=func.type_params,
- attrs=new_attrs,
- )
-
-
def _get_call_node(expr: relax.Expr, op_name: str) -> Optional[relax.Call]:
node = None
@@ -1047,52 +876,6 @@ def compile_cutlass_module(c_source_module, options):
return tvm.runtime.load_static_library(lib_path, function_names)
-@register_func("relay.ext.cutlass.compile_for_cutlass")
-def compile_for_cutlass(mod, cutlass_target):
- """Given an IRModule with at least one Compiler='cutlass' Relay function, return a
- LibraryModule with all such functions compiled into their PackedFunc-compatible form.
- - First runs CUTLASS tuning to decide on the best kernels, which itself requires the
- repeated compilation and execution of CUDA code using nvcc. The results of this
- is captured as annotation on each relevant function. Kernel performance is cached
- overall all functions.
- - Then generates a single CSourceModule containing C code implementing all the
- Compiler='cutlass' Relay functions, accounting for the tuning done above.
- - Then compiles that CSourceModule with the appropriate nvcc arguments to yield
- a static .o library. An export_library step will be required on the final runtime
- module to link that library into the overall .so library.
- See CompileForCutlass in src/relay/backend/contrib/cutlass/codegen.cc for where this
- helper function is used to implement the RelayToTIR pass hook for CUTLASS."""
-
- # Recover options from the current 'cutlass' Target
- assert cutlass_target.kind.name == "cutlass"
- tuning_config = {
- key: cutlass_target.attrs.get(key)
- for key in [
- "sm",
- "use_3xtf32",
- "split_k_slices",
- "profile_all_alignments",
- "find_first_valid",
- "use_multiprocessing",
- ]
- }
- compile_config = {
- key: cutlass_target.attrs.get(key) for key in ["sm", "threads", "use_fast_math"]
- }
- tmp_dir = cutlass_target.attrs.get("tmp_dir")
- compile_config["tmp_dir"] = tmp_dir
-
- # Tune
- logger.info("Tuning for CUTLASS")
- mod, _ = tune_cutlass_kernels(mod, tmp_dir=tmp_dir, **tuning_config)
-
- # Compile
- logger.info("Creating CSource module for CUTLASS")
- create_c_source_module = tvm._ffi.get_global_func("relay.ext.cutlass.create_c_source_module")
- c_module = create_c_source_module(mod)
- return compile_cutlass_module(c_module, compile_config)
-
-
def finalize_modules(lib, lib_path="compile.so", tmp_dir="./tmp"):
"""Returns lib with any C source, LLVM and static library modules complied and linked in ready
for use by the graph or AOT executors. This method is not specific to CUTLASS, however it does
@@ -1102,7 +885,7 @@ def finalize_modules(lib, lib_path="compile.so", tmp_dir="./tmp"):
Parameters
----------
lib : runtime.Module
- The output from relay.build.
+ The output from build.
lib_path : string
The path to a shared library which will be generated as the result of the build process.
@@ -1119,38 +902,3 @@ def finalize_modules(lib, lib_path="compile.so", tmp_dir="./tmp"):
lib_path = os.path.join(tmp_dir, lib_path)
lib.export_library(lib_path, workspace_dir=tmp_dir, cc="nvcc")
return runtime.load_module(lib_path)
-
-
-def finalize_modules_vm(vm_exec, lib_path="compile.so", vmcode_path="vmcode.ro", tmp_dir="./tmp"):
- """Returns vm_exec with any C source, LLVM and static library modules compiled and linked in
- ready for use by the VM executor. This method is not specific to CUTLASS, however it does
- assume nvcc will be used for final compilation and linking. It is provided here for
- convenience.
-
- Parameters
- ----------
- vm_exec : vm.Executable
- The output from relay.vm.compile containing compiled host code and kernels.
-
- lib_path : string
- The path to a shared library which will be generated as the result of the build process.
-
- vmcode_path : string
- The path where the VM bytecode will be serialized to as a side-effect.
-
- tmp_dir : string
- A temporary directory where intermediate compiled artifacts will be stored.
-
- Returns
- -------
- updated_vm_exec : vm.Executable
- The updated VM executable with all compilation and linking completed.
- """
- code, lib = vm_exec.save()
- lib_path = os.path.join(tmp_dir, lib_path)
- vmcode_path = os.path.join(tmp_dir, vmcode_path)
- lib.export_library(lib_path, workspace_dir=tmp_dir, cc="nvcc")
- with open(vmcode_path, "wb") as fo:
- fo.write(code)
- lib = tvm.runtime.load_module(lib_path)
- return tvm.runtime.vm.Executable.load_exec(code, lib)
diff --git a/python/tvm/contrib/graph_executor.py b/python/tvm/contrib/graph_executor.py
deleted file mode 100644
index 0db7877b5e..0000000000
--- a/python/tvm/contrib/graph_executor.py
+++ /dev/null
@@ -1,549 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Minimum graph executor that executes graph containing TVM PackedFunc."""
-import numpy as np
-import tvm._ffi
-
-from tvm.rpc import _ffi_api as _rpc_ffi_api
-from tvm.rpc import base as rpc_base
-from tvm._ffi.base import string_types
-from tvm._ffi.runtime_ctypes import Device
-
-
-def create(graph_json_str, libmod, device):
- """Create a runtime executor module given a graph and module.
-
- Parameters
- ----------
- graph_json_str : str
- The graph to be deployed in json format output by json graph.
- The graph can contain operator(tvm_op) that points to the name
- of PackedFunc in the libmod.
-
- libmod : tvm.runtime.Module
- The module of the corresponding function
-
- device : Device or list of Device
- The device to deploy the module. It can be local or remote when there
- is only one Device. Otherwise, the first device in the list will
- be used as this purpose. All device should be given for heterogeneous
- execution.
-
- Returns
- -------
- graph_module : GraphModule
- Runtime graph module that can be used to execute the graph.
-
- Note
- ----
- See also :py:class:`tvm.contrib.graph_executor.GraphModule`
- for examples to directly construct a GraphModule from an exported
- relay compiled library.
- """
- assert isinstance(graph_json_str, string_types)
-
- dev, num_rpc_dev, device_type_id = get_device(libmod, device)
-
- if num_rpc_dev == len(dev):
- fcreate = dev[0]._rpc_sess.get_function("tvm.graph_executor.create")
- else:
- fcreate = tvm._ffi.get_global_func("tvm.graph_executor.create")
-
- return GraphModule(fcreate(graph_json_str, libmod, *device_type_id))
-
-
-def get_device(libmod, device):
- """Parse and validate all the device(s).
-
- Parameters
- ----------
- libmod : tvm.runtime.Module
- The module of the corresponding function
-
- device : Device or list of Device
-
- Returns
- -------
- device : list of Device
- num_rpc_dev : Number of rpc devices
- device_type_id : List of device type and device id
- """
-
- if isinstance(device, Device):
- device = [device]
- elif not isinstance(device, (list, tuple)):
- raise ValueError("dev has to be the type of Device or a list of Device")
- for cur_dev in device:
- if not isinstance(cur_dev, Device):
- raise ValueError("dev has to be the type of Device or a list of Device")
-
- # device_type_id[0], device_type_id[1] are used as the primary/fallback
- # device type and id. All other ones are used as device for
- # heterogeneous execution.
- num_rpc_dev = 0
- device_type_id = []
- for cur_dev in device:
- device_type = cur_dev.device_type
- if device_type >= rpc_base.RPC_SESS_MASK:
- assert libmod.type_key == "rpc"
- assert _rpc_ffi_api.SessTableIndex(libmod) == cur_dev._rpc_sess._tbl_index
- num_rpc_dev += 1
- device_type = cur_dev.device_type % rpc_base.RPC_SESS_MASK
- device_type_id.append(device_type)
- device_type_id.append(cur_dev.device_id)
-
- if 0 < num_rpc_dev < len(device):
- raise ValueError("Either all or none of the devices should be rpc.")
- return device, num_rpc_dev, device_type_id
-
-
-class GraphModule(object):
- """Wrapper runtime module.
-
- This is a thin wrapper of the underlying TVM module.
- you can also directly call set_input, run, and get_output
- of underlying module functions
-
- Parameters
- ----------
- module : tvm.runtime.Module
- The internal tvm module that holds the actual graph functions.
-
- Attributes
- ----------
- module : tvm.runtime.Module
- The internal tvm module that holds the actual graph functions.
-
- Examples
- --------
-
- .. code-block:: python
-
- import tvm
- from tvm import relay
- from tvm.contrib import graph_executor
-
- # build the library using graph executor
- lib = relay.build(...)
- lib.export_library("compiled_lib.so")
- # load it back as a runtime
- lib: tvm.runtime.Module = tvm.runtime.load_module("compiled_lib.so")
- # Call the library factory function for default and create
- # a new runtime.Module, wrap with graph module.
- gmod = graph_executor.GraphModule(lib["default"](dev))
- # use the graph module.
- gmod.set_input("x", data)
- gmod.run()
- """
-
- def __init__(self, module):
- self.module = module
- self._set_input = module["set_input"]
-
- # TODO(shingjan): The graph_executor in C doesn't have
- # set_input/output_zero_copy implemented.
- try:
- self._set_input_zero_copy = module["set_input_zero_copy"]
- except AttributeError:
- self._set_input_zero_copy = lambda *_: (_ for _ in ()).throw(
- Exception("set_input_zero_copy is not implemented for C graph executor")
- )
- try:
- self._set_output_zero_copy = module["set_output_zero_copy"]
- except AttributeError:
- self._set_output_zero_copy = lambda *_: (_ for _ in ()).throw(
- Exception("set_output_zero_copy is not implemented for C graph executor")
- )
- self._run = module["run"]
- self._get_output = module["get_output"]
- self._get_input = module["get_input"]
- self._get_num_outputs = module["get_num_outputs"]
- self._get_input_index = module["get_input_index"]
- self._get_output_index = module["get_output_index"]
- self._get_input_info = module["get_input_info"]
- self._get_output_info = module["get_output_info"]
- self._get_num_inputs = module["get_num_inputs"]
- self._load_params = module["load_params"]
- self._share_params = module["share_params"]
-
- def set_input(self, key=None, value=None, **params):
- """Set inputs to the module via kwargs
-
- Parameters
- ----------
- key : int or str
- The input key
-
- value : the input value.
- The input value
-
- params : dict of str to NDArray
- Additional arguments
- """
- if key is not None:
- v = self._get_input(key)
- if v is None:
- raise RuntimeError(f"Could not find '{key}' in graph's inputs")
- v.copyfrom(value)
-
- if params:
- # upload big arrays first to avoid memory issue in rpc mode
- keys = list(params.keys())
- keys.sort(key=lambda x: -np.prod(params[x].shape))
- for k in keys:
- # TODO(zhiics) Skip the weights for submodule in a better way.
- # We should use ConstLoaderModule for initialization and remove
- # params from set_input
- val = self._get_input(k)
- if val:
- self._get_input(k).copyfrom(params[k])
-
- def set_input_zero_copy(self, key=None, value=None, **params):
- """Set inputs to the module via kwargs with zero memory copy
-
- Parameters
- ----------
- key : int or str
- The input key
-
- value : the input value in DLPack
- The input value
-
- params : dict of str to NDArray
- Additional arguments
- """
- if key is not None:
- self._set_input_zero_copy(key, value)
-
- if params:
- keys = list(params.keys())
-
- for k in keys:
- # TODO(zhiics) Skip the weights for submodule in a better way.
- # We should use ConstLoaderModule for initialization and remove
- # params from set_input
- val = self._get_input(k)
- if val:
- self._set_input_zero_copy(k, params[k])
-
- def set_output_zero_copy(self, key, value):
- """Set outputs to the module with zero memory copy
-
- Parameters
- ----------
- key : int or str
- The output key
-
- value : the output value in DLPack
- The output value
- """
- self._set_output_zero_copy(key, value)
-
- def run(self, **input_dict):
- """Run forward execution of the graph
-
- Parameters
- ----------
- input_dict: dict of str to NDArray
- List of input values to be feed to
- """
- if input_dict:
- self.set_input(**input_dict)
- self._run()
-
- def get_num_outputs(self):
- """Get the number of outputs from the graph
-
- Returns
- -------
- count : int
- The number of outputs.
- """
- return self._get_num_outputs()
-
- def get_num_inputs(self):
- """Get the number of inputs to the graph
-
- Returns
- -------
- count : int
- The number of inputs.
- """
- return self._get_num_inputs()
-
- def get_input(self, index, out=None):
- """Get index-th input to out
-
- Parameters
- ----------
- index : int
- The input index
-
- out : NDArray
- The output array container
- """
- if out:
- self._get_input(index).copyto(out)
- return out
-
- return self._get_input(index)
-
- def get_input_index(self, name):
- """Get inputs index via input name.
-
- Parameters
- ----------
- name : str
- The input key name
-
- Returns
- -------
- index: int
- The input index. -1 will be returned if the given input name is not found.
- """
- return self._get_input_index(name)
-
- def get_output_index(self, name):
- """Get outputs index via output name.
-
- Parameters
- ----------
- name : str
- The output key name
-
- Returns
- -------
- index: int
- The output index. -1 will be returned if the given output name is not found.
- """
- return self._get_output_index(name)
-
- def get_input_info(self):
- """Return the 'shape' and 'dtype' dictionaries of the graph.
-
- .. note::
- We can't simply get the input tensors from a TVM graph
- because weight tensors are treated equivalently. Therefore, to
- find the input tensors we look at the 'arg_nodes' in the graph
- (which are either weights or inputs) and check which ones don't
- appear in the params (where the weights are stored). These nodes
- are therefore inferred to be input tensors.
-
- Returns
- -------
- shape_dict : Map
- Shape dictionary - {input_name: tuple}.
- dtype_dict : Map
- dtype dictionary - {input_name: dtype}.
- """
- input_info = self._get_input_info()
- assert "shape" in input_info
- shape_dict = input_info["shape"]
- assert "dtype" in input_info
- dtype_dict = input_info["dtype"]
-
- return shape_dict, dtype_dict
-
- def get_output_info(self):
- """Return the 'shape' and 'dtype' dictionaries of the graph.
-
- Returns
- -------
- shape_dict : Map
- Shape dictionary - {output_name: tuple}.
- dtype_dict : Map
- dtype dictionary - {output_name: dtype}.
- """
- output_info = self._get_output_info()
- assert "shape" in output_info
- shape_dict = output_info["shape"]
- assert "dtype" in output_info
- dtype_dict = output_info["dtype"]
-
- return shape_dict, dtype_dict
-
- def get_output(self, index, out=None):
- """Get index-th output to out
-
- Parameters
- ----------
- index : int
- The output index
-
- out : NDArray
- The output array container
- """
- if out:
- self._get_output(index, out)
- return out
-
- return self._get_output(index)
-
- def debug_get_output(self, node, out):
- """Run graph up to node and get the output to out
-
- Parameters
- ----------
- node : int / str
- The node index or name
-
- out : NDArray
- The output array container
- """
- raise NotImplementedError("Please use debugger.debug_executor as graph_executor instead.")
-
- def load_params(self, params_bytes):
- """Load parameters from serialized byte array of parameter dict.
-
- Parameters
- ----------
- params_bytes : bytearray
- The serialized parameter dict.
- """
- self._load_params(bytearray(params_bytes))
-
- def share_params(self, other, params_bytes):
- """Share parameters from pre-existing GraphExecutor instance.
-
- Parameters
- ----------
- other: GraphExecutor
- The parent GraphExecutor from which this instance should share
- it's parameters.
- params_bytes : bytearray
- The serialized parameter dict (used only for the parameter names).
- """
- self._share_params(other.module, bytearray(params_bytes))
-
- def __getitem__(self, key):
- """Get internal module function
-
- Parameters
- ----------
- key : str
- The key to the module.
- """
- return self.module[key]
-
- def benchmark(
- self,
- device,
- func_name="run",
- repeat=5,
- number=5,
- min_repeat_ms=None,
- limit_zero_time_iterations=100,
- end_to_end=False,
- cooldown_interval_ms=0,
- repeats_to_cooldown=1,
- **kwargs,
- ):
- """Calculate runtime of a function by repeatedly calling it.
-
- Use this function to get an accurate measurement of the runtime of a function. The function
- is run multiple times in order to account for variability in measurements, processor speed
- or other external factors. Mean, median, standard deviation, min and max runtime are all
- reported. On GPUs, CUDA and ROCm specifically, special on-device timers are used so that
- synchonization and data transfer operations are not counted towards the runtime. This allows
- for fair comparison of runtimes across different functions and models. The `end_to_end` flag
- switches this behavior to include data transfer operations in the runtime.
-
- The benchmarking loop looks approximately like so:
-
- .. code-block:: python
-
- for r in range(repeat):
- time_start = now()
- for n in range(number):
- func_name()
- time_end = now()
- total_times.append((time_end - time_start)/number)
-
-
- Parameters
- ----------
- func_name : str
- The function to benchmark. This is ignored if `end_to_end` is true.
-
- repeat : int
- Number of times to run the outer loop of the timing code (see above). The output will
- contain `repeat` number of datapoints.
-
- number : int
- Number of times to run the inner loop of the timing code. This inner loop is run in
- between the timer starting and stopping. In order to amortize any timing overhead,
- `number` should be increased when the runtime of the function is small (less than a 1/10
- of a millisecond).
-
- min_repeat_ms : Optional[int]
- If set, the inner loop will be run until it takes longer than `min_repeat_ms`
- milliseconds. This can be used to ensure that the function is run enough to get an
- accurate measurement.
-
- limit_zero_time_iterations : Optional[int]
- The maximum number of repeats when measured time is equal to 0.
- It helps to avoid hanging during measurements.
-
- end_to_end : bool
- If set, include time to transfer input tensors to the device and time to transfer
- returned tensors in the total runtime. This will give accurate timings for end to end
- workloads.
-
- cooldown_interval_ms: Optional[int]
- The cooldown interval in milliseconds between the number of repeats defined by
- `repeats_to_cooldown`.
-
- repeats_to_cooldown: Optional[int]
- The number of repeats before the cooldown is activated.
-
- kwargs : Dict[str, Object]
- Named arguments to the function. These are cached before running timing code, so that
- data transfer costs are not counted in the runtime.
-
- Returns
- -------
- timing_results : BenchmarkResult
- Runtimes of the function. Use `.mean` to access the mean runtime, use `.results` to
- access the individual runtimes (in seconds).
- """
- min_repeat_ms = 0 if min_repeat_ms is None else min_repeat_ms
- if end_to_end:
- # Have to unpack kwargs into a single list
- args = []
- for k, v in kwargs.items():
- args.append(k)
- args.append(v)
- return self.module.time_evaluator(
- "run_from_inputs",
- device,
- repeat=repeat,
- number=number,
- min_repeat_ms=min_repeat_ms,
- limit_zero_time_iterations=limit_zero_time_iterations,
- )(device.device_type % rpc_base.RPC_SESS_MASK, device.device_id, *args)
- if kwargs:
- self.set_input(**kwargs)
- return self.module.time_evaluator(
- func_name,
- device,
- repeat=repeat,
- number=number,
- min_repeat_ms=min_repeat_ms,
- limit_zero_time_iterations=limit_zero_time_iterations,
- cooldown_interval_ms=cooldown_interval_ms,
- repeats_to_cooldown=repeats_to_cooldown,
- )()
diff --git a/python/tvm/contrib/pipeline_executor.py b/python/tvm/contrib/pipeline_executor.py
deleted file mode 100644
index d6be16653c..0000000000
--- a/python/tvm/contrib/pipeline_executor.py
+++ /dev/null
@@ -1,345 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Pipeline executor that executes a series of modules in a pipeline fashion."""
-import json
-import os
-import time
-from tvm import runtime
-from tvm._ffi import get_global_func
-from tvm.contrib import graph_executor
-
-
-def pipeline_executor_enabled():
- """Check if the pipeline executor is enabled.
-
- Return
- -------
- enable: bool
- Return whether the pipeline executor is enabled.
- """
- return get_global_func("tvm.pipeline_executor.create", allow_missing=True) is not None
-
-
-class PipelineModule(object):
- """Wrapper of runtime module, caller can use this module to set parameters and get outputs.
-
- Parameters
- ----------
- module : Union[PipelineExecutorFactoryModule, Module]
- Common interface for pipeline executor factory modules or Module.
- """
-
- def __init__(self, module):
- if isinstance(module, PipelineExecutorFactoryModule):
- self.module = module.get_pipeline_executor_module()
- else:
- self.module = module
- # Get the packed functions from the pipeline executor.
- self._get_params_group_pipeline_map = self.module["get_params_group_pipeline_map"]
- self._run = self.module["run"]
- self._set_param = self.module["set_param"]
- self._set_input = self.module["set_input"]
- self._get_input = self.module["get_input"]
- self._get_output = self.module["get_output"]
- self._get_num_outputs = self.module["get_num_outputs"]
- self._get_num_inputs = self.module["get_num_inputs"]
- self._get_input_pipeline_map = self.module["get_input_pipeline_map"]
- self._get_pipe_execute_count = self.module["get_execute_count"]
-
- def run(self):
- """Run the pipeline executor."""
- self._run()
-
- def get_input_pipeline_map(self, name):
- """Using the "name" to get the corresponding subgraph index and also get the "input name"
- of the corresponding subgraph interface.
- Returns
- -------
- input map: Array[str]
- Returning the index and "input name" of the subgraph.
- """
- return self._get_input_pipeline_map(name)
-
- def get_params_group_pipeline_map(self, name):
- """Use the name of the parameters group to get the corresponding runtime module index.
-
- Parameters
- ----------
- name: str
- The parameter group name.
-
- Returns
- -------
- module_index: int
- The index of the runtime module.
- """
- return self._get_params_group_pipeline_map(name)
-
- def set_input(self, key, value):
- """Set the input via input name.
-
- Parameters
- ----------
- key : str
- The input name
- value : array_like.
- The input value
- """
- self._set_input(key, value)
-
- def set_params(self, params_group_name, params_data):
- """Set the parameter group value given the parameter group name. Note that the parameter
- group name is declared in the pipeline executor config.
-
- Parameters
- ----------
- params_group_name : str
- The parameters group name.
-
- params_data : Dict[str, NDArray]
- A map from parameter name to data.
- """
- if not params_data:
- raise RuntimeError('"params_data is empty!"')
-
- for key, val in params_data.items():
- self._set_param(params_group_name, key, val)
-
- def get_input(self, key):
- """Get the input via an input name.
- Parameters
- ----------
- key : str
- The input key
- Returns
- -------
- data : NDArray
- The input data.
- """
- return self._get_input(key)
-
- def get_output(self, synchronize=True, sleep_interval=0.001):
- """Get the output.
- Returns
- -------
- data : Array[NDArray]
- A list of output data.
- synchronize : BOOL
- Whether to do a synchronize poll.
- sleep_interval : Float32
- When doing the synchronize loop poll, how many seconds the loop should sleep for yield.
- """
- outputs = []
- if not synchronize:
- outputs = self._get_output()
- else:
- while not outputs:
- outputs = self._get_output()
- time.sleep(sleep_interval)
-
- return outputs
-
- @property
- def num_executing_pipeline(self):
- """Getting the count of running pipeline.
- Returns
- -------
- count : int
- The count of running pipeline.
- """
- return self._get_pipe_execute_count()
-
- @property
- def num_outputs(self):
- """Get the number of outputs.
- Returns
- -------
- count : int
- The number of outputs.
- """
- return self._get_num_outputs()
-
- @property
- def num_inputs(self):
- """Get the number of inputs
- Returns
- -------
- count : int
- The number of inputs
- """
- return self._get_num_inputs()
-
- @staticmethod
- def load_library(config_file_name):
- """Import files to create a pipeline executor.
-
- Parameters
- ----------
- config_file_name : str
- Path and name of the configuration file, the configuration file contains the
- disk path of the parameter file, library file, and JSON file.
- """
- with open(config_file_name, "r") as file_handle:
- config = file_handle.read()
- config = json.loads(config)
- if "load_config" not in config or "pipeline_config" not in config:
- raise RuntimeError(
- f'"load_config" or "pipeline_config" is missing in {config_file_name}'
- )
-
- # The config file used to load library, prameters, and JSON files.
- with open(config["load_config"], "r") as file_handle:
- load_config = file_handle.read()
-
- # The config file used to load pipeline compute config.
- with open(config["pipeline_config"], "r") as file_handle:
- pipeline_config = file_handle.read()
-
- # Load a PipelineExecutor from the disk files.
- load_library = get_global_func("tvm.pipeline_executor.load", allow_missing=False)
- module = load_library(load_config, pipeline_config)
-
- return PipelineModule(module)
-
-
-class PipelineExecutorFactoryModule(object):
- """Common interface for pipeline executor factory modules.
-
- Parameters
- ----------
- pipeline_mods : List[GraphExecutorFactoryModule]
- List of GraphExecutorFactoryModule.
-
- mod_config : Dict[int, Dict[str, Any]]
- Modules dependency configuration information.
-
- """
-
- def __init__(self, pipeline_mods, mods_config):
- self.pipeline_mods = pipeline_mods
- self.mods_config = mods_config
- self.module = None
-
- def get_pipeline_executor_module(self):
- """Get the pipeline executor module.
-
- Returns
- -------
- module : Module
- Common interface for pipeline executor factory Module.
- """
- if not self.module:
- graph_executors, config = self.graph_executor_create(
- self.pipeline_mods, self.mods_config
- )
- self.pipeline_create = get_global_func(
- "tvm.pipeline_executor.create", allow_missing=False
- )
- self.module = self.pipeline_create(graph_executors, config)
- return self.module
-
- def graph_executor_create(self, pipeline_mods, mod_config):
- """Create graph_executor list and return configuration as a json string.
-
- Parameters
- ----------
- pipeline_mods : List[GraphExecutorFactoryModule]
- List of GraphExecutorFactoryModule
-
- mod_config : Dict[str, Any]
- Modules dependency configuration information.
-
- Returns
- -------
- mods : List[Module]
- The Module list.
-
- mod_config : str
- The Modudle configuration.
- """
- # Should store modules in the list named 'mods' in index order.
- mods = [None for _ in range(len(pipeline_mods))]
- for lib_index in pipeline_mods:
- pipeline_lib = pipeline_mods[lib_index]["lib"]
- dev = pipeline_mods[lib_index]["dev"]
- lib = graph_executor.GraphModule(pipeline_lib["default"](dev))
- # Return a module list sorted by lib_index.
- mods[lib_index] = lib.module
-
- return mods, json.dumps(mod_config)
-
- def export_library(self, directory_path):
- """Export the pipeline executor into disk files.
-
- Parameters
- ----------
- directory_path : str
- Export the files to this directory.
- """
- if not self.pipeline_mods:
- raise RuntimeError("The pipeline executor has not been initialized.")
-
- # Check if the directory_path exists.
- if not os.path.exists(directory_path):
- raise RuntimeError("The directory {directory_path} does not exist.")
- # Create an load configuration.
- load_config_file_name = f"{directory_path}/load_config"
- pipeline_config_file_name = f"{directory_path}/pipeline_config"
- config = {}
- config["load_config"] = load_config_file_name
- config["pipeline_config"] = pipeline_config_file_name
- load_config = []
- # Export the library, JSON, and parameter into files, then export these files path
- # into a configuration file.
- for lib_index in self.pipeline_mods:
- mconfig = {}
- mconfig["mod_idx"] = lib_index
- mconfig["lib_name"] = f"{directory_path}/lib{lib_index}.so"
- mconfig["json_name"] = f"{directory_path}/json{lib_index}"
- mconfig["params_name"] = f"{directory_path}/params{lib_index}"
- mconfig["dev"] = (
- f"{self.pipeline_mods[lib_index]['dev'].device_type},"
- f"{self.pipeline_mods[lib_index]['dev'].device_id}"
- )
- # Get the graph, lib, and parameters from GraphExecutorFactoryModule.
- lib = self.pipeline_mods[lib_index]["lib"]
- # Export the lib, graph, and parameters to disk.
- if self.pipeline_mods[lib_index]["export_cc"]:
- lib.export_library(
- mconfig["lib_name"], cc=self.pipeline_mods[lib_index]["export_cc"]
- )
- else:
- lib.export_library(mconfig["lib_name"])
-
- with open(mconfig["json_name"], "w") as file_handle:
- file_handle.write(lib.graph_json)
- with open(mconfig["params_name"], "wb") as file_handle:
- file_handle.write(runtime.save_param_dict(lib.params))
-
- load_config.append(mconfig)
-
- with open(load_config_file_name, "w") as file_handle:
- json.dump(load_config, file_handle)
-
- with open(pipeline_config_file_name, "w") as file_handle:
- json.dump(self.mods_config, file_handle)
-
- config_file_name = f"{directory_path}/config"
- with open(config_file_name, "w") as file_handle:
- json.dump(config, file_handle)
-
- return config_file_name
diff --git a/python/tvm/contrib/pipeline_executor_build.py b/python/tvm/contrib/pipeline_executor_build.py
deleted file mode 100644
index 9a16d1b7af..0000000000
--- a/python/tvm/contrib/pipeline_executor_build.py
+++ /dev/null
@@ -1,674 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=f-string-without-interpolation
-"""Pipeline executor that executes a series of modules in a pipeline fashion."""
-import json
-import os
-import tvm._ffi
-from tvm import relay
-from tvm.relay.transform import InferType
-from tvm.contrib.pipeline_executor import PipelineExecutorFactoryModule
-
-
-def pipeline_executor_build_enabled():
- """Check if the pipeline executor build is enabled.
-
- Return
- -------
- enable: bool
- Return whether the pipeline executor is enabled.
- """
- return tvm.contrib.pipeline_executor.pipeline_executor_enabled()
-
-
-def build(pipe_configs):
- """Build modules used in the pipeline executor, then use these modules and configuration
- to create a pipeline executor.
-
- Parameters
- ----------
- pipe_configs: PipelineConfig
- Build Configuration information.
-
- Returns
- -------
- ret: PipelineExecutorFactoryModule
- Common interface for pipeline executor factory modules.
- """
- libs = {}
- config = pipe_configs.get_config()
- if "module_connection" not in config:
- raise RuntimeError('"module_connection" is missing')
- if "input_connection" not in config:
- raise RuntimeError('"input_connection" is missing')
- if "param_connection" not in config:
- raise RuntimeError('"param_connection" is missing')
-
- mod_n_configs = config["module_connection"]
- config_len = len(mod_n_configs)
- module_string_config = [{} for _ in range(config_len)]
- # Use hardware configurations to build backend modules for each subgraph.
- for ir_mod, mod_config in mod_n_configs.items():
- pipe_config = mod_config["pipeline"].copy()
- mod_idx = pipe_config["mod_idx"]
- dev = mod_config["dev"]
- target = mod_config["target"]
- build_func = relay.build
- # Callers may need to use a customized building function to wrap the pre-building logic
- # and the backend building logic. For example, in order to support a backend which only
- # can do "int8" computation, the caller may need to merge the "quantization" logic
- # into the building logic to creat a customized building function.
- if "build" in mod_config and mod_config["build"]:
- build_func = mod_config["build"]
-
- lib = build_func(
- ir_mod,
- target,
- params=mod_config["params"],
- target_host=mod_config["target_host"],
- mod_name=mod_config["mod_name"],
- )
-
- pipe_config["dev"] = f"{dev.device_type},{dev.device_id}"
- # Use "mod_idx" as the key to create a "module_connection" map which is not only
- # for the module index but also for the module connection used to build the pipeline.
- module_string_config[mod_idx] = pipe_config
- libs[mod_idx] = {
- "lib": lib,
- "dev": dev,
- "fcompile": mod_config["fcompile"],
- "export_cc": mod_config["export_cc"],
- }
-
- # Creating a text form configuration to record the "input_connection" and the
- # "module_connection" information. The "input_connection" is used to record the
- # map of global input and subgraph input, and the "module_connection" is used to
- # record module dependency.
- string_config = {}
- string_config["param_connection"] = config["param_connection"]
- string_config["input_connection"] = config["input_connection"]
- string_config["module_connection"] = module_string_config
-
- return PipelineExecutorFactoryModule(libs, string_config)
-
-
-def export_library(factory, directory_path):
- """Export the pipeline executor into disk files.
-
- Parameters
- ----------
- factory : PipelineExecutorFactoryModule
- The pipeline executor factory
- directory_path : str
- Export the files to this directory.
- """
- if not factory.pipeline_mods:
- raise RuntimeError("The pipeline executor has not been initialized.")
-
- # Check if the directory_path exists.
- if not directory_path or not os.path.exists(directory_path):
- raise RuntimeError("The directory {directory_path} does not exist.")
- # Create an load configuration.
- load_config_file_name = f"{directory_path}/load_config"
- pipeline_config_file_name = f"{directory_path}/pipeline_config"
- config = {}
- config["load_config"] = load_config_file_name
- config["pipeline_config"] = pipeline_config_file_name
- load_config = []
- # Export the library, JSON, and parameter into files, then export these files path
- # into a configuration file.
- for lib_index in factory.pipeline_mods:
- mconfig = {}
- mconfig["mod_idx"] = lib_index
- mconfig["lib_name"] = f"{directory_path}/lib{lib_index}.so"
- mconfig["json_name"] = f"{directory_path}/json{lib_index}"
- mconfig["params_name"] = f"{directory_path}/params{lib_index}"
- lib_config = factory.pipeline_mods[lib_index]
- mconfig["dev"] = f"{lib_config['dev'].device_type}," f"{lib_config['dev'].device_id}"
- fcompile = lib_config["fcompile"]
- if not fcompile:
- fcompile = False
-
- # Get the graph, lib, and parameters from GraphExecutorFactoryModule.
- lib = factory.pipeline_mods[lib_index]["lib"]
- # Export the lib, graph, and parameters to disk.
- lib.export_library(mconfig["lib_name"], fcompile=fcompile)
- with open(mconfig["json_name"], "w") as file_handle:
- file_handle.write(lib.graph_json)
- with open(mconfig["params_name"], "wb") as file_handle:
- file_handle.write(relay.save_param_dict(lib.params))
-
- load_config.append(mconfig)
-
- with open(load_config_file_name, "w") as file_handle:
- json.dump(load_config, file_handle)
-
- with open(pipeline_config_file_name, "w") as file_handle:
- json.dump(factory.mods_config, file_handle)
-
- config_file_name = f"{directory_path}/config"
- with open(config_file_name, "w") as file_handle:
- json.dump(config, file_handle)
-
- return config_file_name
-
-
-class PipelineConfig(object):
- """Pipeline configuration information, this class contains the DAG that expresses
- the dependency of each module involved in a pipeline and the parameters for building
- each module.
- """
-
- class Binding:
- """This class defines the module connections information.
- The type can only be "input" or "output".
-
- Parameters
- ----------
- owner : ModuleWrapper
- The class who owns this interface.
-
- io_type : str
- The I/O type of this interface. It can only be "input" or "output".
-
- name : str/integer
- Name, for input it is string such as "data0", for output it is an integer such as 0.
-
- data_type: TensorType
- The data type of this interface.
- """
-
- def __init__(self, owner, io_type, name, data_type=None):
- self.io_owner = owner
- self.io_type = io_type
- self.name = str(name)
- # Child interfaces that depend on this interface.
- self.bindings = []
- # Parents interfaces that this interface depend on.
- self.parents = []
-
- self.data_type = data_type
-
- def get_name(self):
- # Return name of this interface and the name of owner who owns this interface.
- owner_name = ""
- if isinstance(self.io_owner, PipelineConfig.ModuleWrapper):
- owner_name = self.io_owner.name
-
- return owner_name, self.name
-
- def get_owner_idx(self):
- # If the owner is ModuleWrapper return the owner index, if not return 0.
- if isinstance(self.io_owner, PipelineConfig.ModuleWrapper):
- return self.io_owner.idx
-
- return -1
-
- def is_pipeline_executor_interface(self):
- """The pipeline interface is used to interact with the caller. There are two types
- of interfaces, one is 'input' another is 'output'. The pipeline input interface
- is responsible for passing parameters to the internal module interface, and the
- pipeline output interface is responsible for outputting the results computed by
- the pipeline executor to the caller.
- """
- return not isinstance(self.io_owner, PipelineConfig.ModuleWrapper)
-
- def __repr__(self):
- # Geting the binding information in the form of text.
- str_format = f" |{self.name}: "
- for binding in self.bindings:
- mname, dname = binding.get_name()
- str_format += f"{mname}:{dname} "
-
- return str_format
-
- def check_binding_dict(self, connection_dict):
- """Checking the binding dictionary.
- Parameter
- ---------
- connection_dict : Dict[str, Any]
- It is a dictionary of module connections.
- """
- if "interface_name" not in connection_dict:
- raise RuntimeError('"inteface_name" is missing in global config!"')
- if "connection" not in connection_dict:
- raise RuntimeError(f'"connection" is missing!"')
- # The global interface mapping should be one-to-one.
- if not connection_dict["connection"]:
- raise RuntimeError("The global interface map is empty!")
- if len(connection_dict["connection"]) > 1:
- raise RuntimeError("A global interface maps multiple module interfaces!")
- if "mod_idx" not in connection_dict["connection"][0]:
- raise RuntimeError('"mod_idx" is missing!')
-
- def get_binding_dict(self):
- """Returning the binding information in the form of dictionary.
- Returns
- -------
- data : Dict[str, Any]
- The binding information is in the form of dictionary.
- """
- dict_format = {"interface_name": self.name, "connection": []}
- for binding in self.bindings:
- _, dname = binding.get_name()
- midx = binding.get_owner_idx()
- dict_format["connection"].append({"mod_idx": midx, "interface_name": dname})
-
- self.check_binding_dict(dict_format)
- return dict_format
-
- def check_dag_acyclic(self, start, inputs):
- """This is to check whether the DAG containing these input interfaces is acyclic.
- Parameters
- ----------
- start: ModuleWrapper
- The starting node of the cycle check algorithm.
-
- inputs: Binding
- These interfaces are used to connect to each other to build DAG.
-
- Return
- ------
- Return true if there is no cycle in the DAG.
- """
- for binding in inputs.values():
- if start == binding.io_owner:
- return False
- for p in binding.parents:
- if not self.check_dag_acyclic(start, p.io_owner.input_bindings.bindings):
- return False
-
- return True
-
- def connect(self, binding):
- """Connect the current interface to the destination interface.
- Correct connections are as follows: 1. the pipeline input connected to a module input,
- 2. the module output connected to a pipeline output, 3. the module output connected to
- a module input.
-
- Parameters
- ----------
- binding: Binding
- The destination of this connection.
- """
-
- # Check whether the binding setting is correct or not.
- if self.io_owner == binding.io_owner:
- raise RuntimeError("Can not bind itself.")
-
- if self.io_type == "param" and not self.is_pipeline_executor_interface():
- raise RuntimeError(
- 'The "param" binding can only be used by a pipeline executor interface!'
- )
-
- if not self.is_pipeline_executor_interface() and self.io_type == "input":
- raise RuntimeError("Module can only bind from output interface!")
-
- if self.io_type == "param" and binding.io_type != "param":
- raise RuntimeError(
- 'A global "param" interface can only be bind with a module "param" interface!'
- )
-
- if (
- not self.is_pipeline_executor_interface()
- and not binding.is_pipeline_executor_interface()
- and binding.io_type == "output"
- ):
- raise RuntimeError("Can not bind module output with another module output!")
-
- if (
- not self.is_pipeline_executor_interface()
- and binding.is_pipeline_executor_interface()
- and binding.io_type == "input"
- ):
- raise RuntimeError("Can not bind module output with pipeline input!")
-
- if self.is_pipeline_executor_interface() and self.io_type == "output":
- raise RuntimeError("Global output can not be used as binding start point.")
-
- if (
- self.is_pipeline_executor_interface()
- and self.io_type == "input"
- and binding.io_type != "input"
- ):
- raise RuntimeError("Global input can only bind with module input.")
-
- self.bindings.append(binding)
- if not self.is_pipeline_executor_interface():
- # Check whether the data types of the source and destination are the same.
- if (
- isinstance(binding.io_owner, PipelineConfig.ModuleWrapper)
- and self.data_type != binding.data_type
- ):
- raise RuntimeError(
- f"Illegal type (%s vs. %s): binding type is not same!"
- % (self.data_type, binding.data_type)
- )
-
- binding.parents.append(self)
-
- # Do acyclic check after increasing the in-degree of child node by setting
- # current interface as a parent of the child node.
-
- if not self.check_dag_acyclic(
- binding.io_owner, self.io_owner.input_bindings.bindings
- ):
- raise RuntimeError("Illegal connection: Cause a cycle!")
-
- class BindingList:
- """Container for bindings(input or output interface).
-
- Parameters
- ----------
- owner : ModuleWrapper/PipelineConfig
- The owner of this class can be ModuleWrapper or PipelineConfig.
-
- io_type : str
- The type of this class can be "input" or "output".
- """
-
- def __init__(self, owner, io_type):
- self.bindings = {}
- self.io_owner = owner
- self.binding_type = io_type
-
- def get_binding_data_type(self, key):
- if isinstance(self.io_owner, PipelineConfig.ModuleWrapper):
- return self.io_owner.get_data_type(key, self.binding_type)
- return None
-
- def __getitem__(self, key):
- if key not in self.bindings:
- data_type = self.get_binding_data_type(key)
- if not data_type and isinstance(self.io_owner, PipelineConfig.ModuleWrapper):
- raise RuntimeError(f"Can not find {key} in binding list {self.binding_type}.")
-
- self.bindings[key] = PipelineConfig.Binding(
- self.io_owner, self.binding_type, key, data_type
- )
-
- return self.bindings[key]
-
- class ModuleWrapper:
- """This class is a wrapper representing the module and contains information such as
- module information, binding information and building information.
- """
-
- def __init__(self, mod=None):
- self.target_host = None
- self.build_func = None
- self.params = None
- self.target = None
- self.fcompile = None
- self.name = None
- self.dev = None
- self.export_cc = None
- self.cpu_affinity = ""
- self.idx = None
- self.mod = mod
- self.input_params = InferType()(mod)["main"].params
- self.output_type = InferType()(mod)["main"].checked_type.ret_type
- self.input_bindings = PipelineConfig.BindingList(self, "input")
- self.output_bindings = PipelineConfig.BindingList(self, "output")
- self.param_binding = PipelineConfig.Binding(self, "param", "param")
-
- def __eq__(self, other):
- if isinstance(other, PipelineConfig.ModuleWrapper):
- return self.mod == other.mod
-
- return False
-
- def __getitem__(self, key):
- if isinstance(key, str):
- if key == "input":
- return self.input_bindings
-
- if key == "output":
- return self.output_bindings
-
- if key == "param":
- return self.param_binding
-
- raise RuntimeError(f"{key} not found!")
-
- raise RuntimeError('The data type of "key" is not supported!')
-
- def get_data_type(self, key, interface_type):
- """Get the module interface data type according to the key value and interface type.
- Parameters
- ----------
- key: str
- The interface name.
-
- interface_type:
- The interface type.
-
- Return
- -------
- Return data type.
- """
- if interface_type == "input":
- for param in self.input_params:
- if param.name_hint == key:
- return param._checked_type_
-
- if interface_type == "output":
- if isinstance(self.output_type, tvm.ir.type.TupleType):
- if int(key) < len(self.output_type.fields):
- return self.output_type.fields[int(key)]
- elif int(key) == 0:
- return self.output_type
-
- return None
-
- def set_idx_name(self, idx):
- # Set the index value and generate the module name.
- self.idx = idx
- self.name = f"mod{str(idx)}"
-
- def is_root_mod(self):
- """Check whether this node is the root node in DAG, this function is used
- in topological sort.
- """
- return all([not b.parents for b in self.input_bindings.bindings.values()])
-
- def remove_self_from_bindings(self):
- """Remove the current node from child dependencies to reduce the in-degree
- of child node, this function is used in topological sort.
- """
- for binding in self.output_bindings.bindings.values():
- for child in binding.bindings:
- if binding in child.parents:
- child.parents.remove(binding)
-
- def __init__(self):
- self.mod_wrapper = {}
- self.input_bindings = self.BindingList(self, "input")
- self.output_bindings = self.BindingList(self, "output")
- # There is a map of global parameters group and module index.
- self.param_group_bindings = self.BindingList(self, "param")
-
- def __str__(self):
- # Get configuration information as a string.
-
- # Use topological sort to get correct module order.
- self.dag_topology_sort()
- # Getting the parameters dependencies.
- param_dump = "Params\n"
- for param_name in self.param_group_bindings.bindings:
- inf = self.param_group_bindings.bindings[param_name]
- param_dump += str(inf) + "\n"
- # Get the input dependencies.
- input_dump = "\nInputs\n"
- for input_name in self.input_bindings.bindings:
- inf = self.input_bindings.bindings[input_name]
- input_dump += str(inf) + "\n"
-
- # Get the connections information of each module.
- output = {}
- connections_dump = "\nconnections\n"
- for mod in self.mod_wrapper:
- for interface in self.mod_wrapper[mod].output_bindings.bindings.values():
- if interface.bindings:
- mname, dname = interface.get_name()
- iname = mname + ".output(" + dname + ")->"
- for dep in interface.bindings:
- dep_mname, dep_dname = dep.get_name()
- if isinstance(dep.io_owner, PipelineConfig.ModuleWrapper):
- iname += f" {dep_mname}.{dep_dname}"
- connections_dump += f" |{iname}\n"
- else:
- output[dep_dname] = f"{mname}.output({dname})"
-
- # Get the output dependencies.
- output_dump = "\noutput\n"
- for name in sorted(output.keys()):
- output_dump += f" |output({name}) : {output[name]}\n"
-
- return param_dump + input_dump + output_dump + connections_dump
-
- def __getitem__(self, key):
- if isinstance(key, tvm.ir.module.IRModule):
- if key not in self.mod_wrapper:
- self.mod_wrapper[key] = self.ModuleWrapper(key)
- return self.mod_wrapper[key]
-
- if isinstance(key, str):
- if key == "input":
- return self.input_bindings
- if key == "output":
- return self.output_bindings
- if key == "param_group":
- return self.param_group_bindings
-
- raise RuntimeError(f"{key} not found!")
-
- raise RuntimeError(f'The key type "{type(key)}" is not supported!')
-
- def get_config(self):
- """Get the configuration information in dictionary form, this configuration
- will be used to create pipeline executor.
- """
-
- # Use topological sort to get the correct order of modules.
- self.dag_topology_sort()
- mconfig = {}
- module_connection = {}
- for mod in self.mod_wrapper:
- # Generate pipeline configuration.
- mconf = {}
- output_conf = []
- module = self.mod_wrapper[mod]
- for _, binding in module.output_bindings.bindings.items():
- dep_conf = []
- output = {}
- if binding.bindings:
- for dep in binding.bindings:
- dep_item = {}
- _, dname = dep.get_name()
- if dep.is_pipeline_executor_interface():
- dep_item["global_output_index"] = int(dname)
- else:
- dep_item["mod_idx"] = dep.get_owner_idx()
- dep_item["input_name"] = dname
- dep_conf.append(dep_item)
-
- # The value of output_idx start from 0.
- output["output_idx"] = int(binding.name)
- output["dependencies"] = dep_conf
- output_conf.append(output)
-
- mconf["mod_idx"] = module.idx
- mconf["cpu_affinity"] = module.cpu_affinity
- mconf["output"] = output_conf
-
- module_connection[mod] = {
- "pipeline": mconf,
- "target_host": module.target_host,
- "mod_name": "default",
- "build": module.build_func,
- "params": module.params,
- "target": module.target,
- "fcompile": module.fcompile,
- "dev": module.dev,
- "export_cc": module.export_cc,
- }
-
- # Creating a map including pipeline inputs and subgraph inputs.
- input_connection = []
- for input_name in self.input_bindings.bindings:
- input_dict = self.input_bindings.bindings[input_name].get_binding_dict()
- if "interface_name" not in input_dict["connection"][0]:
- raise RuntimeError("interface_name is missing in connection config!")
- # Creating the map including global interfaces and subgraph interfaces.
- input_map = {
- "global_interface_name": input_dict["interface_name"],
- "mod_idx": input_dict["connection"][0]["mod_idx"],
- "module_interface_name": input_dict["connection"][0]["interface_name"],
- }
- input_connection.append(input_map)
-
- # Create a map including global parameters groups and modules.
- param_connection = []
- for param_name in self.param_group_bindings.bindings:
- param_dict = self.param_group_bindings.bindings[param_name].get_binding_dict()
- param_map = {
- "global_param_name": param_dict["interface_name"],
- "mod_idx": param_dict["connection"][0]["mod_idx"],
- }
- param_connection.append(param_map)
-
- mconfig["module_connection"] = module_connection
- mconfig["input_connection"] = input_connection
- mconfig["param_connection"] = param_connection
- return mconfig
-
- def dag_topology_sort(self):
- """Use topological sort to get order of pipeline modules."""
- mlist = []
- mod_wrapper = self.mod_wrapper.copy()
- while mod_wrapper:
- temp_list = []
- for mod, wrapper in mod_wrapper.items():
- if wrapper.is_root_mod():
- temp_list.append(mod)
- wrapper.remove_self_from_bindings()
-
- for mod in temp_list:
- mod_wrapper.pop(mod, None)
-
- mlist += temp_list
-
- mod_wrapper_sort = {}
- for mod, i in zip(mlist, range(len(mlist))):
- self.mod_wrapper[mod].set_idx_name(i)
- mod_wrapper_sort[mod] = self.mod_wrapper[mod]
-
- self.mod_wrapper = mod_wrapper_sort
-
- def get_mod_idx(self, mod):
- # Return the module index.
- idx = self.mod_wrapper[mod].idx
- return idx
-
- def pipe_input(self, name):
- # Return the input interface according to the name.
- return self.input_bindings[name]
-
- def pipe_output(self, idx):
- # Return the output interface according to the name.
- return self.output_bindings[idx]
diff --git a/python/tvm/contrib/target/coreml.py b/python/tvm/contrib/target/coreml.py
index 8ff9e2210c..d8846ce5f1 100644
--- a/python/tvm/contrib/target/coreml.py
+++ b/python/tvm/contrib/target/coreml.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# pylint: disable=invalid-name, unused-argument, import-outside-toplevel
+# pylint: disable=invalid-name, unused-argument, import-outside-toplevel, missing-function-docstring
"""Utility to compile CoreML models"""
import os
diff --git a/python/tvm/contrib/target/onnx.py b/python/tvm/contrib/target/onnx.py
deleted file mode 100644
index 239bf1e4b1..0000000000
--- a/python/tvm/contrib/target/onnx.py
+++ /dev/null
@@ -1,1110 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=invalid-name, import-self, len-as-condition, unused-argument, too-many-lines, redefined-builtin
-"""Relay to ONNX codegen """
-
-import os
-import struct
-import copy
-import numpy
-import onnx
-import onnx.utils
-from onnx import numpy_helper, OperatorSetIdProto, defs
-from onnx import TensorProto
-import tvm
-from tvm import relay
-import tvm._ffi
-from tvm.relay.expr_functor import ExprVisitor
-from tvm.relay.ty import TupleType, TensorType
-
-ONNX_OPSET_VERSONS_SUPPORTED = [11]
-
-
-def run_onnx_optimizer(onnx_model):
- """Run ONNX's optimization routines.
-
- ONNX Optimizer was moved to an external library in
- version 1.9. Attempt to use the optimizer in onnx if
- it is available, fall back to the standalone
- onnxoptimizer otherwise, and return the model
- unoptimized if neither are available.
-
- """
- try:
- onnx_polish_model = onnx.utils.polish_model
- except AttributeError:
- pass
- else:
- return onnx_polish_model(onnx_model)
-
- try:
- # pylint: disable=import-outside-toplevel
- import onnxoptimizer
- except ImportError:
- pass
- else:
- return onnxoptimizer.optimize(onnx_model)
-
- return onnx_model
-
-
-def tvm_array_to_list(arr):
- return tuple(x.value for x in arr)
-
-
-def get_onnx_version():
- return onnx.__version__
-
-
-def get_node_shape(node):
- return tuple("Any" if isinstance(i, tvm.tir.Any) else int(i) for i in node.shape)
-
-
-def infer_type(node):
- """A method to infer the type of a relay expression."""
- mod = tvm.IRModule.from_expr(node)
- mod = relay.transform.InferType()(mod)
- entry = mod["main"]
- return entry if isinstance(node, relay.Function) else entry.body
-
-
-def call_node_infer_type(node):
- """infer the output types of call node"""
- infer_out = infer_type(node)
- out_type = infer_out._checked_type_
- if isinstance(out_type, TensorType):
- types = [out_type]
- elif isinstance(out_type, TupleType):
- types = list(out_type.fields)
- else:
- raise RuntimeError(f"Unsupported output type {type(out_type)} in operator {node.op.name}")
-
- return types
-
-
-def add_input(data, name, prefix, model_container):
- input_name = f"{prefix}_{name}"
- dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[data.dtype]
- tensor_value_info = onnx.helper.make_tensor_value_info(input_name, dtype, shape=data.shape)
- model_container.add_inputs([tensor_value_info])
- data_tensor = numpy_helper.from_array(data, input_name)
- model_container.add_initializers([data_tensor])
- return input_name
-
-
-class OpConverter(object):
- """Operator converter Base Class."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- """convert Relay attributes to ONNX attributes.
- The derived classes should implement this method
- if attributes are required by the operator
- otherwise by default no attributes are passed
- """
- return {}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
- onnx_node = onnx.helper.make_node(
- cls.__name__, node_entry["input_names"], node_entry["output_names"], **attrs
- )
- model_container.add_nodes([onnx_node])
-
-
-def rename(op_name):
- """This method creates dynamic operator of name op_name with empty attributes"""
- return type(op_name, (OpConverter,), {})
-
-
-class Reshape(object):
- """Operator converter for Reshape."""
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- """Converts Relay operator Reshape to ONNX operator.
- Relay operator accepts shape as attribute but ONNX operator
- accepts it as a input.
- """
- name = node_entry["name"]
- shape = numpy.asarray(
- [a.value for a in node_entry["relay_node"].attrs.newshape], dtype=numpy.int64
- )
-
- input_names = [
- node_entry["input_names"][0],
- add_input(shape, name, "shape", model_container),
- ]
-
- node = onnx.helper.make_node(cls.__name__, input_names, node_entry["output_names"])
- model_container.add_nodes([node])
-
-
-class Conv(OpConverter):
- """Operator converter for Conv."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {
- "group": attrs.get_int("groups"),
- "pads": attrs.get_int_tuple("padding"),
- "strides": attrs.get_int_tuple("strides"),
- "dilations": attrs.get_int_tuple("dilation"),
- "kernel_shape": attrs.get_int_tuple("kernel_size"),
- }
-
-
-class ConvTranspose(OpConverter):
- """Operator converter for ConvTranspose."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {
- "group": attrs.get_int("groups"),
- "pads": attrs.get_int_tuple("padding"),
- "strides": attrs.get_int_tuple("strides"),
- "dilations": attrs.get_int_tuple("dilation"),
- "kernel_shape": attrs.get_int_tuple("kernel_size"),
- "output_padding": attrs.get_int_tuple("output_padding"),
- }
-
-
-class MaxPool(OpConverter):
- """Operator converter for MaxPool."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {
- "pads": attrs.get_int_tuple("padding"),
- "strides": attrs.get_int_tuple("strides"),
- "kernel_shape": attrs.get_int_tuple("pool_size"),
- "ceil_mode": 1 if attrs.ceil_mode else 0,
- }
-
-
-class Transpose(OpConverter):
- """Operator converter for Transpose."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"perm": attrs.get_int_tuple("axes")} if attrs["axes"] else {}
-
-
-class MatMul(OpConverter):
- """Operator converter for MatMul."""
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- inter_output_name = f"inter{node_entry['name']}"
- transpose_node = onnx.helper.make_node(
- Transpose.__name__, [node_entry["input_names"][1]], [inter_output_name], perm=(1, 0)
- )
- model_container.add_nodes([transpose_node])
-
- inputs = [node_entry["input_names"][0], inter_output_name]
- matmul_node = onnx.helper.make_node(cls.__name__, inputs, node_entry["output_names"])
- model_container.add_nodes([matmul_node])
-
-
-class Flatten(OpConverter):
- """Operator converter for Flatten."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"axis": 1}
-
-
-class BatchNormalization(OpConverter):
- """Operator converter for BatchNormalization."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"epsilon": float(attrs.get_str("epsilon")), "axis": float(attrs.get_int("axis"))}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- """Converts Relay operator batch_norm to ONNX operator.
- Relay operator has property axis to handle data in NHWC format.
- """
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
- transpose_out_name = node_entry["input_names"][0]
- inter_output_names = [node_entry["output_names"][0]]
- # axis==3 means channel is specified along the 3rd axis
- if attrs["axis"] == 3:
- transpose_out_name = f"transpose_{node_entry['name']}"
- node_transposed = onnx.helper.make_node(
- Transpose.__name__,
- [node_entry["input_names"][0]],
- [transpose_out_name],
- perm=[0, 3, 1, 2],
- )
- model_container.add_nodes([node_transposed])
- inter_output_names = [f"batch_norm_{node_entry['name']}"]
-
- input_names = [transpose_out_name] + node_entry["input_names"][1:]
- batch_norm_node = onnx.helper.make_node(
- cls.__name__, input_names, inter_output_names, epsilon=attrs["epsilon"]
- )
- model_container.add_nodes([batch_norm_node])
-
- if attrs["axis"] == 3:
- node_transposed = onnx.helper.make_node(
- Transpose.__name__,
- inter_output_names,
- [node_entry["output_names"][0]],
- perm=[0, 2, 3, 1],
- )
- model_container.add_nodes([node_transposed])
-
-
-class Dropout(OpConverter):
- """Operator converter for Dropout."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"ratio": float(attrs.get_str("rate"))}
-
-
-class AveragePool(MaxPool):
- """Operator converter for AveragePool."""
-
-
-class Concat(OpConverter):
- """Operator converter for Concat."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"axis": attrs.get_int("axis")}
-
-
-class BiasAdd(OpConverter):
- """Operator converter for BiasAdd."""
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node_entry can not be a Tuple"
- input_node = input_node[0]
- data_ndim = len(input_node["types"][0].shape)
- axis = node_entry["relay_node"].attrs.get_int("axis")
- if axis < 0:
- axis = axis + data_ndim
- new_axes = data_ndim - axis - 1
- if new_axes:
- inter_output_name = f"inter{node_entry['name']}"
- unsqueeze_node = onnx.helper.make_node(
- "Unsqueeze",
- [node_entry["input_names"][1]],
- [inter_output_name],
- axes=tuple(range(1, new_axes + 1)),
- )
- model_container.add_nodes([unsqueeze_node])
- else:
- inter_output_name = node_entry["input_names"][1]
-
- inputs = [node_entry["input_names"][0], inter_output_name]
- matmul_node = onnx.helper.make_node("Add", inputs, node_entry["output_names"])
- model_container.add_nodes([matmul_node])
-
-
-class ReduceMean(OpConverter):
- """Operator converter for ReduceMean."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {
- "axes": attrs.axis,
- "keepdims": 0 if bool(attrs.get_int("keepdims", 0)) is False else 1,
- }
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- shape = input_node["types"][0].shape
- axis = node_entry["relay_node"].attrs.axis
- axis = list(range(shape.size())) if not axis else tvm_array_to_list(axis)
- exclude = 0 if not bool(node_entry["relay_node"].attrs.exclude) else 1
- keepdims = 0 if not bool(node_entry["relay_node"].attrs.keepdims) else 1
- if exclude:
- all_axis = list(range(len(shape)))
- axis = set(all_axis) - set(axis)
-
- node = onnx.helper.make_node(
- cls.__name__,
- node_entry["input_names"],
- node_entry["output_names"],
- axes=axis,
- keepdims=keepdims,
- )
- model_container.add_nodes([node])
-
-
-class Pad(OpConverter):
- """Operator converter for Pad."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- before = []
- after = []
- for axis_pads in attrs.pad_width:
- before.append(axis_pads[0])
- after.append(axis_pads[1])
- pads = before + after
- pads = numpy.asarray(pads, dtype=pads[0].dtype)
- return {"pads": pads, "mode": attrs.get_str("pad_mode")}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- """Converts Relay operator Pad to ONNX operator.
- Relay operator accepts pads as attribute but ONNX operator
- accepts it as a input.
- """
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- name = node_entry["name"]
- pad_data = numpy.asarray(attrs["pads"], dtype=attrs["pads"][0].dtype).astype(numpy.int64)
-
- input_names = [
- node_entry["input_names"][0],
- add_input(pad_data, name, "pads", model_container),
- node_entry["input_names"][1],
- ]
-
- node = onnx.helper.make_node(
- cls.__name__, input_names, node_entry["output_names"], mode=attrs["mode"]
- )
- model_container.add_nodes([node])
-
-
-class Softmax(OpConverter):
- """Operator converter for SoftMax."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"axis": attrs.axis}
-
-
-class Squeeze(OpConverter):
- """Operator converter for Squeeze."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"axes": attrs.axis}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- shape = input_node["types"][0].shape
- axis = node_entry["relay_node"].attrs.get_int("axis")
- if not axis:
- axis = []
- for axis_idx, val in enumerate(shape):
- if val.value == 1:
- axis.append(axis_idx)
- else:
- axis = node_entry["relay_node"].attrs.get_int_tuple("axis")
-
- node = onnx.helper.make_node(
- cls.__name__, node_entry["input_names"], node_entry["output_names"], axes=axis
- )
- model_container.add_nodes([node])
-
-
-class Slice(OpConverter):
- """Operator converter for Slice."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {
- "starts": attrs.get_int_tuple("begin"),
- "ends": attrs.get_int_tuple("end"),
- "steps": attrs.get_int_tuple("strides"),
- "slice_mode": attrs.get_str("slice_mode"),
- }
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- name = node_entry["name"]
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- shape = input_node["types"][0].shape
-
- starts = list(attrs["starts"])
- ends = list(attrs["ends"])
- steps = list(attrs["steps"])
- starts += [0] * (len(shape) - len(starts))
- ends += [shape[i] + 1 for i in range(len(ends), len(shape))]
- axes = list(range(len(shape)))
-
- if attrs["slice_mode"] == "size":
- ends = [
- starts[i] + (shape[i] + 1 if ends[i] < 0 else ends[i]) for i in range(len(shape))
- ]
- steps = [1] * len(shape)
- else:
- steps += [1] * (len(shape) - len(steps))
-
- starts = numpy.asarray(starts).astype(numpy.int64)
- ends = numpy.asarray(ends).astype(numpy.int64)
- axes = numpy.asarray(axes).astype(numpy.int64)
- steps = numpy.asarray(steps).astype(numpy.int64)
-
- input_names = []
- input_names.append(add_input(starts, name, "starts", model_container))
- input_names.append(add_input(ends, name, "ends", model_container))
- input_names.append(add_input(axes, name, "axes", model_container))
- input_names.append(add_input(steps, name, "steps", model_container))
-
- input_names = [node_entry["input_names"][0]] + input_names
-
- slice_node = onnx.helper.make_node(cls.__name__, input_names, node_entry["output_names"])
- model_container.add_nodes([slice_node])
-
-
-class Split(OpConverter):
- """Operator converter for Split."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- indices_or_sections = attrs["indices_or_sections"]
-
- if isinstance(indices_or_sections, (list, tvm.ir.container.Array)):
- indices_or_sections = attrs.get_int_tuple("indices_or_sections")
- if isinstance(indices_or_sections, tvm.ir.PrimExpr):
- indices_or_sections = indices_or_sections.value
-
- return {"indices_or_section": indices_or_sections, "axis": attrs.get_int("axis")}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- shape = get_node_shape(input_node["types"][0])
-
- indices_or_sect = attrs["indices_or_section"]
- axis = attrs["axis"]
- axis_length = shape[axis]
-
- if isinstance(indices_or_sect, int):
- split = [axis_length // indices_or_sect] * indices_or_sect
- else:
- split = []
- for i in range(len(indices_or_sect) + 1):
- if i == 0:
- split.append(indices_or_sect[0])
- elif i == len(indices_or_sect):
- split.append(axis_length - indices_or_sect[-1])
- else:
- split.append(indices_or_sect[i] - indices_or_sect[i - 1])
-
- slice_node = onnx.helper.make_node(
- cls.__name__,
- node_entry["input_names"],
- node_entry["output_names"],
- split=split,
- axis=axis,
- )
- model_container.add_nodes([slice_node])
-
-
-class LayoutTransform(OpConverter):
- """Operator converter for Layouttransform"""
-
- @classmethod
- def convert_attributes(cls, attrs):
- src_layout = attrs.get_str("src_layout")
- dst_layout = attrs.get_str("dst_layout")
-
- perm = [src_layout.index(c) for c in dst_layout]
- return {"perm": tuple(perm)}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
- onnx_node = onnx.helper.make_node(
- "Transpose", node_entry["input_names"], node_entry["output_names"], **attrs
- )
- model_container.add_nodes([onnx_node])
-
-
-class Clip(OpConverter):
- """Operator converter for Clip."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"min": attrs.a_min, "max": attrs.a_max}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- name = node_entry["name"]
-
- min_val = numpy.asarray(attrs["min"]).astype(numpy.float32)
- max_val = numpy.asarray(attrs["max"]).astype(numpy.float32)
-
- input_names = []
- input_names.append(add_input(min_val, name, "min", model_container))
- input_names.append(add_input(max_val, name, "max", model_container))
-
- input_names = [node_entry["input_names"][0]] + input_names
-
- node = onnx.helper.make_node(cls.__name__, input_names, node_entry["output_names"])
- model_container.add_nodes([node])
-
-
-class Expand(OpConverter):
- """Operator converter for Expand_dims."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"axis": attrs.axis, "num_newaxis": attrs.num_newaxis}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- name = node_entry["name"]
-
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node_entry can not be a Tuple"
- input_node = input_node[0]
- data_shape = input_node["types"][0].shape
- new_shape = list(data_shape)
-
- for _ in range(attrs["num_newaxis"]):
- new_shape.insert(attrs["axis"], 1)
-
- new_shape = numpy.asarray(new_shape).astype(numpy.int64)
- input_names = []
- input_names.append(add_input(new_shape, name, "shape", model_container))
-
- input_names = [node_entry["input_names"][0]] + input_names
-
- node = onnx.helper.make_node(cls.__name__, input_names, node_entry["output_names"])
- model_container.add_nodes([node])
-
-
-class ConstantOfShapeZeros(OpConverter):
- """Operator converter for ConstantOfShape."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"value": 0}
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- dtype = input_node["types"][0].dtype
-
- name = node_entry["name"]
- shape = [val.value for val in input_node["types"][0].shape]
- shape = numpy.asarray(shape).astype(numpy.int64)
-
- input_names = []
- input_names.append(add_input(shape, name, "shape", model_container))
-
- dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[numpy.dtype(dtype)]
- tensor_value = onnx.helper.make_tensor("value", dtype, [1], [attrs["value"]])
-
- node = onnx.helper.make_node(
- "ConstantOfShape", input_names, node_entry["output_names"], value=tensor_value
- )
- model_container.add_nodes([node])
-
-
-class ConstantOfShapeOnes(ConstantOfShapeZeros):
- """Operator converter for ConstantOfShape."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"value": 1}
-
-
-class LRN(OpConverter):
- """Operator converter for LRN."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- """axis attr is not supported as an argument in onnx.
- Onnx only supports axis=1 (channels)."""
- if attrs.get_int("axis") != 1:
- raise RuntimeError(
- f"Unsupported axis {attrs.get_int('axis')} in operator relay lrn operator. "
- f"Only axis = 1 is supported by Onnx."
- )
-
- return {"alpha": attrs.alpha, "beta": attrs.beta, "bias": attrs.bias, "size": attrs.size}
-
-
-class Cast(OpConverter):
- """Operator converter for Cast."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- return {"to": getattr(TensorProto, attrs.dtype.upper())}
-
-
-class Resize(OpConverter):
- """Operator converter for Resize."""
-
- @classmethod
- def convert_attributes(cls, attrs):
- method = attrs.get_str("method")
- if method == "nearest_neighbor":
- mode = b"nearest"
- elif "linear" in method: # linear / bilinear
- mode = b"linear"
- elif "cubic" in method: # cubic / bicubic
- mode = b"cubic"
- else:
- raise RuntimeError(f"Unsupported method {method} in operator Resize")
-
- coord_trans = attrs.get_str("coordinate_transformation_mode")
- if coord_trans == "half_pixel":
- coord_trans = b"half_pixel"
- elif coord_trans == "align_corners":
- coord_trans = b"align_corners"
- elif coord_trans == "asymmetric":
- coord_trans = b"asymmetric"
- else:
- raise RuntimeError(
- f"Unsupported coordinate transform mode {coord_trans} in operator Resize"
- )
-
- rounding_method = attrs.get_str("rounding_method")
- if rounding_method == "round":
- rounding_method = b"round_prefer_ceil"
- elif rounding_method == "floor":
- rounding_method = b"floor"
- elif rounding_method == "ceil":
- rounding_method = b"ceil"
- else:
- raise RuntimeError(f"Unsupported rounding method {rounding_method} in operator Resize")
-
- size = attrs.get_int_tuple("size")
-
- return {
- "mode": mode,
- "coord_trans": coord_trans,
- "size": size,
- "nearest_mode": rounding_method,
- }
-
- @classmethod
- def convert(cls, node_entry, model_container, node_dict):
- attrs = cls.convert_attributes(node_entry["relay_node"].attrs)
-
- name = node_entry["name"]
- input_node = node_dict[node_entry["inputs"][0]]
- assert len(input_node) == 1, "input node can not be a Tuple"
- input_node = input_node[0]
- input_shape = input_node["types"][0].shape
-
- # (TBD) needed in opset 11
- roi = [0] * len(input_shape) + [1] * len(input_shape)
- roi_array = numpy.asarray(roi).astype(numpy.float64)
- roi_node = add_input(roi_array, name, "roi", model_container)
-
- out_size = attrs["size"]
-
- # (onnx) rank of scale / size must match rank of X
- # relay size node contains only spatial dimensions
- # pad with 1s to match rank
- match_rank_pad = len(input_shape) - len(out_size)
- out_size_full_rank = input_shape[:match_rank_pad] + list(out_size)
- out_size_array = numpy.asarray(out_size_full_rank).astype(numpy.int64)
-
- input_size_array = numpy.asarray(list(input_shape)).astype(numpy.int64)
-
- scale_array = numpy.divide(out_size_array, input_size_array).astype(numpy.float32)
- scale_node = add_input(scale_array, name, "scales", model_container)
-
- input_names = [node_entry["input_names"][0], roi_node, scale_node]
-
- resize_node = onnx.helper.make_node(
- cls.__name__,
- input_names,
- node_entry["output_names"],
- mode=attrs["mode"],
- coordinate_transformation_mode=attrs["coord_trans"],
- nearest_mode=attrs["nearest_mode"],
- )
- model_container.add_nodes([resize_node])
-
-
-relay_to_onnx_op_mapping = {
- "reshape": Reshape,
- "nn.conv2d": Conv,
- "nn.conv2d_transpose": ConvTranspose,
- "add": rename("Add"),
- "nn.relu": rename("Relu"),
- "transpose": Transpose,
- "nn.dense": MatMul,
- "nn.max_pool2d": MaxPool,
- "nn.batch_flatten": Flatten,
- "multiply": rename("Mul"),
- "nn.bias_add": BiasAdd,
- "nn.batch_norm": BatchNormalization,
- "nn.global_avg_pool2d": rename("GlobalAveragePool"),
- "concatenate": Concat,
- "nn.dropout": Dropout,
- "nn.avg_pool2d": AveragePool,
- "divide": rename("Div"),
- "mean": ReduceMean,
- "nn.pad": Pad,
- "nn.softmax": Softmax,
- "squeeze": Squeeze,
- "strided_slice": Slice,
- "greater": rename("Greater"),
- "less": rename("Less"),
- "equal": rename("Equal"),
- "zeros_like": ConstantOfShapeZeros,
- "ones_like": ConstantOfShapeOnes,
- "subtract": rename("Sub"),
- "split": Split,
- "exp": rename("Exp"),
- "layout_transform": LayoutTransform,
- "clip": Clip,
- "expand_dims": Expand,
- "nn.lrn": LRN,
- "sigmoid": rename("Sigmoid"),
- "copy": rename("Identity"),
- "round": rename("Round"),
- "cast": Cast,
- "image.resize2d": Resize,
-}
-
-
-class ModelContainer(object):
- """A container class to hold different attributes of ONNX model graph"""
-
- def __init__(self, name, opset_version):
- self._name = name
- self._opset_version = opset_version
- self._inputs = []
- self._outputs = []
- self._nodes = []
- self._initializers = []
-
- def add_inputs(self, inputs):
- self._inputs.extend(inputs)
-
- def add_outputs(self, outputs):
- self._outputs.extend(outputs)
-
- def add_nodes(self, nodes):
- self._nodes.extend(nodes)
-
- def add_initializers(self, initializers):
- self._initializers.extend(initializers)
-
- def _get_opsets(self):
- opsets = []
- imp = OperatorSetIdProto()
- imp.version = self._opset_version
- opsets.append(imp)
- return opsets
-
- def make_model(self):
- """Creates the onnx model from the graph"""
- onnx_graph = onnx.helper.make_graph(
- self._nodes, self._name, self._inputs, self._outputs, self._initializers
- )
- kwargs = {}
- kwargs["opset_imports"] = self._get_opsets()
- kwargs["producer_name"] = "TVM Relay"
- kwargs["producer_version"] = tvm.__version__
-
- return onnx.helper.make_model(onnx_graph, **kwargs)
-
-
-class RelayToONNXConverter(ExprVisitor):
- """A helper class to traverse the Relay graph and convert Relay nodes to ONNX model
-
- Parameters
- ----------
- name : str
- name of the model
-
- params : dict
- dict of the parameter names and NDarray values
-
- opset_version : int
- target onnx opset version
-
- """
-
- def __init__(self, name, params, opset_version):
- super().__init__()
- self._name = name
- self._mc = ModelContainer(name, opset_version)
- self._params = params
- self._node_dict = {}
- self._node_count = 0
- self.last_node = None
-
- @classmethod
- def _get_node_entry(cls, relay_node, name):
- return {
- "relay_node": relay_node,
- "inputs": [relay_node], # inputs in the form of relay nodes
- "types": [], # output types in case of call nodes else self type
- "name": name, # name of the node
- "input_names": [name], # input names in case of call nodes else self name
- "output_names": [name], # output names in case of call nodes else self name
- "op": None, # op name in case of call node else None
- }
-
- def convert_to_onnx(self, func):
- """Traverse Relay graph and generate a ONNX model"""
-
- self.visit(func)
- self._add_output(self._node_dict[self.last_node])
- model = self._mc.make_model()
- return run_onnx_optimizer(model)
-
- def visit(self, expr):
- self._node_count += 1
- super().visit(expr)
-
- def visit_constant(self, const):
- node_index = self._node_count
- name = self._name + "_const_" + str(node_index)
- node_entry = self._get_node_entry(const, name)
- node_entry["types"] = [const.checked_type]
-
- self._add_constant_input(node_entry, node_index)
- self._node_dict[const] = [node_entry]
-
- def visit_var(self, var):
- node_index = self._node_count
- node_entry = self._get_node_entry(var, var.name_hint)
- node_entry["types"] = [var.type_annotation]
-
- self._add_input(node_entry, node_index)
- self._node_dict[var] = [node_entry]
-
- def visit_tuple(self, tup):
- self._node_dict[tup] = []
- for f in tup.fields:
- self.visit(f)
- self._node_dict[tup].extend(self._node_dict[f])
-
- self.last_node = tup
-
- def visit_tuple_getitem(self, t):
- self.visit(t.tuple_value)
- tup_node = self._node_dict[t.tuple_value]
- if len(tup_node) > 1:
- self._node_dict[t] = tup_node[t.index]
- else:
- node_entry = copy.deepcopy(tup_node[0])
- output_names = [node_entry["output_names"][t.index]]
- node_entry["output_names"] = output_names
- self._node_dict[t] = [node_entry]
- self.last_node = t
-
- def visit_call(self, call):
- node_index = self._node_count
- op = call.op
- name = f"{op}_{node_index}"
- node_entry = self._get_node_entry(call, name)
-
- node_entry["op"] = op
- node_entry["input_names"] = []
- node_entry["inputs"] = []
- node_entry["output_names"] = None
- for input_arg in call.args:
- self.visit(input_arg)
- input_names = []
- for arg_node_entry in self._node_dict[input_arg]:
- input_names.extend(arg_node_entry["output_names"])
- node_entry["input_names"].extend(input_names)
- node_entry["inputs"].extend([input_arg])
-
- node_entry["types"] = call_node_infer_type(call)
- node_entry["output_names"] = []
- for i in range(len(node_entry["types"])):
- node_entry["output_names"].append(name + str(i))
- self.last_node = call
- self._add_node(node_entry, node_index)
- self._node_dict[call] = [node_entry]
-
- def _add_node(self, node_entry, idx):
- """Convert Relay operator node to ONNX operator and add it to container nodes list"""
- if node_entry["op"].name not in relay_to_onnx_op_mapping:
- raise NotImplementedError(
- f"Currently the operator '{node_entry['op'].name}' is " "not supported."
- )
- converter = relay_to_onnx_op_mapping[node_entry["op"].name]()
-
- return converter.convert(node_entry, self._mc, self._node_dict)
-
- def _add_params(self, node_entry, idx):
- """Add param value to initializer and name to inputs"""
- param_name = node_entry["name"]
- assert param_name in self._params, (
- f"The parameter {param_name} is not present" "in params dict provided."
- )
- value = self._params[param_name]
- numpy_array = value.numpy()
- tensor = numpy_helper.from_array(numpy_array, param_name)
- self._mc.add_initializers([tensor])
- dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[numpy_array.dtype]
- input = onnx.helper.make_tensor_value_info(param_name, dtype, shape=numpy_array.shape)
- self._mc.add_inputs([input])
-
- def _add_constant_input(self, node_entry, idx):
- """Create named input for constant and add it to container inputs.
- If input is a parameter then add to param
- """
- node = node_entry["relay_node"]
- param_name = node_entry["name"]
- self._params[param_name] = node.data
- self._add_params(node_entry, idx)
-
- def _add_input(self, node_entry, idx):
- """Add input node to container inputs. If input is a parameter then add to param"""
- if node_entry["name"] in self._params:
- self._add_params(node_entry, idx)
- else:
- node_type = node_entry["types"][0]
- dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[numpy.dtype(node_type.dtype)]
- input = onnx.helper.make_tensor_value_info(
- node_entry["name"], dtype, shape=get_node_shape(node_type)
- )
- self._mc.add_inputs([input])
-
- def _add_output(self, node_entries):
- """Add output node to container outputs."""
-
- for node_entry in node_entries:
- for node_type, output_name in zip(node_entry["types"], node_entry["output_names"]):
- dtype = onnx.mapping.NP_TYPE_TO_TENSOR_TYPE[numpy.dtype(node_type.dtype)]
- output = onnx.helper.make_tensor_value_info(
- output_name, dtype, shape=get_node_shape(node_type)
- )
- self._mc.add_outputs([output])
-
-
-def to_onnx(relay_ir, params, name, opset_version=11, path=None):
- """Convert a Relay Function Module into an equivalent ONNX and serialize it to the path
-
- Parameters
- ----------
- relay_ir : tvm.ir.IRModule or tvm.relay.Function
- The relay module object
-
- params : dict
- dict of the parameter names and NDarray values
-
- name : str
- name of the output ONNX graph
-
- opset_version : int
- target onnx opset version
-
- path : str
- The path where ONNX model will be saved
-
- Returns
- -------
- onnx_model : onnx.ModelProto
- converted ONNX model as a ModelProto.
-
- """
-
- if opset_version not in ONNX_OPSET_VERSONS_SUPPORTED:
- raise NotImplementedError("Currently only opset version 11 is supported.")
-
- if opset_version > defs.onnx_opset_version():
- raise Exception(
- f"The ONNX package installed of version {get_onnx_version()} does not support the "
- f"opset version {opset_version}. Upgrade the ONNX package to latest version."
- )
-
- func = relay_ir["main"] if isinstance(relay_ir, tvm.ir.IRModule) else relay_ir
- converter = RelayToONNXConverter(name, params, opset_version)
- onnx_model = converter.convert_to_onnx(func)
-
- if path:
- onnx.save(onnx_model, path)
- return onnx_model
-
-
-@tvm._ffi.register_func("relay.ext.onnx")
-def onnx_compiler(func):
- """Create a runtime module for ONNX from Relay Function
-
- :param func: Relay function
- :return: runtime module for ONNX
- """
-
- assert isinstance(func, tvm.relay.function.Function)
- name = str(func.attrs.global_symbol)
- model = to_onnx(func, {}, name)
- const_vars = [const.name for const in model.graph.initializer]
- name_bytes = bytes(name, "utf-8")
- name_size = struct.pack("I", len(name_bytes))
- model_serialized = model.SerializeToString()
- model_size = struct.pack("I", model.ByteSize())
- data = b"" + name_size + name_bytes + model_size + model_serialized
-
- runtime_func = "runtime.ONNXModuleCreate"
- fcreate = tvm._ffi.get_global_func(runtime_func)
- return fcreate(data.hex(), name, const_vars)
-
-
-@tvm._ffi.register_func("relay.ext.onnx.save_to_file")
-def save_to_file(hex_str, path=None, fmt="onnx"):
- """Store the ONNX subgraphs in the path folder
-
- :param hex_str: Subgrah names and corresponding serialized onnx hex string
- :param path: path to which ONNX files to be stored
- It is assumed that path exists
- :param fmt: extension of the files to be stored
- """
- onnx_ir = bytes.fromhex(hex_str)
-
- offset = 0
- while offset < len(onnx_ir):
- stop = offset + 4
- (name_size,) = struct.unpack("I", onnx_ir[offset:stop])
- name = onnx_ir[stop : stop + name_size].decode("utf-8")
- stop = stop + name_size
- (model_size,) = struct.unpack("I", onnx_ir[stop : stop + 4])
- stop = stop + 4
- model_serialized = onnx_ir[stop : stop + model_size]
- offset = stop + model_size
-
- model_onnx = onnx.load_model_from_string(model_serialized)
- onnx.save(model_onnx, f"{path}{os.path.sep}{name}.{fmt}")
diff --git a/python/tvm/contrib/torch/__init__.py b/python/tvm/contrib/torch/__init__.py
deleted file mode 100644
index c3dd34d470..0000000000
--- a/python/tvm/contrib/torch/__init__.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=wrong-import-position,redefined-builtin,invalid-name
-"""Module container of Pytorch custom class"""
-import os
-import platform
-import warnings
-import torch
-from tvm._ffi import libinfo
-
-
-def _load_platform_specific_library(lib_name):
- system = platform.system()
- if system == "Darwin":
- lib_file_name = lib_name + ".dylib"
- elif system == "Windows":
- lib_file_name = lib_name + ".dll"
- else:
- lib_file_name = lib_name + ".so"
- lib_path = libinfo.find_lib_path()[0]
- lib_dir = os.path.dirname(lib_path)
- lib_file_path = os.path.join(lib_dir, lib_file_name)
- try:
- torch.classes.load_library(lib_file_path)
- except OSError as err:
- errmsg = str(err)
- if errmsg.find("undefined symbol") != -1:
- reason = " ".join(
- (
- "Got undefined symbol error,",
- "which might be due to the CXXABI incompatibility.",
- )
- )
- else:
- reason = errmsg
- warnings.warn(
- f"The library {lib_name} is not built successfully. {reason}",
- RuntimeWarning,
- )
-
-
-_load_platform_specific_library("libpt_tvmdsoop")
-_load_platform_specific_library("libpt_tvmdsoop_new")
-
-from . import module
-
-GraphModule = module.GraphModule
-VMModule = module.VMModule
-TraceTvmModule = module.TraceTvmModule
-
-from . import pytorch_tvm
-
-PyTorchTVMModule = pytorch_tvm.PyTorchTVMModule
-compile = pytorch_tvm.compile
-
-from . import as_torch
-
-TVMScriptIRModule = as_torch.OperatorModuleWrapper
-as_torch = as_torch.as_torch
-
-from . import optimize_torch
-
-GraphExecutorFactoryWrapper = optimize_torch.GraphExecutorFactoryWrapper
-optimize_torch = optimize_torch.optimize_torch
diff --git a/python/tvm/contrib/torch/as_torch.py b/python/tvm/contrib/torch/as_torch.py
deleted file mode 100644
index c78d7bb831..0000000000
--- a/python/tvm/contrib/torch/as_torch.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# pylint: disable=inconsistent-return-statements
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=missing-function-docstring
-"""
-as_torch: a decorator, which is used to wrap the TVMScript code to `torch.nn.module`.
-"""
-import tempfile
-from typing import Callable, List, Optional, Union
-
-# isort: off
-from typing_extensions import Literal
-
-# isort: on
-
-import torch
-import torch.utils.dlpack
-
-import tvm
-from tvm import meta_schedule as ms
-from tvm.target.target import Target
-from tvm.tir import PrimFunc
-
-
-# python wrapper for OperatorModule
-class OperatorModuleWrapper(torch.nn.Module):
- def __init__(
- self,
- module: Union[
- tvm.ir.module.IRModule,
- tvm.tir.function.PrimFunc,
- ],
- ):
- super().__init__()
- self.rt_module = None # runtime module
- self.ir_module = module # IR modules
-
- def tune(
- self,
- target: Union[str, Target] = "cpu",
- max_trials_global: int = 32,
- *,
- num_trials_per_iter: int = 32,
- builder: ms.Builder.BuilderType = "local",
- runner: ms.Runner.RunnerType = "local",
- database: ms.Database.DatabaseType = "json",
- cost_model: ms.CostModel.CostModelType = "xgb",
- measure_callbacks: ms.MeasureCallback.CallbackListType = "default",
- task_scheduler: ms.TaskScheduler.TaskSchedulerType = "round-robin",
- space: ms.SpaceGenerator.SpaceGeneratorType = "post-order-apply",
- strategy: ms.SearchStrategy.SearchStrategyType = "replay-trace",
- num_tuning_cores: Union[Literal["physical", "logical"], int] = "physical",
- seed: Optional[int] = None,
- ) -> None:
- """
- Tune the TVMScript code.
-
- Parameters
- ----------
- config: Optional[TuneConfig]
- The tuning configuration.
-
- target : Optional[str, Target]
- The target to tune for.
- """
- if target == "cpu":
- target = f"llvm --num-cores {ms.utils.cpu_count(logical=False)}"
-
- with tempfile.TemporaryDirectory() as work_dir:
- database = ms.tir_integration.tune_tir(
- mod=self.ir_module,
- target=target,
- work_dir=work_dir,
- max_trials_global=max_trials_global,
- num_trials_per_iter=num_trials_per_iter,
- builder=builder,
- runner=runner,
- database=database,
- cost_model=cost_model,
- measure_callbacks=measure_callbacks,
- task_scheduler=task_scheduler,
- space=space,
- strategy=strategy,
- num_tuning_cores=num_tuning_cores,
- seed=seed,
- )
- sch = ms.tir_integration.compile_tir(database, self.ir_module, target)
- self.ir_module = sch.mod
- self.build(target)
-
- def script(self):
- return self.ir_module.script()
-
- def build(self, target=None):
- runtime_module = tvm.build(self.ir_module, target=target)
- func = tvm.get_global_func("tvmtorch.save_runtime_mod", allow_missing=True)
-
- if func is None:
- raise ValueError('as_torch requires the flag /"USE_PT_TVMDSOOP/" set in config.cmake')
- func(runtime_module)
-
- self.rt_module = torch.classes.tvm_torch.OperatorModuleWrapper()
-
- def forward(self, *torch_inputs: List[torch.Tensor]) -> List[torch.Tensor]:
- if self.rt_module is None:
- if torch_inputs[0].is_cuda:
- self.build(target="cuda")
- elif torch_inputs[0].device.type == "cpu":
- self.build()
- else:
- raise Exception(f"the target {torch_inputs[0].device.type} is not supported yet")
-
- return self.rt_module.forward(torch_inputs)
-
-
-def as_torch(func: Union[tvm.ir.module.IRModule, tvm.tir.function.PrimFunc, Callable]):
- """A decorator of converting TensorIR to PyTorch nn.Module.
-
- Parameters
- ----------
- func: Optional[tvm.ir.module.IRModule, tvm.tir.function.PrimFunc, Callable]
- The function written by TVMScript.
-
- Returns
- -------
- mod : Union[OperatorModuleWrapper, Callable]
- It will return an object, or a templated function of OperatorModuleWrapper,
- which is the subclass of the original nn.Module.
-
- """
- if isinstance(func, (tvm.ir.module.IRModule, PrimFunc)):
- return OperatorModuleWrapper(func)
- if callable(func):
-
- def func_get_param(*args, **kwargs):
- return OperatorModuleWrapper(func(*args, **kwargs))
-
- return func_get_param
diff --git a/python/tvm/contrib/torch/module.py b/python/tvm/contrib/torch/module.py
deleted file mode 100644
index cfa3ad264c..0000000000
--- a/python/tvm/contrib/torch/module.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=invalid-name
-"""Module container of PyTorch custom class"""
-import warnings
-from typing import List
-
-import torch
-
-
-class GraphModule(torch.nn.Module):
- r"""Module container of Pytorch class which wraps exported
- TVM op implementation library to be called on Pytorch side"""
-
- @classmethod
- def shape_repr(cls, input_shapes):
- return torch.ops.tvm_dsoop.tvm_shape_repr(input_shapes)
-
- def __init__(self, num_inputs, num_outputs, device=None):
- warnings.warn(
- "This module will be removed at TVM version 0.11",
- DeprecationWarning,
- stacklevel=2,
- )
- super().__init__()
- self.dummy_param = torch.nn.Parameter(torch.empty(0))
- self.engine = None
-
- if device is not None:
- self.to(device)
- self.engine = torch.classes.tvm_dsoop.TvmGraphModule(num_inputs, num_outputs, self.device)
-
- def init(self, input_shapes, lib_path, graph_path, params_path):
- r"""Load tvm module"""
- self.engine.load_tvm_module(input_shapes, lib_path, graph_path, params_path)
-
- def forward(self, inputs: List[torch.Tensor]):
- r"""Call tvm module to forward"""
- return self.engine.forward(inputs)
-
- @property
- def device(self):
- r"""Get the device string"""
- return str(self.dummy_param.device)
-
- def _apply(self, fn):
- r"""Override to device function, manually move tvm module to desired device"""
- super()._apply(fn)
- if self.engine is not None:
- self.engine.to(self.device)
- return self
-
-
-class VMModule(torch.nn.Module):
- r"""Module container of Pytorch class which wraps exported
- TVM op implementation library to be called on Pytorch side"""
-
- @classmethod
- def shape_repr(cls, input_shapes):
- return torch.ops.tvm_dsoop.tvm_shape_repr(input_shapes)
-
- def __init__(self, num_inputs, num_outputs, device=None):
- warnings.warn(
- "This module will be removed at TVM version 0.11",
- DeprecationWarning,
- stacklevel=2,
- )
- super().__init__()
- self.dummy_param = torch.nn.Parameter(torch.empty(0))
- self.engine = None
-
- if device is not None:
- self.to(device)
- self.engine = torch.classes.tvm_dsoop.TvmVMModule(num_inputs, num_outputs, self.device)
-
- def init(self, input_shapes, lib_path, code_path):
- r"""Load tvm module"""
- self.engine.load_tvm_module(input_shapes, lib_path, code_path)
-
- def forward(self, inputs: List[torch.Tensor]):
- r"""Call tvm module to forward"""
- return self.engine.forward(inputs)
-
- @property
- def device(self):
- r"""Get the device string"""
- return str(self.dummy_param.device)
-
- def _apply(self, fn):
- r"""Override to device function, manually move tvm module to desired device"""
- super()._apply(fn)
- if self.engine is not None:
- self.engine.to(self.device)
- return self
-
-
-class TraceTvmModule(torch.nn.Module):
- r"""Wrapper for trace GraphModule
-
- GraphModule and VMModule only supports List[Tensor] inputs and cannot be traced.
- This is a wrapper class for trace GraphModule or VMModule in order to support
- arbitrary number of inputs
-
- Example:
- import tvm.contrib.torch
- tvm_module = tvm.contrib.torch.GraphModule(1, 1, 'cuda:0')
- tvm_module.init(input_shapes, lib_path, graph_path, params_path)
-
- trace_wrapper = tvm.contrib.torch.TraceGraphModule(torch.jit.script(tvm_module))
- traced = torch.jit.trace(trace_wrapper, example_inputs)
- """
-
- def __init__(self, tvm_module):
- warnings.warn(
- "This module will be removed at TVM version 0.11",
- DeprecationWarning,
- stacklevel=2,
- )
- super().__init__()
- self.tvm_module = tvm_module
-
- def forward(self, *inputs):
- outputs = self.tvm_module(inputs)
- return outputs[0] if len(outputs) == 1 else tuple(outputs)
diff --git a/python/tvm/contrib/torch/optimize_torch.py b/python/tvm/contrib/torch/optimize_torch.py
deleted file mode 100644
index dfe35f2aae..0000000000
--- a/python/tvm/contrib/torch/optimize_torch.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# pylint: disable=inconsistent-return-statements
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=missing-function-docstring
-"""
-optimize_torch: a function similar to `torch.jit.trace`,
-which is used to optimize the `torch.nn.module` by TVM metaSchedule,
-and returns a custom TorchScript operator
-"""
-
-import contextlib
-import tempfile
-from typing import Optional, Tuple, Union
-import base64
-import torch
-import torch.utils.dlpack
-import tvm
-import tvm._ffi
-from tvm._ffi import register_func
-from tvm import meta_schedule as ms
-from tvm import relay
-from tvm._ffi import get_global_func
-from tvm.target import Target
-
-
-class GraphExecutorFactoryWrapper(torch.nn.Module):
- def __init__(self, module: tvm.runtime.Module):
- super().__init__()
- self.inner_module = module
-
- def forward(self, *torch_inputs: Tuple[torch.Tensor]):
- ret = self.inner_module.forward(torch_inputs)
- if len(ret) == 1:
- return ret[0]
- return ret
-
-
-def optimize_torch(
- func,
- example_inputs,
- *,
- max_trials_global: int,
- work_dir=None,
- target: Union[str, Target] = "cpu",
- max_trials_per_task: Optional[int] = None,
- num_trials_per_iter: int = 64,
- builder: ms.Builder.BuilderType = "local",
- runner: ms.Runner.RunnerType = "local",
- database: ms.Database.DatabaseType = "json",
- cost_model: ms.CostModel.CostModelType = "xgb",
- measure_callbacks: ms.MeasureCallback.CallbackListType = "default",
- task_scheduler: ms.TaskScheduler.TaskSchedulerType = "gradient",
- space: ms.SpaceGenerator.SpaceGeneratorType = "post-order-apply",
- strategy: ms.SearchStrategy.SearchStrategyType = "evolutionary",
- seed: Optional[int] = None,
-):
- """Load PyTorch model that could be traced by TorchScript, then optimize it via MetaSchedule.
-
- Parameters
- ----------
- func : callable or torch.nn.Module
- A Python function or nn.Module that could run by TorchScript's trace.
- (ie: torch.jit.trace(model, input))
- example_inputs : tuple or torch.Tensor
- Inputs to `torch.jit.trace`.
- max_trials_global : int
- The maximum number of trials to run globally.
- work_dir : Optional[str]
- The working directory to save intermediate results.
- target : Optional[Union[str, Target]]
- The target of the compilation.
- If user doesn't set the target, the module will be built for the CPU target.
- max_trials_per_task : Optional[int]
- The maximum number of trials to run per task.
- num_trials_per_iter : int
- The number of trials to run per iteration
- builder : Builder.BuilderType
- The builder.
- runner : Runner.RunnerType
- The runner.
- database : Database.DatabaseType
- The database.
- cost_model : CostModel.CostModelType
- The cost model.
- measure_callbacks : MeasureCallback.CallbackListType
- The measure callbacks.
- task_scheduler : TaskScheduler.TaskSchedulerType
- The task scheduler.
- space : SpaceGenerator.SpaceGeneratorType
- The space generator to use.
- strategy : SearchStrategy.SearchStrategyType
- The search strategy to use.
- seed : Optional[int]
- The random seed to use.
-
- Returns
- -------
- mod : GraphExecutorFactoryWrapper
- It will return an object of GraphExecutorFactoryWrapper,
- which is the subclass of the original nn.Module.
- """
-
- if target == "cpu":
- target = f"llvm --num-cores {ms.utils.cpu_count(logical=False)}"
- if not isinstance(target, Target):
- target = Target(target)
-
- # If `func` is already a traced module this statement makes no effect
- jit_mod = torch.jit.trace(func, example_inputs)
- if isinstance(example_inputs, torch.Tensor):
- example_inputs = [example_inputs]
- shape_list = [(f"inp_{idx}", i.shape) for idx, i in enumerate(example_inputs)]
- mod, params = relay.frontend.from_pytorch(jit_mod, shape_list) # IRmodule
-
- if work_dir:
- context_manager = contextlib.nullcontext(work_dir)
- else:
- context_manager = tempfile.TemporaryDirectory()
- with context_manager as work_dir: # pylint: disable=redefined-argument-from-local
- database = ms.relay_integration.tune_relay(
- mod=mod,
- params=params,
- target=target,
- work_dir=work_dir,
- max_trials_global=max_trials_global,
- max_trials_per_task=max_trials_per_task,
- num_trials_per_iter=num_trials_per_iter,
- builder=builder,
- runner=runner,
- database=database,
- cost_model=cost_model,
- measure_callbacks=measure_callbacks,
- task_scheduler=task_scheduler,
- space=space,
- strategy=strategy,
- seed=seed,
- )
- executor_factory = ms.relay_integration.compile_relay(
- database=database,
- mod=mod,
- target=target,
- params=params,
- backend="graph",
- )
-
- save_runtime_mod = get_global_func("tvmtorch.save_runtime_mod", allow_missing=True)
- if save_runtime_mod is None:
- raise ValueError('optimize_torch requires the flag /"USE_PT_TVMDSOOP/" set in config.cmake')
- save_runtime_mod(executor_factory.module)
-
- return GraphExecutorFactoryWrapper(torch.classes.tvm_torch.GraphExecutorFactoryWrapper())
-
-
-@register_func("export_runtime_module")
-def save_to_base64(obj) -> bytes:
- with tempfile.NamedTemporaryFile(suffix=".so") as tmpfile:
- obj.export_library(tmpfile.name)
- with open(tmpfile.name, "rb") as temp_file:
- return base64.b64encode(temp_file.read())
diff --git a/python/tvm/contrib/torch/pytorch_tvm.py b/python/tvm/contrib/torch/pytorch_tvm.py
deleted file mode 100644
index 30b0dd4f8c..0000000000
--- a/python/tvm/contrib/torch/pytorch_tvm.py
+++ /dev/null
@@ -1,292 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=redefined-builtin
-"""`compile` api that convert torch module to torch tvm module"""
-import os
-import warnings
-import tvm
-import tvm.testing
-from tvm import relay, autotvm
-from tvm.runtime import load_module
-from tvm.autotvm.tuner import XGBTuner, GATuner, RandomTuner, GridSearchTuner
-from tvm.contrib import graph_executor
-from tvm.contrib.debugger import debug_executor
-from . import GraphModule
-
-
-def tune_tasks(
- tasks,
- measure_option,
- tuner="xgb",
- n_trial=1000,
- early_stopping=None,
- log_filename="tuning.log",
- use_transfer_learning=True,
-):
- """Tune tasks and generate tuning log to file"""
- # create tmp log file
- tmp_log_file = log_filename + ".tmp"
- if os.path.exists(tmp_log_file):
- os.remove(tmp_log_file)
-
- for i, tsk in enumerate(reversed(tasks)):
- prefix = f"[Task {i + 1:2d}/{len(tasks):2d}] "
-
- # create tuner
- if tuner == "xgb":
- tuner_obj = XGBTuner(tsk, loss_type="reg")
- elif tuner == "xgb_knob":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="knob")
- elif tuner == "xgb_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="itervar")
- elif tuner == "xgb_curve":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="curve")
- elif tuner == "xgb_rank":
- tuner_obj = XGBTuner(tsk, loss_type="rank")
- elif tuner == "xgb_rank_knob":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="knob")
- elif tuner == "xgb_rank_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="itervar")
- elif tuner == "xgb_rank_curve":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="curve")
- elif tuner == "xgb_rank_binary":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary")
- elif tuner == "xgb_rank_binary_knob":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="knob")
- elif tuner == "xgb_rank_binary_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="itervar")
- elif tuner == "xgb_rank_binary_curve":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="curve")
- elif tuner == "ga":
- tuner_obj = GATuner(tsk, pop_size=100)
- elif tuner == "random":
- tuner_obj = RandomTuner(tsk)
- elif tuner == "gridsearch":
- tuner_obj = GridSearchTuner(tsk)
- else:
- raise ValueError("Invalid tuner: " + tuner)
-
- if use_transfer_learning:
- if os.path.isfile(tmp_log_file):
- tuner_obj.load_history(autotvm.record.load_from_file(tmp_log_file))
-
- # do tuning
- tsk_trial = min(n_trial, len(tsk.config_space))
- tuner_obj.tune(
- n_trial=tsk_trial,
- early_stopping=early_stopping,
- measure_option=measure_option,
- callbacks=[
- autotvm.callback.progress_bar(tsk_trial, prefix=prefix),
- autotvm.callback.log_to_file(tmp_log_file),
- ],
- )
-
- # pick best records to a cache file
- if not os.path.exists(log_filename):
- with open(log_filename, "w", encoding="utf-8"):
- pass
- if os.path.exists(tmp_log_file):
- autotvm.record.pick_best(tmp_log_file, log_filename)
- os.remove(tmp_log_file)
-
-
-def get_tuning_opt(log_file="tuning.log", n_trial=200):
- """Returns tuning options"""
- tuning_opt = {
- "log_filename": log_file,
- "tuner": "random",
- "n_trial": n_trial,
- "early_stopping": 60,
- "measure_option": autotvm.measure_option(
- builder=autotvm.LocalBuilder(timeout=10),
- runner=autotvm.LocalRunner(number=20, repeat=3, timeout=4, min_repeat_ms=150),
- ),
- }
- return tuning_opt
-
-
-TVM_ASSETS = ["mod.so", "graph.json", "params"]
-
-
-class PyTorchTVMModule:
- """Helper class for compiling pytorch module to tvm module"""
-
- def __init__(self, target="cuda", device=tvm.cuda(0)) -> None:
- self.script_module = None
- self.input_infos = None
- self.default_dtype = "float32"
- self.mod = None
- self.params = None
- self.tasks = None
- self.target = target
- self.dev = device
- self.log_file = None
- self.tvm_module = None
- self.tvm_graph = None
- self.tvm_lib = None
- self.tvm_params = None
-
- def from_pytorch(self, script_module, input_infos, default_dtype="float32"):
- self.script_module = script_module
- self.input_infos = input_infos
- self.default_dtype = default_dtype
- self.mod, self.params = relay.frontend.from_pytorch(
- script_module, input_infos, default_dtype=default_dtype
- )
-
- def tune_tvm(self, log_file="tuning.log", n_trial=200):
- self.tasks = autotvm.task.extract_from_program(
- self.mod["main"],
- target=self.target,
- params=self.params,
- )
- self.log_file = log_file
- tuning_opt = get_tuning_opt(log_file, n_trial)
- tune_tasks(self.tasks, **tuning_opt)
-
- def build_tvm(self, export_dir, debug_runtime=False):
- tvm_mod = self._build_tvm(debug_runtime)
- self._export_tvm(export_dir)
- return tvm_mod
-
- def _build_tvm(self, debug_runtime=False):
- # compile kernels with history best records
- with autotvm.apply_history_best(self.log_file):
- with tvm.transform.PassContext(opt_level=3):
- self.tvm_graph, self.tvm_lib, self.tvm_params = relay.build(
- self.mod, target=self.target, params=self.params
- )
-
- if not debug_runtime:
- self.tvm_module = graph_executor.create(self.tvm_graph, self.tvm_lib, device=self.dev)
- else:
- self.tvm_module = debug_executor.create(self.tvm_graph, self.tvm_lib, device=self.dev)
- self.tvm_module.set_input(**self.tvm_params)
- return self.tvm_module
-
- def _export_tvm(self, export_dir):
- if not os.path.isdir(export_dir):
- os.makedirs(export_dir)
- self.export_dir = export_dir
- self.tvm_lib.export_library(os.path.join(export_dir, TVM_ASSETS[0]))
- with open(os.path.join(export_dir, TVM_ASSETS[1]), "w", encoding="utf8") as fout:
- fout.write(self.tvm_graph)
- with open(os.path.join(export_dir, TVM_ASSETS[2]), "wb") as fout:
- fout.write(relay.save_param_dict(self.tvm_params))
-
- def load_tvm(self, export_dir):
- """Load tvm module from export directory"""
- self.export_dir = export_dir
- self.tvm_lib = load_module(os.path.join(export_dir, TVM_ASSETS[0]))
- with open(os.path.join(export_dir, TVM_ASSETS[1]), "r", encoding="utf8") as f:
- self.tvm_graph = f.read()
- with open(os.path.join(export_dir, TVM_ASSETS[2]), "rb") as f:
- self.tvm_params = relay.load_param_dict(f.read())
-
- self.tvm_module = graph_executor.create(self.tvm_graph, self.tvm_lib, device=self.dev)
- self.tvm_module.set_input(**self.tvm_params)
- return self.tvm_module
-
- def build_pytorch_module(self, num_inputs, num_outputs, input_infos=None):
- """Build pytorch module containing TVM Graph Module"""
- warnings.warn(
- " ".join(
- (
- "This function will be removed at TVM version 0.11,",
- "we suggest users to use `optimized_torch` for tuning Torch modules instead.",
- )
- ),
- DeprecationWarning,
- stacklevel=2,
- )
- assert self.export_dir, "you must build_tvm or load_tvm before"
- input_infos = input_infos or self.input_infos
- assert input_infos
- assert len(input_infos) == num_inputs
- assets = [os.path.join(self.export_dir, i) for i in TVM_ASSETS]
- input_shapes = [i[1] for i in input_infos]
-
- def _tvm_dev_to_pt_dev(device):
- """convert tvm device to pytorch device string"""
- if tvm.runtime.Device.MASK2STR[device.device_type] == "cpu":
- return "cpu"
- if tvm.runtime.Device.MASK2STR[device.device_type] == "cuda":
- return f"cuda:{device.device_id}"
- raise ValueError(f"unsupported device for pt graph module: {device}")
-
- mod = GraphModule(num_inputs=num_inputs, num_outputs=num_outputs).to(
- _tvm_dev_to_pt_dev(self.dev)
- )
- mod.init(input_shapes, *assets)
- return mod
-
-
-def compile(script_module, option):
- """
- example:
- option = {
- "input_infos": [
- ("x", (1, 3, 244, 244)),
- ],
- "default_dtype": "float16",
- "export_dir": "pytorch_compiled",
- "num_outputs": 1,
- "tuning_n_trials": 20, # set zero to skip tuning
- "tuning_log_file": "tuning.log",
- "target": "llvm",
- "device": tvm.cpu(),
- }
- script_module = torch.jit.script(model)
- pytorch_tvm_module = compile(script_module, option)
- pytorch_tvm_module("model_tvm.pt")
- """
- warnings.warn(
- " ".join(
- (
- "This function will be removed at TVM version 0.11,",
- "we suggest users to use `optimized_torch` for tuning Torch modules instead.",
- )
- ),
- DeprecationWarning,
- stacklevel=2,
- )
- input_infos = option["input_infos"]
- default_dtype = option.get("default_dtype", "float32")
- export_dir = option.get("export_dir", "pytorch_compiled")
- tuning_log_file = option.get("tuning_log_file", "tuning.log")
- tuning_n_trials = option.get("tuning_n_trials", 20)
- num_outputs = option.get("num_outputs", 1)
- target = option.get("target", "cuda")
- device = option.get("device", tvm.cuda(0))
-
- mod = PyTorchTVMModule(target=target, device=device)
- print("Converting...")
-
- mod.log_file = tuning_log_file
- mod.from_pytorch(script_module, input_infos, default_dtype)
-
- if tuning_n_trials > 0:
- print("Tuning...")
- mod.tune_tvm(log_file=tuning_log_file, n_trial=tuning_n_trials)
-
- print("Building...")
- mod.build_tvm(export_dir)
- pytorch_mod = mod.build_pytorch_module(num_inputs=len(input_infos), num_outputs=num_outputs)
- return pytorch_mod
diff --git a/python/tvm/contrib/xcode.py b/python/tvm/contrib/xcode.py
index 2b68600197..d12367330d 100644
--- a/python/tvm/contrib/xcode.py
+++ b/python/tvm/contrib/xcode.py
@@ -16,7 +16,6 @@
# under the License.
# pylint: disable=invalid-name
"""Utility to invoke Xcode compiler toolchain"""
-from __future__ import absolute_import as _abs
import os
import sys
diff --git a/python/tvm/driver/build_module.py b/python/tvm/driver/build_module.py
index f91cc18ef1..fb325de1d3 100644
--- a/python/tvm/driver/build_module.py
+++ b/python/tvm/driver/build_module.py
@@ -293,6 +293,7 @@ def build(
target_host = "llvm" if tvm.runtime.enabled("llvm") else "stackvm"
annotated_mods, target_host = Target.canon_target_map_and_host(annotated_mods, target_host)
+
rt_mod_host = _driver_ffi.tir_to_runtime(annotated_mods, target_host)
annotated_mods, target_host = Target.canon_target_map_and_host(annotated_mods, target_host)
diff --git a/python/tvm/driver/tvmc/__init__.py b/python/tvm/driver/tvmc/__init__.py
deleted file mode 100644
index f7798a8512..0000000000
--- a/python/tvm/driver/tvmc/__init__.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=redefined-builtin,wrong-import-position
-"""
-TVMC - TVM driver command-line interface
-"""
-
-
-class TVMCException(Exception):
- """TVMC Exception"""
-
-
-class TVMCImportError(TVMCException):
- """TVMC TVMCImportError"""
-
-
-from . import runner
-from . import autotuner
-from . import compiler
-from . import result_utils
-from .frontends import load_model as load
-from .compiler import compile_model as compile
-from .runner import run_module as run
-from .autotuner import tune_model as tune
-from .model import TVMCModel, TVMCPackage, TVMCResult
diff --git a/python/tvm/driver/tvmc/__main__.py b/python/tvm/driver/tvmc/__main__.py
deleted file mode 100644
index 55235a6adf..0000000000
--- a/python/tvm/driver/tvmc/__main__.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-TVMC - TVM driver command-line interface
-"""
-
-from tvm.driver import tvmc
-
-if __name__ == "__main__":
- tvmc.main.main()
diff --git a/python/tvm/driver/tvmc/arguments.py b/python/tvm/driver/tvmc/arguments.py
deleted file mode 100644
index 57b6ee2f96..0000000000
--- a/python/tvm/driver/tvmc/arguments.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-TVMC Argument Parsing
-"""
-
-import argparse
-
-from tvm.driver.tvmc import TVMCException
-
-
-class TVMCSuppressedArgumentParser(argparse.ArgumentParser):
- """
- A silent ArgumentParser class.
- This class is meant to be used as a helper for creating dynamic parsers in
- TVMC. It will create a "supressed" parser based on an existing one (parent)
- which does not include a help message, does not print a usage message (even
- when -h or --help is passed) and does not exit on invalid choice parse
- errors but rather throws a TVMCException so it can be handled and the
- dynamic parser construction is not interrupted prematurely.
- """
-
- def __init__(self, parent, **kwargs):
- # Don't add '-h' or '--help' options to the newly created parser. Don't print usage message.
- # 'add_help=False' won't supress existing '-h' and '--help' options from the parser (and its
- # subparsers) present in 'parent'. However that class is meant to be used with the main
- # parser, which is created with `add_help=False` - the help is added only later. Hence it
- # the newly created parser won't have help options added in its (main) root parser. The
- # subparsers in the main parser will eventually have help activated, which is enough for its
- # use in TVMC.
- super().__init__(parents=[parent], add_help=False, usage=argparse.SUPPRESS, **kwargs)
-
- def exit(self, status=0, message=None):
- # Don't exit on error when parsing the command line.
- # This won't catch all the errors generated when parsing tho. For instance, it won't catch
- # errors due to missing required arguments. But this will catch "error: invalid choice",
- # which is what it's necessary for its use in TVMC.
- raise TVMCException()
diff --git a/python/tvm/driver/tvmc/autotuner.py b/python/tvm/driver/tvmc/autotuner.py
deleted file mode 100644
index ad4f8ae616..0000000000
--- a/python/tvm/driver/tvmc/autotuner.py
+++ /dev/null
@@ -1,864 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=unused-argument
-"""
-Provides support to auto-tuning networks using AutoTVM.
-"""
-import os.path
-import logging
-import time
-from copy import deepcopy
-from typing import Any, Optional, Dict, List, Union
-
-from urllib.parse import urlparse
-
-import tvm
-from tvm import autotvm, auto_scheduler
-from tvm.auto_scheduler.search_task import HardwareParams
-from tvm.autotvm.tuner import GATuner
-from tvm.autotvm.tuner import GridSearchTuner
-from tvm.autotvm.tuner import RandomTuner
-from tvm.autotvm.tuner import XGBTuner
-from tvm.target import Target
-
-from . import TVMCException, composite_target, frontends
-from .main import register_parser
-from .model import TVMCModel
-from .target import target_from_cli, generate_target_args, reconstruct_target_args
-from .shape_parser import parse_shape_string
-from .transform import generate_transform_args, parse_graph_transform_args, apply_graph_transforms
-
-
-# pylint: disable=invalid-name
-logger = logging.getLogger("TVMC")
-
-
-@register_parser
-def add_tune_parser(subparsers, _, json_params):
- """Include parser for 'tune' subcommand"""
-
- parser = subparsers.add_parser("tune", help="auto-tune a model")
- parser.set_defaults(func=drive_tune)
- parser.add_argument(
- "--early-stopping",
- type=int,
- help="minimum number of trials before early stopping",
- )
-
- # There is some extra processing required to define the actual default value
- # for --min-repeat-ms. This is done in `tune_model`.
- parser.add_argument(
- "--min-repeat-ms",
- default=None,
- type=int,
- help="minimum time to run each trial, in milliseconds. "
- "Defaults to 0 on x86 and 1000 on all other targets",
- )
- parser.add_argument(
- "--model-format",
- choices=frontends.get_frontend_names(),
- help="specify input model format",
- )
- parser.add_argument(
- "--number",
- default=10,
- type=int,
- help="number of runs a single repeat is made of. "
- "The final number of tuning executions is: "
- "(1 + number * repeat)",
- )
- parser.add_argument(
- "-o",
- "--output",
- required=True,
- help="output file to store the tuning records for the tuning process",
- )
- parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity.")
- parser.add_argument(
- "--parallel",
- default=4,
- type=int,
- help="the maximum number of parallel devices to use when tuning",
- )
- parser.add_argument(
- "--repeat",
- type=int,
- default=1,
- help="how many times to repeat each measurement",
- )
- parser.add_argument(
- "--rpc-key",
- help="the RPC tracker key of the target device. "
- "Required when --rpc-tracker is provided.",
- )
- parser.add_argument(
- "--rpc-tracker",
- help="hostname (required) and port (optional, defaults to 9090) of the RPC tracker, "
- "e.g. '192.168.0.100:9999'",
- )
-
- generate_target_args(parser)
- parser.add_argument(
- "--target-host",
- help="the host compilation target.",
- )
-
- parser.add_argument("--timeout", type=int, default=10, help="compilation timeout, in seconds")
- parser.add_argument(
- "--trials",
- type=int,
- default=1000,
- help="the maximum number of tuning trials to perform",
- )
- parser.add_argument(
- "--tuning-records",
- metavar="PATH",
- help="path to an auto-tuning log file by AutoTVM.",
- )
- generate_transform_args(parser)
- parser.add_argument(
- "--enable-autoscheduler",
- help="enable tuning the graph through the AutoScheduler tuner",
- action="store_true",
- )
- parser.add_argument(
- "--tasks",
- default="all",
- help="which tasks should be tuned, i.e. 0 0,2 3-5 all list",
- )
-
- auto_scheduler_group = parser.add_argument_group(
- "AutoScheduler options",
- "AutoScheduler options, used when --enable-autoscheduler is provided",
- )
-
- auto_scheduler_group.add_argument(
- "--cache-line-bytes",
- type=int,
- help="the size of cache line in bytes. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--num-cores",
- type=int,
- help="the number of device cores. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--vector-unit-bytes",
- type=int,
- help="the width of vector units in bytes. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--max-shared-memory-per-block",
- type=int,
- help="the max shared memory per block in bytes. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--max-local-memory-per-block",
- type=int,
- help="the max local memory per block in bytes. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--max-threads-per-block",
- type=int,
- help="the max number of threads per block. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--max-vthread-extent",
- type=int,
- help="the max vthread extent. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--warp-size",
- type=int,
- help="the thread numbers of a warp. "
- "If not specified, it will be autoset for the current machine.",
- )
- auto_scheduler_group.add_argument(
- "--include-simple-tasks",
- help="whether to extract simple tasks that do not include complicated ops",
- action="store_true",
- )
- auto_scheduler_group.add_argument(
- "--log-estimated-latency",
- help="whether to log the estimated latency to the file after tuning a task",
- action="store_true",
- )
- autotvm_group = parser.add_argument_group(
- "AutoTVM options",
- "AutoTVM options, used when the AutoScheduler is not enabled",
- )
- autotvm_group.add_argument(
- "--tuner",
- choices=[
- "ga",
- "gridsearch",
- "random",
- "xgb",
- "xgb_knob",
- "xgb_itervar",
- "xgb_curve",
- "xgb_rank",
- "xgb_rank_knob",
- "xgb_rank_itervar",
- "xgb_rank_curve",
- "xgb_rank_binary",
- "xgb_rank_binary_knob",
- "xgb_rank_binary_itervar",
- "xgb_rank_binary_curve",
- ],
- default="xgb",
- help="type of tuner to use when tuning with autotvm.",
- )
- # TODO (@leandron) This is a path to a physical file, but
- # can be improved in future to add integration with a modelzoo
- # or URL, for example.
- parser.add_argument("FILE", help="path to the input model file")
- parser.add_argument(
- "--input-shapes",
- help="specify non-generic shapes for model to run, format is "
- '"input_name:[dim1,dim2,...,dimn] input_name2:[dim1,dim2]"',
- type=parse_shape_string,
- )
-
- for one_entry in json_params:
- parser.set_defaults(**one_entry)
-
-
-def drive_tune(args):
- """Invoke auto-tuning with command line arguments
-
- Parameters
- ----------
- args: argparse.Namespace
- Arguments from command line parser.
- """
- if not os.path.isfile(args.FILE):
- raise TVMCException(
- f"Input file '{args.FILE}' doesn't exist, is a broken symbolic link, or a directory."
- )
-
- tvmc_model = frontends.load_model(args.FILE, args.model_format, shape_dict=args.input_shapes)
-
- # Specify hardware parameters, although they'll only be used if autoscheduling.
- hardware_params = auto_scheduler.HardwareParams(
- num_cores=args.num_cores,
- vector_unit_bytes=args.vector_unit_bytes,
- cache_line_bytes=args.cache_line_bytes,
- max_shared_memory_per_block=args.max_shared_memory_per_block,
- max_local_memory_per_block=args.max_local_memory_per_block,
- max_threads_per_block=args.max_threads_per_block,
- max_vthread_extent=args.max_vthread_extent,
- warp_size=args.warp_size,
- target=args.target,
- target_host=args.target_host,
- )
-
- if args.rpc_tracker:
- parsed_url = urlparse("//%s" % args.rpc_tracker)
- rpc_hostname = parsed_url.hostname
- rpc_port = parsed_url.port or 9090
- logger.info("RPC tracker hostname: %s", rpc_hostname)
- logger.info("RPC tracker port: %s", rpc_port)
-
- if not args.rpc_key:
- raise TVMCException("need to provide an RPC tracker key (--rpc-key) for remote tuning")
- else:
- rpc_hostname = None
- rpc_port = None
-
- transform_args = parse_graph_transform_args(args)
-
- tune_model(
- tvmc_model,
- args.target,
- tuning_records=args.output,
- prior_records=args.tuning_records,
- enable_autoscheduler=args.enable_autoscheduler,
- rpc_key=args.rpc_key,
- hostname=rpc_hostname,
- port=rpc_port,
- trials=args.trials,
- target_host=args.target_host,
- tuner=args.tuner,
- min_repeat_ms=args.min_repeat_ms,
- early_stopping=args.early_stopping,
- timeout=args.timeout,
- repeat=args.repeat,
- number=args.number,
- parallel=args.parallel,
- hardware_params=hardware_params,
- include_simple_tasks=args.include_simple_tasks,
- log_estimated_latency=args.log_estimated_latency,
- additional_target_options=reconstruct_target_args(args),
- tasks_filter=args.tasks,
- **transform_args,
- )
-
-
-def filter_tasks(
- tasks: Union[List[auto_scheduler.SearchTask], List[autotvm.task.Task]],
- expr: str,
-):
- """Utility to filter a list of tasks (AutoTVM or AutoScheduler) based on
- a user-supplied string expression.
-
- Parameters
- ----------
- tasks: list
- A list of extracted AutoTVM or AutoScheduler tasks.
- expr: str
- User-supplied expression to be used for filtering.
- """
- assert isinstance(expr, str), "Expected filter expression of string type"
- assert len(expr) > 0, "Got empty filter expression"
-
- # groups of keywords are comma-separated
- splitted = expr.split(",")
-
- do_list = False
- do_filter = False
- selected = []
- for item in splitted:
- if item in ["list", "help"]:
- do_list = True
- elif item in ["all"]:
- selected = list(range(len(tasks)))
- else:
- do_filter = True
- if "-" in item:
- assert item.count("-") == 1, "Malformed range expression"
- assert len(item) > 1, "Missing lhs or rhs for range expression"
- lhs, rhs = item.split("-")[:2]
- lhs = int(lhs) if lhs else 0
- rhs = int(rhs) if rhs else len(tasks) - 1
- assert 0 <= lhs < len(tasks), "Left-hand side expression out of range"
- assert 0 <= rhs < len(tasks), "Right-hand side expression out of range"
- selected.extend(list(range(lhs, rhs + 1)))
- else:
- assert isinstance(item, str)
- idx = int(item)
- assert 0 <= idx < len(tasks), "Task index out of range"
- selected.append(idx)
-
- if do_filter:
- # remove duplicates
- selected = list(set(selected))
- tasks = [task for i, task in enumerate(tasks) if i in selected]
-
- return tasks, do_list
-
-
-def gen_task_list(
- tasks: Union[List[auto_scheduler.SearchTask], List[autotvm.task.Task]],
- enable_autoscheduler: bool,
-):
- """Utility for printing a list of tasks (AutoTVM or AutoScheduler)
- to the terminal.
-
- Parameters
- ----------
- tasks: list
- A list of extracted AutoTVM or AutoScheduler tasks.
- enable_autoscheduler: bool
- Wether the tasks are extracted with AutoScheduler or AutoTVM.
- """
- ret = "Available Tasks for tuning:\n"
-
- def _trunc_helper(text, length):
- return text if len(text) < length else text[: length - 3] + "..."
-
- ret += "\n".join(
- [
- " {}. {}".format(
- i, _trunc_helper("Unnamed" if len(task.desc) == 0 else task.desc, 100)
- )
- if enable_autoscheduler
- else " {}. {} (len={})".format(
- i,
- _trunc_helper(str(task), 100),
- "?" if task.config_space is None else len(task.config_space),
- )
- for i, task in enumerate(tasks)
- ]
- )
- return ret
-
-
-def tune_model(
- tvmc_model: TVMCModel,
- target: str,
- tuning_records: Optional[str] = None,
- prior_records: Optional[str] = None,
- enable_autoscheduler: bool = False,
- rpc_key: Optional[str] = None,
- hostname: Optional[str] = None,
- port: Optional[Union[int, str]] = 9090,
- trials: int = 10000,
- target_host: Optional[str] = None,
- tuner: str = "xgb",
- min_repeat_ms: Optional[int] = None,
- early_stopping: Optional[int] = None,
- timeout: int = 10,
- repeat: int = 1,
- number: int = 10,
- parallel: int = 4,
- hardware_params: Optional[HardwareParams] = None,
- include_simple_tasks: bool = False,
- log_estimated_latency: bool = False,
- additional_target_options: Optional[Dict[str, Dict[str, Any]]] = None,
- tasks_filter: str = "all",
- desired_layout: Optional[str] = None,
- desired_layout_ops: Optional[List[str]] = None,
- mixed_precision: bool = False,
- mixed_precision_ops: Optional[List[str]] = None,
- mixed_precision_calculation_type: Optional[str] = None,
- mixed_precision_acc_type: Optional[str] = None,
-):
- """Use tuning to automatically optimize the functions in a model.
-
- Parameters
- ----------
- tvmc_model : TVMCModel
- The model to be optimized.
- target : str
- Compilation target as plain string, inline JSON or path to a JSON file.
- tuning_records: str, optional
- The path to a file that tuning results will be saved to. If not specified,
- a temporary file will be used.
- prior_records: str, optional
- A path to previous tuning results that will be used to hot-start the tuning
- cost model if provided.
- enable_autoscheduler : bool, optional
- When true, use autoscheduling rather than autotvm. This should produce
- faster kernels for compatible model-target pairs.
- rpc_key : str, optional
- The RPC tracker key of the target device. Required when rpc_tracker is provided.
- hostname : str, optional
- The IP address of an RPC tracker, used when benchmarking remotely.
- port : int or str, optional
- The port of the RPC tracker to connect to. Defaults to 9090.
- trials : int, optional
- The number of schedules to try out for the entire model. Note that the default
- value is chosen as a decent average for most models, but larger models may need
- more trials to reach a good result while smaller models will converge with fewer
- trials.
- tuner : str, optional
- The type of tuner to use when tuning with autotvm. Can be one of
- "ga", "gridsearch", "random", "xgb", "xgb_knob", "xgb_itervar", "xgb_curve",
- "xgb_rank", "xgb_rank_knob", "xgb_rank_itervar", "xgb_rank_binary", "xgb_rank_binary_knob",
- "xgb_rank_binary_itervar" and "xgb_rank_binary_curve".
- min_repeat_ms : int, optional
- Minimum time to run each trial. Defaults to 0 on x86 and 1000 on other targets.
- early_stopping : int, optional
- When specified, stop tuning after this number of trials if results aren't improving.
- timeout : int, optional,
- If a kernel trial lasts longer than this duration in seconds, it will be
- considered a failure.
- repeat : int, optional
- How many times each measurement should be repeated.
- number : int, optional
- The number of runs a single repeat is made of.
- parallel : int, optional
- The maximum number of parallel devices to use when tuning.
- hardware_params : auto_scheduler.HardwareParams, optional
- When using the autoscheduler, this object defines the configuration of the target hardware.
- include_simple_tasks : bool, optional
- Whether to extract simple operations or only computationally intensive ones when using
- the autoscheduler.
- log_estimated_latency : bool, optional
- If using the autoscheduler, write the estimated latency at each step of tuning to file.
- additional_target_options: Optional[Dict[str, Dict[str, Any]]]
- Additional target options in a dictionary to combine with initial Target arguments
- tasks_filter : str, optional
- Filter which tasks should be tuned or output a list of the extracted tasks.
- Examples: 0 0,2 3-5 all list
- desired_layout: str, optional
- Can be one of "NCHW" or "NHWC". When specified, compatible operations in the graph
- will have their layout set to this format. Tasks will then be tuned using this
- specified layout.
- desired_layout_ops: list[str], optional
- The list of operators to be transformed with desired layout.
- mixed_precision: bool
- To enable mixed precision transformation.
- mixed_precision_ops: list[str], optional
- The list of operators to be converted to mixed precision.
- mixed_precision_calculation_type: str
- The calculation dtype to be used while mixed precision.
- mixed_precision_acc_type: str
- The accumulation data type to be used while mixed precision.
-
- Returns
- -------
- tuning_records : str
- The path to the produced tuning log file.
- """
- transform_args = parse_graph_transform_args(locals())
- target, extra_targets = target_from_cli(target, additional_target_options)
- target, target_host = Target.canon_target_and_host(target, target_host)
- # TODO(jwfromm) Remove this deepcopy once AlterOpLayout bug that mutates source
- # model is fixed. For now, creating a clone avoids the issue.
- mod = deepcopy(tvmc_model.mod)
- params = tvmc_model.params
-
- with tvm.transform.PassContext(opt_level=3):
- if tuning_records is None:
- tuning_records = tvmc_model.default_tuning_records_path()
-
- for codegen_from_cli in extra_targets:
- codegen = composite_target.get_codegen_by_target(codegen_from_cli["name"])
- partition_function = codegen["pass_pipeline"]
- mod = partition_function(mod, params, **codegen_from_cli["opts"])
-
- # min_repeat_ms should be:
- # a. the value provided by the user, if any, or
- # b. 0ms in case target is "cpu"; otherwise 1000ms
- if min_repeat_ms is None:
- min_repeat_ms = 0 if target.keys[0] == "cpu" else 1000
- logger.info("Default --min-repeat-ms for this target is %s", min_repeat_ms)
-
- if rpc_key:
- if hostname is None or port is None:
- raise TVMCException(
- "You must provide a hostname and port to connect to a remote RPC device."
- )
- if isinstance(port, str):
- port = int(port)
-
- logger.info("Tuning will be performed on device %s at %s:%d.", rpc_key, hostname, port)
-
- runner_ctor = auto_scheduler.RPCRunner if enable_autoscheduler else autotvm.RPCRunner
- runner = runner_ctor(
- key=rpc_key,
- host=hostname,
- port=port,
- number=number,
- repeat=repeat,
- n_parallel=parallel,
- timeout=timeout,
- min_repeat_ms=min_repeat_ms,
- )
- else:
- logger.info("Starting localhost tuning.")
- runner_ctor = (
- auto_scheduler.LocalRPCMeasureContext
- if enable_autoscheduler
- else autotvm.LocalRunner
- )
- local_server = runner_ctor(
- number=number,
- repeat=repeat,
- timeout=timeout,
- min_repeat_ms=min_repeat_ms,
- )
-
- # For autoscheduling on some devices, we need to maintain a
- # LocalRPCMeasureContext object.
- if enable_autoscheduler:
- runner = local_server.runner
- else:
- runner = local_server
-
- if enable_autoscheduler:
- tasks, weights = autoscheduler_get_tuning_tasks(
- mod=mod,
- params=params,
- target=target,
- transform_args=transform_args,
- hardware_params=hardware_params,
- include_simple_tasks=include_simple_tasks,
- )
- else:
- tasks = autotvm_get_tuning_tasks(
- mod=mod,
- params=params,
- target=target,
- transform_args=transform_args,
- )
-
- # Filter extracted tasks by provided user expression
- if tasks_filter:
- tasks, do_list = filter_tasks(tasks, tasks_filter)
- if do_list:
- print(gen_task_list(tasks, enable_autoscheduler))
- return None
- if len(tasks) == 0:
- logger.info("No tasks have been selected for tuning.")
- return None
- else:
- logger.info("Selected %s tasks for tuning.", len(tasks))
-
- if enable_autoscheduler:
- # Create the autoscheduler tuning options
- tuning_options = auto_scheduler.TuningOptions(
- num_measure_trials=trials,
- measure_callbacks=[auto_scheduler.RecordToFile(tuning_records)],
- runner=runner,
- early_stopping=early_stopping,
- )
-
- logger.info("Autoscheduling with configuration: %s", tuning_options)
-
- # Schedule the tasks (i.e., produce a schedule for each task)
- schedule_tasks(tasks, weights, tuning_options, prior_records, log_estimated_latency)
- else:
- # In autotvm, trials is specified per task. We can convert the per-model input
- # provided to per-task trials by dividing by the number of tasks.
- trials = int(max(1, trials / max(len(tasks), 1)))
- logger.info("Autotuning with %d trials per task.", trials)
-
- tuning_options = {
- "tuner": tuner,
- "trials": trials,
- "early_stopping": early_stopping,
- "measure_option": autotvm.measure_option(
- builder=autotvm.LocalBuilder(build_func="default"), runner=runner
- ),
- "tuning_records": prior_records,
- }
- logger.info("Autotuning with configuration: %s", tuning_options)
-
- tune_tasks(tasks, tuning_records, **tuning_options)
-
- return tuning_records
-
-
-def autotvm_get_tuning_tasks(
- mod: tvm.IRModule,
- params: Dict[str, tvm.nd.NDArray],
- target: str,
- target_host: Optional[str] = None,
- transform_args: Optional[Dict[str, Any]] = None,
-):
- """Get the autotvm tuning tasks for a given relay module.
-
- Parameters
- ----------
- mod : tvm.IRModule
- The relay module from which to extract tuning tasks.
- params : dict
- The params for the relay module.
- target : tvm.target.Target
- The compilation target.
- target_host : str, optional
- The compilation target for the host.
- transform_args: dict, optional
- Graph transformation arguments that are applied to the relay module.
-
- Returns
- -------
- tasks : list of autotvm.Tasks
- list of tasks to be tuned
- """
- target, target_host = Target.canon_target_and_host(target, target_host)
-
- mod = apply_graph_transforms(mod, transform_args, params)
-
- tasks = autotvm.task.extract_from_program(
- mod["main"],
- target=target,
- params=params,
- )
-
- return tasks
-
-
-def autoscheduler_get_tuning_tasks(
- mod: tvm.IRModule,
- params: Dict[str, tvm.nd.NDArray],
- target: str,
- target_host: Optional[str] = None,
- transform_args: Optional[Dict[str, Any]] = None,
- hardware_params: Optional[HardwareParams] = None,
- include_simple_tasks: bool = False,
-):
- """Get the autoscheduler tuning tasks for a given relay module.
-
- Parameters
- ----------
- mod : tvm.IRModule
- The relay module from which to extract tuning tasks.
- params : dict
- The params for the relay module.
- target : tvm.target.Target
- The compilation target.
- target_host : str, optional
- The compilation target for the host.
- transform_args: dict, optional
- Graph transformation arguments that are applied to the relay module.
- hardware_params : Optional[HardwareParams]
- Hardware parameters used for the search tasks
-
- Returns
- -------
- tasks : list of autotvm.Tasks
- list of tasks to be tuned
- weights : List[int]
- the weight (i.e. the number of appearance) of extracted tasks
- """
- target, target_host = Target.canon_target_and_host(target, target_host)
-
- mod = apply_graph_transforms(mod, transform_args, params)
-
- # Extract the tasks
- tasks, task_weights = auto_scheduler.extract_tasks(
- mod["main"],
- params,
- target=target,
- hardware_params=hardware_params,
- include_simple_tasks=include_simple_tasks,
- )
-
- return tasks, task_weights
-
-
-def schedule_tasks(
- tasks: List[auto_scheduler.SearchTask],
- task_weights: List[float],
- tuning_options: auto_scheduler.TuningOptions,
- prior_records: Optional[str] = None,
- log_estimated_latency: bool = False,
-):
- """Generate the schedules for the different tasks (i.e., subgraphs) contained in the module.
- Store the schedules in a json file that will be used later by the compiler.
-
- Parameters
- ----------
- tasks : list
- A list of auto_scheduler.SearchTask to tune.
- task_weights : list
- The weight (i.e. the number of appearance) of extracted tasks
- tuning_options: auto_scheduler.TuningOptions
- The options of tuning
- prior_records : str, optional
- The json file used to preload the autoscheduler
- log_estimated_latency : bool, optional
- If true, writes the estimated runtime of the model during each step of tuning to file.
- """
- if not log_estimated_latency:
- callbacks = [auto_scheduler.task_scheduler.PrintTableInfo()]
- else:
- callbacks = [
- auto_scheduler.task_scheduler.PrintTableInfo(),
- auto_scheduler.task_scheduler.LogEstimatedLatency(("total_latency.tsv")),
- ]
-
- # Create the scheduler
- tuner = auto_scheduler.TaskScheduler(
- tasks, task_weights, load_log_file=prior_records, callbacks=callbacks
- )
-
- # Tune the tasks
- tuner.tune(tuning_options)
-
-
-def tune_tasks(
- tasks: List[autotvm.task.Task],
- log_file: str,
- measure_option: autotvm.measure_option,
- tuner: str,
- trials: int,
- early_stopping: Optional[int] = None,
- tuning_records: Optional[str] = None,
-):
- """Tune a list of tasks and output the history to a log file.
-
- Parameters
- ----------
- tasks : list
- A list of autotvm.Tasks to tune.
- log_file : str
- A file to output the tuning history, in JSON.
- measure_option : autotvm.measure_option
- Options to build and run a tuning task.
- tuner : str
- Which tuner to use.
- trials : int
- The maximum number of tuning trials to perform.
- early_stopping : int, optional
- The minimum number of tuning trials to perform.
- This will be equal to 'trials' if not specified.
- tuning_records: str, optional
- Path to the file produced by the tuning, to be used during
- tuning.
- """
- if not tasks:
- logger.warning("there were no tasks found to be tuned")
- return
-
- if not early_stopping:
- early_stopping = trials
-
- for i, tsk in enumerate(tasks):
- prefix = "[Task %2d/%2d] " % (i + 1, len(tasks))
-
- # Create a tuner
- if tuner == "xgb":
- tuner_obj = XGBTuner(tsk, loss_type="reg")
- elif tuner == "xgb_knob":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="knob")
- elif tuner == "xgb_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="itervar")
- elif tuner == "xgb_curve":
- tuner_obj = XGBTuner(tsk, loss_type="reg", feature_type="curve")
- elif tuner == "xgb_rank":
- tuner_obj = XGBTuner(tsk, loss_type="rank")
- elif tuner == "xgb_rank_knob":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="knob")
- elif tuner == "xgb_rank_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="itervar")
- elif tuner == "xgb_rank_curve":
- tuner_obj = XGBTuner(tsk, loss_type="rank", feature_type="curve")
- elif tuner == "xgb_rank_binary":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary")
- elif tuner == "xgb_rank_binary_knob":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="knob")
- elif tuner == "xgb_rank_binary_itervar":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="itervar")
- elif tuner == "xgb_rank_binary_curve":
- tuner_obj = XGBTuner(tsk, loss_type="rank-binary", feature_type="curve")
- elif tuner == "ga":
- tuner_obj = GATuner(tsk, pop_size=50)
- elif tuner == "random":
- tuner_obj = RandomTuner(tsk)
- elif tuner == "gridsearch":
- tuner_obj = GridSearchTuner(tsk)
- else:
- raise TVMCException("invalid tuner: %s " % tuner)
-
- # If transfer learning is being used, load the existing results
- if tuning_records and os.path.exists(tuning_records):
- logger.info("loading tuning records from %s", tuning_records)
- start_time = time.time()
- tuner_obj.load_history(autotvm.record.load_from_file(tuning_records))
- logging.info("loaded history in %.2f sec(s)", time.time() - start_time)
-
- tuner_obj.tune(
- n_trial=min(trials, len(tsk.config_space)),
- early_stopping=early_stopping,
- measure_option=measure_option,
- callbacks=[
- autotvm.callback.progress_bar(min(trials, len(tsk.config_space)), prefix=prefix),
- autotvm.callback.log_to_file(log_file),
- ],
- )
diff --git a/python/tvm/driver/tvmc/compiler.py b/python/tvm/driver/tvmc/compiler.py
deleted file mode 100644
index 0dbae47294..0000000000
--- a/python/tvm/driver/tvmc/compiler.py
+++ /dev/null
@@ -1,743 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=unused-argument
-"""
-Provides support to compile networks both AOT and JIT.
-"""
-import logging
-import os.path
-import re
-import itertools
-from copy import deepcopy
-from typing import Any, Optional, Dict, List, Union, Callable, Sequence
-from pathlib import Path
-from collections import defaultdict
-
-import tvm
-from tvm import autotvm, auto_scheduler
-from tvm import relay
-from tvm.driver.tvmc.registry import generate_registry_args, reconstruct_registry_entity
-from tvm.ir.instrument import PassInstrument, PassTimingInstrument, PassPrintingInstrument
-from tvm.ir.memory_pools import WorkspaceMemoryPools
-from tvm.target import Target
-from tvm.relay.backend import Executor, Runtime
-from tvm.relay.analysis.operations_distribution import analyze_operations_distribution
-from tvm.relay.transform.suffixes import tag_suffixes
-
-from . import composite_target, frontends, TVMCException
-from .model import TVMCModel, TVMCPackage
-from .main import register_parser
-from .target import target_from_cli, generate_target_args, reconstruct_target_args
-from .pass_config import parse_configs
-from .pass_list import parse_pass_list_str
-from .transform import generate_transform_args, parse_graph_transform_args, apply_graph_transforms
-from .shape_parser import parse_shape_string
-from .workspace_pools import generate_workspace_pools_args, workspace_pools_recombobulate
-
-# pylint: disable=invalid-name
-logger = logging.getLogger("TVMC")
-
-
-@register_parser
-def add_compile_parser(subparsers, _, json_params):
- """Include parser for 'compile' subcommand"""
-
- parser = subparsers.add_parser("compile", help="compile a model.")
- parser.set_defaults(func=drive_compile)
- parser.add_argument(
- "--cross-compiler",
- default="",
- help="the cross compiler to generate target libraries, e.g. 'aarch64-linux-gnu-gcc'.",
- )
- parser.add_argument(
- "--cross-compiler-options",
- default="",
- help="the cross compiler options to generate target libraries, e.g. '-mfpu=neon-vfpv4'.",
- )
- generate_transform_args(parser)
- parser.add_argument(
- "--dump-code",
- metavar="FORMAT",
- default="",
- help="comma separated list of formats to export the input model, e.g. 'asm,ll,tir,relay'.",
- )
- parser.add_argument(
- "--dump-offloads",
- default="",
- help="output a mapping of which operations of the initial Relay "
- "will be transferred to which backend, indicating the composite "
- "that includes those operations, "
- "e.g. '--dump-offloads -' to dump to the console, "
- "e.g. '--dump-offloads <path_to_file>' to dump to the file. "
- "If not presented, no output is done. ",
- )
- parser.add_argument(
- "--model-format",
- choices=frontends.get_frontend_names(),
- help="specify input model format.",
- )
- parser.add_argument(
- "-o",
- "--output",
- default="module.tar",
- help="output the compiled module to a specified archive. Defaults to 'module.tar'.",
- )
- parser.add_argument(
- "-f",
- "--output-format",
- choices=["so"],
- default="so",
- help="output format. Use 'so' for shared object. Defaults to 'so'.",
- )
- parser.add_argument(
- "--pass-config",
- action="append",
- metavar=("name=value"),
- help="configurations to be used at compile time. This option can be provided multiple "
- "times, each one to set one configuration value, "
- "e.g. '--pass-config relay.backend.use_auto_scheduler=0', "
- "e.g. '--pass-config tir.add_lower_pass=opt_level1,pass1,opt_level2,pass2'.",
- )
-
- generate_target_args(parser)
- parser.add_argument(
- "--tuning-records",
- metavar="PATH",
- default="",
- help="path to an auto-tuning log file by AutoTVM. If not presented, "
- "the fallback/tophub configs will be used.",
- )
- generate_registry_args(parser, Executor, "graph")
- generate_registry_args(parser, Runtime, "cpp")
-
- parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity.")
- # TODO (@leandron) This is a path to a physical file, but
- # can be improved in future to add integration with a modelzoo
- # or URL, for example.
- parser.add_argument("FILE", help="path to the input model file.")
- parser.add_argument(
- "-O",
- "--opt-level",
- default=3,
- type=int,
- choices=range(0, 4),
- metavar="[0-3]",
- help="specify which optimization level to use. Defaults to '3'.",
- )
- parser.add_argument(
- "--input-shapes",
- help="specify non-generic shapes for model to run, format is "
- '"input_name:[dim1,dim2,...,dimn] input_name2:[dim1,dim2]".',
- type=parse_shape_string,
- default=None,
- )
- parser.add_argument(
- "--disabled-pass",
- help="disable specific passes, comma-separated list of pass names.",
- type=parse_pass_list_str,
- default="",
- )
- parser.add_argument(
- "--module-name",
- default="default",
- help="The output module name. Defaults to 'default'.",
- )
- parser.add_argument(
- "--print-pass-times",
- action="store_true",
- help="print compilation time per pass",
- )
- parser.add_argument(
- "--print-ir-before",
- help="print IR before each named pass of a comma-separated list of pass names."
- "e.g. '--print-ir-before [tir.SplitHostDevice,tir.ConvertSSA]' ",
- default="",
- )
- parser.add_argument(
- "--print-ir-after",
- help="print IR after each named pass of a comma-separated list of pass names."
- "e.g. '--print-ir-after [tir.SplitHostDevice,tir.ConvertSSA]' ",
- default="",
- )
- for one_entry in json_params:
- parser.set_defaults(**one_entry)
-
- generate_workspace_pools_args(parser)
-
-
-def drive_compile(args):
- """Invoke tvmc.compiler module with command line arguments
-
- Parameters
- ----------
- args: argparse.Namespace
- Arguments from command line parser.
-
- Returns
- -------
- int
- Zero if successfully completed
-
- """
-
- if not os.path.isfile(args.FILE):
- raise TVMCException(
- f"Input file '{args.FILE}' doesn't exist, is a broken symbolic link, or a directory."
- )
-
- tvmc_model = frontends.load_model(args.FILE, args.model_format, args.input_shapes)
-
- dump_code = [x.strip() for x in args.dump_code.split(",")] if args.dump_code else None
-
- dump_offloads = args.dump_offloads if args.dump_offloads else ""
-
- additional_targets = reconstruct_target_args(args)
- workspace_pools_target, extra_targets = target_from_cli(args.target, additional_targets)
- transform_args = parse_graph_transform_args(args)
-
- compile_model(
- tvmc_model,
- args.target,
- opt_level=args.opt_level,
- executor=reconstruct_registry_entity(args, Executor),
- runtime=reconstruct_registry_entity(args, Runtime),
- tuning_records=args.tuning_records,
- package_path=args.output,
- cross=args.cross_compiler,
- cross_options=args.cross_compiler_options,
- output_format=args.output_format,
- dump_code=dump_code,
- dump_offloads=dump_offloads,
- target_host=None,
- disabled_pass=args.disabled_pass,
- pass_context_configs=args.pass_config,
- mod_name=args.module_name,
- additional_target_options=additional_targets,
- workspace_pools=(
- workspace_pools_recombobulate(args, [workspace_pools_target], extra_targets)
- ),
- print_pass_times=args.print_pass_times,
- print_ir_before=args.print_ir_before,
- print_ir_after=args.print_ir_after,
- **transform_args,
- )
-
- return 0
-
-
-def compile_model(
- tvmc_model: TVMCModel,
- target: str,
- opt_level: int = 3,
- executor: Optional[Executor] = Executor("graph"),
- runtime: Optional[Runtime] = Runtime("cpp"),
- tuning_records: Optional[str] = None,
- package_path: Optional[str] = None,
- cross: Optional[Union[str, Callable]] = None,
- cross_options: Optional[str] = None,
- output_format: str = "so",
- dump_code: Optional[List[str]] = None,
- dump_offloads: str = "",
- target_host: Optional[str] = None,
- disabled_pass: Optional[str] = None,
- pass_context_configs: Optional[List[str]] = None,
- additional_target_options: Optional[Dict[str, Dict[str, Any]]] = None,
- use_vm: bool = False,
- mod_name: Optional[str] = "default",
- workspace_pools: Optional[WorkspaceMemoryPools] = None,
- print_pass_times: bool = False,
- print_ir_before: Optional[List[str]] = None,
- print_ir_after: Optional[List[str]] = None,
- instruments: Optional[Sequence[PassInstrument]] = None,
- desired_layout: Optional[str] = None,
- desired_layout_ops: Optional[List[str]] = None,
- mixed_precision: bool = False,
- mixed_precision_ops: Optional[List[str]] = None,
- mixed_precision_calculation_type: Optional[str] = None,
- mixed_precision_acc_type: Optional[str] = None,
-):
- """Compile a model from a supported framework into a TVM module.
-
- This function takes a union of the arguments of both frontends.load_model
- and compiler.compile_relay. The resulting TVM module can be executed using
- the graph executor.
-
- Parameters
- ----------
- tvmc_model : TVMCModel
- The model object that should be compiled.
- target : str
- The target for which to compile. Can be a plain string or
- a path.
- opt_level : int
- The option that controls various sorts of optimizations.
- tuning_records : str
- A path to tuning records produced using tvmc.tune. When provided,
- compilation will use more optimized kernels leading to better results.
- package_path : str, optional
- The path to export the compiled model to. If not provided it will
- be saved in a temporary directory.
- cross : str or callable object, optional
- Function that performs the actual compilation
- cross_options : str, optional
- Command line options to be passed to the cross compiler.
- output_format : str
- What format to use when saving the function library. Must be one of "so" or "tar".
- When compiling for a remote device without a cross compiler, "tar" will likely work better.
- dump_code : list[str], optional
- Dump the generated code for the specified source types, on
- the requested target. Choose from: ["asm", "ll", "tir", "relay"].
- dump_offloads : str
- Dump the information about the partition of input model's layers by external codegen.
- Can be '' to not dump at all, '-' to dump to the console
- or '<path_to_file>' to dump to the specified file.
- target_host : str, optional
- The target of the host machine if host-side code
- needs to be generated.
- disabled_pass: str, optional
- Comma-separated list of passes which needs to be disabled
- during compilation.
- pass_context_configs: list[str], optional
- List of strings containing a set of configurations to be passed to the
- PassContext.
- additional_target_options: Optional[Dict[str, Dict[str, Any]]]
- Additional target options in a dictionary to combine with initial Target arguments
- use_vm: bool
- Whether to use the VM to compile the model as opposed to the graph executor
- mod_name: str, optional
- The module name
- workspace_pools: WorkspaceMemoryPools, optional
- Specification of WorkspacePoolInfo objects to be used as workspace memory in the
- compilation.
- print_pass_times: bool
- To enable printing a breakdown of compilation times by pass. Disabled by default.
- print_ir_before: list[str], optional
- To print IR before each named pass of a comma-separated list of passes.
- print_ir_after: list[str], optional
- To print IR after each named pass of a comma-separated list of passes.
- instruments: Optional[Sequence[PassInstrument]]
- The list of pass instrument implementations.
- desired_layout: str, optional
- Can be one of "NCHW" or "NHWC". When specified, compatible operations in the graph
- will have their layout set to this format. Tasks will then be tuned using this
- specified layout.
- desired_layout_ops: list[str], optional
- The list of operators to be transformed with desired layout.
- mixed_precision: bool
- To enable mixed precision transformation. Disabled by default.
- mixed_precision_ops: list[str], optional
- The list of operators to be converted to mixed precision.
- Set to ["nn.conv2d", "nn.dense"] by default
- mixed_precision_calculation_type: str
- The calculation dtype to be used while mixed precision. Set to "float16" by default.
- mixed_precision_acc_type: str
- The accumulation data type to be used while mixed precision. Set to "float16" by default.
-
- Returns
- -------
- compiled_model : TVMCPackage
- The compiled TVMCModel ready to be run.
-
- """
- mod, params = tvmc_model.mod, tvmc_model.params
-
- if dump_code is None:
- dump_code = []
- if not isinstance(dump_code, list):
- dump_code = [dump_code]
- dumps = {}
-
- config = parse_configs(pass_context_configs)
- if "tir" in dump_code:
- config, dumps = add_tir_to_dumps(config, dumps)
-
- initial_relay = None
- if dump_offloads != "":
- # add suffixes to the span field for calls in Relay
- mod = tag_suffixes(mod)
- # remember initial Relay
- initial_relay = deepcopy(mod)
-
- tvm_target, extra_targets = target_from_cli(target, additional_target_options)
- tvm_target, target_host = Target.canon_target_and_host(tvm_target, target_host)
-
- partition_functions = []
- partition_opts = []
- for codegen_from_cli in extra_targets:
- codegen = composite_target.get_codegen_by_target(codegen_from_cli["name"])
- partition_functions.append(codegen["pass_pipeline"])
- partition_opts.append(codegen_from_cli["opts"])
- if codegen["config_key"] is not None:
- config[codegen["config_key"]] = codegen_from_cli["opts"]
-
- if print_pass_times:
- timing_inst = PassTimingInstrument()
- instruments = [timing_inst] if instruments is None else [timing_inst] + instruments
-
- if print_ir_before or print_ir_after:
- print_ir_instr = PassPrintingInstrument(
- print_before_pass_names=print_ir_before, print_after_pass_names=print_ir_after
- )
- instruments = [print_ir_instr] if instruments is None else [print_ir_instr] + instruments
-
- with tvm.transform.PassContext(
- opt_level=opt_level,
- config=config,
- disabled_pass=disabled_pass,
- instruments=instruments,
- ):
- transform_args = parse_graph_transform_args(locals())
- mod = apply_graph_transforms(mod, transform_args, params)
-
- for partition_function, opts in zip(partition_functions, partition_opts):
- mod = partition_function(mod, params, mod_name=mod_name, **opts)
-
- if initial_relay:
- # dump which operations are offloaded to which backend
- dump_operation_offloads(mod, initial_relay, dump_offloads)
-
- if tuning_records and os.path.exists(tuning_records):
- logger.debug("tuning records file provided: %s", tuning_records)
-
- use_autoscheduler = True
- try:
- auto_scheduler.load_records(tuning_records)
- except tvm._ffi.base.TVMError:
- use_autoscheduler = False
-
- if use_autoscheduler:
- with auto_scheduler.ApplyHistoryBest(tuning_records):
- config["relay.backend.use_auto_scheduler"] = True
- logger.debug("building relay graph with autoscheduler")
- graph_module = build(
- mod,
- tvm_target=tvm_target,
- executor=executor,
- runtime=runtime,
- params=params,
- use_vm=use_vm,
- mod_name=mod_name,
- workspace_pools=workspace_pools,
- )
- else:
- with autotvm.apply_history_best(tuning_records):
- logger.debug("building relay graph with tuning records")
- graph_module = build(
- mod,
- tvm_target=tvm_target,
- executor=executor,
- runtime=runtime,
- params=params,
- use_vm=use_vm,
- mod_name=mod_name,
- workspace_pools=workspace_pools,
- )
- else:
- logger.debug("building relay graph (no tuning records provided)")
- graph_module = build(
- mod,
- tvm_target=tvm_target,
- executor=executor,
- runtime=runtime,
- params=params,
- use_vm=use_vm,
- mod_name=mod_name,
- workspace_pools=workspace_pools,
- )
-
- # Generate output dump files with sources
- for source_type in dump_code:
- if source_type == "relay":
- dumps[source_type] = str(mod)
- elif source_type == "tir":
- dumps[source_type] = "\n".join(dumps[source_type])
- else:
- lib = graph_module.lib if use_vm else graph_module.get_lib()
- # TODO lib.get_source call have inconsistent behavior for unsupported
- # formats (@leandron).
- try:
- dumps[source_type] = lib.get_source(source_type)
- except tvm.TVMError:
- pass
- for smod in lib.imported_modules:
- try:
- if smod.type_key not in dumps:
- dumps[smod.type_key] = ""
- else:
- dumps[smod.type_key] += "\n"
- dumps[smod.type_key] += smod.get_source()
- except tvm.TVMError:
- print(f"Imported module {smod.type_key} doesn't support source dump")
-
- # Create a new tvmc model package object from the graph definition.
- package_path = tvmc_model.export_package(
- graph_module, package_path, cross, cross_options, output_format
- )
-
- # Write dumps to file.
- if dumps:
- save_dumps(package_path, dumps)
-
- # Print compilation times per pass
- if print_pass_times:
- print("Compilation time breakdown by pass:")
- print(timing_inst.render())
-
- return TVMCPackage(package_path)
-
-
-def build(
- mod: tvm.IRModule,
- tvm_target: str,
- executor: Executor,
- runtime: Runtime,
- params: Dict[str, tvm.nd.NDArray],
- use_vm: bool,
- mod_name: str,
- workspace_pools: Optional[WorkspaceMemoryPools],
-):
- """
- Builds the model with the provided executor.
-
- Parameters
- ----------
- mod : tvm.IRModule
- The relay module corresponding to this model.
- tvm_target : str
- The target for which to compile. Can be a plain string or
- a path.
- executor : Executor
- The graph executor to build the model if use_vm is not True
- runtime : Runtime
- The runtime configuration.
- params : dict
- A parameter dictionary for the model.
- use_vm: bool
- Whether to use the VM to compile the model as opposed to the graph executor
- mod_name: str
- The module name
-
- """
- if use_vm:
- logger.debug("building with vm compile")
- return relay.vm.compile(mod, target=tvm_target, params=params)
- logger.debug("building with relay build")
- return relay.build(
- mod,
- target=tvm_target,
- executor=executor,
- runtime=runtime,
- params=params,
- mod_name=mod_name,
- workspace_memory_pools=workspace_pools,
- )
-
-
-def add_tir_to_dumps(config, dumps):
- """
- Creates a debug pass that dumps TIR functions as a list of strings.
- """
- key = "tir"
- phase = 3 # final TIR phase before codegen
- dumps[key] = []
-
- @tvm.tir.transform.prim_func_pass(opt_level=0)
- def _dump_tir_pass(tir_func, _, __):
- dumps[key].append(str(tir_func))
- return tir_func
-
- tir_lower_passes = config.get("tir.add_lower_pass", [])
- tir_lower_passes.append((phase, _dump_tir_pass))
- config["tir.add_lower_pass"] = tir_lower_passes
-
- return config, dumps
-
-
-def save_dumps(module_name: str, dumps: Dict[str, str], dump_root: str = "."):
- """
- Serialize dump files to the disk.
-
- Parameters
- ----------
- module_name : str
- File name, referring to the module that generated
- the dump contents
- dumps : dict
- The output contents to be saved into the files
- dump_root : str, optional
- Path in which dump files will be created
- """
-
- for dump_format in dumps:
- dump_name = module_name + "." + dump_format
- with open(Path(dump_root, dump_name), "w") as f:
- f.write(dumps[dump_format])
-
-
-def dump_operation_offloads(mod: tvm.ir.IRModule, initial_mod: tvm.ir.IRModule, dump_path: str):
- """This helper function forms a line-by-line output of the initial Relay lines,
- indicating which operations are ported to which target,
- and indicating the composite that includes those operations;
- the 'generic' target refers to operations uploaded to the host, e.g
- 'target1 <- target1.qnn_conv2d'
- 'target1 <- %0 = qnn.conv2d(%tfl.quantize, %v_param_1, ...'
- 'target1 <- %1 = nn.bias_add(%0, %v_param_2, axis=3);'
- 'target1 <- %2 = qnn.requantize(%1, meta[relay.Constant]...'
- 'target2 <- target2.reshape'
- 'target2 <- %3 = reshape(%2, newshape=[1, 1001]);'
- 'generic <- %4 = nn.pad(%3, -128f, pad_width=[[0, 0], [1, 1]...'
-
- Parameters
- ----------
- mod : tvm.ir.IRModule
- The partitioned IRModule with external global functions.
- initial_mod : tvm.ir.IRModule
- The initial IRModule that gets generated from a relay frontend.
- dump_path: str
- Value of the "dump_offloads" compiler atribute.
- Could be dash ("-") or file path or empty string for
- printing to console, file or doing nothing respectively.
- """
- print_to_console = dump_path == "-"
- save_to_file = all([dump_path != "-", dump_path != ""])
-
- if print_to_console or save_to_file:
- operations_distribution = analyze_operations_distribution(mod)
-
- def annotate_f(x):
- ret = ""
- if isinstance(x, relay.Call):
- # if there is no x.span.source_name.name in operations_distribution,
- # this could mean that the span was not copied during the application of passes
- # to the Relay, in which case we can not associate the initial Relay string
- # with the resulting Relay call
- source_name = x.span.source_name.name
- suffix = tvm.relay.transform.suffixes.SUFFIX_STRING
- result = re.search(r"(.*)(" + suffix + r")(.*)", source_name)
- func_id = result.group(1)
- if func_id in operations_distribution:
- compiler_name, op_name = operations_distribution[func_id]
- ret = (
- f", compiler_name: {compiler_name}, op_name: {op_name}, "
- f"func_id: {func_id}"
- )
- else:
- ret = ", compiler_name: unknown, op_name: unknown, func_id: unknown"
- elif isinstance(x, (relay.Tuple, relay.TupleGetItem)):
- ret = ", compiler_name: none, op_name: none, func_id: none"
-
- return ret
-
- initial_relay_astext = initial_mod.astext(show_meta_data=False, annotate=annotate_f).split(
- "\n"
- )
-
- # funcs_list is a list of internal composite/function IDs.
- # funcs_list helps keep the order of lines from the initial Relay.
- funcs_list = []
-
- # target_statistic is a mapping of the target name to the
- # number of initial Relay calls offloaded on the target
- target_statistic = defaultdict(int)
-
- # funcs_dict is a mapping of the generated analyze_operations_distribution
- # internal composite/function IDs to a list, where:
- # 1st element is
- # (1a): "generic"|"unknown"|"none"* or
- # (1b): specific target name, like "ethos-u" or "cmsis-nn"
- # 2nd element is
- # (2a): corresponding initial Relay line for the case (1a) or
- # (2b): the name of the target composite functon in the other case (1b)
- # 3rd element or subsequent ones are presented only for the case (2b)
- # and are the initial Relay's lines included in the corresponding
- # target composite functon
- #
- # *Description of what is meant by "generic"|"unknown"|"none":
- # "generic" means that operation will be run on a host
- # "unknown" means that unique identifier of this Relay line not found in the partitioned
- # Relay and therefore not present in the operations_distribution dictionary
- # "none" means that this Relay line is not relay.Call
- funcs_dict = {}
-
- # Here we group together initial Relay lines from the one composite
- counter = itertools.count()
- for s in initial_relay_astext:
- result = re.search(
- r"(compiler_name: )(.*)(, op_name: )(.*)(, func_id: )((.*)(?=;)|(.*))", s
- )
- if result:
- target_name = result.group(2)
- op_name = result.group(4)
- func_id = result.group(6)
- if target_name != "none":
- target_statistic[target_name] += 1
-
- # create an identifier for each "unknown" or "none" case to keep the lines order
- if func_id == "unknown" or func_id == "none" or target_name == "generic":
- func_id = str(next(counter) * -1)
-
- if func_id not in funcs_dict:
- funcs_list.append(func_id)
- funcs_dict[func_id] = [target_name]
- if target_name not in ["unknown", "generic", "none"]:
- funcs_dict[func_id].append(op_name)
-
- s = re.sub(r", compiler_name: (.*)", "", s).lstrip()
- funcs_dict[func_id].append(s)
-
- # Here we prepare the output for printing.
- # The output in most cases keeps the original order of the Relay lines
- # but some lines are moved to be in the corresponding composite group
- output = []
- total = 0
- output.append("Total number of operators and distribution by targets")
- output.append("Total:")
- for target, statistic in target_statistic.items():
- total += statistic
- output.append(f"{target}: {statistic}")
- output[1] += f" {total}"
- output[len(target_statistic) + 1] += "\n"
-
- for func_id in funcs_list:
- _list = funcs_dict[func_id]
-
- if _list[0] != "none":
- output.append(f"{_list[0]:<15}<-{' ':5}{_list[1]}")
- else:
- output.append(f"{' ':>22}{_list[1]}")
-
- if _list[0] == "unknown":
- output.append(
- "Warning: The above line means that some pass(es) \
- in Relay partitioning"
- )
- output.append("do not copy the span when the call is recreated")
- output.append(
- "and a line from initial Relay could not be associated \
- with the resulting Relay"
- )
- for el in _list[2:]:
- output.append(f"{_list[0]:<15}<-{' ':10}{el}")
-
- if print_to_console:
- print("\n" + "\n".join(output))
- if save_to_file:
- file_path = os.path.abspath(dump_path)
- os.makedirs(os.path.dirname(file_path), exist_ok=True)
- with open(file_path, "w") as f:
- f.write("\n".join(output))
- f.write("\n")
diff --git a/python/tvm/driver/tvmc/composite_target.py b/python/tvm/driver/tvmc/composite_target.py
deleted file mode 100644
index 68f544f06a..0000000000
--- a/python/tvm/driver/tvmc/composite_target.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Provides support to composite target on TVMC.
-"""
-import logging
-
-# Make sure Vitis AI codegen is registered
-import tvm.contrib.target.vitis_ai # pylint: disable=unused-import
-
-from tvm.relay.op.contrib.arm_compute_lib import partition_for_arm_compute_lib
-from tvm.relay.op.contrib.bnns import partition_for_bnns
-from tvm.relay.op.contrib.vitis_ai import partition_for_vitis_ai
-from tvm.relay.op.contrib.clml import partition_for_clml
-from tvm.relay.op.contrib.mrvl import partition_for_mrvl
-
-
-from tvm.driver.tvmc import TVMCException
-
-
-# pylint: disable=invalid-name
-logger = logging.getLogger("TVMC")
-
-
-# Global dictionary to map targets
-#
-# Options
-# -------
-# config_key : str
-# The configuration key to be used in the PassContext (if any).
-# pass_pipeline : Callable
-# A function to transform a Module before compilation, mainly used
-# for partitioning for the target currently.
-REGISTERED_CODEGEN = {
- "compute-library": {
- "config_key": None,
- "pass_default": False,
- "default_target": None,
- "pass_pipeline": partition_for_arm_compute_lib,
- },
- "bnns": {
- "config_key": None,
- "pass_default": False,
- "default_target": None,
- "pass_pipeline": partition_for_bnns,
- },
- "vitis-ai": {
- "config_key": "relay.ext.vitis_ai.options",
- "pass_default": False,
- "default_target": None,
- "pass_pipeline": partition_for_vitis_ai,
- },
- "clml": {
- "config_key": None,
- "pass_default": False,
- "default_target": None,
- "pass_pipeline": partition_for_clml,
- },
- "mrvl": {
- "config_key": "relay.ext.mrvl.options",
- "pass_default": True,
- "default_target": "llvm",
- "pass_pipeline": partition_for_mrvl,
- },
-}
-
-
-def get_codegen_names():
- """Return a list of all registered codegens.
-
- Returns
- -------
- list of str
- all registered targets
- """
- return list(REGISTERED_CODEGEN.keys())
-
-
-def get_codegen_by_target(name):
- """Return a codegen entry by name.
-
- Parameters
- ----------
- name : str
- The name of the target for which the codegen info should be retrieved.
-
- Returns
- -------
- dict
- requested target codegen information
- """
- try:
- return REGISTERED_CODEGEN[name]
- except KeyError:
- raise TVMCException("Composite target %s is not defined in TVMC." % name)
diff --git a/python/tvm/driver/tvmc/config_options.py b/python/tvm/driver/tvmc/config_options.py
deleted file mode 100644
index 6ba89d650e..0000000000
--- a/python/tvm/driver/tvmc/config_options.py
+++ /dev/null
@@ -1,210 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-manipulate json config file to work with TVMC
-"""
-import os
-import json
-
-from tvm._ffi import libinfo
-from tvm.driver.tvmc import TVMCException
-
-CONFIGS_JSON_DIR = None
-
-
-class ConfigsJsonNotFoundError(TVMCException):
- """Raised when the JSON configs dirtree cannot be found."""
-
-
-def get_configs_json_dir() -> str:
- """Find the 'configs' directory, containing the JSON files used to configure tvmc
- with persistent argument settings.
-
- Returns
- -------
- str :
- The path to the 'configs' directory
- """
- global CONFIGS_JSON_DIR
- if CONFIGS_JSON_DIR is None:
-
- # If a non-default location for the build directory is used, e.g. set via TVM_LIBRARY_PATH
- # we need to provide the user a way to overwrite CONFIGS_JSON_DIR as well.
- if os.environ.get("TVM_CONFIGS_JSON_DIR", None):
- user_config_dir = os.environ["TVM_CONFIGS_JSON_DIR"]
- if os.path.isdir(user_config_dir):
- CONFIGS_JSON_DIR = user_config_dir
- return CONFIGS_JSON_DIR
-
- candidate_paths = []
- candidate_paths.extend(libinfo.find_lib_path())
- # When running from source, the configs directory will be located one directory above the
- # native libraries, so covering that case.
- candidate_paths.extend(
- [os.path.abspath(os.path.join(lib_path, "..")) for lib_path in libinfo.find_lib_path()]
- )
- candidate_paths.extend(
- [
- os.path.abspath(os.path.join(lib_path, "../.."))
- for lib_path in libinfo.find_lib_path()
- ]
- )
- for path in candidate_paths:
- configs_path = os.path.join(os.path.dirname(path), "configs")
- if os.path.isdir(configs_path):
- CONFIGS_JSON_DIR = configs_path
- break
-
- else:
- raise ConfigsJsonNotFoundError()
-
- return CONFIGS_JSON_DIR
-
-
-def find_json_file(name, path):
- """search for json file given file name a path
-
- Parameters
- ----------
- name: string
- the file name need to be searched
- path: string
- path to search at
-
- Returns
- -------
- string
- the full path to that file
-
- """
- match = ""
- for root, _dirs, files in os.walk(path):
- if name in files:
- match = os.path.join(root, name)
- break
-
- return match
-
-
-def read_and_convert_json_into_dict(config_args):
- """Read json configuration file and return a dictionary with all parameters
-
- Parameters
- ----------
- args: argparse.Namespace
- Arguments from command line parser holding the json file path.
-
- Returns
- -------
- dictionary
- dictionary with all the json arguments keys and values
-
- """
- try:
- if ".json" not in config_args.config:
- config_args.config = config_args.config.strip() + ".json"
- if os.path.isfile(config_args.config):
- json_config_file = config_args.config
- else:
- config_dir = get_configs_json_dir()
- json_config_file = find_json_file(config_args.config, config_dir)
- return json.load(open(json_config_file, "rb"))
-
- except FileNotFoundError:
- raise TVMCException(
- f"File {config_args.config} does not exist at {config_dir} or is wrong format."
- )
-
-
-def parse_target_from_json(one_target, command_line_list):
- """parse the targets out of the json file struct
-
- Parameters
- ----------
- one_target: dict
- dictionary with all target's details
- command_line_list: list
- list to update with target parameters
- """
- target_kind, *sub_type = [
- one_target[key] if key == "kind" else (key, one_target[key]) for key in one_target
- ]
-
- internal_dict = {}
- if sub_type:
- sub_target_type = sub_type[0][0]
- target_value = sub_type[0][1]
- internal_dict[f"target_{target_kind}_{sub_target_type}"] = target_value
- command_line_list.append(internal_dict)
-
- return target_kind
-
-
-def convert_config_json_to_cli(json_params):
- """convert all configuration keys & values from dictionary to cli format
-
- Parameters
- ----------
- args: dictionary
- dictionary with all configuration keys & values.
-
- Returns
- -------
- int
- list of configuration values in cli format
-
- """
- command_line_list = []
- for param_key in json_params:
- if param_key == "targets":
- target_list = [
- parse_target_from_json(one_target, command_line_list)
- for one_target in json_params[param_key]
- ]
-
- internal_dict = {}
- internal_dict["target"] = ", ".join(map(str, target_list))
- command_line_list.append(internal_dict)
-
- elif param_key in ("executor", "runtime"):
- for key, value in json_params[param_key].items():
- if key == "kind":
- kind = f"{value}_"
- new_dict_key = param_key
- else:
- new_dict_key = f"{param_key}_{kind}{key}"
-
- internal_dict = {}
- internal_dict[new_dict_key.replace("-", "_")] = value
- command_line_list.append(internal_dict)
-
- elif isinstance(json_params[param_key], dict):
- internal_dict = {}
- modify_param_key = param_key.replace("-", "_")
- internal_dict[modify_param_key] = []
- for key, value in json_params[param_key].items():
- internal_dict[modify_param_key].append(f"{key}={value}")
- command_line_list.append(internal_dict)
-
- else:
- internal_dict = {}
- internal_dict[param_key.replace("-", "_")] = json_params[param_key]
- command_line_list.append(internal_dict)
-
- return command_line_list
diff --git a/python/tvm/driver/tvmc/fmtopt.py b/python/tvm/driver/tvmc/fmtopt.py
deleted file mode 100644
index 7f27826d77..0000000000
--- a/python/tvm/driver/tvmc/fmtopt.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Utils to format help text for project options.
-"""
-from textwrap import TextWrapper
-
-
-# Maximum column length for accommodating option name and its choices.
-# Help text is placed after it in a new line.
-MAX_OPTNAME_CHOICES_TEXT_COL_LEN = 80
-
-
-# Maximum column length for accommodating help text.
-# 0 turns off formatting for the help text.
-MAX_HELP_TEXT_COL_LEN = 0
-
-
-# Justification of help text placed below option name + choices text.
-HELP_TEXT_JUST = 2
-
-
-def format_option(option_text, help_text, default_text, required=True):
- """Format option name, choices, and default text into a single help text.
-
- Parameters
- ----------
- options_text: str
- String containing the option name and option's choices formatted as:
- optname={opt0, opt1, ...}
-
- help_text: str
- Help text string.
-
- default_text: str
- Default text string.
-
- required: bool
- Flag that controls if a "(required)" text mark needs to be added to the final help text to
- inform if the option is a required one.
-
- Returns
- -------
- help_text_just: str
- Single justified help text formatted as:
- optname={opt0, opt1, ... }
- HELP_TEXT. "(required)" | "Defaults to 'DEFAULT'."
-
- """
- optname, choices_text = option_text.split("=", 1)
-
- # Prepare optname + choices text chunck.
-
- optname_len = len(optname)
- wrapper = TextWrapper(width=MAX_OPTNAME_CHOICES_TEXT_COL_LEN - optname_len)
- choices_lines = wrapper.wrap(choices_text)
-
- # Set first choices line which merely appends to optname string.
- # No justification is necessary for the first line since first
- # line was wrapped based on MAX_OPTNAME_CHOICES_TEXT_COL_LEN - optname_len,
- # i.e. considering optname_len, hence only append justified choices_lines[0] line.
- choices_just_lines = [optname + "=" + choices_lines[0]]
-
- # Justify the remaining lines based on first optname + '='.
- for line in choices_lines[1:]:
- line_len = len(line)
- line_just = line.rjust(
- optname_len + 1 + line_len
- ) # add 1 to align after '{' in the line above
- choices_just_lines.append(line_just)
-
- choices_text_just_chunk = "\n".join(choices_just_lines)
-
- # Prepare help text chunck.
-
- help_text = help_text[0].lower() + help_text[1:]
- if MAX_HELP_TEXT_COL_LEN > 0:
- wrapper = TextWrapper(width=MAX_HELP_TEXT_COL_LEN)
- help_text_lines = wrapper.wrap(help_text)
- else:
- # Don't format help text.
- help_text_lines = [help_text]
-
- help_text_just_lines = []
- for line in help_text_lines:
- line_len = len(line)
- line_just = line.rjust(HELP_TEXT_JUST + line_len)
- help_text_just_lines.append(line_just)
-
- help_text_just_chunk = "\n".join(help_text_just_lines)
-
- # An option might be required for one method but optional for another one.
- # If the option is required for one method it means there is no default for
- # it when used in that method, hence suppress default text in that case.
- if default_text and not required:
- help_text_just_chunk += " " + default_text
-
- if required:
- help_text_just_chunk += " (required)"
-
- help_text_just = choices_text_just_chunk + "\n" + help_text_just_chunk
- return help_text_just
diff --git a/python/tvm/driver/tvmc/frontends.py b/python/tvm/driver/tvmc/frontends.py
deleted file mode 100644
index 2da5483564..0000000000
--- a/python/tvm/driver/tvmc/frontends.py
+++ /dev/null
@@ -1,478 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Provides support to parse models from different frameworks into Relay networks.
-
-Frontend classes do lazy-loading of modules on purpose, to reduce time spent on
-loading the tool.
-"""
-import logging
-import os
-import sys
-import re
-import importlib
-from abc import ABC
-from abc import abstractmethod
-from typing import Optional, List, Dict
-from pathlib import Path
-
-import numpy as np
-
-from tvm import relay
-from tvm import parser
-from tvm.driver.tvmc import TVMCException, TVMCImportError
-from tvm.driver.tvmc.model import TVMCModel
-
-
-# pylint: disable=invalid-name
-logger = logging.getLogger("TVMC")
-
-
-class Frontend(ABC):
- """Abstract class for command line driver frontend.
-
- Provide a unified way to import models (as files), and deal
- with any required preprocessing to create a TVM module from it."""
-
- @staticmethod
- @abstractmethod
- def name():
- """Frontend name"""
-
- @staticmethod
- @abstractmethod
- def suffixes():
- """File suffixes (extensions) used by this frontend"""
-
- @abstractmethod
- def load(self, path, shape_dict=None, **kwargs):
- """Load a model from a given path.
-
- Parameters
- ----------
- path: str
- Path to a file
- shape_dict: dict, optional
- Mapping from input names to their shapes.
-
- Returns
- -------
- mod : tvm.IRModule
- The produced relay module.
- params : dict
- The parameters (weights) for the relay module.
-
- """
-
-
-def lazy_import(pkg_name, from_pkg_name=None, hide_stderr=False):
- """Lazy import a frontend package or subpackage"""
- try:
- return importlib.import_module(pkg_name, package=from_pkg_name)
- except ImportError as error:
- raise TVMCImportError(pkg_name) from error
- finally:
- if hide_stderr:
- sys.stderr = stderr
-
-
-class KerasFrontend(Frontend):
- """Keras frontend for TVMC"""
-
- @staticmethod
- def name():
- return "keras"
-
- @staticmethod
- def suffixes():
- return ["h5"]
-
- def load(self, path, shape_dict=None, **kwargs):
- # pylint: disable=C0103
- tf = lazy_import("tensorflow")
- keras = lazy_import("keras", from_pkg_name="tensorflow")
-
- # tvm build currently imports keras directly instead of tensorflow.keras
- try:
- model = keras.models.load_model(path)
- except ValueError as err:
- raise TVMCException(str(err))
-
- # There are two flavours of keras model, sequential and
- # functional, TVM expects a functional model, so convert
- # if required:
- if self.is_sequential_p(model):
- model = self.sequential_to_functional(model)
-
- in_shapes = []
- for layer in model._input_layers:
- if tf.executing_eagerly():
- in_shapes.append(tuple(dim if dim is not None else 1 for dim in layer.input.shape))
- else:
- in_shapes.append(
- tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape)
- )
-
- inputs = [np.random.uniform(size=shape, low=-1.0, high=1.0) for shape in in_shapes]
- input_shapes = {name: x.shape for (name, x) in zip(model.input_names, inputs)}
- if shape_dict is not None:
- input_shapes.update(shape_dict)
- kwargs.setdefault("layout", "NHWC")
- return relay.frontend.from_keras(model, input_shapes, **kwargs)
-
- def is_sequential_p(self, model):
- keras = lazy_import("keras", from_pkg_name="tensorflow")
- return isinstance(model, keras.models.Sequential)
-
- def sequential_to_functional(self, model):
- keras = lazy_import("keras", from_pkg_name="tensorflow")
- assert self.is_sequential_p(model)
- input_layer = keras.layers.Input(batch_shape=model.layers[0].input_shape)
- prev_layer = input_layer
- for layer in model.layers:
- prev_layer = layer(prev_layer)
- model = keras.models.Model([input_layer], [prev_layer])
- return model
-
-
-class OnnxFrontend(Frontend):
- """ONNX frontend for TVMC"""
-
- @staticmethod
- def name():
- return "onnx"
-
- @staticmethod
- def suffixes():
- return ["onnx"]
-
- def load(self, path, shape_dict=None, **kwargs):
- onnx = lazy_import("onnx")
-
- # pylint: disable=E1101
- model = onnx.load(path)
-
- return relay.frontend.from_onnx(model, shape=shape_dict, **kwargs)
-
-
-class TensorflowFrontend(Frontend):
- """TensorFlow frontend for TVMC"""
-
- @staticmethod
- def name():
- return "pb"
-
- @staticmethod
- def suffixes():
- return ["pb"]
-
- def load(self, path, shape_dict=None, **kwargs):
- tf = lazy_import("tensorflow")
- tf_testing = lazy_import("tvm.relay.testing.tf")
-
- with tf.io.gfile.GFile(path, "rb") as tf_graph:
- content = tf_graph.read()
-
- graph_def = tf.compat.v1.GraphDef()
- graph_def.ParseFromString(content)
- graph_def = tf_testing.ProcessGraphDefParam(graph_def)
-
- logger.debug("parse TensorFlow model and convert into Relay computation graph")
- return relay.frontend.from_tensorflow(graph_def, shape=shape_dict, **kwargs)
-
-
-class TFLiteFrontend(Frontend):
- """TFLite frontend for TVMC"""
-
- @staticmethod
- def name():
- return "tflite"
-
- @staticmethod
- def suffixes():
- return ["tflite"]
-
- def load(self, path, shape_dict=None, **kwargs):
- model = lazy_import("tflite.Model")
-
- with open(path, "rb") as tf_graph:
- content = tf_graph.read()
-
- # tflite.Model.Model is tflite.Model in 1.14 and 2.1.0
- try:
- tflite_model = model.Model.GetRootAsModel(content, 0)
- except AttributeError:
- tflite_model = model.GetRootAsModel(content, 0)
-
- try:
- version = tflite_model.Version()
- logger.debug("tflite version %s", version)
- except Exception:
- raise TVMCException("input file not tflite")
-
- if version != 3:
- raise TVMCException("input file not tflite version 3")
-
- logger.debug("parse TFLite model and convert into Relay computation graph")
- mod, params = relay.frontend.from_tflite(tflite_model, shape_dict=shape_dict, **kwargs)
- return mod, params
-
-
-class PyTorchFrontend(Frontend):
- """PyTorch frontend for TVMC"""
-
- @staticmethod
- def name():
- return "pytorch"
-
- @staticmethod
- def suffixes():
- # Torch Script is a zip file, but can be named pth
- return ["pth", "zip"]
-
- def load(self, path, shape_dict=None, **kwargs):
- torch = lazy_import("torch")
-
- if shape_dict is None:
- raise TVMCException("--input-shapes must be specified for %s" % self.name())
-
- traced_model = torch.jit.load(path)
- traced_model.eval() # Switch to inference mode
-
- # Convert shape dictionary to list for Pytorch frontend compatibility
- input_shapes = list(shape_dict.items())
-
- logger.debug("parse Torch model and convert into Relay computation graph")
- return relay.frontend.from_pytorch(
- traced_model, input_shapes, keep_quantized_weight=True, **kwargs
- )
-
-
-class PaddleFrontend(Frontend):
- """PaddlePaddle frontend for TVMC"""
-
- @staticmethod
- def name():
- return "paddle"
-
- @staticmethod
- def suffixes():
- return ["pdmodel"]
-
- def load(self, path, shape_dict=None, **kwargs):
- # pylint: disable=C0415
- import paddle
-
- paddle.enable_static()
- paddle.disable_signal_handler()
-
- if not os.path.exists(path):
- raise TVMCException("File {} is not exist.".format(path))
- if not path.endswith(".pdmodel"):
- raise TVMCException("Path of model file should be endwith suffixes '.pdmodel'.")
- prefix = "".join(path.strip().split(".")[:-1])
- params_file_path = prefix + ".pdiparams"
- if not os.path.exists(params_file_path):
- raise TVMCException("File {} is not exist.".format(params_file_path))
-
- # pylint: disable=E1101
- exe = paddle.static.Executor(paddle.CPUPlace())
- prog, _, _ = paddle.static.load_inference_model(prefix, exe)
-
- return relay.frontend.from_paddle(prog, shape_dict=shape_dict, **kwargs)
-
-
-class RelayFrontend(Frontend):
- """Relay frontend for TVMC"""
-
- @staticmethod
- def name():
- return "relay"
-
- @staticmethod
- def suffixes():
- return ["relay"]
-
- def load(self, path, shape_dict=None, **kwargs):
- with open(path, "r", encoding="utf-8") as relay_text:
- text = relay_text.read()
- if shape_dict is None:
- logger.warning(
- "Specify --input-shapes to ensure that model inputs "
- "will not be considered as constants."
- )
-
- def _validate_text(text):
- """Check the provided file contents.
- The relay.txt artifact contained in the MLF is missing the version header and
- the metadata which is required to use meta[relay.Constant]."""
-
- if re.compile(r".*\#\[version\.*").match(text) is None:
- raise TVMCException(
- "The relay model does not include the required version information."
- )
- if re.compile(r".*meta\[.+\].*", re.DOTALL).match(text):
- if "#[metadata]" not in text:
- raise TVMCException(
- "The relay model does not include the required #[metadata] section. "
- "Use ir_mod.astext(show_meta_data=True) to export compatible code."
- )
-
- _validate_text(text)
-
- ir_mod = parser.fromtext(text)
-
- if shape_dict:
- input_names = shape_dict.keys()
- else:
- input_names = []
-
- def _gen_params(ir_mod, skip_names=None):
- """Populate the all the params in the mode with ones."""
- main_func = ir_mod["main"]
- shape_dict = {p.name_hint: p.checked_type.concrete_shape for p in main_func.params}
- type_dict = {p.name_hint: p.checked_type.dtype for p in main_func.params}
- params = {}
- for name, shape in shape_dict.items():
- if skip_names and name in skip_names:
- continue
-
- if "int" in type_dict[name]:
- data = np.random.randint(128, size=shape, dtype=type_dict[name])
- else:
- data = np.random.uniform(-1, 1, size=shape).astype(type_dict[name])
- params[name] = data
- return params
-
- params = _gen_params(ir_mod, skip_names=input_names)
-
- return ir_mod, params
-
-
-ALL_FRONTENDS = [
- KerasFrontend,
- OnnxFrontend,
- TensorflowFrontend,
- TFLiteFrontend,
- PyTorchFrontend,
- PaddleFrontend,
- RelayFrontend,
-]
-
-
-def get_frontend_names():
- """Return the names of all supported frontends
-
- Returns
- -------
- list : list of str
- A list of frontend names as strings
-
- """
- return [frontend.name() for frontend in ALL_FRONTENDS]
-
-
-def get_frontend_by_name(name: str):
- """
- This function will try to get a frontend instance, based
- on the name provided.
-
- Parameters
- ----------
- name : str
- the name of a given frontend
-
- Returns
- -------
- frontend : tvm.driver.tvmc.Frontend
- An instance of the frontend that matches with
- the file extension provided in `path`.
-
- """
-
- for frontend in ALL_FRONTENDS:
- if name == frontend.name():
- return frontend()
-
- raise TVMCException(
- "unrecognized frontend '{0}'. Choose from: {1}".format(name, get_frontend_names())
- )
-
-
-def guess_frontend(path: str):
- """
- This function will try to imply which framework is being used,
- based on the extension of the file provided in the path parameter.
-
- Parameters
- ----------
- path : str
- The path to the model file.
-
- Returns
- -------
- frontend : tvm.driver.tvmc.Frontend
- An instance of the frontend that matches with
- the file extension provided in `path`.
-
- """
-
- suffix = Path(path).suffix.lower()
- if suffix.startswith("."):
- suffix = suffix[1:]
-
- for frontend in ALL_FRONTENDS:
- if suffix in frontend.suffixes():
- return frontend()
-
- raise TVMCException("failed to infer the model format. Please specify --model-format")
-
-
-def load_model(
- path: str,
- model_format: Optional[str] = None,
- shape_dict: Optional[Dict[str, List[int]]] = None,
- **kwargs,
-):
- """Load a model from a supported framework and convert it
- into an equivalent relay representation.
-
- Parameters
- ----------
- path : str
- The path to the model file.
- model_format : str, optional
- The underlying framework used to create the model.
- If not specified, this will be inferred from the file type.
- shape_dict : dict, optional
- Mapping from input names to their shapes.
-
- Returns
- -------
- tvmc_model : TVMCModel
- The produced model package.
-
- """
-
- if model_format is not None:
- frontend = get_frontend_by_name(model_format)
- else:
- frontend = guess_frontend(path)
-
- mod, params = frontend.load(path, shape_dict, **kwargs)
-
- return TVMCModel(mod, params)
diff --git a/python/tvm/driver/tvmc/main.py b/python/tvm/driver/tvmc/main.py
deleted file mode 100644
index b5d039c75a..0000000000
--- a/python/tvm/driver/tvmc/main.py
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=use-a-generator
-"""
-TVMC - TVM driver command-line interface
-"""
-import argparse
-import logging
-import sys
-
-import tvm
-
-from tvm.driver.tvmc import TVMCException, TVMCImportError
-from tvm.driver.tvmc.config_options import (
- read_and_convert_json_into_dict,
- convert_config_json_to_cli,
-)
-
-REGISTERED_PARSER = []
-
-
-def register_parser(make_subparser):
- """
- Utility function to register a subparser for tvmc.
-
- Functions decorated with `tvm.driver.tvmc.main.register_parser` will be invoked
- with a parameter containing the subparser instance they need to add itself to,
- as a parser.
-
- Example
- -------
-
- @register_parser
- def _example_parser(main_subparser):
- subparser = main_subparser.add_parser('example', help='...')
- ...
-
- """
- REGISTERED_PARSER.append(make_subparser)
- return make_subparser
-
-
-def _main(argv):
- """TVM command line interface."""
-
- parser = argparse.ArgumentParser(
- prog="tvmc",
- formatter_class=argparse.RawDescriptionHelpFormatter,
- description="TVM compiler driver",
- epilog=__doc__,
- # Help action will be added later, after all subparsers are created,
- # so it doesn't interfere with the creation of the dynamic subparsers.
- add_help=False,
- )
-
- parser.add_argument("--config", default="default", help="configuration json file")
- config_arg, argv = parser.parse_known_args(argv)
-
- json_param_dict = read_and_convert_json_into_dict(config_arg)
- json_config_values = convert_config_json_to_cli(json_param_dict)
-
- parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity")
- parser.add_argument("--version", action="store_true", help="print the version and exit")
-
- subparser = parser.add_subparsers(title="commands")
- for make_subparser in REGISTERED_PARSER:
- make_subparser(subparser, parser, json_config_values)
-
- # Finally, add help for the main parser.
- parser.add_argument("-h", "--help", action="help", help="show this help message and exit.")
-
- args = parser.parse_args(argv)
- args.verbose = min(args.verbose, 3)
-
- # See the meaning of the logging levels at
- # https://docs.python.org/3/library/logging.html#logging-levels
- logging.basicConfig(stream=sys.stdout)
- logging.getLogger("TVMC").setLevel(40 - args.verbose * 10)
-
- if args.version:
- sys.stdout.write("%s\n" % tvm.__version__)
- return 0
-
- if not hasattr(args, "func"):
- # In case no valid subcommand is provided, show usage and exit
- parser.print_help(sys.stderr)
- return 1
-
- try:
- return args.func(args)
- except TVMCImportError as err:
- sys.stderr.write(
- f'Package "{err}" is not installed. ' f'Hint: "pip install tlcpack[tvmc]".'
- )
- return 5
- except TVMCException as err:
- sys.stderr.write("Error: %s\n" % err)
- return 4
-
-
-def main():
- sys.exit(_main(sys.argv[1:]))
-
-
-if __name__ == "__main__":
- main()
diff --git a/python/tvm/driver/tvmc/model.py b/python/tvm/driver/tvmc/model.py
deleted file mode 100644
index 73cc8da71c..0000000000
--- a/python/tvm/driver/tvmc/model.py
+++ /dev/null
@@ -1,467 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# pylint: disable=consider-using-with,broad-exception-raised,consider-using-from-import
-"""
-This file contains the definition of a set of classes that wrap the outputs
-of TVMC functions to create a simpler and more intuitive API.
-
-There is one class for each required stage of a TVM workflow.
-The TVMCModel represents the result of importing a model into TVM, it
-contains the precompiled graph definition and parameters that define
-what the model does.
-
-Compiling a TVMCModel produces a TVMCPackage, which contains the generated
-artifacts that allow the model to be run on the target hardware.
-
-Running a TVMCPackage produces a TVMCResult, which contains the outputs of
-the model and the measured runtime.
-
-Examples
---------
-The following code shows a full lifecycle for a model using tvmc, first the
-model is imported from an exterior framework, in this case onnx, then it
-is tuned to find the best schedules on CPU, then compiled into a TVMCPackage,
-and finally run.
-
-.. code-block:: python
- tvmc_model = tvmc.load("my_model.onnx")
- tuning_records = tvmc.tune(tvmc_model, target="llvm")
- tvmc_package = tvmc.compile(tvmc_model, target="llvm", tuning_records=tuning_records)
- result = tvmc.run(tvmc_package, device="cpu")
- print(result)
-"""
-import os
-import tarfile
-from typing import Optional, Union, Dict, Callable, TextIO
-import numpy as np
-
-import tvm
-import tvm.contrib.cc
-from tvm import relay
-from tvm.contrib import utils
-from tvm.driver.tvmc import TVMCException
-from tvm.relay.backend.executor_factory import GraphExecutorFactoryModule
-from tvm.runtime.module import BenchmarkResult
-from tvm.runtime.vm import Executable
-
-
-class TVMCModel(object):
- """Initialize a TVMC model from a relay model definition or a saved file.
-
- Parameters
- ----------
- mod : tvm.IRModule, optional
- The relay module corresponding to this model.
- params : dict, optional
- A parameter dictionary for the model.
- model_path: str, optional
- An alternative way to load a TVMCModel, the path to a previously
- saved model.
- """
-
- def __init__(
- self,
- mod: Optional[tvm.IRModule] = None,
- params: Optional[Dict[str, tvm.nd.NDArray]] = None,
- model_path: Optional[str] = None,
- ):
- if (mod is None or params is None) and (model_path is None):
- raise TVMCException(
- "Either mod and params must be provided "
- "or a path to a previously saved TVMCModel"
- )
- self._tmp_dir = utils.tempdir()
- if model_path is not None:
- self.load(model_path)
- else:
- self.mod = mod
- self.params = params if params else {}
-
- def save(self, model_path: str):
- """Save the TVMCModel to disk.
-
- Note that this saves the graph representation,
- the parameters, and the tuning records if applicable. It will not save any
- compiled artifacts.
-
- Parameters
- ----------
- model_path : str
- A full path to save this TVMCModel to including the output file name.
- The file will be saved as a tar file so using a ".tar" extension is advised.
- """
- temp = self._tmp_dir
-
- # Save relay graph
- relay_name = "model.json"
- relay_path = temp.relpath(relay_name)
- with open(relay_path, "w") as relay_file:
- relay_file.write(tvm.ir.save_json(self.mod))
-
- # Save params
- params_name = "model.params"
- params_path = temp.relpath(params_name)
- with open(params_path, "wb") as params_file:
- params_file.write(relay.save_param_dict(self.params))
-
- # Create a tar file.
- with tarfile.open(model_path, "w") as tar:
- tar.add(relay_path, relay_name)
... 50751 lines suppressed ...