You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2023/06/01 23:47:22 UTC

[tvm] 01/01: [MERGE] Merge main into unity 2023-06-01

This is an automated email from the ASF dual-hosted git repository.

tqchen pushed a commit to branch unity-staging
in repository https://gitbox.apache.org/repos/asf/tvm.git

commit 153f70aa60b82c4f7670ee32dceee859a431fdd0
Merge: f360556a0f 7f02606af2
Author: tqchen <ti...@gmail.com>
AuthorDate: Thu Jun 1 19:46:25 2023 -0400

    [MERGE] Merge main into unity 2023-06-01
    
    Updated the runtime to latest
    cutlass conflict resolutions

 CMakeLists.txt                                     |  21 +-
 apps/dso_plugin_module/plugin_module.cc            |   3 +-
 apps/hexagon_launcher/README.md                    |  14 +-
 apps/ios_rpc/tests/ios_rpc_mobilenet.py            |  27 +-
 apps/ios_rpc/tests/ios_rpc_test.py                 |  13 +-
 apps/topi_recipe/broadcast/test_broadcast_map.py   |  12 +-
 apps/topi_recipe/conv/depthwise_conv2d_test.py     |  13 +-
 apps/topi_recipe/conv/test_conv2d_hwcn_map.py      |   9 +-
 apps/topi_recipe/reduce/test_reduce_map.py         |  10 +-
 apps/topi_recipe/rnn/lstm.py                       |   9 +-
 apps/topi_recipe/rnn/matexp.py                     |  13 +-
 ci/scripts/github/update_branch.py                 |  19 +-
 cmake/config.cmake                                 |   4 +-
 cmake/modules/HexagonSDK.cmake                     |   9 +-
 cmake/modules/LibInfo.cmake                        |   1 +
 docker/install/ubuntu2204_install_llvm.sh          |   2 +-
 docker/install/ubuntu_install_python_package.sh    |   3 +-
 include/tvm/ir/si_builder.h                        | 103 ++++++
 include/tvm/ir/source_map.h                        |  46 ++-
 include/tvm/runtime/data_type.h                    |  34 ++
 include/tvm/runtime/logging.h                      |   6 +-
 include/tvm/runtime/module.h                       |  19 +-
 include/tvm/runtime/packed_func.h                  |   2 +-
 include/tvm/runtime/registry.h                     |  12 +-
 include/tvm/runtime/relax_vm/executable.h          |   6 +-
 include/tvm/runtime/vm/executable.h                |   4 +-
 include/tvm/runtime/vm/vm.h                        |   2 +-
 include/tvm/target/target.h                        |  13 +
 include/tvm/tir/op.h                               |   3 +-
 include/tvm/tir/transform.h                        |  52 +++
 jvm/core/src/test/scripts/test_add_gpu.py          |   4 +-
 python/gen_requirements.py                         |   1 +
 python/tvm/_ffi/runtime_ctypes.py                  |  22 ++
 python/tvm/contrib/clang.py                        |   6 +-
 python/tvm/contrib/cudnn.py                        |   6 +-
 python/tvm/contrib/cutlass/build.py                |  29 +-
 python/tvm/contrib/cutlass/conv2d_operation.py     |  11 +-
 python/tvm/contrib/cutlass/gemm_operation.py       |  22 +-
 python/tvm/contrib/cutlass/gen_conv2d.py           |   2 +-
 python/tvm/contrib/cutlass/gen_gemm.py             |   2 +-
 python/tvm/contrib/cutlass/gen_tensor_op.py        |  49 +--
 python/tvm/contrib/cutlass/library.py              |  23 +-
 python/tvm/contrib/graph_executor.py               |   2 +-
 .../contrib/hexagon/profiling/process_lwp_data.py  |   2 +-
 python/tvm/contrib/hexagon/pytest_plugin.py        |   2 +-
 python/tvm/contrib/hexagon/tools.py                |   8 +-
 python/tvm/contrib/nvcc.py                         |  29 +-
 python/tvm/contrib/peak.py                         |   4 +-
 python/tvm/contrib/pickle_memoize.py               |   4 +-
 python/tvm/contrib/pipeline_executor.py            |  20 +-
 python/tvm/contrib/pipeline_executor_build.py      |  22 +-
 python/tvm/contrib/rocm.py                         |   4 +-
 python/tvm/contrib/sdaccel.py                      |  10 +-
 python/tvm/contrib/sparse.py                       |   8 +-
 python/tvm/contrib/tar.py                          |   2 +-
 python/tvm/contrib/target/coreml.py                |  23 +-
 python/tvm/contrib/target/onnx.py                  |  79 ++--
 python/tvm/contrib/target/vitis_ai.py              |   2 +-
 python/tvm/contrib/tf_op/module.py                 |   2 +-
 python/tvm/contrib/utils.py                        |   2 +-
 python/tvm/contrib/xcode.py                        |   6 +-
 python/tvm/ir/__init__.py                          |   1 +
 python/tvm/ir/base.py                              |  17 +
 python/tvm/relay/__init__.py                       |   1 +
 .../tvm/relay/backend/contrib/ethosu/legalize.py   |  11 +-
 python/tvm/relay/base.py                           |   2 +-
 python/tvm/relay/frontend/keras.py                 |   4 +-
 python/tvm/relay/frontend/pytorch.py               |  26 +-
 python/tvm/relay/op/contrib/ethosu.py              |  27 +-
 python/tvm/relay/op/strategy/adreno.py             |  47 ++-
 python/tvm/rpc/base.py                             |   6 +-
 python/tvm/rpc/client.py                           |  23 +-
 python/tvm/rpc/minrpc.py                           |   4 +-
 python/tvm/rpc/proxy.py                            |  22 +-
 python/tvm/rpc/server.py                           |  12 +-
 python/tvm/rpc/testing.py                          |   4 +-
 python/tvm/rpc/tracker.py                          |  16 +-
 python/tvm/runtime/container.py                    |  14 +-
 python/tvm/runtime/executor/aot_executor.py        |   2 +-
 python/tvm/runtime/module.py                       |  38 +-
 python/tvm/runtime/ndarray.py                      |  36 +-
 python/tvm/runtime/object.py                       |   7 +-
 python/tvm/runtime/object_generic.py               |   6 +-
 python/tvm/runtime/vm.py                           |  15 +-
 python/tvm/script/ir_builder/tir/ir.py             |  24 +-
 python/tvm/script/parser/tir/parser.py             |  14 +-
 python/tvm/target/target.py                        |   8 +-
 python/tvm/tir/op.py                               |   2 +-
 python/tvm/tir/transform/transform.py              |  84 +++++
 src/driver/driver_api.cc                           |   5 +
 src/ir/expr.cc                                     |  14 +-
 src/ir/si_builder.cc                               | 325 +++++++++++++++++
 src/ir/source_map.cc                               |  60 ++++
 src/relay/backend/aot_executor_codegen.cc          |   2 +-
 src/relay/backend/build_module.cc                  |   2 +-
 src/relay/backend/contrib/ethosu/source_module.cc  |   8 +-
 src/relay/backend/graph_executor_codegen.cc        |   2 +-
 src/relay/backend/vm/compiler.cc                   |   2 +-
 src/relay/backend/vm/compiler.h                    |   2 +-
 src/relay/printer/model_library_format_printer.cc  |   2 +-
 src/runtime/aot_executor/aot_executor.cc           |   3 +-
 src/runtime/aot_executor/aot_executor.h            |   2 +-
 src/runtime/aot_executor/aot_executor_factory.cc   |   2 +-
 src/runtime/aot_executor/aot_executor_factory.h    |   2 +-
 src/runtime/const_loader_module.cc                 |   2 +-
 src/runtime/contrib/coreml/coreml_runtime.h        |   2 +-
 src/runtime/contrib/coreml/coreml_runtime.mm       |   3 +-
 src/runtime/contrib/dnnl/dnnl_json_runtime.cc      |   2 +-
 src/runtime/contrib/ethosn/ethosn_runtime.cc       |   5 +-
 src/runtime/contrib/ethosn/ethosn_runtime.h        |   4 +-
 src/runtime/contrib/json/json_runtime.h            |   4 +-
 src/runtime/contrib/libtorch/libtorch_runtime.cc   |   4 +-
 src/runtime/contrib/onnx/onnx_module.cc            |   6 +-
 src/runtime/contrib/tflite/tflite_runtime.cc       |   3 +-
 src/runtime/contrib/tflite/tflite_runtime.h        |   2 +-
 src/runtime/contrib/vitis_ai/vitis_ai_runtime.cc   |   3 +-
 src/runtime/contrib/vitis_ai/vitis_ai_runtime.h    |   2 +-
 src/runtime/cuda/cuda_module.cc                    |  11 +-
 .../cuda_graph/graph_runtime_cuda_graph.cc         |   4 +-
 .../graph_executor/debug/graph_executor_debug.cc   |   2 +-
 .../graph_executor/debug/graph_executor_debug.h    |   2 +-
 src/runtime/graph_executor/graph_executor.cc       |   3 +-
 src/runtime/graph_executor/graph_executor.h        |   2 +-
 .../graph_executor/graph_executor_factory.cc       |   2 +-
 .../graph_executor/graph_executor_factory.h        |   2 +-
 src/runtime/hexagon/README.md                      |   9 +-
 src/runtime/hexagon/hexagon_module.cc              |   6 +-
 src/runtime/hexagon/hexagon_module.h               |   6 +-
 src/runtime/hexagon/rpc/simulator/session.cc       |  30 ++
 src/runtime/library_module.cc                      |   5 +-
 src/runtime/metadata.cc                            |   2 +-
 src/runtime/metal/metal_module.mm                  |   9 +-
 src/runtime/module.cc                              |  15 +-
 src/runtime/opencl/opencl_common.h                 |   8 +-
 src/runtime/opencl/opencl_module.cc                |  14 +-
 src/runtime/opencl/opencl_module_spirv.cc          |   6 +-
 src/runtime/opencl/sdaccel/sdaccel_module.cc       |   2 +-
 src/runtime/pipeline/pipeline_executor.cc          |   2 +-
 src/runtime/pipeline/pipeline_executor.h           |   2 +-
 src/runtime/registry.cc                            |  18 +-
 src/runtime/relax_vm/executable.cc                 |   6 +-
 src/runtime/relax_vm/ndarray_cache_support.cc      |   4 +-
 src/runtime/relax_vm/vm.cc                         |   6 +-
 src/runtime/rocm/rocm_module.cc                    |   9 +-
 src/runtime/rpc/rpc_module.cc                      |   4 +-
 src/runtime/stackvm/stackvm_module.cc              |   9 +-
 src/runtime/static_library.cc                      |   4 +-
 src/runtime/vm/executable.cc                       |   6 +-
 src/runtime/vm/profiler/vm.cc                      |   2 +-
 src/runtime/vm/profiler/vm.h                       |   2 +-
 src/runtime/vm/vm.cc                               |   3 +-
 src/runtime/vulkan/vulkan_module.cc                |   2 +-
 src/runtime/vulkan/vulkan_wrapped_func.cc          |   6 +-
 src/runtime/vulkan/vulkan_wrapped_func.h           |   6 +-
 src/script/printer/tir/block.cc                    |   3 +-
 src/script/printer/tir/buffer.cc                   |  29 +-
 src/script/printer/tir/function.cc                 |   6 +-
 src/script/printer/tir/stmt.cc                     |  24 +-
 src/script/printer/tir/utils.h                     |  28 +-
 src/support/ffi_testing.cc                         |   4 +-
 src/support/libinfo.cc                             |   1 +
 src/support/scalars.h                              |  12 +
 src/target/llvm/codegen_amdgpu.cc                  |  10 +-
 src/target/llvm/codegen_cpu.cc                     |  51 +--
 src/target/llvm/codegen_cpu.h                      |   2 +-
 src/target/llvm/codegen_hexagon.cc                 |   4 +-
 src/target/llvm/codegen_llvm.cc                    | 110 +++++-
 src/target/llvm/codegen_llvm.h                     |  61 +++-
 src/target/llvm/codegen_nvptx.cc                   |  14 +-
 src/target/llvm/llvm_module.cc                     |  33 +-
 src/target/opt/build_cuda_on.cc                    |   4 +-
 src/target/source/codegen_aocl.cc                  |   2 +-
 src/target/source/codegen_cuda.cc                  |  17 +
 src/target/source/codegen_cuda.h                   |   5 +-
 src/target/source/codegen_metal.cc                 |   7 +-
 src/target/source/codegen_opencl.cc                |  12 +-
 src/target/source/codegen_vhls.cc                  |   5 +-
 src/target/source/codegen_webgpu.cc                |   8 +-
 src/target/source/interface_c.cc                   |   4 +-
 src/target/source/source_module.cc                 |  28 +-
 src/target/spirv/spirv_utils.cc                    |   2 +-
 src/target/target.cc                               |  15 +
 src/tir/ir/stmt.cc                                 |   3 +
 src/tir/op/op.cc                                   |   7 +
 src/tir/schedule/state.cc                          |   2 +-
 src/tir/transforms/annotate_device_regions.cc      |  81 +++++
 src/tir/transforms/combine_context_call.cc         |   7 +-
 src/tir/transforms/dtype_conversion.cc             | 101 ++++++
 src/tir/transforms/dtype_conversion.h              | 165 +++++++++
 src/tir/transforms/install_debug_spans.cc          |  36 +-
 src/tir/transforms/ir_utils.cc                     |  32 +-
 src/tir/transforms/ir_utils.h                      |  13 +
 src/tir/transforms/lower_device_kernel_launch.cc   | 305 ++++++++++++++++
 src/tir/transforms/lower_thread_allreduce.cc       |   3 +-
 src/tir/transforms/make_packed_api.cc              | 160 ++++++---
 src/tir/transforms/make_unpacked_api.cc            | 113 +++++-
 src/tir/transforms/primfunc_utils.cc               |  30 +-
 src/tir/transforms/split_host_device.cc            | 272 ++++----------
 ...6_legalize.cc => unsupported_dtype_legalize.cc} | 264 +++++++++-----
 tests/cpp/si_builder_test.cc                       | 399 +++++++++++++++++++++
 tests/python/ci/test_ci.py                         |  49 +--
 tests/python/contrib/test_ethosu/test_codegen.py   |  32 ++
 tests/python/contrib/test_ethosu/test_legalize.py  | 211 +++++++++++
 tests/python/contrib/test_hexagon/README.md        |   2 +-
 tests/python/frontend/keras/test_forward.py        |   2 +-
 tests/python/frontend/pytorch/test_forward.py      |  29 +-
 tests/python/integration/test_ewise.py             |   5 +-
 tests/python/integration/test_ewise_fpga.py        |   3 +-
 .../opencl_texture/test_conv2d_nchw_texture.py     |  30 ++
 .../opencl_texture/test_conv2d_nhwc_texture.py     |  30 ++
 .../test_depthwise_conv2d_nchw_texture.py          |  31 ++
 .../test_depthwise_conv2d_nhwc_texture.py          |  31 ++
 tests/python/tir/test_debug_info.py                |  50 ++-
 tests/python/unittest/test_datatype_nv_fp8.py      | 104 ++++++
 tests/python/unittest/test_target_codegen_llvm.py  |  89 ++++-
 tests/python/unittest/test_target_codegen_metal.py |  30 +-
 .../python/unittest/test_target_codegen_opencl.py  |  80 ++---
 .../unittest/test_target_texture_codegen_opencl.py |  38 +-
 .../unittest/test_te_schedule_bound_inference.py   |  56 ++-
 .../test_tir_transform_annotate_device_regions.py  |  58 +++
 .../unittest/test_tir_transform_bf16_legalize.py   |   5 +-
 .../test_tir_transform_combine_context_call.py     | 102 ++++--
 .../unittest/test_tir_transform_convert_ssa.py     | 253 +++++++++++++
 .../test_tir_transform_device_kernel_launch.py     | 193 ++++++++++
 .../unittest/test_tir_transform_fp8_legalize.py    | 224 ++++++++++++
 .../python/unittest/test_tir_transform_helpers.py  | 112 ++++++
 .../test_tir_transform_inject_ptx_async_copy.py    |   4 +-
 .../python/unittest/test_tir_transform_ir_utils.py |  40 ---
 .../test_tir_transform_lower_warp_memory.py        |  37 +-
 .../unittest/test_tir_transform_make_packed_api.py | 128 ++++++-
 .../test_tir_transform_make_unpacked_api.py        | 158 +++++++-
 .../test_tir_transform_split_host_device.py        | 129 ++++++-
 .../unittest/test_tir_transform_thread_sync.py     |   5 +-
 .../python/unittest/test_tvmscript_printer_tir.py  |   7 +-
 tests/python/unittest/test_tvmscript_roundtrip.py  |  80 +++++
 .../python/unittest/test_tvmscript_syntax_sugar.py |  35 ++
 tests/scripts/setup-adreno-env.sh                  |  13 +-
 tests/scripts/task_config_build_cpu.sh             |   3 -
 tests/scripts/task_config_build_i386.sh            |   3 -
 tests/scripts/task_python_adreno.sh                |  13 +-
 tests/scripts/task_python_vta_fsim.sh              |   3 +
 web/emcc/webgpu_runtime.cc                         |   6 +-
 242 files changed, 5684 insertions(+), 1394 deletions(-)

diff --cc include/tvm/runtime/module.h
index deb3deae42,60e3535319..9b67b12579
--- a/include/tvm/runtime/module.h
+++ b/include/tvm/runtime/module.h
@@@ -217,11 -216,7 +216,11 @@@ class TVM_DLL ModuleNode : public Objec
     * \param name name of the function.
     * \return The corresponding function.
     */
-   const PackedFunc* GetFuncFromEnv(const std::string& name);
+   const PackedFunc* GetFuncFromEnv(const String& name);
 +
 +  /*! \brief Clear all imports of the module. */
 +  void ClearImports() { imports_.clear(); }
 +
    /*! \return The module it imports from */
    const std::vector<Module>& imports() const { return imports_; }
  
diff --cc include/tvm/runtime/relax_vm/executable.h
index 5833cb4718,0000000000..d3bf9d356e
mode 100644,000000..100644
--- a/include/tvm/runtime/relax_vm/executable.h
+++ b/include/tvm/runtime/relax_vm/executable.h
@@@ -1,213 -1,0 +1,213 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +/*!
 + * \file tvm/runtime/relax_vm/executable.h
 + */
 +#ifndef TVM_RUNTIME_RELAX_VM_EXECUTABLE_H_
 +#define TVM_RUNTIME_RELAX_VM_EXECUTABLE_H_
 +
 +#include <tvm/runtime/container/closure.h>
 +#include <tvm/runtime/object.h>
 +#include <tvm/runtime/registry.h>
 +
 +#include <string>
 +#include <unordered_map>
 +#include <vector>
 +
 +#include "./bytecode.h"
 +
 +namespace tvm {
 +namespace runtime {
 +namespace relax_vm {
 +
 +/*!
 + * \brief Information entry in executable function table.
 + *
 + * Contains metadata about the compiled function, as
 + * well as the compiled VM instructions.
 + */
 +struct VMFuncInfo {
 +  /*! \brief kind of the function. */
 +  enum class FuncKind : int {
 +    /*! \brief system level packed function */
 +    kPackedFunc = 0,
 +    /*! \brief VM function. */
 +    kVMFunc = 1,
 +    /*! \brief VMTIR function. */
 +    kVMTIRFunc = 2,
 +  };
 +  /*! \brief The kind of function. */
 +  FuncKind kind;
 +  /*! \brief The function's name, global symbol */
 +  std::string name;
 +  /*! \brief The start instruction index of the function. */
 +  Index start_instr = 0;
 +  /*! \brief The end instruction index of the function. */
 +  Index end_instr = 0;
 +  /*! \brief The number of arguments of the function. */
 +  Index num_args = 0;
 +  /*! \brief The register file size of the function. */
 +  Index register_file_size = 0;
 +  /*! \brief The function parameter names.*/
 +  std::vector<std::string> param_names;
 +
 +  // defined customized loader save
 +  void Save(dmlc::Stream* writer) const;
 +  bool Load(dmlc::Stream* reader);
 +};
 +
 +/*!
 + * \brief The executable emitted by the VM compiler.
 + *
 + * The executable contains information (e.g. data in different memory regions)
 + * to run in a virtual machine.
 + */
 +class Executable : public runtime::ModuleNode {
 + public:
 +  /*!
 +   * \brief Get a PackedFunc from the executable module.
 +   * \param name the name of the function.
 +   * \param sptr_to_self The shared_ptr that points to this module node.
 +   * \return PackedFunc or nullptr when it is not available.
 +   */
-   PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final;
++  PackedFunc GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) final;
 +
 +  /*! \brief Get the property of the runtime module .*/
 +  int GetPropertyMask() const final { return ModulePropertyMask::kBinarySerializable; };
 +
 +  /*!
 +   * \brief Print the detailed statistics of the given code, i.e. number of
 +   * globals and constants, etc.
 +   * \return The statistics represented by a string.
 +   */
 +  std::string Stats() const;
 +  /*!
 +   * \brief Get the i-th instruction from the executable.
 +   * \param i The index of the instruction to be fetched.
 +   * \return The instruction.
 +   */
 +  Instruction GetInstruction(Index i) const;
 +  /*!
 +   * \brief Set j-th byte data of i-th instruction to val.
 +   * \param i The index of the instruction to be updated.
 +   * \param j The index of the byte data of the instruction to be updated.
 +   * \param val The value to be set
 +   */
 +  void SetInstructionData(Index i, Index j, ExecWord val);
 +  /*!
 +   * \brief Print the instructions as text format.
 +   * \return The text format of the instructions.
 +   */
 +  String AsText() const;
 +  /*!
 +   * \brief Print the instructions as python program.
 +   * \return The python program of the instructions, represented by a string.
 +   */
 +  String AsPython() const;
 +  /*!
 +   * \brief Write the Executable to the binary stream in serialized form.
 +   * \param stream The binary stream to save the executable to.
 +   */
 +  void SaveToBinary(dmlc::Stream* stream) final;
 +  /*!
 +   * \brief Load Executable from the binary stream in serialized form.
 +   * \param stream The binary stream that load the executable from.
 +   * \return The loaded executable, in the form of a `runtime::Module`.
 +   */
 +  static Module LoadFromBinary(void* stream);
 +  /*!
 +   * \brief Write the Executable to the provided path as a file containing its serialized content.
 +   * \param file_name The name of the file to write the serialized data to.
 +   * \param format The target format of the saved file.
 +   */
-   void SaveToFile(const std::string& file_name, const std::string& format) final;
++  void SaveToFile(const String& file_name, const String& format) final;
 +  /*!
 +   * \brief Load Executable from the file.
 +   * \param file_name The path of the file that load the executable from.
 +   * \return The loaded executable, in the form of a `runtime::Module`.
 +   */
-   static Module LoadFromFile(const std::string& file_name);
++  static Module LoadFromFile(const String& file_name);
 +
 +  /*! \brief The virtual machine's function table. */
 +  std::vector<VMFuncInfo> func_table;
 +  /*! \brief A map from globals (as strings) to their index in the function map. */
 +  std::unordered_map<std::string, Index> func_map;
 +  /*! \brief The global constant pool. */
 +  std::vector<TVMRetValue> constants;
 +  /*! \brief The offset of instruction. */
 +  std::vector<Index> instr_offset;
 +  /*! \brief The byte data of instruction. */
 +  std::vector<ExecWord> instr_data;
 +
 +  virtual ~Executable() {}
 +
 +  const char* type_key() const final { return "relax.Executable"; }
 +
 + private:
 +  /*!
 +   * \brief Save the globals.
 +   * \param strm The input stream.
 +   */
 +  void SaveGlobalSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Save the constant pool.
 +   * \param strm The input stream.
 +   */
 +  void SaveConstantSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Save the instructions.
 +   * \param strm The input stream.
 +   */
 +  void SaveCodeSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Save the packed functions.
 +   * \param strm The input stream.
 +   */
 +  void SavePackedFuncNames(dmlc::Stream* strm);
 +  /*!
 +   * \brief Load the globals.
 +   * \param strm The input stream.
 +   */
 +  void LoadGlobalSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Load the constant pool.
 +   * \param strm The input stream.
 +   */
 +  void LoadConstantSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Load the instructions.
 +   * \param strm The input stream.
 +   */
 +  void LoadCodeSection(dmlc::Stream* strm);
 +  /*!
 +   * \brief Save the packed functions.
 +   * \param strm The input stream.
 +   */
 +  void LoadPackedFuncNames(dmlc::Stream* strm);
 +};
 +
 +}  // namespace relax_vm
 +}  // namespace runtime
 +}  // namespace tvm
 +
 +namespace dmlc {
 +DMLC_DECLARE_TRAITS(has_saveload, ::tvm::runtime::relax_vm::VMFuncInfo, true);
 +}  // namespace dmlc
 +#endif  // TVM_RUNTIME_RELAX_VM_EXECUTABLE_H_
diff --cc python/tvm/contrib/cutlass/build.py
index 07583a4851,0fddee54e0..f25d5eeb95
--- a/python/tvm/contrib/cutlass/build.py
+++ b/python/tvm/contrib/cutlass/build.py
@@@ -72,9 -63,8 +70,9 @@@ def _get_cutlass_compile_options(sm, th
          "-Xcompiler=-fno-strict-aliasing",
          "-O3",
          "-std=c++17",
-         "-I" + cutlass_include,
-         "-I" + cutlass_util_include,
-         "-I" + cutlass_attention_include,
+         f"-I{cutlass_include}",
+         f"-I{cutlass_util_include}",
++        f"-I{cutlass_attention_include}",
      ]
      if use_fast_math:
          kwargs["options"].append("-DCUTLASS_USE_TANH_FOR_SIGMOID")
diff --cc python/tvm/contrib/cutlass/gen_conv2d.py
index 9e9e16426b,3887fc2e2e..3a9cbf1e84
--- a/python/tvm/contrib/cutlass/gen_conv2d.py
+++ b/python/tvm/contrib/cutlass/gen_conv2d.py
@@@ -185,13 -179,9 +185,13 @@@ class CutlassConv2DProfiler
      def __init__(self, sm, cutlass_path, binary_path):
          self.gemm_profiler = CutlassGemmProfiler(sm, cutlass_path, binary_path)
          self.sm = sm
-         assert sm in GENERATOR_FUNC_TABLE, "sm%d not supported yet." % sm
+         assert sm in GENERATOR_FUNC_TABLE, f"sm{sm} not supported yet."
          self.engine = ProfilerEngine(sm, cutlass_path, binary_path)
 -        self.cache = {}
 +        self.cache_path = os.path.join(binary_path, "cutlass_conv2d_cache.pickle")
 +        if os.path.exists(self.cache_path):
 +            self.cache = pickle.load(open(self.cache_path, "rb"))
 +        else:
 +            self.cache = {}
  
      def get_default(
          self,
diff --cc python/tvm/contrib/cutlass/gen_gemm.py
index 0ea6231b81,78e19f510d..a3204791f0
--- a/python/tvm/contrib/cutlass/gen_gemm.py
+++ b/python/tvm/contrib/cutlass/gen_gemm.py
@@@ -194,24 -149,13 +194,24 @@@ class CutlassGemmProfiler
      """Profile all candidate kernels and select the best one."""
  
      def __init__(self, sm, cutlass_path, binary_path):
-         assert sm in GENERATOR_FUNC_TABLE and sm in DEFAULT_KERNELS, "sm%d not supported yet." % sm
+         assert sm in GENERATOR_FUNC_TABLE and sm in DEFAULT_KERNELS, f"sm{sm} not supported yet."
          self.engine = ProfilerEngine(sm, cutlass_path, binary_path)
          self.sm = sm
 -        self.cache = {}
 +        self.cache_path = os.path.join(binary_path, "cutlass_gemm_cache.pickle")
 +        if os.path.exists(self.cache_path):
 +            self.cache = pickle.load(open(self.cache_path, "rb"))
 +        else:
 +            self.cache = {}
  
      def get_default(
 -        self, op_type, out_dtype, arg0_dtype, arg1_dtype, use_3xtf32=True, batched=False
 +        self,
 +        op_type,
 +        out_dtype,
 +        arg0_dtype,
 +        arg1_dtype,
 +        use_3xtf32=True,
 +        batched=False,
 +        layout_b=LayoutType.ColumnMajor,
      ):
          """Return the default kernel for the requested architecture.
          For now, the default kernel was picked arbitrary.
diff --cc python/tvm/contrib/cutlass/gen_tensor_op.py
index 0caf62043e,855d8dc2d1..be3bb289cf
--- a/python/tvm/contrib/cutlass/gen_tensor_op.py
+++ b/python/tvm/contrib/cutlass/gen_tensor_op.py
@@@ -96,14 -85,11 +90,12 @@@ def generate_sm50_simt(out_dtype, arg0_
                  DataType.f32,
                  DataType.f32,
                  DataType.f32,
 +                DataType.f32,
                  OpcodeClass.Simt,
                  MathOperation.multiply_add,
-             ),
-         ]
-         alignment_constraints = [
-             1,
+             )
          ]
+         alignment_constraints = [1]
          tile_descriptions = [
              ([128, 128, 8], 2, [4, 2, 1], min_cc, max_cc),
              ([128, 64, 8], 2, [2, 2, 1], min_cc, max_cc),
@@@ -687,100 -575,9 +668,100 @@@ def instantiate_template(func_name, ann
              attrs["split_k_slices"] = str(re.search(r"splitk(\d+)", op_name).group(1))
          else:
              attrs["split_k_mode"] = "kSerial"
 -            attrs["split_k_slices"] = "1"
 +            attrs["split_k_slices"] = 1
 +
 +        if "residual_shape" in annotations:
 +            attrs["residual_shape"] = annotations["residual_shape"]
 +
 +        code = instantiate_conv2d_template(attrs)
 +        return CodegenResult(code, headers)
  
 -        code = instantiate_conv2d_template(attrs, func_args)
 +    elif "attention" in func_name:
 +        headers.append("kernel_forward.h")
 +        data_type = dtype_map[annotations["arg0_dtype"]]
 +        attrs["data_type"] = DataTypeTag[data_type]
 +        attrs["num_batches"] = b = annotations["num_batches"]
 +        attrs["num_queries"] = s = annotations["num_queries"]
 +        attrs["num_keys"] = annotations["num_keys"]
 +        attrs["num_heads"] = n = annotations["num_heads"]
 +        attrs["head_dim"] = h = annotations["head_dim"]
 +        attrs["head_dim_value"] = h_v = annotations["head_dim_value"]
 +        data_type_size = DataTypeSize[data_type]
 +        if (data_type_size * h // 8) % 16 == 0 and (data_type_size * h_v // 8) % 16 == 0:
 +            attrs["kIsAligned"] = True
 +        elif (h % 4 == 0) and (h_v % 4 == 0):
 +            attrs["kIsAligned"] = False
 +        else:
 +            raise NotImplementedError()
 +        if h_v > 64:
 +            attrs["kQueriesPerBlock"] = 32
 +            attrs["kKeysPerBlock"] = 128
 +            attrs["kSingleValueIteration"] = h_v <= 128
 +        else:
 +            attrs["kQueriesPerBlock"] = 64
 +            attrs["kKeysPerBlock"] = 64
 +            attrs["kSingleValueIteration"] = True
 +        attrs["output_size"] = b * s * n * h_v
 +        attrs["scale"] = (
 +            float(1 / math.sqrt(h.value)) if annotations["scale"] is None else annotations["scale"]
 +        )
 +        attrs["custom_mask_type"] = annotations["custom_mask_type"]
 +
 +        assert (
 +            attrs["scale"] > 0 or attrs["scale"] < 0
 +        ), "Cutlass may generate nan occasionally when scale == 0.0"
 +        attrs["arch"] = "cutlass::arch::Sm{}".format(annotations["arch"])
 +        attrs["kSupportsDropout"] = False
 +        attrs["qkv_layout"] = annotations["qkv_layout"]
 +
 +        for arg in func_args:
 +            if "workspace" in arg:
 +                attrs["workspace"] = arg
 +
 +        if attrs["qkv_layout"] == "default":
 +            attrs["query"] = func_args[0]
 +            attrs["key"] = func_args[1]
 +            attrs["value"] = func_args[2]
 +            if len(func_args) > 4:  # +1 for workspace, the last arg
 +                attrs["bias"] = func_args[3]
 +        elif attrs["qkv_layout"] == "qkv_stacked":
 +            attrs["qkv"] = func_args[0]
 +            if len(func_args) > 5:  # +1 for workspace, the last arg
 +                attrs["bias"] = func_args[4]
 +        else:
 +            raise NotImplementedError()
 +        if "bias" in attrs:
 +            attrs["kSupportsBias"] = True
 +            if len(annotations["bias_shape"]) == 4:
 +                strides = "p.num_keys"
 +                if annotations["bias_shape"][2] == 1:
 +                    attrs["bias_strideM"] = 0
 +                else:
 +                    attrs["bias_strideM"] = strides
 +                    strides = f"p.num_queries * {strides}"
 +                if annotations["bias_shape"][1] == 1:
 +                    attrs["bias_strideH"] = 0
 +                else:
 +                    attrs["bias_strideH"] = strides
 +                    strides = f"p.num_heads * {strides}"
 +                if annotations["bias_shape"][0] == 1:
 +                    attrs["bias_strideB"] = 0
 +                else:
 +                    attrs["bias_strideB"] = strides
 +            else:
 +                raise NotImplementedError()
 +        else:
 +            # To support negative scale in current Cutlass implementation,
 +            # kSupportsBias should be set true, or there are nan's as result.
 +            attrs["kSupportsBias"] = attrs["scale"] < 0
 +        code = instantiate_attention_template(attrs)
 +        return CodegenResult(code, headers)
 +    elif "layer_norm" in func_name:
 +        headers.append("cutlass/util/device_layernorm.h")
 +        headers.append("cutlass/layout/matrix.h")
 +        attrs = {"input": func_args[0], "gamma": func_args[1], "beta": func_args[2]}
 +        attrs.update(dict(annotations))
 +        code = instantiate_layer_norm_template(attrs)
          return CodegenResult(code, headers)
  
-     raise ValueError("Do not have a template for {}".format(func_name))
+     raise ValueError(f"Do not have a template for {func_name}")
diff --cc python/tvm/contrib/cutlass/library.py
index ead5804b59,dff166fb58..2b8bb830e1
--- a/python/tvm/contrib/cutlass/library.py
+++ b/python/tvm/contrib/cutlass/library.py
@@@ -145,13 -139,7 +141,13 @@@ def substitute_template(template, value
      while changed:
          changed = False
          for key, value in values.items():
 +            if isinstance(value, (int, IntImm)):
 +                value = str(int(value))
 +            if isinstance(value, (float, FloatImm)):
 +                value = str(float(value))
 +            elif isinstance(value, bool):
 +                value = str(value).lower()
-             regex = "\\$\\{%s\\}" % key
+             regex = f"\\$\\{{{key}\\}}"
              newtext = re.sub(regex, value, text)
              if newtext != text:
                  changed = True
diff --cc python/tvm/rpc/proxy.py
index a6709a3421,687d47264b..b14b02eb56
--- a/python/tvm/rpc/proxy.py
+++ b/python/tvm/rpc/proxy.py
@@@ -267,14 -252,9 +265,14 @@@ class ProxyServerHandler(object)
                  )
                  logging.info("Serving RPC index html page at http://localhost:%d", web_port)
              resource_files = resource_files if resource_files else []
 -            for fname in resource_files:
 +            for item in resource_files:
 +                prefix, fname = item
 +                if not prefix.endswith("/"):
 +                    prefix += "/"
 +                if not prefix.startswith("/"):
 +                    prefix = "/" + prefix
                  basename = os.path.basename(fname)
-                 pair = (r"%s%s" % (prefix, basename), RequestHandler, {"file_path": fname})
 -                pair = (rf"/{basename}", RequestHandler, {"file_path": fname})
++                pair = (rf"{prefix}{basename}", RequestHandler, {"file_path": fname})
                  handlers.append(pair)
                  logging.info(pair)
              self.app = tornado.web.Application(handlers)
diff --cc src/runtime/relax_vm/executable.cc
index 2090a3b254,0000000000..5abed9bbbb
mode 100644,000000..100644
--- a/src/runtime/relax_vm/executable.cc
+++ b/src/runtime/relax_vm/executable.cc
@@@ -1,583 -1,0 +1,583 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +/*!
 + * \file src/runtime/relax_vm/executable.cc
 + */
 +
 +#include <dmlc/memory_io.h>
 +#include <tvm/runtime/logging.h>
 +#include <tvm/runtime/relax_vm/executable.h>
 +#include <tvm/runtime/relax_vm/vm.h>
 +
 +#include <functional>
 +#include <sstream>
 +
 +#include "../file_utils.h"
 +
 +namespace tvm {
 +namespace runtime {
 +namespace relax_vm {
 +
 +/*! \brief The magic number for the serialized VM bytecode file  */
 +constexpr uint64_t kTVMVMBytecodeMagic = 0xD225DE2F4214151D;
 +
 +/*! \brief Possible types in the constant pool */
 +enum ConstantType : int {
 +  kNDArray = 0,
 +  kDLDataType = 1,
 +  kShapeTuple = 2,
 +  kString = 3,
 +  kInt = 4,
 +};
 +
 +#define STREAM_CHECK(val, section)                                          \
 +  ICHECK(val) << "Invalid VM file format in the " << section << " section." \
 +              << "\n";
 +
- PackedFunc Executable::GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) {
++PackedFunc Executable::GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) {
 +  if (name == "stats") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->Stats(); });
 +  } else if (name == "as_text") {
 +    return PackedFunc(
 +        [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->AsText(); });
 +  } else if (name == "as_python") {
 +    return PackedFunc(
 +        [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->AsPython(); });
 +  } else if (name == "vm_load_executable") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      ObjectPtr<VirtualMachine> vm = VirtualMachine::Create();
 +      ICHECK(sptr_to_self.get() == this);
 +      vm->LoadExecutable(GetObjectPtr<Executable>(this));
 +      *rv = Module(vm);
 +    });
 +  } else if (name == "vm_profiler_load_executable") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      ObjectPtr<VirtualMachine> vm = VirtualMachine::CreateProfiler();
 +      ICHECK(sptr_to_self.get() == this);
 +      vm->LoadExecutable(GetObjectPtr<Executable>(this));
 +      *rv = Module(vm);
 +    });
 +  }
 +  return nullptr;
 +}
 +
 +std::string Executable::Stats() const {
 +  std::ostringstream oss;
 +  oss << "Relax VM executable statistics:" << std::endl;
 +
 +  // Get the number of constants.
 +  // If the constant is an NDArray, get the shape of each of them.
 +  // If the constant is an DLDataType, get the data type of each of them.
 +  oss << "  Constant pool (# " << constants.size() << "): [";
 +  for (const auto& it : constants) {
 +    if (it.IsObjectRef<runtime::NDArray>()) {
 +      const auto ndarray = it.operator tvm::runtime::NDArray();
 +      const auto& shape = ndarray.Shape();
 +      // Scalar
 +      if (shape.empty()) {
 +        oss << "scalar, ";
 +        continue;
 +      }
 +      oss << "[";
 +      for (auto s : shape) {
 +        oss << s << ", ";
 +      }
 +      oss.seekp(-2, oss.cur);
 +      oss << "], ";
 +    } else if (it.IsObjectRef<ShapeTuple>()) {
 +      ShapeTuple shape = it.operator ShapeTuple();
 +      oss << "shapetuple[";
 +      for (size_t i = 0; i < shape.size(); ++i) {
 +        oss << shape.at(i) << ", ";
 +      }
 +      oss.seekp(-2, oss.cur);
 +      oss << "], ";
 +    } else if (it.IsObjectRef<String>()) {
 +      std::string f = it.AsObjectRef<tvm::runtime::String>().operator std::string();
 +      oss << "\"";
 +      oss << f;
 +      oss << "\", ";
 +    } else if (it.type_code() == kDLInt) {
 +      oss << static_cast<int64_t>(it);
 +      oss << ", ";
 +    } else {
 +      try {
 +        DLDataType dtype = it.operator DLDataType();
 +        oss << dtype;
 +        oss << ", ";
 +      } catch (std::exception& exc) {
 +        LOG(FATAL) << "Constant pool can only contain NDArray and DLDataType, but got "
 +                   << ArgTypeCode2Str(it.type_code());
 +      }
 +    }
 +  }
 +  if (!constants.empty()) oss.seekp(-2, oss.cur);
 +  oss << "]" << std::endl;
 +
 +  // Get the number of globals and the name of each of them.
 +  oss << "  Globals (#" << func_table.size() << "): [";
 +  for (const auto& it : func_table) {
 +    oss << it.name << ", ";
 +  }
 +  if (!func_map.empty()) oss.seekp(-2, oss.cur);
 +  oss << "]" << std::endl;
 +
 +  return oss.str();
 +}
 +
 +void Executable::SetInstructionData(Index i, Index j, ExecWord val) {
 +  ICHECK_LT(i, instr_offset.size());
 +  Index instr_idx = instr_offset[i];
 +  ICHECK_LT(instr_idx + j, instr_data.size());
 +  instr_data[instr_idx + j] = val;
 +}
 +
 +Instruction Executable::GetInstruction(Index i) const {
 +  Index offset = instr_offset[i];
 +  Opcode op = static_cast<Opcode>(instr_data[offset]);
 +  switch (op) {
 +    case Opcode::Call: {
 +      RegName dst = instr_data[offset + 1];
 +      Index func_idx = instr_data[offset + 2];
 +      Index num_args = instr_data[offset + 3];
 +      ExecWord* args = const_cast<ExecWord*>(&instr_data[offset + 4]);
 +      return Instruction::Call(func_idx, num_args, reinterpret_cast<Instruction::Arg*>(args), dst);
 +    }
 +    case Opcode::Ret: {
 +      RegName result = instr_data[offset + 1];
 +      return Instruction::Ret(result);
 +    }
 +    case Opcode::Goto: {
 +      Index pc_offset = instr_data[offset + 1];
 +      return Instruction::Goto(pc_offset);
 +    }
 +    case Opcode::If: {
 +      RegName cond = instr_data[offset + 1];
 +      Index false_offset = instr_data[offset + 2];
 +      return Instruction::If(cond, false_offset);
 +    }
 +    default:
 +      LOG(FATAL) << "should never hit this case: " << static_cast<int>(op);
 +      break;
 +  }
 +  return Instruction();
 +}
 +
 +void SaveHeader(dmlc::Stream* strm) {
 +  uint64_t header = kTVMVMBytecodeMagic;
 +  strm->Write(header);
 +  std::string version = TVM_VERSION;
 +  strm->Write(version);
 +}
 +
 +void LoadHeader(dmlc::Stream* strm) {
 +  // Check header.
 +  uint64_t header;
 +  STREAM_CHECK(strm->Read(&header), "header");
 +  STREAM_CHECK(header == kTVMVMBytecodeMagic, "header");
 +
 +  // Check version.
 +  std::string version;
 +  STREAM_CHECK(strm->Read(&version), "version");
 +  STREAM_CHECK(version == TVM_VERSION, "version");
 +}
 +
 +void Executable::SaveToBinary(dmlc::Stream* stream) {
 +  std::string code;
 +  // Initialize the stream object.
 +  dmlc::MemoryStringStream strm(&code);
 +
 +  // Save header
 +  SaveHeader(&strm);
 +
 +  // Global section.
 +  SaveGlobalSection(&strm);
 +
 +  // Constant section.
 +  SaveConstantSection(&strm);
 +
 +  // Code section.
 +  SaveCodeSection(&strm);
 +
 +  stream->Write(code);
 +}
 +
- void Executable::SaveToFile(const std::string& file_name, const std::string& format) {
++void Executable::SaveToFile(const String& file_name, const String& format) {
 +  std::string data;
 +  dmlc::MemoryStringStream writer(&data);
 +  dmlc::SeekStream* strm = &writer;
 +  Executable::SaveToBinary(strm);
 +  runtime::SaveBinaryToFile(file_name, data);
 +}
 +
 +Module Executable::LoadFromBinary(void* stream) {
 +  std::string code;
 +  static_cast<dmlc::Stream*>(stream)->Read(&code);
 +  dmlc::MemoryStringStream strm(&code);
 +
 +  ObjectPtr<Executable> exec = make_object<Executable>();
 +
 +  // Load header.
 +  LoadHeader(&strm);
 +
 +  // Global section.
 +  exec->LoadGlobalSection(&strm);
 +
 +  // Constant section.
 +  exec->LoadConstantSection(&strm);
 +
 +  // Code section.
 +  exec->LoadCodeSection(&strm);
 +
 +  return Module(exec);
 +}
 +
 +TVM_REGISTER_GLOBAL("runtime.module.loadbinary_relax.Executable")
 +    .set_body_typed(Executable::LoadFromBinary);
 +
- Module Executable::LoadFromFile(const std::string& file_name) {
++Module Executable::LoadFromFile(const String& file_name) {
 +  std::string data;
 +  runtime::LoadBinaryFromFile(file_name, &data);
 +  dmlc::MemoryStringStream reader(&data);
 +  dmlc::Stream* strm = &reader;
 +  return Executable::LoadFromBinary(reinterpret_cast<void*>(strm));
 +}
 +
 +TVM_REGISTER_GLOBAL("runtime.module.loadfile_relax.Executable")
 +    .set_body_typed(Executable::LoadFromFile);
 +
 +void VMFuncInfo::Save(dmlc::Stream* strm) const {
 +  int32_t temp_kind = static_cast<int32_t>(kind);
 +  strm->Write(temp_kind);
 +  strm->Write(name);
 +  strm->Write(start_instr);
 +  strm->Write(end_instr);
 +  strm->Write(num_args);
 +  strm->Write(register_file_size);
 +  strm->Write(param_names);
 +}
 +
 +bool VMFuncInfo::Load(dmlc::Stream* strm) {
 +  int32_t temp_kind;
 +  if (!strm->Read(&temp_kind)) return false;
 +  this->kind = static_cast<VMFuncInfo::FuncKind>(temp_kind);
 +  if (!strm->Read(&name)) return false;
 +  if (!strm->Read(&start_instr)) return false;
 +  if (!strm->Read(&end_instr)) return false;
 +  if (!strm->Read(&num_args)) return false;
 +  if (!strm->Read(&register_file_size)) return false;
 +  if (!strm->Read(&param_names)) return false;
 +  return true;
 +}
 +
 +void Executable::SaveGlobalSection(dmlc::Stream* strm) { strm->Write(func_table); }
 +
 +void Executable::SaveConstantSection(dmlc::Stream* strm) {
 +  strm->Write(static_cast<uint64_t>(this->constants.size()));
 +  for (const auto& it : this->constants) {
 +    if (it.IsObjectRef<runtime::NDArray>()) {
 +      strm->Write(ConstantType::kNDArray);
 +      runtime::SaveDLTensor(strm, it.operator DLTensor*());
 +    } else if (it.IsObjectRef<ShapeTuple>()) {
 +      ShapeTuple shape = it.operator ShapeTuple();
 +      strm->Write(ConstantType::kShapeTuple);
 +      strm->Write(shape.size());
 +      for (size_t i = 0; i < shape.size(); ++i) {
 +        strm->Write(shape.at(i));
 +      }
 +    } else if (it.IsObjectRef<String>()) {
 +      String str = it.operator String();
 +      strm->Write(ConstantType::kString);
 +      strm->Write(str.size());
 +      for (size_t i = 0; i < str.size(); ++i) {
 +        strm->Write(str.at(i));
 +      }
 +    } else if (it.type_code() == kDLInt) {
 +      strm->Write(ConstantType::kInt);
 +      strm->Write(it.value());
 +    } else {
 +      try {
 +        strm->Write(ConstantType::kDLDataType);
 +        strm->Write(it.operator DLDataType());
 +      } catch (std::exception& exc) {
 +        LOG(FATAL) << "Constant pool can only contain NDArray, DLDataType, and Integers but got "
 +                   << ArgTypeCode2Str(it.type_code());
 +      }
 +    }
 +  }
 +}
 +
 +void Executable::SaveCodeSection(dmlc::Stream* strm) {
 +  strm->Write(instr_offset);
 +  strm->Write(instr_data);
 +}
 +
 +void Executable::LoadGlobalSection(dmlc::Stream* strm) {
 +  STREAM_CHECK(strm->Read(&func_table), "Global Section");
 +  // setup func map
 +  for (size_t i = 0; i < func_table.size(); ++i) {
 +    this->func_map[func_table[i].name] = i;
 +  }
 +}
 +
 +void Executable::LoadConstantSection(dmlc::Stream* strm) {
 +  uint64_t sz;
 +  // Load the number of constants.
 +  STREAM_CHECK(strm->Read(&sz, sizeof(sz)), "constant");
 +
 +  size_t size = static_cast<size_t>(sz);
 +  runtime::NDArray ndarray;
 +  DLDataType dtype;
 +  // Load each of the constants.
 +  for (size_t i = 0; i < size; i++) {
 +    int constant_type;
 +    STREAM_CHECK(strm->Read(&constant_type, sizeof(constant_type)), "constant");
 +    if (constant_type == ConstantType::kNDArray) {
 +      ndarray.Load(strm);
 +      TVMRetValue cell;
 +      cell = ndarray;
 +      this->constants.push_back(cell);
 +    } else if (constant_type == ConstantType::kShapeTuple) {
 +      uint64_t size;
 +      strm->Read(&size);
 +      std::vector<ShapeTuple::index_type> data(size);
 +      for (size_t i = 0; i < size; ++i) {
 +        strm->Read(&(data[i]));
 +      }
 +      TVMRetValue cell;
 +      cell = ShapeTuple(data);
 +      this->constants.push_back(cell);
 +    } else if (constant_type == ConstantType::kDLDataType) {
 +      strm->Read(&dtype);
 +      TVMRetValue cell;
 +      cell = dtype;
 +      this->constants.push_back(cell);
 +    } else if (constant_type == ConstantType::kString) {
 +      uint64_t size;
 +      strm->Read(&size);
 +      std::vector<char> data(size);
 +      for (size_t i = 0; i < size; ++i) {
 +        strm->Read(&(data[i]));
 +      }
 +      TVMRetValue cell;
 +      cell = String(std::string(data.begin(), data.end()));
 +      this->constants.push_back(cell);
 +    } else if (constant_type == ConstantType::kInt) {
 +      int64_t value;
 +      strm->Read(&value);
 +      TVMRetValue cell;
 +      cell = value;
 +      this->constants.push_back(cell);
 +    } else {
 +      LOG(FATAL) << "Constant pool can only contain NDArray and DLDataType, but got "
 +                 << ArgTypeCode2Str(constant_type) << " when loading the VM constant pool.";
 +    }
 +  }
 +}
 +
 +void Executable::LoadCodeSection(dmlc::Stream* strm) {
 +  STREAM_CHECK(strm->Read(&(this->instr_offset)), "instr offset");
 +  STREAM_CHECK(strm->Read(&(this->instr_data)), "instr data");
 +}
 +
 +template <typename T>
 +std::string StrJoin(T* items, int offset, int cnt, std::string delim = ", ",
 +                    std::function<std::string(T)> repr = std::to_string) {
 +  if (cnt == 0) {
 +    return "";
 +  }
 +  std::ostringstream oss;
 +  oss << repr(items[offset]);
 +  for (int i = 1; i < cnt; ++i) {
 +    oss << delim << repr(items[offset + i]);
 +  }
 +  return oss.str();
 +}
 +
 +std::string RegNameToStr(RegName reg) {
 +  if (reg == Instruction::kVoidRegister) {
 +    return "%void";
 +  }
 +  if (reg == Instruction::kVMRegister) {
 +    return "%vm";
 +  }
 +  return "%" + std::to_string(reg);
 +}
 +
 +String Executable::AsText() const {
 +  auto get_func_name = [&](Index index) -> std::string {
 +    if (static_cast<size_t>(index) < func_table.size()) {
 +      return func_table[index].name;
 +    } else {
 +      return "unknown_func_index(" + std::to_string(index) + ")";
 +    }
 +  };
 +
 +  auto instr_to_str = [&](Instruction::Arg arg) -> std::string {
 +    // only for argument
 +    switch (arg.kind()) {
 +      case Instruction::ArgKind::kRegister:
 +        return RegNameToStr(arg.value());
 +      case Instruction::ArgKind::kImmediate:
 +        return "i" + std::to_string(arg.value());
 +      case Instruction::ArgKind::kConstIdx:
 +        return "c[" + std::to_string(arg.value()) + "]";
 +      case Instruction::ArgKind::kFuncIdx:
 +        return "f[" + get_func_name(arg.value()) + "]";
 +      default:
 +        LOG(FATAL) << "Wrong instruction kind: " << static_cast<int>(arg.kind());
 +        return "";
 +    }
 +  };
 +
 +  // print the text format
 +  std::ostringstream os;
 +  for (size_t fidx = 0; fidx < this->func_table.size(); ++fidx) {
 +    const VMFuncInfo& gfunc = this->func_table[fidx];
 +    if (gfunc.kind == VMFuncInfo::FuncKind::kPackedFunc) {
 +      os << "@" << gfunc.name << " packed_func;\n\n";
 +      continue;
 +    }
 +    if (gfunc.kind == VMFuncInfo::FuncKind::kVMTIRFunc) {
 +      os << "@" << gfunc.name << " num_inputs=" << gfunc.num_args << " vm_tir_func;\n\n";
 +      continue;
 +    }
 +    ICHECK(gfunc.kind == VMFuncInfo::FuncKind::kVMFunc);
 +    os << "@" << gfunc.name << ":\n";
 +    size_t start_instr = gfunc.start_instr;
 +    size_t end_instr = gfunc.end_instr;
 +
 +    for (size_t idx = start_instr; idx < end_instr; ++idx) {
 +      os << "  ";
 +      Instruction instr = this->GetInstruction(idx);
 +      switch (instr.op) {
 +        case Opcode::Call: {
 +          os << std::setw(6) << std::left << "call" << std::setw(16) << std::left
 +             << get_func_name(instr.func_idx) << " in: " << std::setw(12) << std::left
 +             << StrJoin<Instruction::Arg>(instr.args, 0, instr.num_args, ", ", instr_to_str)
 +             << " dst: " << RegNameToStr(instr.dst) << "\n";
 +          break;
 +        }
 +        case Opcode::Ret: {
 +          os << std::setw(6) << std::left << "ret " << RegNameToStr(instr.result) << "\n";
 +          break;
 +        }
 +        case Opcode::Goto: {
 +          os << std::setw(6) << std::left << "goto" << instr.pc_offset << "\n";
 +          break;
 +        }
 +        case Opcode::If: {
 +          os << std::setw(6) << std::left << "If" << RegNameToStr(instr.cond) << ", "
 +             << instr.false_offset << "\n";
 +          break;
 +        }
 +        default:
 +          LOG(FATAL) << "should never hit this case: " << static_cast<int>(instr.op);
 +          break;
 +      }
 +    }
 +    os << "\n";
 +  }
 +  return String(os.str());
 +}
 +
 +String Executable::AsPython() const {
 +  auto get_func_name = [&](Index index) -> std::string {
 +    if (static_cast<size_t>(index) < func_table.size()) {
 +      return "\"" + func_table[index].name + "\"";
 +    } else {
 +      return "ib.unknown_func_index(" + std::to_string(index) + ")";
 +    }
 +  };
 +
 +  auto arg_to_py_str = [&](Instruction::Arg arg) -> std::string {
 +    switch (arg.kind()) {
 +      case Instruction::ArgKind::kRegister:
 +        if (arg.value() == Instruction::kVMRegister) {
 +          return "ib.r(vm)";
 +        }
 +        return "ib.r(" + std::to_string(arg.value()) + ")";
 +      case Instruction::ArgKind::kImmediate:
 +        return "ib.imm(" + std::to_string(arg.value()) + ")";
 +      case Instruction::ArgKind::kConstIdx:
 +        return "ib.c(" + std::to_string(arg.value()) + ")";
 +      case Instruction::ArgKind::kFuncIdx: {
 +        return "ib.f(" + get_func_name(arg.value()) + ")";
 +      }
 +      default:
 +        LOG(FATAL) << "Wrong instruction kind: " << static_cast<int>(arg.kind());
 +        return "";
 +    }
 +  };
 +
 +  // print the python format
 +  std::ostringstream os;
 +  os << "ib = rx.Builder()\n";
 +  for (size_t fidx = 0; fidx < this->func_table.size(); ++fidx) {
 +    const VMFuncInfo& gfunc = this->func_table[fidx];
 +    if (gfunc.kind == VMFuncInfo::FuncKind::kPackedFunc) {
 +      continue;
 +    }
 +    if (gfunc.kind == VMFuncInfo::FuncKind::kVMTIRFunc) {
 +      continue;
 +    }
 +    ICHECK(gfunc.kind == VMFuncInfo::FuncKind::kVMFunc);
 +
 +    os << "with ib.function(\"" << gfunc.name << "\", num_inputs=" << gfunc.num_args << "):\n";
 +    size_t start_instr = gfunc.start_instr;
 +    size_t end_instr = gfunc.end_instr;
 +
 +    for (size_t idx = start_instr; idx < end_instr; ++idx) {
 +      Instruction instr = this->GetInstruction(idx);
 +      switch (instr.op) {
 +        case Opcode::Call: {
 +          os << "    ib.emit_call(" << get_func_name(instr.func_idx) << ", args=["
 +             << StrJoin<Instruction::Arg>(instr.args, 0, instr.num_args, ", ", arg_to_py_str)
 +             << "]";
 +          if (instr.dst != Instruction::kVoidRegister) os << ", dst=ib.r(" << instr.dst << ")";
 +          os << ")\n";
 +          break;
 +        }
 +        case Opcode::Ret: {
 +          os << "    ib.emit_ret(ib.r(" << instr.result << "))\n";
 +          break;
 +        }
 +        case Opcode::Goto: {
 +          os << "    ib.emit_goto(" << instr.pc_offset << ")\n";
 +          break;
 +        }
 +        case Opcode::If: {
 +          os << "    ib.emit_if(ib.r(" << instr.cond << "), " << instr.false_offset << ")\n";
 +          break;
 +        }
 +        default:
 +          LOG(FATAL) << "should never hit this case: " << static_cast<int>(instr.op);
 +          break;
 +      }
 +    }
 +  }
 +  return String(os.str());
 +}
 +
 +TVM_REGISTER_GLOBAL("relax.ExecutableLoadFromFile").set_body_typed(Executable::LoadFromFile);
 +
 +}  // namespace relax_vm
 +}  // namespace runtime
 +}  // namespace tvm
diff --cc src/runtime/relax_vm/ndarray_cache_support.cc
index 410859d7e1,0000000000..7d1d1fc59e
mode 100644,000000..100644
--- a/src/runtime/relax_vm/ndarray_cache_support.cc
+++ b/src/runtime/relax_vm/ndarray_cache_support.cc
@@@ -1,233 -1,0 +1,233 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +/*!
 + * \file src/runtime/relax_vm/ndarray_cache_support.cc
 + * \brief Runtime to support ndarray cache file loading.
 + *
 + * This file provides a minimum support for ndarray cache file loading.
 + *
 + * The main focus of this implementation is to enable loading
 + * with minimum set of intermediate files while also being
 + * compatible to some of the multi-shard files that are more
 + * friendly in some of the environments.
 + *
 + * NDArray cache also provides a way to do system-wide
 + * parameter sharing across multiple VMs.
 + *
 + * There are likely other ways to load the parameters ndarray-ache.
 + * We will keep the impact minimum by puting it as a private
 + * runtime builtin provide as in this file.
 + */
 +#define PICOJSON_USE_INT64
 +
 +#include <picojson.h>
 +#include <tvm/runtime/ndarray.h>
 +#include <tvm/runtime/registry.h>
 +
 +#include <sstream>
 +#include <string>
 +#include <vector>
 +
 +#include "../../support/utils.h"
 +#include "../file_utils.h"
 +
 +namespace tvm {
 +namespace runtime {
 +namespace relax_vm {
 +
 +/*!
 + * A NDArray cache to store pre-loaded arrays in the system.
 + */
 +class NDArrayCache {
 + public:
 +  static NDArrayCache* Global() {
 +    static NDArrayCache* inst = new NDArrayCache();
 +    return inst;
 +  }
 +
 +  static void Update(String name, NDArray arr, bool override) {
 +    NDArrayCache* pool = Global();
 +    if (!override) {
 +      ICHECK_EQ(pool->pool_.count(name), 0) << "Name " << name << " already exists in the cache";
 +    }
 +    pool->pool_.Set(name, arr);
 +  }
 +
 +  static Optional<NDArray> Get(String name) {
 +    NDArrayCache* pool = Global();
 +    auto it = pool->pool_.find(name);
 +    if (it != pool->pool_.end()) {
 +      return (*it).second;
 +    } else {
 +      return NullOpt;
 +    }
 +  }
 +
 +  static void Remove(String name) {
 +    NDArrayCache* pool = Global();
 +    pool->pool_.erase(name);
 +  }
 +
 +  static void Clear() { Global()->pool_.clear(); }
 +
 +  /*!
 +   * \brief Load parameters from path and append them.
 +   *
 +   * \param cache_path The cache to path.
 +   * \param device_type The type of device to be loaded.
 +   * \param device_id The device id.
 +   */
 +  static void Load(const std::string& cache_path, int device_type, int device_id) {
 +    DLDevice device{static_cast<DLDeviceType>(device_type), device_id};
 +    std::string json_str;
 +    LoadBinaryFromFile(cache_path + "/ndarray-cache.json", &json_str);
 +    picojson::value json_info;
 +    picojson::parse(json_info, json_str);
 +    auto shard_records = json_info.get<picojson::object>()["records"].get<picojson::array>();
 +
 +    Map<String, NDArray> result;
 +    std::string raw_data;
 +    Optional<NDArray> staging_buffer;
 +
 +    auto fcopy_param_from_bytes = [&](NDArray param, void* data, size_t nbytes) {
 +      if (device_type != kDLOpenCL) {
 +        param.CopyFromBytes(data, nbytes);
 +      }
 +      // special handle OpenCL
 +      // OpenCL runtime can create a host side memory mirror
 +      // for every cl_mem that tries to copy data from host
 +      // which can cause memory issue.
 +      // We use a single staging buffer here
 +      // that get de-allocated later
 +      if (staging_buffer.defined()) {
 +        size_t curr_size = runtime::GetDataSize(*(staging_buffer.value().operator->()));
 +        if (curr_size < nbytes) {
 +          staging_buffer = NullOpt;
 +        }
 +      }
 +      if (!staging_buffer.defined()) {
 +        staging_buffer = NDArray::Empty(param.Shape(), param->dtype, param->device);
 +      }
 +      NDArray staging_view = staging_buffer.value().CreateView(param.Shape(), param->dtype);
 +      staging_view.CopyFromBytes(data, nbytes);
 +      param.CopyFrom(staging_view);
 +      TVMSynchronize(device_type, device_id, nullptr);
 +    };
 +
 +    for (auto shard_item : shard_records) {
 +      auto shard_rec = shard_item.get<picojson::object>();
 +      ICHECK(shard_rec["dataPath"].is<std::string>());
 +      std::string data_path = shard_rec["dataPath"].get<std::string>();
 +
 +      LoadBinaryFromFile(cache_path + "/" + data_path, &raw_data);
 +      CHECK_EQ(shard_rec["format"].get<std::string>(), "raw-shard");
 +      int64_t raw_nbytes = shard_rec["nbytes"].get<int64_t>();
 +      CHECK_EQ(raw_nbytes, raw_data.length())
 +          << "ValueError: Parameters are not loaded properly. Please check your parameter shards "
 +             "and git lfs installation";
 +
 +      for (auto nd_item : shard_rec["records"].get<picojson::array>()) {
 +        auto nd_rec = nd_item.get<picojson::object>();
 +        CHECK(nd_rec["name"].is<std::string>());
 +        String name = nd_rec["name"].get<std::string>();
 +
 +        std::vector<int64_t> shape;
 +        for (auto value : nd_rec["shape"].get<picojson::array>()) {
 +          shape.push_back(value.get<int64_t>());
 +        }
 +
 +        DataType dtype(String2DLDataType(nd_rec["dtype"].get<std::string>()));
 +        std::string encode_format = nd_rec["format"].get<std::string>();
 +        int64_t offset = nd_rec["byteOffset"].get<int64_t>();
 +        int64_t nbytes = nd_rec["nbytes"].get<int64_t>();
 +        NDArray arr = NDArray::Empty(ShapeTuple(shape.begin(), shape.end()), dtype, device);
 +
 +        if (dtype == DataType::Float(32) && encode_format == "f32-to-bf16") {
 +          // decode bf16 to f32
 +          std::vector<uint16_t> buffer(nbytes / 2);
 +          std::vector<uint32_t> decoded(nbytes / 2);
 +          std::memcpy(buffer.data(), raw_data.data() + offset, nbytes);
 +          for (size_t i = 0; i < buffer.size(); ++i) {
 +            decoded[i] = static_cast<uint32_t>(buffer[i]) << 16;
 +          }
 +          fcopy_param_from_bytes(arr, decoded.data(), decoded.size() * sizeof(uint32_t));
 +        } else {
 +          fcopy_param_from_bytes(arr, raw_data.data() + offset, nbytes);
 +        }
 +        Update(name, arr, true);
 +      }
 +    }
 +  }
 +
 + private:
 +  Map<String, NDArray> pool_;
 +};
 +
 +TVM_REGISTER_GLOBAL("vm.builtin.ndarray_cache.get").set_body_typed(NDArrayCache::Get);
 +TVM_REGISTER_GLOBAL("vm.builtin.ndarray_cache.update").set_body_typed(NDArrayCache::Update);
 +TVM_REGISTER_GLOBAL("vm.builtin.ndarray_cache.remove").set_body_typed(NDArrayCache::Remove);
 +TVM_REGISTER_GLOBAL("vm.builtin.ndarray_cache.clear").set_body_typed(NDArrayCache::Clear);
 +TVM_REGISTER_GLOBAL("vm.builtin.ndarray_cache.load").set_body_typed(NDArrayCache::Load);
 +
 +// This param module node can be useful to get param dict in RPC mode
 +// when the remote already have loaded parameters from file.
 +class ParamModuleNode : public runtime::ModuleNode {
 + public:
 +  const char* type_key() const final { return "param_module"; }
 +
-   PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final {
++  PackedFunc GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) final {
 +    if (name == "get_params") {
 +      auto params = params_;
 +      return PackedFunc([params](TVMArgs args, TVMRetValue* rv) { *rv = params; });
 +    } else {
 +      return PackedFunc();
 +    }
 +  }
 +
-   static Array<NDArray> GetParams(const std::string& prefix, int num_params) {
++  static Array<NDArray> GetParams(const String& prefix, int num_params) {
 +    Array<NDArray> params;
 +    for (int i = 0; i < num_params || num_params == -1; ++i) {
 +      std::string name = prefix + "_" + std::to_string(i);
 +      auto opt = NDArrayCache::Get(name);
 +      if (opt) {
 +        params.push_back(opt.value());
 +      } else {
 +        if (num_params == -1) return params;
 +        LOG(FATAL) << "Cannot find " << name << " in cache";
 +      }
 +    }
 +    return params;
 +  }
 +
 +  static Module Create(const std::string& prefix, int num_params) {
 +    auto n = make_object<ParamModuleNode>();
 +    n->params_ = GetParams(prefix, num_params);
 +    return Module(n);
 +  }
 +
 + private:
 +  Array<NDArray> params_;
 +};
 +
 +TVM_REGISTER_GLOBAL("vm.builtin.param_module_from_cache").set_body_typed(ParamModuleNode::Create);
 +TVM_REGISTER_GLOBAL("vm.builtin.param_array_from_cache").set_body_typed(ParamModuleNode::GetParams);
 +
 +}  // namespace relax_vm
 +}  // namespace runtime
 +}  // namespace tvm
diff --cc src/runtime/relax_vm/vm.cc
index 01497fdd7c,0000000000..9c25ca5307
mode 100644,000000..100644
--- a/src/runtime/relax_vm/vm.cc
+++ b/src/runtime/relax_vm/vm.cc
@@@ -1,1038 -1,0 +1,1038 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +
 +/*!
 + * \file src/runtime/relax_vm/vm.cc
 + */
 +
 +#include <tvm/runtime/packed_func.h>
 +#include <tvm/runtime/profiling.h>
 +#include <tvm/runtime/relax_vm/vm.h>
 +
 +#include <optional>
 +
 +namespace tvm {
 +namespace runtime {
 +namespace relax_vm {
 +
 +//---------------------------------------------
 +// VM Closure object
 +//---------------------------------------------
 +TVM_REGISTER_OBJECT_TYPE(VMClosureObj);
 +
 +VMClosure::VMClosure(String func_name, PackedFunc impl) {
 +  auto ptr = make_object<VMClosureObj>();
 +  ptr->func_name = func_name;
 +  ptr->impl = std::move(impl);
 +  data_ = std::move(ptr);
 +}
 +
 +/*!
 + * \brief Create another PackedFunc with last arguments already bound to last_args.
 + * \param func The input func, can be a VMClosure or PackedFunc.
 + * \param last_args The arguments to bound to in the end of the function.
 + * \note The new function takes in arguments and append the last_args in the end.
 + */
 +PackedFunc VMClosure::BindLastArgs(PackedFunc func, std::vector<TVMRetValue> last_args) {
 +  return PackedFunc([func, last_args](TVMArgs args, TVMRetValue* rv) {
 +    std::vector<TVMValue> values(args.size() + last_args.size());
 +    std::vector<int> tcodes(args.size() + last_args.size());
 +    runtime::TVMArgsSetter setter(values.data(), tcodes.data());
 +    std::copy(args.values, args.values + args.size(), values.data());
 +    std::copy(args.type_codes, args.type_codes + args.size(), tcodes.data());
 +    for (size_t i = 0; i < last_args.size(); ++i) {
 +      setter(i + args.size(), last_args[i]);
 +    }
 +    func.CallPacked(TVMArgs(values.data(), tcodes.data(), values.size()), rv);
 +  });
 +}
 +
 +//-----------------------------------------------------------
 +// Utility functions.
 +//-----------------------------------------------------------
 +// Use the args after `starting_arg_idx` as a series of indices into `obj`,
 +// indexing into nested ADTs and returning the final indexed object.
 +ObjectRef IndexIntoNestedObject(ObjectRef obj, TVMArgs args, int starting_arg_idx) {
 +  for (int i = starting_arg_idx; i < args.size(); i++) {
 +    // the object must be an ADT to be able to index into it
 +    if (!obj.as<ArrayNode>()) {
 +      LOG(FATAL) << "ValueError: Attempted to index into an object that is not an ADT.";
 +    }
 +    int index = args[i];
 +    auto arr = Downcast<Array<ObjectRef>>(obj);
 +    // make sure the index is in bounds
 +    if (index >= static_cast<int>(arr.size())) {
 +      LOG(FATAL) << "IndexError: Invalid index (" << index << " >= " << arr.size() << ").";
 +    }
 +    obj = arr[index];
 +  }
 +  return obj;
 +}
 +
 +NDArray ConvertNDArrayToDevice(NDArray src, const DLDevice& dev, Allocator* alloc) {
 +  if (src->device.device_type == dev.device_type && src->device.device_id == dev.device_id) {
 +    return src;
 +  } else {
 +    auto res = alloc->Empty(src.Shape(), src->dtype, dev);
 +    res.CopyFrom(src);
 +    return res;
 +  }
 +}
 +
 +ObjectRef ConvertObjectToDevice(ObjectRef src, const Device& dev, Allocator* alloc) {
 +  if (src->IsInstance<NDArray::ContainerType>()) {
 +    return ConvertNDArrayToDevice(Downcast<NDArray>(src), dev, alloc);
 +  } else if (src->IsInstance<ArrayNode>()) {
 +    std::vector<ObjectRef> ret;
 +    auto arr = Downcast<Array<ObjectRef>>(src);
 +    for (size_t i = 0; i < arr.size(); i++) {
 +      ret.push_back(ConvertObjectToDevice(arr[i], dev, alloc));
 +    }
 +    return Array<ObjectRef>(ret.begin(), ret.end());
 +  } else {
 +    return src;
 +  }
 +}
 +
 +TVMRetValue ConvertArgToDevice(TVMArgValue input, Device dev, Allocator* alloc) {
 +  // NOTE: NDArray::FromExternalDLTensor is not safe
 +  // in terms of memory-behavior.
 +  // To be extra careful, we copy DLTensor.
 +  // The developer can still explicitly allocate NDArray
 +  // in TVM Native API or NDArray::FromDLPack to regain zero copy behavior.
 +  TVMRetValue ret;
 +
 +  if (input.type_code() == kTVMDLTensorHandle) {
 +    DLTensor* tensor = input;
 +    std::vector<int64_t> shape(tensor->shape, tensor->shape + tensor->ndim);
 +    auto dst = alloc->Empty(shape, tensor->dtype, dev);
 +    dst.CopyFrom(tensor);
 +    ret = dst;
 +  } else if (input.IsObjectRef<ObjectRef>()) {
 +    ret = ConvertObjectToDevice(input.operator ObjectRef(), dev, alloc);
 +  } else {
 +    ret = input;
 +  }
 +  return ret;
 +}
 +
 +TVMRetValue ConvertRegToDevice(TVMRetValue input, Device dev, Allocator* alloc) {
 +  TVMRetValue ret;
 +  if (input.IsObjectRef<ObjectRef>()) {
 +    ret = ConvertObjectToDevice(input.operator ObjectRef(), dev, alloc);
 +  } else {
 +    ret = input;
 +  }
 +  return ret;
 +}
 +
 +//-----------------------------------------------------------
 +// VM implementations.
 +//-----------------------------------------------------------
 +/*!
 + * \brief The register type.
 + */
 +using RegType = TVMRetValue;
 +
 +/*!
 + * \brief A representation of a stack frame.
 + *
 + * A stack frame is a record containing the information needed
 + * to restore the caller's virtual machine state after returning
 + * from a function call.
 + */
 +struct VMFrame {
 +  /*! \brief The return program counter. */
 +  Index return_pc;
 +  /*! \brief Statically allocated space for objects */
 +  std::vector<RegType> register_file;
 +  /*! \brief Register in caller's frame to put return value */
 +  RegName caller_return_register;
 +  // The following fields are used for PackedFunc call within
 +  // a single function scope. The space is reused across multiple
 +  // packed func calls to increase cache locality and avoid re-allocation
 +  /*! \brief Temporary argument value stack for packed func call. */
 +  std::vector<TVMValue> call_arg_values;
 +  /*! \brief Temporary argument tcode stack for packed func call. */
 +  std::vector<int> call_arg_tcodes;
 +
 +  VMFrame(Index pc, Index register_file_size)
 +      : return_pc(pc), register_file(register_file_size), caller_return_register(0) {}
 +};
 +
 +class VirtualMachineImpl : public VirtualMachine {
 + public:
 +  //---------------------------------------------------
 +  // Public facing functions overloading
 +  //---------------------------------------------------
 +  void LoadExecutable(ObjectPtr<Executable> exec) final;
 +
 +  void Init(const std::vector<Device>& devices,
 +            const std::vector<AllocatorType>& alloc_types) final;
 +
-   PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) override;
++  PackedFunc GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) override;
 +
 +  VMClosure GetClosure(const String& func_name) final {
 +    return this->GetClosureInternal(func_name, false).value();
 +  }
 +
 +  void InvokeClosurePacked(const ObjectRef& closure_or_packedfunc, TVMArgs args,
 +                           TVMRetValue* rv) final;
 +
 +  void SetInstrument(PackedFunc instrument) final { this->instrument_ = instrument; }
 +
 +  //--------------------------------------------------
 +  // Additional support arguments functions for VM
 +  //--------------------------------------------------
 +  /*!
 +   * \brief Internal implementation of GetClosure which also allow none.
 +   * \param func_name The name of the function.
 +   * \param allow_missing Whether none is allowed.
 +   * \return The result
 +   */
 +  Optional<VMClosure> GetClosureInternal(const String& func_name, bool allow_missing);
 +
 +  /*!
 +   * \brief Set inputs to a function.
 +   * \param func_name The function name.
 +   * \param args args[offset:] are arguments to the function. If the arguments are not of the
 +   * correct device for the function, they will be copied to the device.
 +   * \param offset Starting offset of the arguments in \p args.
 +   * \param with_param_module If set to true, the last argument will be a module and can be invoked
 +   *        to get the argument, this is mainly used for debugging purposes and setting composite
 +   * objects. \note This interface works when using VM over RPC by internally converting NDArray in
 +   * the arguments to DLTensor, which is supported in RPC where remote could only have a minimal C
 +   * runtime.
 +   */
 +  void SetInput(std::string func_name, TVMArgs args, int offset, bool with_param_module = false);
 +
 +  /*!
 +   * \brief Look up whether the VM has a function by the given name.
 +   * \param func_name the function's name
 +   * \return The function, if it exists. Logs a fatal error if not.
 +   */
 +  VMFuncInfo LookupVMFuncInfo(const std::string& func_name);
 +
 +  /*!
 +   * \brief Look up whether the VM has outputs for the given function.
 +   * \param func_name the function's name
 +   * \return The output, if it exists. Logs a fatal error if not.
 +   */
 +  RegType LookupVMOutput(const std::string& func_name);
 +
 +  /*!
 +   * \brief Fully bind the argument of a global function and save it in the env.
 +   * \param func_name The global function name to be saved.
 +   * \param save_name The saved name of the function.
 +   * \param include_return Whether forward the return value, set it to false allows
 +   *        us to ignore forwarding return value, which can be helpful to do benchmarking
 +   *        in RPC environment when return value is complicated ADT.
 +   *
 +   * \param args The arguments to bound to the function.
 +   * \note This function is used by RPC server to help benchmarking.
 +   */
 +  void SaveClosure(const String& func_name, const String& save_name, bool include_return,
 +                   TVMArgs args);
 +  /*!
 +   * \brief Internal function to invoke a closure.
 +   * \param closure_or_packed The closure to be invoked.
 +   * \param args The arguments to the function.
 +   * \return The result value.
 +   */
 +  RegType InvokeClosureInternal(const ObjectRef& closure_or_packed,
 +                                const std::vector<RegType>& args);
 +  /*!
 +   * \brief Invoke a VM function by interpreting bytecode.
 +   * \param fidx The function index.
 +   * \param args The arguments to the function.
 +   * \return The object representing the result.
 +   */
 +  RegType InvokeBytecode(Index fidx, const std::vector<RegType>& args);
 +
 + protected:
 +  /*!
 +   * \brief Get function by querying all of the current module's imports.
 +   * \param name The name of the function.
 +   * \return The result function, can return PackedFunc(nullptr) if nothing is found.
 +   */
 +  PackedFunc GetFuncFromImports(const String& name) {
 +    for (auto& lib : this->imports_) {
 +      PackedFunc func = lib->GetFunction(name, true);
 +      if (func.defined()) return func;
 +    }
 +    return PackedFunc(nullptr);
 +  }
 +  /*!
 +   * \brief Initialize function pool.
 +   */
 +  void InitFuncPool();
 +  //-------------------------------------------------
 +  // Instruction interpretations.
 +  //-------------------------------------------------
 +  /*!
 +   * \brief Push a call frame onto the call stack.
 +   * \param ret_pc The program counter to return to.
 +   * \param vm_func The function to be pushed to the call stack.
 +   */
 +  void PushFrame(Index ret_pc, const VMFuncInfo& vm_func) {
 +    frames_.emplace_back(std::make_unique<VMFrame>(ret_pc, vm_func.register_file_size));
 +  }
 +  /*!
 +   * \brief Pop a frame off the call stack.
 +   */
 +  void PopFrame() {
 +    ICHECK_GT(frames_.size(), 0);
 +    pc_ = frames_.back()->return_pc;
 +    frames_.pop_back();
 +  }
 +  /*!
 +   * \brief Write to a VM register.
 +   * \param frame current vm frame.
 +   * \param reg The register to write to.
 +   * \param obj The object to write to.
 +   */
 +  void WriteRegister(VMFrame* frame, RegName reg, const RegType& obj) {
 +    ICHECK_LT(reg, frame->register_file.size());
 +    frame->register_file[reg] = obj;
 +  }
 +  /*!
 +   * \brief Read a VM register.
 +   * \param frame current vm frame.
 +   * \param reg The register to read from.
 +   * \return The value of the register.
 +   */
 +  RegType ReadRegister(VMFrame* frame, RegName reg) {
 +    if (reg < Instruction::kBeginSpecialReg) {
 +      return frame->register_file[reg];
 +    }
 +    RegType ret;
 +    if (reg == Instruction::kVoidRegister) {
 +      ret = nullptr;
 +    } else {
 +      ICHECK_EQ(reg, Instruction::kVMRegister);
 +      // per convention, ctx ptr must be VirtualMachine* casted to void.
 +      // this and VirtualMachine* may or may not be the same
 +      // do first cast to VirtualMachine* then to void*
 +      ret = static_cast<void*>(static_cast<VirtualMachine*>(this));
 +    }
 +    return ret;
 +  }
 +  /*!
 +   * \brief Run call instruction.
 +   * \param curr_frame The current frame.
 +   * \param inst The call instruction.
 +   */
 +  virtual void RunInstrCall(VMFrame* curr_frame, Instruction inst);
 +
 +  /*! \brief Run VM dispatch loop. */
 +  void RunLoop();
 +
 +  /*!
 +   * \brief Retrieve the name of the function identified by the given index.
 +   * \param idx The index into the VM executable function table.
 +   * \return The name of the function.
 +   */
 +  const std::string& GetFuncName(int idx) { return exec_->func_table[idx].name; }
 +
 +  /*!
 +   * \brief Retrieve the inputs for a function.
 +   * \param func_name The name of the function.
 +   * \return The function inputs.
 +   */
 +  const std::vector<RegType>& GetInputsFor(const std::string& func_name) {
 +    return inputs_[func_name];
 +  }
 +
 +  void ClearInputsFor(const std::string& func_name) { inputs_.erase(func_name); }
 +
 +  //--------------------------------------------------------
 +  // Internal states for execution.
 +  //--------------------------------------------------------
 +  /*! \brief The loaded executable. */
 +  ObjectPtr<Executable> exec_;
 +  /*! \brief The global constant pool */
 +  std::vector<TVMRetValue> const_pool_;
 +  /*!
 +   * \brief Function pool to cache functions in func_table
 +   */
 +  std::vector<TVMRetValue> func_pool_;
 +  //--------------------------------------------------------
 +  // Executor interface support
 +  //--------------------------------------------------------
 +  /*! \brief The function name to input register mapping. */
 +  std::unordered_map<std::string, std::vector<RegType>> inputs_;
 +  /*! \brief The function name to output register. */
 +  std::unordered_map<std::string, RegType> outputs_;
 +  /*! \brief A store of closures created by `save_function`. */
 +  std::unordered_map<std::string, VMClosure> saved_closures_;
 +  //------------------------------------------------------------
 +  // VM Instruction execution.
 +  //------------------------------------------------------------
 +  /*!
 +   * \brief The current stack of call frames.
 +   * \note: Use unique ptr to avoid re-allocation and copy when frames_ get resized.
 +   */
 +  std::vector<std::unique_ptr<VMFrame>> frames_;
 +  /*! \brief The virtual machine PC. */
 +  Index pc_{0};
 +  /*! \brief The special return register. */
 +  RegType return_value_;
 +  /*!\ brief instrument function. */
 +  PackedFunc instrument_ = nullptr;
 +};
 +
 +void VirtualMachineImpl::LoadExecutable(ObjectPtr<Executable> exec) {
 +  this->exec_ = exec;
 +  this->imports_ = exec_->imports();
 +}
 +
 +void VirtualMachineImpl::Init(const std::vector<Device>& devices,
 +                              const std::vector<AllocatorType>& alloc_types) {
 +  // TODO(@yuchen): support multi-device heterogeneous execution
 +  ICHECK_LT(devices.size(), 3)
 +      << "Currently relax vm only supports at most 2 devices (host + device)";
 +  ICHECK_EQ(devices.size(), alloc_types.size());
 +
 +  this->devices.reserve(devices.size());
 +  this->allocators.reserve(alloc_types.size());
 +  for (size_t i = 0; i < devices.size(); i++) {
 +    auto alloc = MemoryManager::GetOrCreateAllocator(devices[i], alloc_types[i]);
 +    this->devices.push_back(devices[i]);
 +    this->allocators.push_back(alloc);
 +  }
 +  // Setup constant sections.
 +  this->const_pool_.reserve(exec_->constants.size());
 +  for (const auto& constant : exec_->constants) {
 +    if (constant.type_code() != kTVMNDArrayHandle) {
 +      this->const_pool_.push_back(constant);
 +    } else {
 +      this->const_pool_.push_back(ConvertRegToDevice(constant, devices[0], allocators[0]));
 +    }
 +  }
 +  // Setup function sections.
 +  this->InitFuncPool();
 +}
 +
 +VMFuncInfo VirtualMachineImpl::LookupVMFuncInfo(const std::string& func_name) {
 +  ICHECK(exec_) << "The executable is not created yet.";
 +  auto it = this->exec_->func_map.find(func_name);
 +  CHECK(it != this->exec_->func_map.end()) << "ValueError: Unknown function: " << func_name;
 +
 +  return exec_->func_table[it->second];
 +}
 +
 +RegType VirtualMachineImpl::LookupVMOutput(const std::string& func_name) {
 +  if (!outputs_.count(func_name)) {
 +    LOG(FATAL) << "ValueError: No output saved for call of \"" << func_name
 +               << "\"; use `invoke_stateful` to call it first.";
 +  }
 +  return outputs_[func_name];
 +}
 +
- PackedFunc VirtualMachineImpl::GetFunction(const std::string& name,
++PackedFunc VirtualMachineImpl::GetFunction(const String& name,
 +                                           const ObjectPtr<Object>& sptr_to_self) {
 +  if (name == "vm_initialization") {
 +    // initialize the VirtualMachine, takes variable-length arguments
 +    // first argument is a runtime::Module, followed by one or more device_type, device_id,
 +    // and the AllocatorType associated with the device.
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      ICHECK_EQ(args.size() % 3, 0);
 +      std::vector<Device> devices;
 +      std::vector<AllocatorType> alloc_types;
 +      for (int i = 0; i < args.size(); i += 3) {
 +        Device dev;
 +        int device_type = args[i];
 +        dev.device_type = DLDeviceType(device_type);
 +        dev.device_id = args[i + 1];
 +        int type = args[i + 2];
 +        devices.push_back(dev);
 +        alloc_types.push_back(AllocatorType(type));
 +      }
 +      this->Init(devices, alloc_types);
 +    });
 +  } else if (name == "save_function") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      ICHECK_GE(args.size(), 3);
 +      this->SaveClosure(args[0], args[1], args[2],
 +                        TVMArgs(args.values + 3, args.type_codes + 3, args.size() - 3));
 +    });
 +  } else if (name == "invoke_closure") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      VMClosure clo = args[0];
 +      this->InvokeClosurePacked(clo, TVMArgs(args.values + 1, args.type_codes + 1, args.size() - 1),
 +                                rv);
 +    });
 +  } else if (name == "set_instrument") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      PackedFunc func;
 +      if (args[0].type_code() != kTVMPackedFuncHandle) {
 +        String func_name = args[0];
 +        const PackedFunc* factory = Registry::Get(func_name);
 +        ICHECK(factory != nullptr) << "Cannot find factory " << func_name;
 +        TVMRetValue rv;
 +        factory->CallPacked(TVMArgs(args.values + 1, args.type_codes + 1, args.num_args - 1), &rv);
 +        func = rv;
 +      } else {
 +        func = args[0];
 +      }
 +      this->SetInstrument(func);
 +    });
 +  } else if (name == "invoke_stateful") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      std::string func_name = args[0];
 +      const auto& m = this->exec_->func_map;
 +      if (m.find(func_name) == m.end()) {
 +        LOG(FATAL) << "ValueError: Unknown function: " << func_name;
 +      }
 +      Index gf_idx = m.at(func_name);
 +      if (!inputs_.count(func_name)) {
 +        LOG(FATAL) << "ValueError: No inputs set for stateful call of " << func_name
 +                   << "; use `set_input` first.";
 +        return;
 +      }
 +      outputs_[func_name] = this->InvokeClosureInternal(func_pool_[gf_idx], inputs_[func_name]);
 +    });
 +  } else if (name == "get_output_arity") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      std::string func_name = args[0];
 +      RegType out = LookupVMOutput(func_name);
 +      // use remaining args as indices
 +      ObjectRef obj = IndexIntoNestedObject(out.AsObjectRef<ObjectRef>(), args, 1);
 +      // after chasing through the indices, examine the final object
 +      if (const auto* arr = obj.as<ArrayNode>()) {
 +        *rv = static_cast<int>(arr->size());
 +      } else {
 +        *rv = -1;
 +      }
 +    });
 +  } else if (name == "get_output") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      std::string func_name = args[0];
 +      RegType out = LookupVMOutput(func_name);
 +      // use remaining args as indices
 +      ObjectRef obj = IndexIntoNestedObject(out.AsObjectRef<ObjectRef>(), args, 1);
 +      if (obj.as<ArrayNode>()) {
 +        LOG(FATAL) << "ValueError: `get_output` cannot return a tuple for RPC compatibility. "
 +                      "Please specify another index argument.";
 +        return;
 +      }
 +      *rv = obj;
 +    });
 +  } else if (name == "set_input") {
 +    return PackedFunc(
 +        [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { SetInput(args[0], args, 1); });
 +  } else if (name == "set_input_with_param_module") {
 +    return PackedFunc(
 +        [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { SetInput(args[0], args, 1, true); });
 +  } else if (name == "get_function_arity") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      std::string func_name = args[0];
 +      const VMFuncInfo& vm_func = LookupVMFuncInfo(func_name);
 +      *rv = static_cast<int>(vm_func.param_names.size());
 +    });
 +  } else if (name == "get_function_param_name") {
 +    return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +      std::string func_name = args[0];
 +      int index = args[1];
 +      const VMFuncInfo& vm_func = LookupVMFuncInfo(func_name);
 +      if (static_cast<size_t>(index) >= vm_func.param_names.size()) {
 +        LOG(FATAL) << "ValueError: Invalid index for " << func_name << " (" << index << " out of "
 +                   << vm_func.param_names.size() << ")";
 +      }
 +      *rv = vm_func.param_names[index];
 +    });
 +  } else {
 +    // default case, look up closure in VM.
 +    if (Optional<VMClosure> opt = this->GetClosureInternal(name, true)) {
 +      auto clo = opt.value();
 +      return PackedFunc([sptr_to_self, this, clo](TVMArgs args, TVMRetValue* rv) {
 +        this->InvokeClosurePacked(clo, args, rv);
 +      });
 +    } else {
 +      return PackedFunc(nullptr);
 +    }
 +  }
 +}
 +
 +void VirtualMachineImpl::SetInput(std::string func_name, TVMArgs args, int offset,
 +                                  bool with_param_module) {
 +  const auto& m = exec_->func_map;
 +  if (m.find(func_name) != m.end()) {
 +    Index gf_idx = m.at(func_name);
 +    const VMFuncInfo& vm_func = exec_->func_table[gf_idx];
 +    size_t params_num = vm_func.num_args;
 +    ICHECK_EQ(args.size() - offset, params_num)
 +        << "The number of provided parameters doesn't match the number of arguments for";
 +    std::vector<RegType> func_args(params_num);
 +
 +    for (int i = offset; i < args.size(); ++i) {
 +      int index = i - offset;
 +      if (with_param_module && i == args.size() - 1) {
 +        // call param func to get the arguments(usually corresponds to param pack.)
 +        func_args[index] = (args[i].operator Module()).GetFunction("get_params")();
 +      } else {
 +        func_args[index] = ConvertArgToDevice(args[i], devices[0], allocators[0]);
 +      }
 +    }
 +    inputs_[func_name] = func_args;
 +  } else {
 +    LOG(FATAL) << "ValueError: Unknown function: " << func_name;
 +  }
 +}
 +
 +//------------------------------------------
 +// Closure handling
 +//------------------------------------------
 +void VirtualMachineImpl::InvokeClosurePacked(const ObjectRef& closure_or_packedfunc, TVMArgs args,
 +                                             TVMRetValue* rv) {
 +  // run packed call if it is a packed func.
 +  if (auto* packed = closure_or_packedfunc.as<PackedFunc::ContainerType>()) {
 +    packed->CallPacked(args, rv);
 +    return;
 +  }
 +  // run closure call.
 +  auto* clo = closure_or_packedfunc.as<VMClosureObj>();
 +  ICHECK(clo != nullptr) << "Function expects a closure or PackedFunc ";
 +
 +  std::vector<TVMValue> values(args.size() + 1);
 +  std::vector<int> tcodes(args.size() + 1);
 +  runtime::TVMArgsSetter setter(values.data(), tcodes.data());
 +  // per convention, ctx ptr must be VirtualMachine* casted to void.
 +  // this and VirtualMachine* may or maynot be the same
 +  // do first cast to VirtualMachine* then to void*
 +  setter(0, static_cast<void*>(static_cast<VirtualMachine*>(this)));
 +  std::copy(args.values, args.values + args.size(), values.begin() + 1);
 +  std::copy(args.type_codes, args.type_codes + args.size(), tcodes.begin() + 1);
 +  clo->impl.CallPacked(TVMArgs(values.data(), tcodes.data(), args.size() + 1), rv);
 +}
 +
 +// internal variant version of invoke closurepacked
 +RegType VirtualMachineImpl::InvokeClosureInternal(const ObjectRef& closure_or_packed,
 +                                                  const std::vector<RegType>& args) {
 +  RegType ret;
 +  auto* packed = closure_or_packed.as<PackedFunc::ContainerType>();
 +  auto* clo = closure_or_packed.as<VMClosureObj>();
 +  int clo_offset = clo != nullptr ? 1 : 0;
 +  std::vector<TVMValue> values(args.size() + clo_offset);
 +  std::vector<int> tcodes(args.size() + clo_offset);
 +  runtime::TVMArgsSetter setter(values.data(), tcodes.data());
 +
 +  if (clo != nullptr) {
 +    setter(0, static_cast<void*>(static_cast<VirtualMachine*>(this)));
 +  }
 +  for (size_t i = 0; i < args.size(); ++i) {
 +    setter(i + clo_offset, args[i]);
 +  }
 +
 +  if (packed != nullptr) {
 +    packed->CallPacked(TVMArgs(values.data(), tcodes.data(), values.size()), &ret);
 +  } else {
 +    ICHECK(clo != nullptr);
 +    clo->impl.CallPacked(TVMArgs(values.data(), tcodes.data(), values.size()), &ret);
 +  }
 +  return ret;
 +}
 +
 +void VirtualMachineImpl::SaveClosure(const String& func_name, const String& save_name,
 +                                     bool include_return, TVMArgs args) {
 +  VMClosure clo = this->GetClosure(func_name);
 +  std::vector<RegType> inputs(args.size());
 +  for (int i = 0; i < args.size(); ++i) {
 +    inputs[i] = ConvertArgToDevice(args[i], this->devices[0], this->allocators[0]);
 +  }
 +  PackedFunc impl = VMClosure::BindLastArgs(clo->impl, inputs);
 +  if (!include_return) {
 +    impl = PackedFunc([impl](TVMArgs args, TVMRetValue* rv) {
 +      TVMRetValue temp;
 +      impl.CallPacked(args, &temp);
 +    });
 +  }
 +  saved_closures_[save_name] = VMClosure(save_name, impl);
 +}
 +
 +Optional<VMClosure> VirtualMachineImpl::GetClosureInternal(const String& func_name,
 +                                                           bool allow_missing) {
 +  // look up saved closures.
 +  auto saved_it = saved_closures_.find(func_name);
 +  if (saved_it != saved_closures_.end()) {
 +    return saved_it->second;
 +  }
 +  auto it = exec_->func_map.find(func_name);
 +  if (it == exec_->func_map.end()) {
 +    if (allow_missing) return NullOpt;
 +    LOG(FATAL) << "ValueError: Unknown function: " << func_name;
 +  }
 +
 +  Index gf_idx = it->second;
 +  const VMFuncInfo& finfo = exec_->func_table[gf_idx];
 +
 +  if (finfo.kind == VMFuncInfo::FuncKind::kVMFunc) {
 +    // NOTE: should not capture strong ref to self and avoid cyclic ref.
 +    auto impl = PackedFunc([gf_idx](TVMArgs args, TVMRetValue* rv) {
 +      // Per convention, ctx ptr is a VirtualMachine*
 +      VirtualMachine* ctx_ptr = static_cast<VirtualMachine*>(args[0].operator void*());
 +
 +      std::vector<RegType> inputs(args.size() - 1);
 +      for (size_t i = 0; i < inputs.size(); ++i) {
 +        inputs[i] = args[i + 1];
 +      }
 +      *rv = static_cast<VirtualMachineImpl*>(ctx_ptr)->InvokeBytecode(gf_idx, inputs);
 +    });
 +    return VMClosure(func_name, impl);
 +  } else {
 +    ICHECK(finfo.kind == VMFuncInfo::FuncKind::kVMTIRFunc)
 +        << "Cannot support closure with function kind " << static_cast<int>(finfo.kind);
 +    PackedFunc tir_func = GetFuncFromImports("__vmtir__" + finfo.name);
 +    ICHECK(tir_func != nullptr) << "Cannot find underlying compiled tir function of VMTIRFunc "
 +                                << finfo.name;
 +    auto impl = PackedFunc([this, finfo, tir_func](TVMArgs args, TVMRetValue* rv) {
 +      // Per convention, ctx ptr is a VirtualMachine*
 +      VirtualMachine* ctx_ptr = static_cast<VirtualMachine*>(args[0].operator void*());
 +      ICHECK(ctx_ptr == this);
 +      ICHECK_EQ(args.size() - 1, finfo.num_args)
 +          << "Function " << finfo.name << " expects " << finfo.num_args << " arguments";
 +      ICHECK_GE(finfo.register_file_size, finfo.num_args + 1);
 +      std::vector<TVMRetValue> reg_file(finfo.register_file_size);
 +      for (int64_t i = 0; i < finfo.num_args; ++i) {
 +        reg_file[i] = args[i + 1];
 +      }
 +      void* reg_anylist_handle = reg_file.data();
 +      void* const_anylist_handle = this->const_pool_.data();
 +      void* func_anylist_handle = this->func_pool_.data();
 +      tir_func(static_cast<void*>(ctx_ptr), reg_anylist_handle, const_anylist_handle,
 +               func_anylist_handle);
 +      // Return value always stored after inputs.
 +      *rv = reg_file[finfo.num_args];
 +    });
 +    return VMClosure(func_name, impl);
 +  }
 +}
 +
 +//--------------------------------------------------------------------
 +// Instruction interpretations.
 +//--------------------------------------------------------------------
 +RegType VirtualMachineImpl::InvokeBytecode(Index gf_idx, const std::vector<RegType>& args) {
 +  const VMFuncInfo& gfunc = exec_->func_table[gf_idx];
 +  ICHECK(gfunc.kind == VMFuncInfo::FuncKind::kVMFunc);
 +
 +  // Get the curr instr which might be a potential caller.
 +  Instruction curr_instr = exec_->GetInstruction(pc_);
 +  PushFrame(this->pc_, gfunc);
 +  // Get new frame and set the caller info.
 +  VMFrame* curr_frame = frames_.back().get();
 +  if (curr_instr.op == Opcode::Call) {
 +    curr_frame->caller_return_register = curr_instr.dst;
 +  }
 +
 +  // load arguments to the register file
 +  ICHECK_EQ(static_cast<size_t>(gfunc.num_args), args.size())
 +      << "ValueError: Invoking function " << gfunc.name << " requires " << gfunc.num_args
 +      << " inputs but only " << args.size() << " inputs are provided.";
 +  for (size_t i = 0; i < args.size(); ++i) {
 +    WriteRegister(frames_.back().get(), i, args[i]);
 +  }
 +  // set program counter
 +  pc_ = gfunc.start_instr;
 +  RunLoop();
 +  return return_value_;
 +}
 +
 +void VirtualMachineImpl::InitFuncPool() {
 +  func_pool_.resize(exec_->func_table.size());
 +
 +  for (size_t func_index = 0; func_index < exec_->func_table.size(); ++func_index) {
 +    const VMFuncInfo& info = exec_->func_table[func_index];
 +    if (info.kind == VMFuncInfo::FuncKind::kPackedFunc) {
 +      // only look through imports first
 +      PackedFunc func = GetFuncFromImports(info.name);
 +      if (!func.defined()) {
 +        const PackedFunc* p_func = Registry::Get(info.name);
 +        if (p_func != nullptr) func = *(p_func);
 +      }
 +      ICHECK(func.defined())
 +          << "Error: Cannot find PackedFunc " << info.name
 +          << " in either Relax VM kernel library, or in TVM runtime PackedFunc registry, or in "
 +             "global Relax functions of the VM executable";
 +      func_pool_[func_index] = func;
 +
 +    } else {
 +      ICHECK(info.kind == VMFuncInfo::FuncKind::kVMFunc ||
 +             info.kind == VMFuncInfo::FuncKind::kVMTIRFunc);
 +      auto clo = this->GetClosure(info.name);
 +      func_pool_[func_index] = clo;
 +    }
 +  }
 +}
 +
 +void VirtualMachineImpl::RunInstrCall(VMFrame* curr_frame, Instruction instr) {
 +  DLOG(INFO) << "\n  pc = " << pc_ << ", execute: " << GetFuncName(instr.func_idx);
 +  int args_begin_offset = instrument_ != nullptr ? 4 : 0;
 +  // Use the call arg stack from the current frame to increase reuse
 +  // and avoid re-allocation
 +  curr_frame->call_arg_values.resize(args_begin_offset + instr.num_args);
 +  curr_frame->call_arg_tcodes.resize(args_begin_offset + instr.num_args);
 +
 +  // NOTE: no changes and resize to those vector ref(otherwise can leads to segfault)
 +  //       in the remainder part of the function.
 +  std::vector<TVMValue>& values = curr_frame->call_arg_values;
 +  std::vector<int>& tcodes = curr_frame->call_arg_tcodes;
 +
 +  runtime::TVMArgsSetter setter(values.data(), tcodes.data());
 +  for (Index i = 0; i < instr.num_args; ++i) {
 +    Instruction::Arg arg = instr.args[i];
 +    int arg_index = args_begin_offset + i;
 +    switch (arg.kind()) {
 +      case Instruction::ArgKind::kRegister: {
 +        setter(arg_index, ReadRegister(curr_frame, arg.value()));
 +        break;
 +      }
 +      case Instruction::ArgKind::kImmediate: {
 +        setter(arg_index, arg.value());
 +        break;
 +      }
 +      case Instruction::ArgKind::kConstIdx: {
 +        setter(arg_index, this->const_pool_[arg.value()]);
 +        break;
 +      }
 +      case Instruction::ArgKind::kFuncIdx: {
 +        ICHECK_LT(static_cast<size_t>(arg.value()), this->func_pool_.size());
 +        setter(arg_index, this->func_pool_[arg.value()]);
 +        break;
 +      }
 +      default: {
 +        LOG(FATAL) << "ValueError: Unknown argument kind: " << int(arg.kind());
 +      }
 +    }
 +  }
 +  TVMArgs args(values.data() + args_begin_offset, tcodes.data() + args_begin_offset,
 +               instr.num_args);
 +  TVMRetValue ret;
 +
 +  ICHECK_LT(static_cast<size_t>(instr.func_idx), this->func_pool_.size());
 +
 +  if (instrument_ == nullptr) {
 +    this->InvokeClosurePacked(func_pool_[instr.func_idx], args, &ret);
 +  } else {
 +    // insert light-weight instrument callback
 +    setter(0, func_pool_[instr.func_idx]);
 +    setter(1, GetFuncName(instr.func_idx));
 +    setter(2, true);
 +    setter(3, nullptr);
 +    TVMRetValue rv;
 +    // store dtype to str since py callback cannot handle dtype atm.
 +    std::vector<std::unique_ptr<std::string>> temp_dtype;
 +    for (int i = 0; i < instr.num_args; ++i) {
 +      if (tcodes[i + args_begin_offset] == kTVMDataType) {
 +        std::string str_dtype = args[i];
 +        temp_dtype.emplace_back(std::make_unique<std::string>(str_dtype));
 +        setter(i + args_begin_offset, *temp_dtype.back());
 +      }
 +    }
 +    int ret_kind = static_cast<int>(VMInstrumentReturnKind::kNoOp);
 +    instrument_.CallPacked(TVMArgs(values.data(), tcodes.data(), values.size()), &rv);
 +    if (rv.type_code() == kDLInt) {
 +      ret_kind = rv;
 +    }
 +    if (ret_kind != static_cast<int>(VMInstrumentReturnKind::kSkipRun)) {
 +      this->InvokeClosurePacked(func_pool_[instr.func_idx], args, &ret);
 +      setter(2, false);
 +      setter(3, ret);
 +      instrument_.CallPacked(TVMArgs(values.data(), tcodes.data(), values.size()), &rv);
 +    }
 +  }
 +
 +  // save the return value to the register
 +  // saving to special register is a NOP
 +  if (instr.dst < Instruction::kBeginSpecialReg) {
 +    WriteRegister(curr_frame, instr.dst, ret);
 +  }
 +  // increment pc
 +  pc_++;
 +}
 +
 +void VirtualMachineImpl::RunLoop() {
 +  VMFrame* curr_frame = frames_.back().get();
 +
 +  while (true) {
 +    ICHECK_LT(static_cast<size_t>(pc_), exec_->instr_offset.size()) << "run into invalid section";
 +    Instruction instr = exec_->GetInstruction(pc_);
 +    switch (instr.op) {
 +      case Opcode::Call: {
 +        this->RunInstrCall(curr_frame, instr);
 +        break;
 +      }
 +      case Opcode::Ret: {
 +        // If we have hit the point from which we started
 +        // running, we should return to the caller breaking
 +        // the dispatch loop.
 +        return_value_ = ReadRegister(curr_frame, instr.result);
 +        RegName caller_return_register = curr_frame->caller_return_register;
 +        PopFrame();
 +        if (frames_.size() == 0) {
 +          // directly return if no frame in the call stack.
 +        } else {
 +          // return from a local call.
 +          // Update the current frame to be the parent frame.
 +          curr_frame = frames_.back().get();
 +          WriteRegister(curr_frame, caller_return_register, return_value_);
 +        }
 +        return;
 +      }
 +      case Opcode::Goto: {
 +        pc_ += instr.pc_offset;
 +        break;
 +      }
 +      case Opcode::If: {
 +        int64_t cond_val = ReadRegister(curr_frame, instr.cond);
 +        if (cond_val != 0) {
 +          pc_++;
 +        } else {
 +          ICHECK_GT(instr.false_offset, 1);
 +          pc_ += instr.false_offset;
 +        }
 +        break;
 +      }
 +    }
 +  }
 +}
 +
 +ObjectPtr<VirtualMachine> VirtualMachine::Create() { return make_object<VirtualMachineImpl>(); }
 +
 +//----------------------------------------------------------------
 +// Profiler can be optionally disabled via a macro to reduce dep.
 +//----------------------------------------------------------------
 +#if TVM_RELAX_VM_ENABLE_PROFILER
 +
 +/*!
 + * \brief An extension of VirtualMachineImpl to support per-op profiling
 + * It overrides RunInstrCall to add instrumentations around it.
 + */
 +class VirtualMachineProfiler : public VirtualMachineImpl {
 + public:
-   PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) override {
++  PackedFunc GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) override {
 +    if (name == "profile") {
 +      return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
 +        std::string f_name = args[0];
 +        VMClosure clo = this->GetClosure(f_name);
 +
 +        std::vector<Device> devices;
 +        for (auto dev : this->devices) {
 +          if (dev.device_type > 0) {
 +            devices.push_back(dev);
 +          }
 +        }
 +
 +        prof_ = profiling::Profiler(devices, {}, {{String("Executor"), String("VM")}});
 +
 +        auto inputs = GetInputsFor(f_name);
 +
 +        bool clear_inputs = false;
 +        if (inputs.size() == 0) {
 +          ICHECK(args.num_args > 1) << "No input is provided";
 +          TVMArgs f_args(args.values + 1, args.type_codes + 1, args.num_args - 1);
 +          SetInput(f_name, args, 1);
 +          inputs = GetInputsFor(f_name);
 +          clear_inputs = true;
 +        } else {
 +          ICHECK_EQ(args.num_args, 1) << "Inputs are already provided by set_input.";
 +        }
 +
 +        // warmup
 +        this->InvokeClosureInternal(clo, inputs);
 +
 +        prof_->Start();
 +        this->InvokeClosureInternal(clo, inputs);
 +        prof_->Stop();
 +
 +        // Return the report as json, since profiling::Report object is not supported by RPC
 +        std::string report_json = prof_->Report()->AsJSON();
 +        *rv = report_json;
 +
 +        prof_ = std::nullopt;  // releases hardware counters
 +        if (clear_inputs) {
 +          // SetInput modifies the internal states of VM. Undo the change after profiling.
 +          ClearInputsFor(f_name);
 +        }
 +      });
 +    } else {
 +      return VirtualMachineImpl::GetFunction(name, sptr_to_self);
 +    }
 +  }
 +
 + protected:
 +  void RunInstrCall(VMFrame* curr_frame, Instruction inst) override {
 +    bool profiling = false;
 +    if (prof_ && prof_->IsRunning()) {
 +      auto f_name = GetFuncName(inst.func_idx);
 +      std::optional<Device> dev;
 +      std::vector<NDArray> arrs;
 +
 +      auto f_check_ndarray_arg = [&dev, &arrs](const RegType& arg) {
 +        if (arg.type_code() == kTVMNDArrayHandle) {
 +          NDArray arr = arg;
 +          dev = arr->device;
 +          arrs.push_back(arr);
 +        }
 +      };
 +
 +      for (Index i = 0; i < inst.num_args; ++i) {
 +        Instruction::Arg arg = inst.args[i];
 +        if (arg.kind() == Instruction::ArgKind::kRegister) {
 +          auto reg = ReadRegister(curr_frame, arg.value());
 +          f_check_ndarray_arg(reg);
 +        } else if (arg.kind() == Instruction::ArgKind::kConstIdx) {
 +          const auto& const_val = this->const_pool_[arg.value()];
 +          f_check_ndarray_arg(const_val);
 +        }
 +      }
 +
 +      std::unordered_map<std::string, ObjectRef> metrics;
 +      metrics["Argument Shapes"] = profiling::ShapeString(arrs);
 +
 +      // If a suitable device is found, enable profiling.
 +      if (dev) {
 +        profiling = true;
 +        prof_->StartCall(f_name, *dev, metrics);
 +      }
 +    }
 +
 +    VirtualMachineImpl::RunInstrCall(curr_frame, inst);
 +
 +    if (profiling) {
 +      prof_->StopCall();
 +    }
 +  }
 +
 + private:
 +  std::optional<profiling::Profiler> prof_;
 +};
 +
 +ObjectPtr<VirtualMachine> VirtualMachine::CreateProfiler() {
 +  return make_object<VirtualMachineProfiler>();
 +}
 +
 +#else
 +ObjectPtr<VirtualMachine> VirtualMachine::CreateProfiler() {
 +  LOG(FATAL) << "Profiler support is disabled";
 +  return nullptr;
 +}
 +#endif  // TVM_RELAX_VM_ENABLE_PROFILER
 +}  // namespace relax_vm
 +}  // namespace runtime
 +}  // namespace tvm
diff --cc src/target/source/codegen_webgpu.cc
index 538acb0a35,4d1d834c7f..1702699ac2
--- a/src/target/source/codegen_webgpu.cc
+++ b/src/target/source/codegen_webgpu.cc
@@@ -663,18 -502,12 +659,18 @@@ class WebGPUSourceModuleNode final : pu
      stream->Write(smap_);
    }
  
-   std::string GetSource(const std::string& format) final {
+   String GetSource(const String& format) final {
 -    std::ostringstream os;
 -    for (auto kv : smap_) {
 -      os << kv.second;
 +    if (format == "func_info") {
 +      std::ostringstream stream;
 +      dmlc::JSONWriter(&stream).Write(fmap_);
 +      return stream.str();
 +    } else {
 +      std::ostringstream os;
 +      for (auto kv : smap_) {
 +        os << kv.second;
 +      }
 +      return os.str();
      }
 -    return os.str();
    }
  
   private:
diff --cc web/emcc/webgpu_runtime.cc
index 6c27a8207e,a4facc8f77..d0ec39efa1
--- a/web/emcc/webgpu_runtime.cc
+++ b/web/emcc/webgpu_runtime.cc
@@@ -159,35 -159,7 +159,35 @@@ class WebGPUModuleNode final : public r
  
    const char* type_key() const final { return "webgpu"; }
  
-   PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final {
+   PackedFunc GetFunction(const String& name, const ObjectPtr<Object>& sptr_to_self) final {
 +    // special function
 +    if (name == "webgpu.get_fmap") {
 +      return PackedFunc([this](TVMArgs args, TVMRetValue* rv) {
 +        std::ostringstream os;
 +        dmlc::JSONWriter writer(&os);
 +        writer.Write(fmap_);
 +        *rv = os.str();
 +      });
 +    } else if (name == "webgpu.get_shader") {
 +      return PackedFunc([this](TVMArgs args, TVMRetValue* rv) {
 +        std::string name = args[0];
 +        auto it = smap_.find(name);
 +        ICHECK(it != smap_.end()) << "Cannot find code " << name;
 +        *rv = it->second;
 +      });
 +    } else if (name == "webgpu.update_prebuild") {
 +      return PackedFunc([this](TVMArgs args, TVMRetValue* rv) {
 +        std::string name = args[0];
 +        PackedFunc func = args[1];
 +        prebuild_[name] = func;
 +      });
 +    }
 +    // check prebuild cache
 +    auto prebuild_it = prebuild_.find(name);
 +    if (prebuild_it != prebuild_.end()) {
 +      return prebuild_it->second;
 +    }
 +
      auto it = smap_.find(name);
      if (it != smap_.end()) {
        FunctionInfo info = fmap_.at(name);
@@@ -201,9 -173,7 +201,9 @@@
      }
    }
  
 -  void SaveToFile(const String& file_name, const String& format) final {
 +  int GetPropertyMask() const final { return ModulePropertyMask::kBinarySerializable; };
 +
-   void SaveToFile(const std::string& file_name, const std::string& format) final {
++  void SaveToFile(const String& file_name, const std::string& format) final {
      LOG(FATAL) << "Not implemented";
    }