You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2021/02/15 23:06:18 UTC
[incubator-mxnet] branch master updated: Disable onnx tests and
pipeline until we forward-port onnx work. (#19898)
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 417accc Disable onnx tests and pipeline until we forward-port onnx work. (#19898)
417accc is described below
commit 417accccc104ce074727558b0f0095278b971451
Author: Joe Evans <gi...@250hacks.net>
AuthorDate: Mon Feb 15 15:04:06 2021 -0800
Disable onnx tests and pipeline until we forward-port onnx work. (#19898)
Co-authored-by: Joe Evans <jo...@amazon.com>
---
ci/docker/runtime_functions.sh | 10 +-
ci/jenkins/Jenkinsfile_unix_cpu | 2 +
tests/python/unittest/onnx/README.md | 50 ------
tests/python/unittest/onnx/backend.py | 130 ---------------
tests/python/unittest/onnx/backend_rep.py | 137 ----------------
tests/python/unittest/onnx/backend_test.py | 93 -----------
tests/python/unittest/onnx/mxnet_export_test.py | 138 ----------------
tests/python/unittest/onnx/test_cases.py | 130 ---------------
tests/python/unittest/onnx/test_node.py | 210 ------------------------
tests/python/unittest/onnx/test_onnxruntime.py | 165 -------------------
10 files changed, 7 insertions(+), 1058 deletions(-)
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index cfff031..34245bb 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -881,12 +881,12 @@ integrationtest_ubuntu_cpu_onnx() {
set -ex
export PYTHONPATH=./python/
export MXNET_SUBGRAPH_VERBOSE=0
- export DMLC_LOG_STACK_TRACE_DEPTH=100
+ export DMLC_LOG_STACK_TRACE_DEPTH=100
python3 tests/python/unittest/onnx/backend_test.py
- OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py
- OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py
- OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py
- OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py
+ #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py
+ #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py
+ #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py
+ #OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py
}
integrationtest_ubuntu_cpu_dist_kvstore() {
diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
index 67d5afc..2b2f5bc 100644
--- a/ci/jenkins/Jenkinsfile_unix_cpu
+++ b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -48,7 +48,9 @@ core_logic: {
custom_steps.test_unix_python3_mkl_cpu('cpu_mkl'),
custom_steps.test_unix_python3_mkldnn_cpu('mkldnn_cpu'),
custom_steps.test_unix_python3_mkldnn_mkl_cpu('mkldnn_mkl_cpu'),
+ /* disable onnx tests for now, until onnx work is forwarded-ported to master
custom_steps.test_unix_onnx_cpu('cpu'),
+ */
/* Disabled due to master build failure:
* http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
* https://github.com/apache/incubator-mxnet/issues/11801
diff --git a/tests/python/unittest/onnx/README.md b/tests/python/unittest/onnx/README.md
deleted file mode 100644
index b9b497b..0000000
--- a/tests/python/unittest/onnx/README.md
+++ /dev/null
@@ -1,50 +0,0 @@
-<!--- Licensed to the Apache Software Foundation (ASF) under one -->
-<!--- or more contributor license agreements. See the NOTICE file -->
-<!--- distributed with this work for additional information -->
-<!--- regarding copyright ownership. The ASF licenses this file -->
-<!--- to you under the Apache License, Version 2.0 (the -->
-<!--- "License"); you may not use this file except in compliance -->
-<!--- with the License. You may obtain a copy of the License at -->
-<!--- -->
-<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
-<!--- -->
-<!--- Unless required by applicable law or agreed to in writing, -->
-<!--- software distributed under the License is distributed on an -->
-<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
-<!--- KIND, either express or implied. See the License for the -->
-<!--- specific language governing permissions and limitations -->
-<!--- under the License. -->
-
-# ONNX tests
-
-## Directory structure:
-
-```bash
-.
-├── README.md
-├── backend.py
-├── backend_rep.py
-├── backend_test.py
-├── gluon_backend_test.py
-├── mxnet_backend_test.py
-├── mxnet_export_test.py
-├── test_cases.py
-├── test_models.py
-└── test_node.py
-```
-
-* `backend.py` - MXNetBackend. This file contains prepare(). \
-This class can be used for both, MXNet and Gluon backend.
-* `backend_rep.py` - MXNetBackendRep and GluonBackendRep for running inference
-* `backend_test.py` - prepare tests by including tests from `test_cases.py`
-* `gluon_backend_test.py` - Set backend as gluon and execute ONNX tests for ONNX->Gluon import.
-* `mxnet_backend_test.py` - Set backend as gluon and add tests for ONNX->MXNet import/export.
-Since MXNetBackend for export, tests both import and export, the test list in this file is
-a union of tests that execute for import and export, export alone, and import alone.
-* `mxnet_export_test.py` - Execute unit tests for testing MXNet export code - this is not specific to
-any operator.
-* `test_cases.py` - list of test cases for operators/models that are supported
-for "both", import and export, "import" alone, or "export" alone.
-* `test_models.py` - custom tests for models
-* `test_node.py` - custom tests for operators. These tests are written independent of ONNX tests, in case
-ONNX doesn't have tests yet or for MXNet specific operators.
diff --git a/tests/python/unittest/onnx/backend.py b/tests/python/unittest/onnx/backend.py
deleted file mode 100644
index eb803f7..0000000
--- a/tests/python/unittest/onnx/backend.py
+++ /dev/null
@@ -1,130 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-"""MXNet/Gluon backend wrapper for onnx test infrastructure"""
-
-from mxnet.contrib.onnx.onnx2mx.import_onnx import GraphProto
-from mxnet.contrib.onnx.mx2onnx.export_onnx import MXNetGraph
-import mxnet as mx
-import numpy as np
-
-try:
- from onnx import helper, TensorProto, mapping
- from onnx.backend.base import Backend
- from onnx.defs import onnx_opset_version
-except ImportError:
- raise ImportError("Onnx and protobuf need to be installed. Instructions to"
- + " install - https://github.com/onnx/onnx#installation")
-from backend_rep import MXNetBackendRep, GluonBackendRep
-
-
-# MXNetBackend class will take an ONNX model with inputs, perform a computation,
-# and then return the output.
-# Implemented by following onnx docs guide:
-# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md
-
-class MXNetBackend(Backend):
- """MXNet/Gluon backend for ONNX"""
-
- backend = 'mxnet'
- operation = 'import'
-
- @classmethod
- def set_params(cls, backend, operation):
- cls.backend = backend
- cls.operation = operation
-
- @staticmethod
- def perform_import_export(sym, arg_params, aux_params, input_shape):
- """ Import ONNX model to mxnet model and then export to ONNX model
- and then import it back to mxnet for verifying the result"""
- graph = GraphProto()
-
- params = {}
- params.update(arg_params)
- params.update(aux_params)
- # use the latest opset version supported by the onnx library
- opset_version = onnx_opset_version()
- # exporting to onnx graph proto format
- converter = MXNetGraph()
- graph_proto = converter.create_onnx_graph_proto(sym, params, in_shape=input_shape,
- in_type=mapping.NP_TYPE_TO_TENSOR_TYPE[np.dtype('float32')],
- opset_version=opset_version)
-
- # importing back to MXNET for verifying result.
- sym, arg_params, aux_params = graph.from_onnx(graph_proto, opset_version)
-
- return sym, arg_params, aux_params
-
- @classmethod
- def prepare(cls, model, device='CPU', **kwargs):
- """For running end to end model(used for onnx test backend)
-
- Parameters
- ----------
- model : onnx ModelProto object
- loaded onnx graph
- device : 'CPU'
- specifying device to run test on
- kwargs :
- other arguments
-
- Returns
- -------
- MXNetBackendRep : object
- Returns object of MXNetBackendRep class which will be in turn
- used to run inference on the input model and return the result for comparison.
- """
- backend = kwargs.get('backend', cls.backend)
- operation = kwargs.get('operation', cls.operation)
-
- graph = GraphProto()
- if device == 'CPU':
- ctx = mx.cpu()
- else:
- raise NotImplementedError("ONNX tests are run only for CPU context.")
-
- # determine opset version model uses
- model_opset_version = max([x.version for x in model.opset_import])
-
- if backend == 'mxnet':
- sym, arg_params, aux_params = graph.from_onnx(model.graph, model_opset_version)
- if operation == 'export':
- metadata = graph.get_graph_metadata(model.graph)
- input_data = metadata['input_tensor_data']
- input_shape = [data[1] for data in input_data]
- sym, arg_params, aux_params = MXNetBackend.perform_import_export(sym, arg_params, aux_params,
- input_shape)
-
- return MXNetBackendRep(sym, arg_params, aux_params, device)
- elif backend == 'gluon':
- if operation == 'import':
- net = graph.graph_to_gluon(model.graph, ctx, model_opset_version)
- return GluonBackendRep(net, device)
- elif operation == 'export':
- raise NotImplementedError("Gluon->ONNX export not implemented.")
-
- @classmethod
- def supports_device(cls, device):
- """Supports only CPU for testing"""
- return device == 'CPU'
-
-
-prepare = MXNetBackend.prepare
-
-supports_device = MXNetBackend.supports_device
diff --git a/tests/python/unittest/onnx/backend_rep.py b/tests/python/unittest/onnx/backend_rep.py
deleted file mode 100644
index 597b1c7..0000000
--- a/tests/python/unittest/onnx/backend_rep.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-"""MXNet backend rep for onnx test infrastructure"""
-try:
- from onnx.backend.base import BackendRep
-except ImportError:
- raise ImportError("Onnx and protobuf need to be installed. Instructions to"
- + " install - https://github.com/onnx/onnx#installation")
-import numpy as np
-import mxnet as mx
-from mxnet import nd
-
-# Using these functions for onnx test infrastructure.
-# Implemented by following onnx docs guide:
-# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md
-# MXNetBackendRep object will be returned by MXNetBackend's prepare method which is used to
-# execute a model repeatedly.
-# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and
-# retrieve the corresponding results for comparison to the onnx backend.
-# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py.
-
-class MXNetBackendRep(BackendRep):
- """Running model inference on mxnet engine and return the result
- to onnx test infrastructure for comparison."""
- def __init__(self, symbol, arg_params, aux_params, device):
- self.symbol = symbol
- self.arg_params = arg_params
- self.aux_params = aux_params
- self.device = device
-
- def run(self, inputs, **kwargs):
- """Run model inference and return the result
-
- Parameters
- ----------
- inputs : numpy array
- input to run a layer on
-
- Returns
- -------
- params : numpy array
- result obtained after running the inference on mxnet
- """
- # create module, passing cpu context
- if self.device == 'CPU':
- ctx = mx.cpu()
- else:
- raise NotImplementedError("ONNX tests are run only for CPU context.")
-
- # To fetch the data names of the input to the model we list the inputs of the symbol graph
- # and exclude the argument and auxiliary parameters from the list
- data_names = [graph_input for graph_input in self.symbol.list_inputs()
- if graph_input not in self.arg_params and graph_input not in self.aux_params]
-
- data_forward = []
- for idx, input_name in enumerate(data_names):
- val = inputs[idx]
- data_forward.append(mx.nd.array(val))
-
- if self.arg_params:
- for idx, input_name in enumerate(self.arg_params):
- val = self.arg_params[input_name]
- data_names.append(input_name)
- data_forward.append(mx.nd.array(val))
-
- args = dict(zip(data_names, data_forward))
- exe = self.symbol._bind(ctx, args=args, aux_states=self.aux_params)
- exe.forward(is_train=False)
- result = []
- for output in exe.outputs:
- result.append(output.asnumpy())
- return result
-
-
-# GluonBackendRep object will be returned by GluonBackend's prepare method which is used to
-# execute a model repeatedly.
-# Inputs will be passed to the run method of MXNetBackendRep class, it will perform computation and
-# retrieve the corresponding results for comparison to the onnx backend.
-# https://github.com/onnx/onnx/blob/master/onnx/backend/test/runner/__init__.py.
-# Implemented by following onnx docs guide:
-# https://github.com/onnx/onnx/blob/master/docs/ImplementingAnOnnxBackend.md
-
-class GluonBackendRep(BackendRep):
- """Running model inference on gluon backend and return the result
- to onnx test infrastructure for comparison."""
- def __init__(self, net, device):
- self.net = net
- self.device = device
-
- def run(self, inputs, **kwargs):
- """Run model inference and return the result
-
- Parameters
- ----------
- inputs : numpy array
- input to run a layer on
-
- Returns
- -------
- params : numpy array
- result obtained after running the inference on mxnet
- """
- # create module, passing cpu context
- if self.device == 'CPU':
- ctx = mx.cpu()
- else:
- raise NotImplementedError("ONNX tests are run only for CPU context.")
-
- # run inference
- net_inputs = [nd.array(input_data, ctx=ctx) for input_data in inputs]
- net_outputs = self.net(*net_inputs)
- results = []
- if isinstance(net_outputs, list):
- for output in net_outputs:
- results.append(output.asnumpy())
- result = results
- else:
- results.extend([o for o in net_outputs.asnumpy()])
- result = [np.array(results)]
-
- return result
diff --git a/tests/python/unittest/onnx/backend_test.py b/tests/python/unittest/onnx/backend_test.py
deleted file mode 100644
index cf3cdc1..0000000
--- a/tests/python/unittest/onnx/backend_test.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""ONNX test backend wrapper"""
-
-def build_test_suite(backend_tests): # type: () -> unittest.TestSuite
- '''
- TestSuite that can be run by TestRunner
- This has been borrowed from onnx/onnx/backend/test/runner/__init__.py,
- since Python3 cannot sort objects of type 'Type' as Runner.test_suite()
- expects.
- '''
- suite = unittest.TestSuite()
- for case in backend_tests.test_cases.values():
- suite.addTests(unittest.defaultTestLoader.loadTestsFromTestCase(case))
- return suite
-
-
-def prepare_tests(backend, oper):
- """
- Prepare the test list
- :param backend: mxnet/gluon backend
- :param oper: str. export or import
- :return: backend test list
- """
- BACKEND_TESTS = onnx.backend.test.BackendTest(backend, __name__)
- implemented_ops = test_cases.IMPLEMENTED_OPERATORS_TEST.get('both', []) + \
- test_cases.IMPLEMENTED_OPERATORS_TEST.get(oper, [])
-
- for op_test in implemented_ops:
- BACKEND_TESTS.include(op_test)
-
- basic_models = test_cases.BASIC_MODEL_TESTS.get('both', []) + \
- test_cases.BASIC_MODEL_TESTS.get(oper, [])
-
- for basic_model_test in basic_models:
- BACKEND_TESTS.include(basic_model_test)
-
- std_models = test_cases.STANDARD_MODEL.get('both', []) + \
- test_cases.STANDARD_MODEL.get(oper, [])
-
- for std_model_test in std_models:
- BACKEND_TESTS.include(std_model_test)
-
- # Tests for scalar ops are in test_node.py
- BACKEND_TESTS.exclude('.*scalar.*')
-
- return BACKEND_TESTS
-
-
-if __name__ == '__main__':
- try:
- import onnx.backend.test
- except ImportError:
- raise ImportError("Onnx and protobuf need to be installed")
-
- import test_cases
- import unittest
- import backend as mxnet_backend
- import logging
-
- operations = ['import', 'export']
- backends = ['mxnet', 'gluon']
- # This is a pytest magic variable to load extra plugins
- pytest_plugins = "onnx.backend.test.report",
-
- for bkend in backends:
- for operation in operations:
- log = logging.getLogger(bkend + operation)
- if bkend == 'gluon' and operation == 'export':
- log.warning('Gluon->ONNX export not implemented. Skipping tests...')
- continue
- log.info('Executing tests for ' + bkend + ' backend: ' + operation)
- mxnet_backend.MXNetBackend.set_params(bkend, operation)
- BACKEND_TESTS = prepare_tests(mxnet_backend, operation)
- unittest.TextTestRunner().run(build_test_suite(BACKEND_TESTS.enable_report()))
diff --git a/tests/python/unittest/onnx/mxnet_export_test.py b/tests/python/unittest/onnx/mxnet_export_test.py
deleted file mode 100644
index baa7476..0000000
--- a/tests/python/unittest/onnx/mxnet_export_test.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# pylint: disable=too-many-locals,wrong-import-position,import-error
-import os, sys
-import unittest
-import logging
-import tempfile
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.insert(0, os.path.join(curr_path, '..'))
-from mxnet import nd, sym
-from mxnet.test_utils import set_default_context
-from mxnet.gluon import nn
-from mxnet.gluon import HybridBlock
-import mxnet as mx
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-def _assert_sym_equal(lhs, rhs):
- assert lhs.list_inputs() == rhs.list_inputs() # input names must be identical
- assert len(lhs.list_outputs()) == len(rhs.list_outputs()) # number of outputs must be identical
-
-
-def _force_list(output):
- if isinstance(output, nd.NDArray):
- return [output]
- return list(output)
-
-
-def _optional_group(symbols, group=False):
- if group:
- return sym.Group(symbols)
- else:
- return symbols
-
-
-def _check_onnx_export(net, group_outputs=False, shape_type=tuple, extra_params={}):
- net.initialize()
- data = nd.random.uniform(0, 1, (1, 1024))
- output = _force_list(net(data)) # initialize weights
- net_sym = _optional_group(net(sym.Variable('data')), group_outputs)
- net_params = {param.var().name: param._reduce() for param in net.collect_params().values()}
- net_params.update(extra_params)
- with tempfile.TemporaryDirectory() as tmpdirname:
- onnx_file_path = os.path.join(tmpdirname, 'net.onnx')
- export_path = mx.contrib.onnx.export_model(
- sym=net_sym,
- params=net_params,
- input_shape=[shape_type(data.shape)],
- onnx_file_path=onnx_file_path)
- assert export_path == onnx_file_path
- # Try importing the model to symbol
- _assert_sym_equal(net_sym, mx.contrib.onnx.import_model(export_path)[0])
-
- # Try importing the model to gluon
- imported_net = mx.contrib.onnx.import_to_gluon(export_path, ctx=None)
- _assert_sym_equal(net_sym, _optional_group(imported_net(sym.Variable('data')), group_outputs))
-
- # Confirm network outputs are the same
- imported_net_output = _force_list(imported_net(data))
- for out, imp_out in zip(output, imported_net_output):
- mx.test_utils.assert_almost_equal(out, imp_out, atol=1e-5, rtol=1e-5)
-
-
-class SplitConcatBlock(HybridBlock):
- """Block which creates two splits and later concatenates them"""
- def __init__(self):
- super(SplitConcatBlock, self).__init__()
-
- def hybrid_forward(self, F, x):
- splits = F.split(x, axis=1, num_outputs=2)
- return F.concat(*splits)
-
-
-class TestExport(unittest.TestCase):
- """ Tests ONNX export.
- """
-
- def setUp(self):
- set_default_context(mx.cpu(0))
-
- def test_onnx_export_single_output(self):
- net = nn.HybridSequential()
- net.add(nn.Dense(100, activation='relu'), nn.Dense(10))
- _check_onnx_export(net)
-
- def test_onnx_export_multi_output(self):
- class MultiOutputBlock(nn.HybridBlock):
- def __init__(self):
- super(MultiOutputBlock, self).__init__()
- self.net = nn.HybridSequential()
- for i in range(10):
- self.net.add(nn.Dense(100 + i * 10, activation='relu'))
-
- def hybrid_forward(self, F, x):
- out = tuple(block()(x) for block in self.net._children.values())
- return out
-
- net = MultiOutputBlock()
- assert len(sym.Group(net(sym.Variable('data'))).list_outputs()) == 10
- _check_onnx_export(net, group_outputs=True)
-
- def test_onnx_export_list_shape(self):
- net = nn.HybridSequential()
- net.add(nn.Dense(100, activation='relu'), nn.Dense(10))
- _check_onnx_export(net, shape_type=list)
-
- def test_onnx_export_extra_params(self):
- net = nn.HybridSequential()
- net.add(nn.Dense(100, activation='relu'), nn.Dense(10))
- _check_onnx_export(net, extra_params={'extra_param': nd.array([1, 2])})
-
- def test_onnx_export_slice(self):
- net = nn.HybridSequential()
- net.add(nn.Dense(100, activation='relu'), SplitConcatBlock(), nn.Dense(10))
- _check_onnx_export(net)
-
- def test_onnx_export_slice_changing_shape(self):
- net = nn.HybridSequential()
- net.add(nn.Dense(100, activation='relu'), SplitConcatBlock(),
- nn.Dense(50, activation='relu'), SplitConcatBlock(), nn.Dense(10))
- _check_onnx_export(net)
diff --git a/tests/python/unittest/onnx/test_cases.py b/tests/python/unittest/onnx/test_cases.py
deleted file mode 100644
index 9323102..0000000
--- a/tests/python/unittest/onnx/test_cases.py
+++ /dev/null
@@ -1,130 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-IMPLEMENTED_OPERATORS_TEST = {
- 'both': ['test_add',
- 'test_sub',
- 'test_mul',
- 'test_div',
- 'test_neg',
- 'test_abs',
- 'test_sum',
- 'test_tanh',
- 'test_ceil',
- 'test_floor',
- 'test_concat',
- 'test_identity',
- 'test_sigmoid',
- 'test_relu',
- 'test_constant_pad',
- 'test_edge_pad',
- 'test_reflect_pad',
- 'test_softmax_example',
- 'test_softmax_large_number',
- 'test_softmax_axis_2',
- 'test_transpose',
- 'test_globalmaxpool',
- 'test_globalaveragepool',
- 'test_reciprocal',
- 'test_sqrt',
- 'test_pow',
- 'test_exp_',
- 'test_argmax',
- 'test_argmin',
- 'test_min',
- # pytorch operator tests
- 'test_exp_',
- 'test_operator_maxpool',
- 'test_operator_params',
- 'test_operator_permute2',
- 'test_cos[^h]',
- 'test_sin[^h]',
- 'test_tan',
- 'test_acos[^h]',
- 'test_asin[^h]',
- 'test_atan[^h]',
- 'test_squeeze',
- 'test_matmul_',
- 'test_depthtospace',
- 'test_hardsigmoid',
- 'test_instancenorm',
- 'test_shape',
- 'test_cast((?!STRING).)*$',
- 'test_clip',
- 'test_size',
- 'test_dropout',
- 'test_unsqueeze',
- 'test_log_',
- 'test_flatten_default_axis',
- 'test_leakyrelu',
- 'test_selu_default',
- 'test_elu',
- 'test_max_',
- 'test_softplus',
- 'test_reduce_',
- 'test_split_equal',
- 'test_gather'
- ],
- 'import': ['test_softsign',
- 'test_mean',
- 'test_averagepool_1d',
- 'test_averagepool_2d_pads_count_include_pad',
- 'test_averagepool_2d_precomputed_pads_count_include_pad',
- 'test_averagepool_2d_precomputed_strides',
- 'test_averagepool_2d_strides',
- 'test_averagepool_3d',
- 'test_hardmax'
- ],
- 'export': ['test_random_uniform',
- 'test_random_normal',
- 'test_reduce_min',
- 'test_reduce_max',
- 'test_reduce_mean',
- 'test_reduce_prod',
- 'test_reduce_sum_d',
- 'test_reduce_sum_keepdims_random',
- 'test_lrn'
- ]
-}
-
-BASIC_MODEL_TESTS = {
- 'both': ['test_AvgPool2D',
- 'test_BatchNorm',
- 'test_ConstantPad2d'
- 'test_Conv2d',
- 'test_MaxPool',
- 'test_PReLU',
- 'test_Softmax',
- 'test_softmax_functional',
- 'test_softmax_lastdim',
- ],
- 'export': []
-}
-
-STANDARD_MODEL = {
- 'both': ['test_bvlc_alexnet',
- 'test_densenet121',
- # 'test_inception_v1',
- # 'test_inception_v2',
- 'test_resnet50',
- # 'test_shufflenet',
- 'test_squeezenet',
- 'test_vgg19'
- ],
- 'import': ['test_zfnet512'],
- 'export': ['test_vgg16']
-}
diff --git a/tests/python/unittest/onnx/test_node.py b/tests/python/unittest/onnx/test_node.py
deleted file mode 100644
index fabba1a..0000000
--- a/tests/python/unittest/onnx/test_node.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Tests for individual operators
-This module contains operator tests which currently do not exist on
-ONNX backend test framework. Once we have PRs on the ONNX repo and get
-those PRs merged, this file will get EOL'ed.
-"""
-# pylint: disable=too-many-locals,wrong-import-position,import-error
-import sys
-import os
-import unittest
-import logging
-import tarfile
-from collections import namedtuple
-import numpy as np
-import numpy.testing as npt
-from onnx import checker, numpy_helper, helper, load_model
-from onnx import TensorProto
-from mxnet.test_utils import download
-import mxnet as mx
-import backend
-
-CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-sys.path.insert(0, os.path.join(CURR_PATH, '../../python/unittest'))
-
-logger = logging.getLogger()
-logger.setLevel(logging.DEBUG)
-
-
-def get_rnd(shape, low=-1.0, high=1.0, dtype=np.float32):
- if dtype == np.float32:
- return (np.random.uniform(low, high,
- np.prod(shape)).reshape(shape).astype(np.float32))
- elif dtype == np.int32:
- return (np.random.randint(low, high,
- np.prod(shape)).reshape(shape).astype(np.float32))
- elif dtype == np.bool_:
- return np.random.choice(a=[False, True], size=shape).astype(np.float32)
-
-
-def _fix_attributes(attrs, attribute_mapping):
- new_attrs = attrs
- attr_modify = attribute_mapping.get('modify', {})
- for k, v in attr_modify.items():
- new_attrs[v] = new_attrs.pop(k, None)
-
- attr_add = attribute_mapping.get('add', {})
- for k, v in attr_add.items():
- new_attrs[k] = v
-
- attr_remove = attribute_mapping.get('remove', [])
- for k in attr_remove:
- if k in new_attrs:
- del new_attrs[k]
-
- return new_attrs
-
-
-def get_input_tensors(input_data):
- input_tensor = []
- input_names = []
- input_sym = []
- for idx, ip in enumerate(input_data):
- name = "input" + str(idx + 1)
- input_sym.append(mx.sym.Variable(name))
- input_names.append(name)
- input_tensor.append(helper.make_tensor_value_info(name,
- TensorProto.FLOAT, shape=np.shape(ip)))
- return input_names, input_tensor, input_sym
-
-
-def get_onnx_graph(testname, input_names, inputs, output_name, output_shape, attr):
- outputs = [helper.make_tensor_value_info("output", TensorProto.FLOAT, shape=output_shape)]
-
- nodes = [helper.make_node(output_name, input_names, ["output"], **attr)]
-
- graph = helper.make_graph(nodes, testname, inputs, outputs)
-
- model = helper.make_model(graph)
- return model
-
-class TestNode(unittest.TestCase):
- """ Tests for models.
- Tests are dynamically added.
- Therefore edit test_models to add more tests.
- """
- def test_imports(self):
- for bk in ['mxnet', 'gluon']:
- for test in import_test_cases:
- test_name, onnx_name, inputs, np_op, attrs = test
- with self.subTest(test_name):
- names, input_tensors, inputsym = get_input_tensors(inputs)
- np_out = [np_op(*inputs, **attrs)]
- output_shape = np.shape(np_out)
- onnx_model = get_onnx_graph(test_name, names, input_tensors, onnx_name, output_shape, attrs)
- bkd_rep = backend.prepare(onnx_model, operation='import', backend=bk)
- mxnet_out = bkd_rep.run(inputs)
- npt.assert_almost_equal(np_out, mxnet_out, decimal=4)
-
- def test_exports(self):
- for test in export_test_cases:
- test_name, onnx_name, mx_op, input_shape, attrs = test
- input_sym = mx.sym.var('data')
- if isinstance(mx_op, type) and issubclass(mx_op, (mx.gluon.HybridBlock, mx.gluon.SymbolBlock)):
- mx_op = mx_op(**attrs)
- mx_op.initialize()
- mx_op(mx.nd.zeros(input_shape))
- params = {p.var().name: p.data() for p in mx_op.collect_params().values()}
- outsym = mx_op(input_sym)
- else:
- params = {}
- outsym = mx_op(input_sym, **attrs)
- converted_model = mx.contrib.onnx.export_model(outsym, params, [input_shape], np.float32,
- onnx_file_path=outsym.name + ".onnx")
- model = load_model(converted_model)
- checker.check_model(model)
-
-
-# test_case = ("test_case_name", mxnet op, "ONNX_op_name", [input_list], attribute map, MXNet_specific=True/False,
-# fix_attributes = {'modify': {mxnet_attr_name: onnx_attr_name},
-# 'remove': [attr_name],
-# 'add': {attr_name: value},
-# check_value=True/False, check_shape=True/False)
-test_cases = [
- ("test_equal", mx.sym.broadcast_equal, "Equal", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True,
- False),
- ("test_greater", mx.sym.broadcast_greater, "Greater", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True,
- False),
- ("test_less", mx.sym.broadcast_lesser, "Less", [get_rnd((1, 3, 4, 5)), get_rnd((1, 5))], {}, False, {}, True,
- False),
- ("test_and", mx.sym.broadcast_logical_and, "And",
- [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False),
- ("test_xor", mx.sym.broadcast_logical_xor, "Xor",
- [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False),
- ("test_or", mx.sym.broadcast_logical_or, "Or",
- [get_rnd((3, 4, 5), dtype=np.bool_), get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False),
- ("test_not", mx.sym.logical_not, "Not", [get_rnd((3, 4, 5), dtype=np.bool_)], {}, False, {}, True, False),
- ("test_square", mx.sym.square, "Pow", [get_rnd((2, 3), dtype=np.int32)], {}, True, {}, True, False),
- ("test_spacetodepth", mx.sym.space_to_depth, "SpaceToDepth", [get_rnd((1, 1, 4, 6))],
- {'block_size': 2}, False, {}, True, False),
- ("test_fullyconnected", mx.sym.FullyConnected, "Gemm", [get_rnd((4, 3)), get_rnd((4, 3)), get_rnd(4)],
- {'num_hidden': 4, 'name': 'FC'}, True, {}, True, False),
- ("test_lppool1", mx.sym.Pooling, "LpPool", [get_rnd((2, 3, 20, 20))],
- {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 1, 'pool_type': 'lp'}, False,
- {'modify': {'kernel': 'kernel_shape', 'pad': 'pads', 'stride': 'strides', 'p_value': 'p'},
- 'remove': ['pool_type']}, True, False),
- ("test_lppool2", mx.sym.Pooling, "LpPool", [get_rnd((2, 3, 20, 20))],
- {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 2, 'pool_type': 'lp'}, False,
- {'modify': {'kernel': 'kernel_shape', 'pad': 'pads', 'stride': 'strides', 'p_value': 'p'},
- 'remove': ['pool_type']}, True, False),
- ("test_globallppool1", mx.sym.Pooling, "GlobalLpPool", [get_rnd((2, 3, 20, 20))],
- {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 1, 'pool_type': 'lp', 'global_pool': True}, False,
- {'modify': {'p_value': 'p'},
- 'remove': ['pool_type', 'kernel', 'pad', 'stride', 'global_pool']}, True, False),
- ("test_globallppool2", mx.sym.Pooling, "GlobalLpPool", [get_rnd((2, 3, 20, 20))],
- {'kernel': (4, 5), 'pad': (0, 0), 'stride': (1, 1), 'p_value': 2, 'pool_type': 'lp', 'global_pool': True}, False,
- {'modify': {'p_value': 'p'},
- 'remove': ['pool_type', 'kernel', 'pad', 'stride', 'global_pool']}, True, False),
- ("test_roipool", mx.sym.ROIPooling, "MaxRoiPool",
- [[[get_rnd(shape=(8, 6), low=1, high=100, dtype=np.int32)]], [[0, 0, 0, 4, 4]]],
- {'pooled_size': (2, 2), 'spatial_scale': 0.7}, False,
- {'modify': {'pooled_size': 'pooled_shape'}}, True, False),
-
- # since results would be random, checking for shape alone
- ("test_multinomial", mx.sym.sample_multinomial, "Multinomial",
- [np.array([0, 0.1, 0.2, 0.3, 0.4]).astype("float32")],
- {'shape': (10,)}, False, {'modify': {'shape': 'sample_size'}}, False, True),
- ("test_random_normal", mx.sym.random_normal, "RandomNormal", [],
- {'shape': (2, 2), 'loc': 0, 'scale': 1}, False, {'modify': {'loc': 'mean'}}, False, True),
- ("test_random_uniform", mx.sym.random_uniform, "RandomUniform", [],
- {'shape': (2, 2), 'low': 0.5, 'high': 1.0}, False, {}, False, True)
-]
-
-test_scalar_ops = ['Add', 'Sub', 'rSub' 'Mul', 'Div', 'Pow']
-
-# test_case = ("test_case_name", "ONNX_op_name", [input_list], np_op, attribute map)
-import_test_cases = [
- ("test_lpnormalization_default", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':2, 'axis':-1}),
- ("test_lpnormalization_ord1", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':1, 'axis':-1}),
- ("test_lpnormalization_ord2", "LpNormalization", [get_rnd([5, 3, 3, 2])], np.linalg.norm, {'ord':2, 'axis':1})
-]
-
-# test_case = ("test_case_name", "ONNX_op_name", mxnet_op, input_shape, attribute map)
-export_test_cases = [
- ("test_expand", "Expand", mx.sym.broadcast_to, (2,1,3,1), {'shape': (2,1,3,1)}),
- ("test_tile", "Tile", mx.sym.tile, (2,1,3,1), {'reps': (2,3)}),
- ("test_topk", "TopK", mx.sym.topk, (2, 10, 2), {'k': 3, 'axis': 1, 'ret_typ': 'both', 'dtype': np.int64}),
- ("test_slice_axis", "Slice", mx.sym.slice_axis, (2, 10, 2), {'begin': 3, 'end': 7, 'axis': 1}),
- ("test_LSTM", "LSTM", mx.gluon.rnn.LSTM, (3,1,2), {'hidden_size': 3}),
- ("test_BiLSTM", "LSTM", mx.gluon.rnn.LSTM, (3,1,2), {'hidden_size': 3, 'bidirectional': True}),
-]
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/python/unittest/onnx/test_onnxruntime.py b/tests/python/unittest/onnx/test_onnxruntime.py
deleted file mode 100644
index 3737b7a..0000000
--- a/tests/python/unittest/onnx/test_onnxruntime.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import mxnet as mx
-import numpy as np
-import onnxruntime
-
-import json
-import os
-import shutil
-
-import pytest
-
-
-def run_cv_model_test(model, tmpdir):
- def get_gluon_cv_model(model_name, tmp):
- tmpfile = os.path.join(tmp, model_name)
- ctx = mx.cpu(0)
- model = mx.gluon.model_zoo.vision.get_model(model_name, pretrained=True, ctx=ctx, root=tmp)
- model.hybridize()
- data = mx.nd.zeros((2,3,224,224), dtype='float32', ctx=ctx)
- model(data)
- model.export(tmpfile, epoch=0)
- sym_file = tmpfile + '-symbol.json'
- params_file = tmpfile + '-0000.params'
- return sym_file, params_file
-
- def export_model_to_onnx(sym_file, params_file):
- input_shape = (1,3,224,224)
- onnx_file = os.path.join(os.path.dirname(sym_file), "model.onnx")
- converted_model_path = mx.contrib.onnx.export_model(sym_file, params_file, [input_shape],
- np.float32, onnx_file)
- return onnx_file
-
- def normalize_image(imgfile):
- image = mx.image.imread(imgfile).asnumpy()
- image_data = np.array(image).transpose(2, 0, 1)
- img_data = image_data.astype('float32')
- mean_vec = np.array([0.485, 0.456, 0.406])
- stddev_vec = np.array([0.229, 0.224, 0.225])
- norm_img_data = np.zeros(img_data.shape).astype('float32')
- for i in range(img_data.shape[0]):
- norm_img_data[i,:,:] = (img_data[i,:,:]/255 - mean_vec[i]) / stddev_vec[i]
- return norm_img_data.reshape(1, 3, 224, 224).astype('float32')
-
- def get_prediction(model, image):
- pass
-
- def softmax(x):
- x = x.reshape(-1)
- e_x = np.exp(x - np.max(x))
- return e_x / e_x.sum(axis=0)
-
- def load_imgnet_labels(tmpdir):
- tmpfile = os.path.join(tmpdir, 'image_net_labels.json')
- mx.test_utils.download('https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/image_net_labels.json',
- fname=tmpfile)
- return np.array(json.load(open(tmpfile, 'r')))
-
- def download_test_images(tmpdir):
- test_images = [
- ['dog.jpg',['boxer']],
- ['apron.jpg', ['apron', 'maillot']],
- ['dolphin.jpg', ['great white shark','grey whale']],
- ['hammerheadshark.jpg', ['tiger shark']],
- ['lotus.jpg', ['pinwheel','pot']]
- ]
- for f,_ in test_images:
- mx.test_utils.download('https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/onnx/images/'+f+'?raw=true',
- fname=os.path.join(tmpdir, f))
- return test_images
-
- labels = load_imgnet_labels(tmpdir)
- test_images = download_test_images(tmpdir)
- sym_file, params_file = get_gluon_cv_model(model, tmpdir)
- onnx_file = export_model_to_onnx(sym_file, params_file)
- #print("exported onnx file: ",onnx_file)
-
- # create onnxruntime session using the generated onnx file
- ses_opt = onnxruntime.SessionOptions()
- ses_opt.log_severity_level = 3
- session = onnxruntime.InferenceSession(onnx_file, ses_opt)
- input_name = session.get_inputs()[0].name
-
- for img,classes in test_images:
- img_data = normalize_image(os.path.join(tmpdir, img))
- raw_result = session.run([], {input_name: img_data})
- res = softmax(np.array(raw_result)).tolist()
- class_idx = np.argmax(res)
- #print("Image top classification:",labels[class_idx])
- sort_idx = np.flip(np.squeeze(np.argsort(res)))
- #print("\tTop labels: " + ",".join(labels[sort_idx[:5]]))
- correct_classification = False
- for label in labels[sort_idx[:5]]:
- for c in classes:
- if c in label:
- correct_classification = True
- assert correct_classification == True
-
- # cleanup
- shutil.rmtree(tmpdir)
-
-@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580")
-def test_cv_model_inference_onnxruntime_mobilenet0_5(tmp_path):
- run_cv_model_test('mobilenet0.5', tmp_path)
-
-def test_cv_model_inference_onnxruntime_mobilenetv2_1_0(tmp_path):
- run_cv_model_test('mobilenetv2_1.0', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet18_v1(tmp_path):
- run_cv_model_test('resnet18_v1', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet18_v2(tmp_path):
- run_cv_model_test('resnet18_v2', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet101_v1(tmp_path):
- run_cv_model_test('resnet101_v1', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet101_v2(tmp_path):
- run_cv_model_test('resnet101_v2', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet152_v1(tmp_path):
- run_cv_model_test('resnet152_v1', tmp_path)
-
-def test_cv_model_inference_onnxruntime_resnet152_v2(tmp_path):
- run_cv_model_test('resnet152_v2', tmp_path)
-
-@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580")
-def test_cv_model_inference_onnxruntime_squeezenet1_0(tmp_path):
- run_cv_model_test('squeezenet1.0', tmp_path)
-
-@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580")
-def test_cv_model_inference_onnxruntime_squeezenet1_1(tmp_path):
- run_cv_model_test('squeezenet1.1', tmp_path)
-
-@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580")
-def test_cv_model_inference_onnxruntime_vgg11(tmp_path):
- run_cv_model_test('vgg11', tmp_path)
-
-@pytest.mark.skip(reason="Older gluon models are not supported, tracked with #19580")
-def test_cv_model_inference_onnxruntime_vgg11_bn(tmp_path):
- run_cv_model_test('vgg11_bn', tmp_path)
-
-def test_cv_model_inference_onnxruntime_vgg19(tmp_path):
- run_cv_model_test('vgg19', tmp_path)
-
-def test_cv_model_inference_onnxruntime_vgg19_bn(tmp_path):
- run_cv_model_test('vgg19_bn', tmp_path)
-
-
-