You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2020/07/10 02:43:24 UTC
[GitHub] [incubator-tvm] FrozenGene commented on a change in pull request #6000: Introduce Caffe frontend to TVM
FrozenGene commented on a change in pull request #6000:
URL: https://github.com/apache/incubator-tvm/pull/6000#discussion_r452590381
##########
File path: python/tvm/relay/frontend/caffe.py
##########
@@ -0,0 +1,1188 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name, unused-argument, too-many-lines, import-outside-toplevel, eval-used, unused-variable, no-else-continue, logging-format-interpolation, no-else-continue, no-else-return
+"""Caffe frontend."""
+from __future__ import absolute_import as _abs
+import logging
+
+import numpy as np
+import tvm
+from tvm.ir import IRModule
+from .. import analysis
+from .. import expr as _expr
+from .. import function as _function
+from .. import op as _op
+from ... import nd as _nd
+from .common import ExprTable
+from .common import infer_shape as _infer_shape
+
+__all__ = ['from_caffe']
+
+
+class OperatorConverter(object):
+ """ Operator Converted for converting Caffe ops to Relay ops """
+ def __init__(self, init_layer_dict, predict_layer, exp_tab):
+ self.init_layer_dict = init_layer_dict
+ self.predict_layer = predict_layer
+ self.exp_tab = exp_tab
+ self.new_bn = {}
+ self.changed_layers = None
+
+ self.convert_map = {
+ 'BatchNorm': self.bn,
+ 'Concat': self.concat,
+ 'Convolution': self.conv,
+ 'Crop': self.crop,
+ 'Deconvolution': self.deconv,
+ 'Dropout': self.dropout,
+ 'Eltwise': self.eltwise,
+ 'Flatten': self.flatten,
+ 'InnerProduct': self.innerproduct,
+ 'Input': None,
+ 'LRN': self.lrn,
+ 'Normalize': self.normalize,
+ 'Permute': self.permute,
+ 'Pooling': self.pooling,
+ 'PReLU': self.prelu,
+ 'PriorBox': self.priorbox,
+ 'proposal': self.proposal,
+ 'PSROIPooling': self.psroipooling,
+ 'Python': self.python_layer,
+ 'ReLU': self.relu,
+ 'Reshape': self.reshape,
+ 'Resize': self.resize,
+ 'ROIPooling': self.roipooling,
+ 'Scale': self.scale,
+ 'Sigmoid': self.sigmoid,
+ 'Slice': self._slice,
+ 'Softmax': self.softmax,
+ 'TanH': self.tanh,
+ 'Upsample': self.upsample
+ }
+
+ def resize(self, op):
+ """ Convert Resize layer """
+ inputs = op.bottom
+
+ # obtain layer params
+ resize_param = op.img_size_param
+ x_scale = float(resize_param.x_scaling)
+ y_scale = float(resize_param.y_scaling)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+
+ # set tvm op params
+ params = dict()
+ params['layout'] = 'NCHW'
+ params['method'] = 'bilinear'
+ params['align_corners'] = False
+
+ out = _op.nn.upsampling(in_expr, y_scale, x_scale, **params)
+
+ return out
+
+ def upsample(self, op):
+ """ Convert Unsample layer """
+ inputs = op.bottom
+ upsample_param = op.upsample_param
+
+ scale = float(upsample_param.scale)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ # set params
+ scale_h_expr = scale
+ scale_w_expr = scale
+ # scale_h_expr = self.exp_tab.new_const(scale, dtype='float32')
+ # scale_w_expr = self.exp_tab.new_const(scale, dtype='float32')
+ params = dict()
+ params['layout'] = 'NCHW'
+ if len(inputs) == 1:
+ params['method'] = 'nearest_neighbor'
+ params['align_corners'] = False
+ out = _op.nn.upsampling(in_expr, scale_h_expr, scale_w_expr,
+ **params)
+ elif len(inputs) == 2:
+ # params['pad_out_h'] = 0
+ # params['pad_out_w'] = 0
Review comment:
remove
##########
File path: python/tvm/relay/frontend/caffe.py
##########
@@ -0,0 +1,1188 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name, unused-argument, too-many-lines, import-outside-toplevel, eval-used, unused-variable, no-else-continue, logging-format-interpolation, no-else-continue, no-else-return
+"""Caffe frontend."""
+from __future__ import absolute_import as _abs
+import logging
+
+import numpy as np
+import tvm
+from tvm.ir import IRModule
+from .. import analysis
+from .. import expr as _expr
+from .. import function as _function
+from .. import op as _op
+from ... import nd as _nd
+from .common import ExprTable
+from .common import infer_shape as _infer_shape
+
+__all__ = ['from_caffe']
+
+
+class OperatorConverter(object):
+ """ Operator Converted for converting Caffe ops to Relay ops """
+ def __init__(self, init_layer_dict, predict_layer, exp_tab):
+ self.init_layer_dict = init_layer_dict
+ self.predict_layer = predict_layer
+ self.exp_tab = exp_tab
+ self.new_bn = {}
+ self.changed_layers = None
+
+ self.convert_map = {
+ 'BatchNorm': self.bn,
+ 'Concat': self.concat,
+ 'Convolution': self.conv,
+ 'Crop': self.crop,
+ 'Deconvolution': self.deconv,
+ 'Dropout': self.dropout,
+ 'Eltwise': self.eltwise,
+ 'Flatten': self.flatten,
+ 'InnerProduct': self.innerproduct,
+ 'Input': None,
+ 'LRN': self.lrn,
+ 'Normalize': self.normalize,
+ 'Permute': self.permute,
+ 'Pooling': self.pooling,
+ 'PReLU': self.prelu,
+ 'PriorBox': self.priorbox,
+ 'proposal': self.proposal,
+ 'PSROIPooling': self.psroipooling,
+ 'Python': self.python_layer,
+ 'ReLU': self.relu,
+ 'Reshape': self.reshape,
+ 'Resize': self.resize,
+ 'ROIPooling': self.roipooling,
+ 'Scale': self.scale,
+ 'Sigmoid': self.sigmoid,
+ 'Slice': self._slice,
+ 'Softmax': self.softmax,
+ 'TanH': self.tanh,
+ 'Upsample': self.upsample
+ }
+
+ def resize(self, op):
+ """ Convert Resize layer """
+ inputs = op.bottom
+
+ # obtain layer params
+ resize_param = op.img_size_param
+ x_scale = float(resize_param.x_scaling)
+ y_scale = float(resize_param.y_scaling)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+
+ # set tvm op params
+ params = dict()
+ params['layout'] = 'NCHW'
+ params['method'] = 'bilinear'
+ params['align_corners'] = False
+
+ out = _op.nn.upsampling(in_expr, y_scale, x_scale, **params)
+
+ return out
+
+ def upsample(self, op):
+ """ Convert Unsample layer """
+ inputs = op.bottom
+ upsample_param = op.upsample_param
+
+ scale = float(upsample_param.scale)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ # set params
+ scale_h_expr = scale
+ scale_w_expr = scale
+ # scale_h_expr = self.exp_tab.new_const(scale, dtype='float32')
+ # scale_w_expr = self.exp_tab.new_const(scale, dtype='float32')
+ params = dict()
+ params['layout'] = 'NCHW'
+ if len(inputs) == 1:
+ params['method'] = 'nearest_neighbor'
+ params['align_corners'] = False
+ out = _op.nn.upsampling(in_expr, scale_h_expr, scale_w_expr,
+ **params)
+ elif len(inputs) == 2:
+ # params['pad_out_h'] = 0
+ # params['pad_out_w'] = 0
+ mask_expr = self.exp_tab.get_expr(inputs[1])
+
+ out = _op.vision.unpooling(in_expr, mask_expr, int(scale_h_expr),
+ int(scale_w_expr), **params)
+
+ return out
+
+ def python_layer(self, op):
+ """ Convert Python layer """
+ inputs = op.bottom
+ python_params = op.python_param
+
+ curr_layer_name = python_params.layer.lower()
+ if curr_layer_name == 'proposallayer':
+ param_dict = {}
+ param_str = python_params.param_str
+ if '{' in param_str:
+ param_dict = eval(param_str)
+ else:
+ param_str = '{' + param_str + '}'
+ param_dict = eval(param_str)
+ # get input expr
+ rpn_cls_prob_expr = self.exp_tab.get_expr(inputs[0])
+ rpn_bbox_pred_expr = self.exp_tab.get_expr(inputs[1])
+ im_info_expr = self.exp_tab.get_expr(inputs[2])
+
+ # set tvm proposal params
+ proposal_params_tvm = dict()
+ proposal_params_tvm['scales'] = param_dict[
+ 'scales'] if 'scales' in param_dict else [8., 16., 32.]
+ proposal_params_tvm['ratios'] = param_dict[
+ 'ratios'] if 'ratios' in param_dict else [0.5, 1., 2.]
+ proposal_params_tvm['feature_stride'] = param_dict[
+ 'feat_stride'] if 'feat_stride' in param_dict else 16
+ proposal_params_tvm['rpn_pre_nms_top_n'] = param_dict[
+ 'rpn_pre_nms_top_n'] if 'rpn_pre_nms_top_n' in param_dict else 6000
+ proposal_params_tvm['rpn_post_nms_top_n'] = param_dict[
+ 'rpn_post_nms_top_n'] if 'rpn_post_nms_top_n' in param_dict else 300
+ proposal_params_tvm['threshold'] = param_dict[
+ 'rpn_nms_thresh'] if 'rpn_nms_thresh' in param_dict else 0.7
+ proposal_params_tvm['rpn_min_size'] = param_dict[
+ 'rpn_min_size'] if 'rpn_min_size' in param_dict else 16
+ proposal_params_tvm['iou_loss'] = param_dict[
+ 'iou_loss'] if 'iou_loss' in param_dict else False
+
+ out = _op.vision.proposal(rpn_cls_prob_expr, rpn_bbox_pred_expr,
+ im_info_expr, **proposal_params_tvm)
+ else:
+ tvm.error.OpNotImplemented(
+ 'Python.{} has not been supported!'.format(curr_layer_name))
+ return out
+
+ def psroipooling(self, op):
+ """ Convert PSROIPooling layer """
+ inputs = op.bottom
+ psroi_pooling_param = op.psroi_pooling_param
+
+ # get inputs expr
+ rfcn_cls_expr = self.exp_tab.get_expr(inputs[0])
+ rois_expr = self.exp_tab.get_expr(inputs[1])
+
+ # set tvm params
+ params = dict()
+ params['spatial_scale'] = psroi_pooling_param.spatial_scale
+ params['output_dim'] = psroi_pooling_param.output_dim
+ params['group_size'] = psroi_pooling_param.group_size
+
+ out = _op.vision.psroipooling(rfcn_cls_expr, rois_expr, **params)
+ return out
+
+ def permute(self, op):
+ """ Convert Permute layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ permute_params = op.permute_param.order
+
+ out = _op.transpose(in_expr, permute_params)
+
+ return out
+
+ def normalize(self, op):
+ """ Convert Normalize layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ in_shape = _infer_shape(in_expr)
+ norm_params = op.norm_param
+
+ across_spatial = norm_params.across_spatial
+ channel_shared = norm_params.channel_shared
+ eps = norm_params.eps
+
+ if channel_shared:
+ scale = np.asarray(norm_params.scale_filler.value, np.float32)
+ scale_expr = self.exp_tab.new_const(scale, dtype='float32')
+ else:
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+ scale = np.asarray(weight_bias_blobs[0].data, np.float32).reshape(
+ (in_shape[0], in_shape[1], 1, 1))
+ scale_expr = self.exp_tab.new_const(scale, dtype='float32')
+ if across_spatial:
+ out = _op.nn.l2_normalize(in_expr, eps)
+ else:
+ out = _op.nn.l2_normalize(in_expr, eps, axis=[1])
+
+ out = _op.multiply(out, scale_expr)
+
+ return out
+
+ def add_box(self, top_data, x, y, width, height, img_width, img_height):
+ """ Generate box coordinate """
+ # xmin
+ top_data.append((x - width / 2.) / img_width)
+ # ymin
+ top_data.append((y - height / 2.) / img_height)
+ # xmax
+ top_data.append((x + width / 2.) / img_width)
+ # ymax
+ top_data.append((y + height / 2.) / img_height)
+ return top_data
+
+ def priorbox(self, op):
+ """ Convert PriorBox layer """
+ inputs = op.bottom
+
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ in_expr_img = self.exp_tab.get_expr(inputs[1])
+ pre_n, pre_c, pre_h, pre_w = _infer_shape(in_expr)
+
+ img_n, img_c, img_h, img_w = _infer_shape(in_expr_img)
+
+ priorbox_params = op.prior_box_param
+ flip = priorbox_params.flip
+ variance = np.asarray(priorbox_params.variance)
+ ratios = [1]
+ for r in priorbox_params.aspect_ratio:
+ if r != 1:
+ ratios.append(r)
+ if flip:
+ ratios.append(1 / r)
+
+ min_size = priorbox_params.min_size
+ max_size = priorbox_params.max_size
+
+ steps = priorbox_params.step
+ if steps:
+ step_h = step_w = steps
+ else:
+ step_h = img_h / pre_h
+ step_w = img_w / pre_w
+
+ offsets = priorbox_params.offset
+ clip = priorbox_params.clip
+ num_priors_ = len(ratios)
+ if max_size:
+ for i in range(len(max_size)):
+ num_priors_ += 1
+
+ dim = pre_h * pre_w * num_priors_ * 4
+ out = []
+ for h in range(pre_h):
+ for w in range(pre_w):
+ center_x = (w + offsets) * step_w
+ center_y = (h + offsets) * step_h
+ for s, _ in enumerate(min_size):
+ min_size_ = min_size[s]
+ box_width = box_height = min_size_
+ out = self.add_box(out, center_x, center_y, box_width,
+ box_height, img_w, img_h)
+ if max_size:
+ assert len(max_size) == len(
+ min_size), "max_size and min_size should be same"
+ max_size_ = max_size[s]
+ box_width = box_height = np.sqrt(min_size_ * max_size_)
+ out = self.add_box(out, center_x, center_y, box_width,
+ box_height, img_w, img_h)
+ for r, _ in enumerate(ratios):
+ ar = ratios[r]
+ if ar != 1:
+ box_width = min_size_ * np.sqrt(ar)
+ box_height = min_size_ / np.sqrt(ar)
+ out = self.add_box(out, center_x, center_y,
+ box_width, box_height, img_w,
+ img_h)
+ if clip:
+ out = np.clip(out, 0, 1)
+ if len(variance) == 1:
+ variance_ = np.full(dim, variance[0], dtype='float32')
+ else:
+ variance_ = np.tile(variance, (int(dim / 4), 1))
+
+ variance_ = variance_.reshape(-1)
+ out = np.append(out, variance_)
+ out = np.asarray(out, dtype='float32')
+ out = self.exp_tab.new_const(out, dtype='float32')
+ out = _op.reshape(out, (img_n, 2, -1))
+
+ return out
+
+ def flatten(self, op):
+ """ Convert Flatten layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ in_shape = _infer_shape(in_expr)
+
+ flatten_params = op.flatten_param.axis
+ assert flatten_params == 1, "flatten axis should be 1"
+ out = _op.nn.batch_flatten(in_expr)
+
+ return out
+
+ def eltwise(self, op):
+ """ Convert Eltwise layer """
+ inputs = op.bottom
+ assert len(inputs) == 2, "input tensors length should be 2"
+
+ lhs_expr = self.exp_tab.get_expr(inputs[0])
+ rhs_expr = self.exp_tab.get_expr(inputs[1])
+
+ lhs_shape = _infer_shape(lhs_expr)
+ rhs_shape = _infer_shape(rhs_expr)
+
+ assert lhs_shape == rhs_shape, "input tensors shape should be equal"
+
+ eltwise_params = op.eltwise_param
+ eltwise_type_dict = ['PROD', 'SUM', 'MAX']
+ eltwise_type = eltwise_params.operation
+ coeff = list(eltwise_params.coeff)
+
+ if eltwise_type_dict[eltwise_type] == 'PROD':
+ out = _op.multiply(lhs_expr, rhs_expr)
+ elif eltwise_type_dict[eltwise_type] == 'SUM':
+ if coeff:
+ left_coeff_expr = self.exp_tab.new_const(
+ np.asarray(coeff[0], np.float32))
+ right_coeff_expr = self.exp_tab.new_const(
+ np.asarray(coeff[1], np.float32))
+ lhs_expr_scale = _op.multiply(lhs_expr, left_coeff_expr)
+ rhs_expr_scale = _op.multiply(rhs_expr, right_coeff_expr)
+ out = _op.add(lhs_expr_scale, rhs_expr_scale)
+ else:
+ out = _op.add(lhs_expr, rhs_expr)
+ elif eltwise_type_dict[eltwise_type] == 'MAX':
+ out = _op.maximum(lhs_expr, rhs_expr)
+ else:
+ raise tvm.error.OpNotImplemented(
+ "eltwise_type {} is not supported for frontend Caffe.".format(
+ eltwise_type))
+
+ return out
+
+ def _parse_conv_params(self, op):
+ """ Parse the parameters of Convolution and Deconvolution layer """
+ nonzone = lambda val, pos, dflt: val[pos] if pos < len(val) else dflt
+
+ conv_params = op.convolution_param
+
+ params = dict()
+ # parse kernel size
+ if conv_params.kernel_h > 0 or conv_params.kernel_w > 0:
+ params['kernel_size'] = (conv_params.kernel_h,
+ conv_params.kernel_w)
+ else:
+ ksize_h = nonzone(conv_params.kernel_size, 0, 1)
+ ksize_w = nonzone(conv_params.kernel_size, 1, ksize_h)
+ params['kernel_size'] = (ksize_h, ksize_w)
+
+ # parse padding size
+ if conv_params.pad_h > 0 or conv_params.pad_w > 0:
+ params['padding'] = (conv_params.pad_h, conv_params.pad_w)
+ else:
+ pad_h = nonzone(conv_params.pad, 0, 0)
+ pad_w = nonzone(conv_params.pad, 1, pad_h)
+ params['padding'] = (pad_h, pad_w)
+
+ # parse stride size
+ if conv_params.stride_h > 0 or conv_params.stride_w > 0:
+ params['strides'] = (conv_params.stride_h, conv_params.stride_w)
+ else:
+ stride_h = nonzone(conv_params.stride, 0, 1)
+ stride_w = nonzone(conv_params.stride, 1, stride_h)
+ params['strides'] = (stride_h, stride_w)
+
+ # parse dilation size
+ if hasattr(conv_params, 'dilation') and len(conv_params.dilation) > 0:
+ dilation = ' '.join(str(d) for d in conv_params.dilation)
+ dilation = tuple(map(int, dilation.split(' ')))
+ params['dilation'] = dilation
+ if len(dilation) == 1:
+ params['dilation'] = (dilation[0], dilation[0])
+
+ params['kernel_layout'] = 'OIHW'
+ params['data_layout'] = 'NCHW'
+ params['groups'] = conv_params.group
+ params['channels'] = conv_params.num_output
+ return params
+
+ def bn(self, op):
+ """ Convert BatchNorm layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ n, c, h, w = _infer_shape(in_expr)
+
+ if op.name in self.new_bn:
+ mean, var, eps, gamma, beta = self.new_bn[op.name]
+ mean_expr = self.exp_tab.new_const(mean, dtype='float32')
+ var_expr = self.exp_tab.new_const(var, dtype='float32')
+ gamma_expr = self.exp_tab.new_const(gamma, dtype='float32')
+ beta_expr = self.exp_tab.new_const(beta, dtype='float32')
+ out = _op.nn.batch_norm(in_expr,
+ gamma_expr,
+ beta_expr,
+ mean_expr,
+ var_expr,
+ epsilon=eps,
+ scale=True)
+
+ else:
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+ mean = np.asarray(weight_bias_blobs[0].data, np.float32)
+ var = np.asarray(weight_bias_blobs[1].data, np.float32)
+ if len(weight_bias_blobs) == 2:
+ mean = np.repeat(mean, h * w).reshape((c, h, w))
+ mean = np.expand_dims(mean, 0).repeat(n, axis=0)
+ mean_expr = self.exp_tab.new_const(mean, dtype='float32')
+
+ var = np.repeat(var, h * w).reshape((c, h, w))
+ var = np.expand_dims(var, 0).repeat(n, axis=0)
+ var_expr = self.exp_tab.new_const(var, dtype='float32')
+
+ tmp_out = _op.multiply(in_expr, mean_expr)
+ out = _op.add(tmp_out, var_expr)
+
+ return out
+ else:
+ scale = np.asarray(weight_bias_blobs[2].data, np.float32)
+ if scale:
+ scale = 1 / scale
+ mean_expr = self.exp_tab.new_const(mean * scale, dtype='float32')
+ var_expr = self.exp_tab.new_const(var * scale, dtype='float32')
+
+ #caffe bn layer not support scale
+ gamma_expr = self.exp_tab.new_const(np.ones(mean.shape,
+ dtype=np.float32),
+ dtype='float32')
+ beta_expr = self.exp_tab.new_const(np.zeros(mean.shape,
+ dtype=np.float32),
+ dtype='float32')
+
+ bn_params = op.batch_norm_param.eps
+ out = _op.nn.batch_norm(in_expr,
+ gamma_expr,
+ beta_expr,
+ mean_expr,
+ var_expr,
+ epsilon=bn_params,
+ scale=False)
+
+ return out[0]
+
+ def scale(self, op):
+ """ Convert Scale layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+
+ params = dict()
+ params['bias'] = op.scale_param.bias_term
+ params['axis'] = op.scale_param.axis
+
+ gamma = np.asarray(weight_bias_blobs[0].data, np.float32)
+ gamma_expr = self.exp_tab.new_const(gamma, dtype='float32')
+ if params['bias']:
+ beta = np.asarray(weight_bias_blobs[1].data, np.float32)
+ beta_expr = self.exp_tab.new_const(beta, dtype='float32')
+ else:
+ beta_expr = self.exp_tab.new_const(np.zeros(gamma.shape,
+ dtype=np.float32),
+ dtype='float32')
+
+ n, c, h, w = _infer_shape(in_expr)
+ gamma_expr = _op.reshape(gamma_expr, newshape=(1, c, 1, 1))
+ beta_expr = _op.reshape(beta_expr, newshape=(1, c, 1, 1))
+ out = _op.multiply(in_expr, gamma_expr)
+ out = _op.add(out, beta_expr)
+
+ return out
+
+ def concat(self, op):
+ """ Convert Concat layer """
+ inputs = op.bottom
+ in_expr = (self.exp_tab.get_expr(inputs[i])
+ for i in range(len(inputs)))
+
+ params = dict()
+ params['axis'] = op.concat_param.axis
+ out = _op.concatenate(in_expr, axis=params['axis'])
+
+ return out
+
+ def reshape(self, op):
+ """ Convert Reshape layer """
+ inputs = op.bottom
+ input_name = inputs[0]
+
+ reshape_param = op.reshape_param
+ dims = list(reshape_param.shape.dim)
+
+ in_expr = self.exp_tab.get_expr(input_name)
+ input_shape = list(_infer_shape(in_expr))
+
+ start_axis = int(reshape_param.axis)
+ if start_axis < 0:
+ start_axis = len(input_shape) + start_axis + 1
+ num_axes = int(reshape_param.num_axes)
+ end_axis = len(input_shape)
+ if num_axes != -1:
+ end_axis = start_axis + num_axes
+
+ left_shape = input_shape[:start_axis]
+ if end_axis == len(input_shape):
+ center_shape = input_shape[start_axis:]
+ right_shape = []
+ else:
+ center_shape = input_shape[start_axis:end_axis]
+ right_shape = input_shape[end_axis:]
+
+ for idx, dim in enumerate(dims):
+ if dim == 0:
+ dims[idx] = center_shape[idx]
+
+ tmp = np.random.rand(*center_shape)
+ tmp = np.reshape(tmp, dims)
+ center_shape = list(tmp.shape)
+
+ newshape = left_shape + center_shape + right_shape
+
+ out = _op.reshape(in_expr, newshape=newshape)
+ return out
+
+ def softmax(self, op):
+ """ Convert Softmax layer """
+ inputs = op.bottom
+ assert len(inputs) == 1, "input tensors length should be 1"
+
+ input_name = inputs[0]
+ in_expr = self.exp_tab.get_expr(input_name)
+
+ softmax_param = op.softmax_param
+ parmas = {'axis': softmax_param.axis}
+
+ out = _op.nn.softmax(in_expr, **parmas)
+
+ return out
+
+ def conv(self, op):
+ """ Convert Convolution layer """
+ params = self._parse_conv_params(op)
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+ conv_params = op.convolution_param
+ inputs = op.bottom
+ # process weight and bias blobs
+ weight, bias = None, None
+ if len(weight_bias_blobs) > 1:
+ weight = weight_bias_blobs[0]
+ bias = weight_bias_blobs[1]
+ else:
+ weight = weight_bias_blobs[0]
+ if weight:
+ kh, kw = params['kernel_size']
+ weight_shape = [conv_params.num_output, -1, kh, kw]
+ weight_value = np.asarray(weight.data, np.float32)
+ weight_value = np.reshape(weight_value, weight_shape)
+ else:
+ raise Exception('No weight value of layer {} in caffemodel'.format(
+ op.name))
+
+ weight_expr = self.exp_tab.new_const(weight_value, dtype='float32')
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ out = _op.nn.conv2d(data=in_expr, weight=weight_expr, **params)
+ if bias:
+ bias_value = np.asarray(bias.data, np.float32)
+ bias_expr = self.exp_tab.new_const(bias_value, dtype='float32')
+ out = _op.nn.bias_add(out, bias_expr)
+ return out
+
+ def pooling(self, op):
+ """ Convert Pooling layer """
+ inputs = op.bottom
+ input_name = inputs[0]
+
+ pool_params = op.pooling_param
+ pool_type_dict = ['MAX', 'AVE', 'STOCHASTIC']
+
+ params = dict()
+ # parse pool type: 0: MAX, 1: AVE, 2: STOCHASTIC
+ pool_type = pool_params.pool
+ # parse kernel size
+ if pool_params.kernel_h > 0 or pool_params.kernel_w > 0:
+ params['pool_size'] = (pool_params.kernel_h, pool_params.kernel_w)
+ else:
+ params['pool_size'] = (pool_params.kernel_size,
+ pool_params.kernel_size)
+
+ # parse padding size
+ if pool_params.pad_h > 0 or pool_params.pad_w > 0:
+ params['padding'] = (pool_params.pad_h, pool_params.pad_w)
+ else:
+ params['padding'] = (pool_params.pad, pool_params.pad)
+
+ # parse stride size
+ if pool_params.stride_h > 0 or pool_params.stride_w > 0:
+ params['strides'] = (pool_params.stride_h, pool_params.stride_w)
+ else:
+ params['strides'] = (pool_params.stride, pool_params.stride)
+
+ params['ceil_mode'] = True
+ if hasattr(pool_params, 'ceil_mode'):
+ params['ceil_mode'] = pool_params.ceil_mode
+
+ in_expr = self.exp_tab.get_expr(input_name)
+
+ if pool_type_dict[pool_type] == 'MAX':
+ if pool_params.global_pooling:
+ out = _op.nn.global_max_pool2d(in_expr)
+ else:
+ if len(op.top) == 1:
+ out = _op.nn.max_pool2d(in_expr, **params)
+ elif len(op.top) == 2:
+ out1 = _op.nn.max_pool2d_with_argmax(in_expr, **params)
+ out2 = _op.vision.max_pool2d_location(in_expr, **params)
+ return _expr.Tuple((out1, out2))
+
+ elif pool_type_dict[pool_type] == 'AVE': # AVE
+ if pool_params.global_pooling:
+ out = _op.nn.global_avg_pool2d(in_expr)
+ else:
+ params['count_include_pad'] = True
+ out = _op.nn.avg_pool2d(in_expr, **params)
+ else:
+ raise tvm.error.OpNotImplemented(
+ "Operator {} is not supported for frontend Caffe.".format(
+ pool_type_dict[pool_type] + ' pool'))
+
+ return out
+
+ def lrn(self, op):
+ """ Convert LRN layer """
+ inputs = op.bottom
+ input_name = inputs[0]
+
+ params = dict()
+ lrn_params = op.lrn_param
+ params['size'] = lrn_params.local_size
+ params['bias'] = lrn_params.k
+ params['alpha'] = lrn_params.alpha
+ params['beta'] = lrn_params.beta
+
+ in_expr = self.exp_tab.get_expr(input_name)
+ out = _op.nn.lrn(in_expr, **params)
+ return out
+
+ def innerproduct(self, op):
+ """ Convert InnerProduct layer """
+ inputs = op.bottom
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+ dense_params = op.inner_product_param
+
+ params = dict()
+ params["num_output"] = dense_params.num_output
+ params["bias"] = dense_params.bias_term
+ params["axis"] = dense_params.axis
+ if params["axis"] != 1:
+ raise Exception("Only support 2D InnerProduct")
+
+ # process weight and bias blobs
+ weight, bias = None, None
+ if params["bias"]:
+ weight = weight_bias_blobs[0]
+ bias = weight_bias_blobs[1]
+ else:
+ weight = weight_bias_blobs[0]
+
+ if weight:
+ weight_value = np.asarray(weight.data, np.float32)
+ weight_value = np.reshape(weight_value, (params["num_output"], -1))
+ weight_shape = weight_value.shape
+ else:
+ raise Exception('No weight value of layer {} in caffemodel'.format(
+ op.name))
+
+ weight_expr = self.exp_tab.new_const(weight_value, dtype='float32')
+
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ in_reshape = _op.reshape(data=in_expr, newshape=(-1, weight_shape[-1]))
+
+ out = _op.nn.dense(data=in_reshape, weight=weight_expr)
+
+ if bias:
+ bias_value = np.asarray(bias.data, np.float32)
+ bias_expr = self.exp_tab.new_const(bias_value, dtype='float32')
+ out = _op.nn.bias_add(out, bias_expr, axis=params["axis"])
+ return out
+
+ def dropout(self, op):
+ """ Convert Dropout layer """
+ inputs = op.bottom
+ input_name = inputs[0]
+
+ params = dict()
+ dropout_params = op.dropout_param
+
+ params['rate'] = dropout_params.dropout_ratio
+
+ in_expr = self.exp_tab.get_expr(input_name)
+ out = _op.nn.dropout(in_expr, **params)
+ return out
+
+ def relu(self, op):
+ """ Convert ReLU layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ negative_slope = op.relu_param.negative_slope
+ if negative_slope:
+ out = _op.nn.leaky_relu(in_expr, negative_slope)
+ return out
+
+ out = _op.nn.relu(in_expr)
+ return out
+
+ def prelu(self, op):
+ """ Convert PReLU layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+
+ alpha = self.init_layer_dict[op.name].blobs[0].data
+ alpha = np.asarray(alpha, np.float32)
+ alpha = self.exp_tab.new_const(alpha, dtype='float32')
+ axis = 1
+ out = _op.nn.prelu(in_expr, alpha, axis=axis)
+ return out
+
+ def deconv(self, op):
+ """ Convert Deconvolution layer """
+ params = self._parse_conv_params(op)
+ weight_bias_blobs = self.init_layer_dict[op.name].blobs
+ conv_params = op.convolution_param
+ inputs = op.bottom
+
+ # process weight and bias blobs
+ weight, bias = None, None
+ if len(weight_bias_blobs) > 1:
+ weight = weight_bias_blobs[0]
+ bias = weight_bias_blobs[1]
+ else:
+ weight = weight_bias_blobs[0]
+ if weight:
+ kh, kw = params['kernel_size']
+ weight_shape = [-1, conv_params.num_output, kh, kw]
+ weight_value = np.asarray(weight.data, np.float32)
+ weight_value = np.reshape(weight_value, weight_shape)
+ else:
+ raise Exception('No weight value of layer {} in caffemodel'.format(
+ op.name))
+
+ weight_expr = self.exp_tab.new_const(weight_value, dtype='float32')
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ out = _op.nn.conv2d_transpose(data=in_expr,
+ weight=weight_expr,
+ **params)
+
+ if bias:
+
+ bias_value = np.asarray(bias.data, np.float32)
+ bias_expr = self.exp_tab.new_const(bias_value, dtype='float32')
+ out = _op.nn.bias_add(out, bias_expr)
+ return out
+
+ def _slice(self, op):
+ """ Convert Slice layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+
+ output_num = len(op.top)
+
+ slice_params = op.slice_param
+ axis = int(slice_params.axis)
+ indices_or_sections = list([int(s) for s in slice_params.slice_point])
+ if len(indices_or_sections) == 0:
+ indices_or_sections = output_num
+ else:
+ indices_or_sections = sorted(indices_or_sections)
+
+ out = _op.split(in_expr,
+ indices_or_sections=indices_or_sections,
+ axis=axis)
+ return out
+
+ def sigmoid(self, op):
+ """ Convert Sigmoid layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ out = _op.sigmoid(in_expr)
+ return out
+
+ def tanh(self, op):
+ """ Convert TanH layer """
+ inputs = op.bottom
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ out = _op.tanh(in_expr)
+ return out
+
+ def crop(self, op):
+ """ Convert Crop layer """
+ inputs = op.bottom
+ assert len(inputs) == 2, "Need two inputs of Crop layer"
+ in_expr_a = self.exp_tab.get_expr(inputs[0])
+ in_expr_b = self.exp_tab.get_expr(inputs[1])
+
+ # parse crop params
+ crop_params = op.crop_param
+ axis = int(getattr(crop_params, 'axis', 2))
+ offset = list(getattr(crop_params, 'offset', 0))
+
+ # expand offset to (offset1, offset2, ...)
+ in_a_shape = _infer_shape(in_expr_a)
+ num_to_crop = len(in_a_shape) - axis
+ if not offset:
+ offset = [0] * num_to_crop
+ if len(offset) == 1:
+ offset = offset * num_to_crop
+ elif len(offset) != num_to_crop:
+ raise Exception("No matching the number between axis and offset!")
+
+ slice_end = in_a_shape
+ slice_start = [0] * len(in_a_shape)
+ for i in range(num_to_crop):
+ slice_start[i + axis] = offset[i]
+
+ to_crop_axis = list(range(len(in_a_shape)))
+ to_crop_axis = to_crop_axis[axis:]
+
+ # secondly, crop in_expr_a by in_expr_b
+ in_expr_a_stride = _op.strided_slice(in_expr_a, slice_start, slice_end)
+ out = _op.slice_like(in_expr_a_stride, in_expr_b, axes=to_crop_axis)
+ return out
+
+ def proposal(self, op):
+ """ Convert proposal layer"""
+ inputs = op.bottom
+ assert len(inputs) == 2, "Need two inputs of proposal layer"
+ rpn_cls_prob_expr = self.exp_tab.get_expr(inputs[0])
+ rpn_bbox_pred = self.exp_tab.get_expr(inputs[1])
+
+ model_input = self.predict_layer[0].top[0]
+ n, c, h, w = _infer_shape(self.exp_tab.get_expr(model_input))
+ im_info = np.array([h, w, 1], dtype=np.float32)
+ im_info = np.tile(im_info, (n, 1))
+ im_info_expr = self.exp_tab.new_const(im_info, dtype='float32')
+
+ proposal_params = op.proposal_param
+ params = dict()
+ if hasattr(proposal_params,
+ 'scale') and len(proposal_params.scale) > 0:
+ params['scales'] = list(float(s) for s in proposal_params.scale)
+ if hasattr(proposal_params,
+ 'ratio') and len(proposal_params.ratio) > 0:
+ params['ratios'] = list(float(r) for r in proposal_params.ratio)
+ if hasattr(proposal_params, 'base_size'):
+ pass
+ if hasattr(proposal_params, 'feat_stride'):
+ params['feature_stride'] = int(proposal_params.feat_stride)
+ if hasattr(proposal_params, 'pre_nms_topn'):
+ params['rpn_pre_nms_top_n'] = int(proposal_params.pre_nms_topn)
+ if hasattr(proposal_params, 'post_nms_topn'):
+ params['rpn_post_nms_top_n'] = int(proposal_params.post_nms_topn)
+ if hasattr(proposal_params, 'nms_thresh'):
+ params['threshold'] = float(proposal_params.nms_thresh)
+ if hasattr(proposal_params, 'min_size'):
+ params['rpn_min_size'] = int(proposal_params.min_size)
+ params['iou_loss'] = False
+
+ out = _op.vision.proposal(rpn_cls_prob_expr, rpn_bbox_pred,
+ im_info_expr, **params)
+ out_score = _op.zeros((n, 1), dtype='float32')
+ return out, out_score
+
+ def roipooling(self, op):
+ """ Convert ROIPooling layer """
+ inputs = op.bottom
+ conv_feature_expr = self.exp_tab.get_expr(inputs[0])
+ proposal_expr = self.exp_tab.get_expr(inputs[1])
+
+ roipooling_params = op.roi_pooling_param
+ params = dict()
+ params['pooled_size'] = (int(roipooling_params.pooled_h),
+ int(roipooling_params.pooled_w))
+ params['spatial_scale'] = float(roipooling_params.spatial_scale)
+
+ out = _op.vision.roi_pool(conv_feature_expr, proposal_expr, **params)
+ return out
+
+ def check_unsupported_ops(self):
+ """Check unsupported Caffe ops in our converter."""
+ logging.debug("check unsupported ops")
+ unsupported_ops_set = set()
+
+ include_layer = dict()
+ for pl in self.predict_layer:
+ if pl.type not in include_layer:
+ include_layer[pl.type] = 1
+ else:
+ include_layer[pl.type] = include_layer[pl.type] + 1
+ logging.debug("include layers: {}".format(include_layer.items()))
+
+ for pl in self.predict_layer:
+ op_name = pl.type
+ if op_name not in self.convert_map:
+ unsupported_ops_set.add(op_name)
+
+ if unsupported_ops_set:
+ msg = 'The following operators are not supported in frontend ' \
+ 'Caffe: {}'
+ ops = str(list(unsupported_ops_set)).strip('[,]')
+ raise tvm.error.OpNotImplemented(msg.format(ops))
+
+ def fuse_op(self, layers):
+ """ Fusing the BatchNorm and Scale layer """
+ bn, scale = layers["bn"], layers["scale"]
+
+ # bn params
+ bn_weight_bias_blobs = self.init_layer_dict[bn.name].blobs
+ bn_scale = np.asarray(bn_weight_bias_blobs[2].data, np.float32)
+ if bn_scale:
+ bn_scale = 1 / bn_scale
+ bn_mean = np.asarray(bn_weight_bias_blobs[0].data,
+ np.float32) * bn_scale
+ bn_var = np.asarray(bn_weight_bias_blobs[1].data,
+ np.float32) * bn_scale
+ bn_eps = bn.batch_norm_param.eps
+
+ # scale params
+ scale_weight_bias_blobs = self.init_layer_dict[scale.name].blobs
+ scale_gamma = np.asarray(scale_weight_bias_blobs[0].data, np.float32)
+ scale_bias = scale.scale_param.bias_term
+ if scale_bias:
+ scale_beta = np.asarray(scale_weight_bias_blobs[1].data,
+ np.float32)
+ else:
+ scale_beta = np.zeros(scale_gamma.shape, dtype=np.float32)
+
+ # new params
+ self.new_bn[bn.name] = [
+ bn_mean, bn_var, bn_eps, scale_gamma, scale_beta
+ ]
+ return bn
+
+ def op_fuse(self):
+ """fuse bn and scale """
+ logging.debug("Caffe:fuse bn and scale")
+ new_layers = []
+ temp_layers = {}
+ changed_layers = {}
+
+ for index, pl in enumerate(self.predict_layer):
+ op_type = pl.type
+ if op_type == "Input":
+ new_layers.append(pl)
+ continue
+ elif op_type == "BatchNorm":
+ if (index != len(self.predict_layer) - 1) and (
+ self.predict_layer[index + 1].type == "Scale"):
+ temp_layers["bn"] = pl
+ continue
+ else:
+ new_layers.append(pl)
+ temp_layers.clear()
+ elif op_type == "Scale":
+ if self.predict_layer[index - 1].type == "BatchNorm":
+ temp_layers["scale"] = pl
+ else:
+ new_layers.append(pl)
+ temp_layers.clear()
+ else:
+ temp_layers.clear()
+
+ if len(temp_layers) == 2:
+ layer = self.fuse_op(temp_layers)
+ new_layers.append(layer)
+ changed_layers[
+ temp_layers["scale"].name] = temp_layers['bn'].name
+
+ for idx, plt in enumerate(pl.bottom):
+ if plt in changed_layers:
+ pl.bottom[idx] = changed_layers[plt]
+
+ if op_type not in ['BatchNorm', 'Scale']:
+ new_layers.append(pl)
+
+ self.predict_layer = new_layers
+ self.changed_layers = changed_layers
+
+ def convert_op_to_relay(self):
+ """Convert Caffe ops to relay ops"""
+ logging.debug("convert op to relay")
+
+ for pl in self.predict_layer:
+ op_type = pl.type
+ if op_type == "Input":
+ continue
+ output_tensors = pl.top
+
+ ret = self.convert_map[op_type](pl)
+
+ if len(output_tensors) == 1:
+ self.exp_tab.set_expr(output_tensors[0], ret)
+ logging.debug(
+ "layer_name:{}, type:{}, output_name:{}, shape:{}".format(
+ pl.name, pl.type, output_tensors[0],
+ _infer_shape(ret)))
Review comment:
don't add this debug information
##########
File path: python/tvm/relay/frontend/caffe.py
##########
@@ -0,0 +1,1188 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name, unused-argument, too-many-lines, import-outside-toplevel, eval-used, unused-variable, no-else-continue, logging-format-interpolation, no-else-continue, no-else-return
+"""Caffe frontend."""
+from __future__ import absolute_import as _abs
+import logging
+
+import numpy as np
+import tvm
+from tvm.ir import IRModule
+from .. import analysis
+from .. import expr as _expr
+from .. import function as _function
+from .. import op as _op
+from ... import nd as _nd
+from .common import ExprTable
+from .common import infer_shape as _infer_shape
+
+__all__ = ['from_caffe']
+
+
+class OperatorConverter(object):
+ """ Operator Converted for converting Caffe ops to Relay ops """
+ def __init__(self, init_layer_dict, predict_layer, exp_tab):
+ self.init_layer_dict = init_layer_dict
+ self.predict_layer = predict_layer
+ self.exp_tab = exp_tab
+ self.new_bn = {}
+ self.changed_layers = None
+
+ self.convert_map = {
+ 'BatchNorm': self.bn,
+ 'Concat': self.concat,
+ 'Convolution': self.conv,
+ 'Crop': self.crop,
+ 'Deconvolution': self.deconv,
+ 'Dropout': self.dropout,
+ 'Eltwise': self.eltwise,
+ 'Flatten': self.flatten,
+ 'InnerProduct': self.innerproduct,
+ 'Input': None,
+ 'LRN': self.lrn,
+ 'Normalize': self.normalize,
+ 'Permute': self.permute,
+ 'Pooling': self.pooling,
+ 'PReLU': self.prelu,
+ 'PriorBox': self.priorbox,
+ 'proposal': self.proposal,
+ 'PSROIPooling': self.psroipooling,
+ 'Python': self.python_layer,
+ 'ReLU': self.relu,
+ 'Reshape': self.reshape,
+ 'Resize': self.resize,
+ 'ROIPooling': self.roipooling,
+ 'Scale': self.scale,
+ 'Sigmoid': self.sigmoid,
+ 'Slice': self._slice,
+ 'Softmax': self.softmax,
+ 'TanH': self.tanh,
+ 'Upsample': self.upsample
+ }
+
+ def resize(self, op):
+ """ Convert Resize layer """
+ inputs = op.bottom
+
+ # obtain layer params
+ resize_param = op.img_size_param
+ x_scale = float(resize_param.x_scaling)
+ y_scale = float(resize_param.y_scaling)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+
+ # set tvm op params
+ params = dict()
+ params['layout'] = 'NCHW'
+ params['method'] = 'bilinear'
+ params['align_corners'] = False
+
+ out = _op.nn.upsampling(in_expr, y_scale, x_scale, **params)
+
+ return out
+
+ def upsample(self, op):
+ """ Convert Unsample layer """
+ inputs = op.bottom
+ upsample_param = op.upsample_param
+
+ scale = float(upsample_param.scale)
+
+ # get input expr
+ in_expr = self.exp_tab.get_expr(inputs[0])
+ # set params
+ scale_h_expr = scale
+ scale_w_expr = scale
+ # scale_h_expr = self.exp_tab.new_const(scale, dtype='float32')
+ # scale_w_expr = self.exp_tab.new_const(scale, dtype='float32')
Review comment:
remove
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org