You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2021/12/02 15:54:16 UTC
[GitHub] [tvm] manupa-arm commented on a change in pull request #9456: [6/10] Code generation for fully connected layer via CMSIS-NN
manupa-arm commented on a change in pull request #9456:
URL: https://github.com/apache/tvm/pull/9456#discussion_r761229235
##########
File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc
##########
@@ -234,6 +222,114 @@ class RelayToTIRVisitor : public MixedModeMutator {
context_buffer_size);
}
+ void EmitFullyConnected(const GlobalVar& global_var, const Expr& expr) {
+ const CallNode* clip_call = nullptr;
+ const CallNode* requantize_call = nullptr;
+ const CallNode* bias_add_call = nullptr;
+ const CallNode* fc_call = nullptr;
+ const CallNode* final_call = expr.as<CallNode>();
+ const OpNode* final_op = final_call->op.as<OpNode>();
+ if (final_op->name == "clip") {
+ clip_call = final_call;
+ requantize_call = clip_call->args[0].as<CallNode>();
+ } else {
+ requantize_call = final_call;
+ }
+ const CallNode* requantize_input = requantize_call->args[0].as<CallNode>();
+ const OpNode* requantize_input_op = requantize_input->op.as<OpNode>();
+ if (requantize_input_op->name == "nn.bias_add") {
+ bias_add_call = requantize_input;
+ fc_call = bias_add_call->args[0].as<CallNode>();
+ } else {
+ fc_call = requantize_input;
+ }
+
+ // TIR variables are created in the order they appear in the Relay partitioned function
+ // %1 = qnn.dense(%input, %weight_const_0, input_zero_point_scalar, kernel_zero_point_scalar,
+ // %input_scale_scalar, %kernel_scale_scalar)
+ // %2 = nn.bias_add(%1, %bias_const_1, axis=1)
+ // %3 = qnn.requantize(%2, %req_input_scale_scalar, %req_input_zero_point_scalar,
+ // %output_scale_scalar, %output_zero_point_scalar)
+ // clip(%3, a_min=%min_scalar, a_max=%max_scalar)
+ tir::Var input("input", DataType::Handle(8));
+ tir::Var filter("filter", DataType::Handle(8));
+ tir::Var bias("bias", DataType::Handle(32));
+ tir::Var output("output", DataType::Handle(8));
+
+ // Individual arguments to the structs arguments of the CMSIS-NN API are filled into call_extern
+ // https://github.com/ARM-software/CMSIS_5/blob/def6f800f95661eb3451d317f7d0dde504f6020d/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c#L50
+
+ // prepare cmsis_nn_fc_params
+ const DenseAttrs* dense_attrs = fc_call->attrs.as<DenseAttrs>();
+ int32_t input_offset = -GetScalarFromConstant<int32_t>(fc_call->args[2]);
+ int32_t filter_offset = -GetScalarFromConstant<int32_t>(fc_call->args[3]);
+ int32_t output_offset = GetScalarFromConstant<int32_t>(requantize_call->args[4]);
+ float input_scale = GetScalarFromConstant<float>(requantize_call->args[1]);
+ float output_scale = GetScalarFromConstant<float>(requantize_call->args[3]);
+ int32_t out_channels = qnn::get_const_int(dense_attrs->units);
+ int32_t clip_min, clip_max;
+ if (clip_call) {
+ const ClipAttrs* clip_attrs = clip_call->attrs.as<ClipAttrs>();
+ clip_min = clip_attrs->a_min;
+ clip_max = clip_attrs->a_max;
+ } else {
+ clip_min = -128;
+ clip_max = 127;
+ }
+
+ double quantized_multiplier =
+ static_cast<double>(input_scale) / static_cast<double>(output_scale);
+ auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(quantized_multiplier);
+ int32_t multiplier = std::get<0>(mult_shift_pair);
+ int32_t shift = std::get<1>(mult_shift_pair);
+
+ tvm::Array<PrimExpr> scalar_args = {
+ ToArg(input_offset), ToArg(filter_offset), ToArg(output_offset), ToArg(clip_min),
+ ToArg(clip_max), ToArg(multiplier), ToArg(shift)};
+
+ // cmsis_nn_dims *input_dims
+ Array<PrimExpr> input_shape = fc_call->args[0]->type_as<TensorTypeNode>()->shape;
+ int32_t batch_size = qnn::get_const_int(input_shape[0]);
+ int32_t in_channels = qnn::get_const_int(input_shape[1]);
+ Array<PrimExpr> cmsisnn_input_shape{input_shape[0], 1, 1, input_shape[1]};
+
+ // cmsis_nn_dims *filter_dims
+ Array<PrimExpr> cmsisnn_filter_shape{in_channels, 1, 1, out_channels};
+
+ // cmsis_nn_dims *bias_dims
+ Array<PrimExpr> bias_shape{1, 1, 1, out_channels};
+
+ // cmsis_nn_dims *output_dims
+ Array<PrimExpr> cmsisnn_output_shape{batch_size, 1, 1, out_channels};
+
+ std::string cmsisnn_api = "arm_fully_connected_s8";
+ tvm::Array<PrimExpr> call_ext_args = {tir::StringImm(cmsisnn_api), input, filter};
+ if (bias_add_call) {
+ call_ext_args.push_back(bias);
+ }
+ call_ext_args.push_back(output);
+
+ int context_buffer_size = 0;
+ std::string context_buffer_name = "NULL";
+ tvm::Array<PrimExpr> context_buffer_args = {tir::StringImm(context_buffer_name),
+ ToArg(context_buffer_size)};
+
+ scalar_args = tvm::runtime::Concat(context_buffer_args, scalar_args);
+ scalar_args = tvm::runtime::Concat(scalar_args, cmsisnn_input_shape);
+ scalar_args = tvm::runtime::Concat(scalar_args, cmsisnn_filter_shape);
+ scalar_args = tvm::runtime::Concat(scalar_args, bias_shape);
+ scalar_args = tvm::runtime::Concat(scalar_args, cmsisnn_output_shape);
+ call_ext_args = tvm::runtime::Concat(call_ext_args, scalar_args);
Review comment:
nit : Would it be possible to keep the definitions of sub-catergories that get concatenated closer to here ? So it is easy to follow ?
##########
File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc
##########
@@ -234,6 +222,114 @@ class RelayToTIRVisitor : public MixedModeMutator {
context_buffer_size);
}
+ void EmitFullyConnected(const GlobalVar& global_var, const Expr& expr) {
+ const CallNode* clip_call = nullptr;
+ const CallNode* requantize_call = nullptr;
+ const CallNode* bias_add_call = nullptr;
+ const CallNode* fc_call = nullptr;
+ const CallNode* final_call = expr.as<CallNode>();
+ const OpNode* final_op = final_call->op.as<OpNode>();
+ if (final_op->name == "clip") {
+ clip_call = final_call;
+ requantize_call = clip_call->args[0].as<CallNode>();
+ } else {
+ requantize_call = final_call;
+ }
+ const CallNode* requantize_input = requantize_call->args[0].as<CallNode>();
+ const OpNode* requantize_input_op = requantize_input->op.as<OpNode>();
+ if (requantize_input_op->name == "nn.bias_add") {
+ bias_add_call = requantize_input;
+ fc_call = bias_add_call->args[0].as<CallNode>();
+ } else {
+ fc_call = requantize_input;
+ }
+
+ // TIR variables are created in the order they appear in the Relay partitioned function
+ // %1 = qnn.dense(%input, %weight_const_0, input_zero_point_scalar, kernel_zero_point_scalar,
+ // %input_scale_scalar, %kernel_scale_scalar)
+ // %2 = nn.bias_add(%1, %bias_const_1, axis=1)
+ // %3 = qnn.requantize(%2, %req_input_scale_scalar, %req_input_zero_point_scalar,
+ // %output_scale_scalar, %output_zero_point_scalar)
+ // clip(%3, a_min=%min_scalar, a_max=%max_scalar)
+ tir::Var input("input", DataType::Handle(8));
+ tir::Var filter("filter", DataType::Handle(8));
+ tir::Var bias("bias", DataType::Handle(32));
+ tir::Var output("output", DataType::Handle(8));
+
+ // Individual arguments to the structs arguments of the CMSIS-NN API are filled into call_extern
+ // https://github.com/ARM-software/CMSIS_5/blob/def6f800f95661eb3451d317f7d0dde504f6020d/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c#L50
+
+ // prepare cmsis_nn_fc_params
+ const DenseAttrs* dense_attrs = fc_call->attrs.as<DenseAttrs>();
+ int32_t input_offset = -GetScalarFromConstant<int32_t>(fc_call->args[2]);
+ int32_t filter_offset = -GetScalarFromConstant<int32_t>(fc_call->args[3]);
+ int32_t output_offset = GetScalarFromConstant<int32_t>(requantize_call->args[4]);
+ float input_scale = GetScalarFromConstant<float>(requantize_call->args[1]);
+ float output_scale = GetScalarFromConstant<float>(requantize_call->args[3]);
+ int32_t out_channels = qnn::get_const_int(dense_attrs->units);
+ int32_t clip_min, clip_max;
+ if (clip_call) {
+ const ClipAttrs* clip_attrs = clip_call->attrs.as<ClipAttrs>();
+ clip_min = clip_attrs->a_min;
+ clip_max = clip_attrs->a_max;
+ } else {
+ clip_min = -128;
+ clip_max = 127;
+ }
+
+ double quantized_multiplier =
+ static_cast<double>(input_scale) / static_cast<double>(output_scale);
+ auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(quantized_multiplier);
+ int32_t multiplier = std::get<0>(mult_shift_pair);
+ int32_t shift = std::get<1>(mult_shift_pair);
+
+ tvm::Array<PrimExpr> scalar_args = {
+ ToArg(input_offset), ToArg(filter_offset), ToArg(output_offset), ToArg(clip_min),
+ ToArg(clip_max), ToArg(multiplier), ToArg(shift)};
+
+ // cmsis_nn_dims *input_dims
Review comment:
What do these comments mean ?
##########
File path: src/relay/backend/contrib/cmsisnn/relay_to_tir.cc
##########
@@ -234,6 +222,114 @@ class RelayToTIRVisitor : public MixedModeMutator {
context_buffer_size);
}
+ void EmitFullyConnected(const GlobalVar& global_var, const Expr& expr) {
+ const CallNode* clip_call = nullptr;
+ const CallNode* requantize_call = nullptr;
+ const CallNode* bias_add_call = nullptr;
+ const CallNode* fc_call = nullptr;
+ const CallNode* final_call = expr.as<CallNode>();
+ const OpNode* final_op = final_call->op.as<OpNode>();
+ if (final_op->name == "clip") {
+ clip_call = final_call;
+ requantize_call = clip_call->args[0].as<CallNode>();
+ } else {
+ requantize_call = final_call;
+ }
+ const CallNode* requantize_input = requantize_call->args[0].as<CallNode>();
+ const OpNode* requantize_input_op = requantize_input->op.as<OpNode>();
+ if (requantize_input_op->name == "nn.bias_add") {
+ bias_add_call = requantize_input;
+ fc_call = bias_add_call->args[0].as<CallNode>();
+ } else {
+ fc_call = requantize_input;
+ }
+
+ // TIR variables are created in the order they appear in the Relay partitioned function
+ // %1 = qnn.dense(%input, %weight_const_0, input_zero_point_scalar, kernel_zero_point_scalar,
+ // %input_scale_scalar, %kernel_scale_scalar)
+ // %2 = nn.bias_add(%1, %bias_const_1, axis=1)
+ // %3 = qnn.requantize(%2, %req_input_scale_scalar, %req_input_zero_point_scalar,
+ // %output_scale_scalar, %output_zero_point_scalar)
+ // clip(%3, a_min=%min_scalar, a_max=%max_scalar)
+ tir::Var input("input", DataType::Handle(8));
+ tir::Var filter("filter", DataType::Handle(8));
+ tir::Var bias("bias", DataType::Handle(32));
+ tir::Var output("output", DataType::Handle(8));
+
+ // Individual arguments to the structs arguments of the CMSIS-NN API are filled into call_extern
+ // https://github.com/ARM-software/CMSIS_5/blob/def6f800f95661eb3451d317f7d0dde504f6020d/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c#L50
+
+ // prepare cmsis_nn_fc_params
+ const DenseAttrs* dense_attrs = fc_call->attrs.as<DenseAttrs>();
+ int32_t input_offset = -GetScalarFromConstant<int32_t>(fc_call->args[2]);
+ int32_t filter_offset = -GetScalarFromConstant<int32_t>(fc_call->args[3]);
+ int32_t output_offset = GetScalarFromConstant<int32_t>(requantize_call->args[4]);
+ float input_scale = GetScalarFromConstant<float>(requantize_call->args[1]);
+ float output_scale = GetScalarFromConstant<float>(requantize_call->args[3]);
+ int32_t out_channels = qnn::get_const_int(dense_attrs->units);
+ int32_t clip_min, clip_max;
+ if (clip_call) {
+ const ClipAttrs* clip_attrs = clip_call->attrs.as<ClipAttrs>();
+ clip_min = clip_attrs->a_min;
+ clip_max = clip_attrs->a_max;
+ } else {
+ clip_min = -128;
+ clip_max = 127;
+ }
+
+ double quantized_multiplier =
+ static_cast<double>(input_scale) / static_cast<double>(output_scale);
+ auto mult_shift_pair = tvm::relay::qnn::GetFixedPointMultiplierShift(quantized_multiplier);
+ int32_t multiplier = std::get<0>(mult_shift_pair);
+ int32_t shift = std::get<1>(mult_shift_pair);
+
+ tvm::Array<PrimExpr> scalar_args = {
+ ToArg(input_offset), ToArg(filter_offset), ToArg(output_offset), ToArg(clip_min),
+ ToArg(clip_max), ToArg(multiplier), ToArg(shift)};
+
+ // cmsis_nn_dims *input_dims
+ Array<PrimExpr> input_shape = fc_call->args[0]->type_as<TensorTypeNode>()->shape;
+ int32_t batch_size = qnn::get_const_int(input_shape[0]);
+ int32_t in_channels = qnn::get_const_int(input_shape[1]);
+ Array<PrimExpr> cmsisnn_input_shape{input_shape[0], 1, 1, input_shape[1]};
+
+ // cmsis_nn_dims *filter_dims
+ Array<PrimExpr> cmsisnn_filter_shape{in_channels, 1, 1, out_channels};
+
+ // cmsis_nn_dims *bias_dims
+ Array<PrimExpr> bias_shape{1, 1, 1, out_channels};
+
+ // cmsis_nn_dims *output_dims
+ Array<PrimExpr> cmsisnn_output_shape{batch_size, 1, 1, out_channels};
+
+ std::string cmsisnn_api = "arm_fully_connected_s8";
Review comment:
nit : I think we can just use a immediate value to extern call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org