You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by GitBox <gi...@apache.org> on 2022/07/27 18:28:09 UTC
[GitHub] [tvm] quic-sanirudh opened a new pull request, #12204: [Hexagon] Add Hand written HVX conv2d
quic-sanirudh opened a new pull request, #12204:
URL: https://github.com/apache/tvm/pull/12204
This PR adds an initial version of a hand-coded HVX intrinsics implementation of conv2d for Hexagon target.
@kparzysz-quic is a co-author to the patch as he has written many of the utility functions for this patch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945443671
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
Review Comment:
I replied to @cconvey's query about the difference above. I've also added the meaning of chunkify in the comments to make it more obvious for people going through the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r951797972
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+inline uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
Review Comment:
Done, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r951797734
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
Review Comment:
I've renamed the file and I've made that function `static inline` for now so that the symbol is not visible outside as its not used outside anyway for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949262003
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
Would [Horner's algorithm](https://en.wikipedia.org/wiki/Horner%27s_method) be a good technique for computing `p`'s index? It's potentially faster and potentially easier to read.
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+inline uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
Review Comment:
This function doesn't appear to be used anywhere in the PR. Is that intentional?
Usually for TVM PRs, we try to ensure thorough test coverage of the new code ([link](https://tvm.apache.org/docs/contribute/code_review.html#factors-to-consider-about-code-quality)).
[A code-comment below](https://github.com/apache/tvm/pull/12204/files#diff-1a3e6ccd9b3b65439dfe0e4ce36f0b04e186203504a2250b6cc3c7ce1172513cR35) states that bias isn't currently accepted. Given that, would it make sense to just remove this function from the PR?
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+inline uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
Review Comment:
Is there a reasonable way to ensure that this `static_cast` is actually valid for `b`?
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
Review Comment:
Is there a reasonable way to ensure that these `static_cast` is actually valid for `a` and `f`?
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
The `DLTensor` struct has two members that don't seem to be considered here: [`strides` and `byte_offset`](https://github.com/dmlc/dlpack/blob/ddeb264880a1fa7e7be238ab3901a810324fbe5f/include/dlpack/dlpack.h#L159-L161).
Is it truly safe to ignore those members here? If so, it might be helpful to (ideally) add an `assert` and/or a comment about why they can be ignored.
If the reason we can safely ignore `strides` and `byte_offset` is that we know they were set by `SDLTensor`'s constructor, then would it make sense to have these two functions explicitly operate on `SDLTensor` objects rather than `DLTensor` objects? (And if so, then any reason they shouldn't be class members rather than free-standing functions?)
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
Review Comment:
This function's name and argument list suggest that it handles tensors with a variety of element dtypes. But its body and return type assume `uint16_t`.
Is there some reason that's a save assumption in this context?
If yes, then it might be helpful to add some comments explaining why. If not, then I'd suggest some form of disambiguation, e.g.:
- rename this to `getElementPtr_u16`, or
- convert this to a template that's parameterized by dtype.
Even if this `uint16_t` is being used as a stand-in for qfloat16 or IEEE-754 half-precision floats, it's not (immediately) obvious that this code will only ever be used for _16-bit_ HVX values.
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
These functions seem to assume that `a.ndim >= 4`. Would it make sense to ensure that's true, by e.g.:
- `assert(a.n >= 4)` or
- converting these to template functions, and replace the first argument with `const SDLTensor<4> & a`?
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
Review Comment:
The roles of `data_ptr` and `data_space` are explained in [this PR comment](https://github.com/apache/tvm/pull/12204#discussion_r945441218), but I don't see that same information in the current code.
Could we add put that same information into one or more code comments?
Also, even if that PR comment is right about there being no risk of dangling pointers, that isn't _obviously_ true just from normal-level-of-effort reading of the code. Is there some way we can address that?
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
Review Comment:
If `uint16_t` is a stand-in for qfloat16, is there a typename supplied by the Hexagon SDK that might be more descriptive?
This might not be a valid example, but I'm thinking of something _like_ `qhl_q0_t`.
Of if the SDK provides no such type name, could it make sense to just define one here? E.g., `using qfloat16_ptr = uint16_t*;`
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_float Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
Review Comment:
Could we move the code-comments that explain "chunked" and "blockized" to just before the first usage of those terms, i.e. near the top of `conv2d.h` or into a separate `README.rst` file? It took me a while to notice this explanation because of where it's placed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r954355275
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_flat Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
+ *
+ * @param out_ptr Base pointer table to be filled with the list of pointers to the first addresses
+ * of the "chunked" weights
+ * @param out_ptr_size The number of chunks
+ * @param out Pointer to pre-allocated output memory
+ * @param inp Pointer to flat input data
+ * @param height
+ * @param width
+ * @param idepth
+ * @param odepth
+ */
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
+ if (copy_data) {
+ blockize_hwc_16b(nhwc_vtcm, nhwc_flat->data, nhwc_flat->shape[1], nhwc_flat->shape[2],
+ nhwc_flat->shape[3]);
+ }
+
+ return SDLTensor<4>(nhwc_vtcm, nhwc_flat->dtype, nhwc_vtcm, {n, h / 8, w / 4, c / 32});
+}
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate one block for filter data. We will need to create our own
+ // pointer table. The reason is that filter chunks cannot be padded
+ // height- or width-wise, so filter chunks may have different sizes.
+ // A filter chunk is a block of size HxWx32x32, where H, W are at most
+ // height and width of a block respectively.
+ int h = hwio_flat->shape[0];
+ int w = hwio_flat->shape[1];
+ int i = round_up(hwio_flat->shape[2], 32);
+ int o = round_up(hwio_flat->shape[3], 32);
+ int64_t shape_1d[] = {h * w * i * o};
+ void* hwio_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 1, shape_1d, hwio_flat->dtype, vtcm_scope);
Review Comment:
Where is this freed?
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_flat Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
+ *
+ * @param out_ptr Base pointer table to be filled with the list of pointers to the first addresses
+ * of the "chunked" weights
+ * @param out_ptr_size The number of chunks
+ * @param out Pointer to pre-allocated output memory
+ * @param inp Pointer to flat input data
+ * @param height
+ * @param width
+ * @param idepth
+ * @param odepth
+ */
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
Review Comment:
Where is this freed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r967374511
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
Review Comment:
@csullivan I've added the unit tests with gtest. Could you please check out the test cases are review the patch again when you get a chance, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on PR #12204:
URL: https://github.com/apache/tvm/pull/12204#issuecomment-1221040317
FYI, due to work schedules, @csullivan will follow up with any issues / suggestions I raised in my reviews. I'm happy to for this to be merged once @csullivan Approves.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945431836
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
Review Comment:
Done thanks for the suggestion, it makes sense to me. BTW, sorry it took a bit of time to address them
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945445274
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
Review Comment:
I added these so that I can print debug info for a few extra variables, that are not directly used in the computation, but makes it easy to debug. This `DEBUG_BLOCK` allows me to create unused variables, without warnings during compilation.
I took the idea from `LLVM_DEBUG` macro used in LLVM, which allows addition of whole blocks of debug code into a source, but if this is not idiomatic of TVM, or if there's any other solution you have in mind, please let me know and I can make those changes
Thanks.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r942612681
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
Review Comment:
Here also as below
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
Review Comment:
"splatted" == broadcast?
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
Review Comment:
`blockized` means the data has been packed into an array of non-contiguous allocations
`chunkified` means weights are in a contiguous packed layout
For chunkified maybe we just refer to `packed` instead, and define the meaning of `blocked` somewhere in the docstring.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
Review Comment:
Use the runtime logging system, LOG(INFO).
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
Review Comment:
```suggestion
```
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "conv2d.h"
+
+namespace detail {
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ debug(
+ "cy: %d, cx: %d, cx0: %d, ci: %d, co: %d, max_x: %d, y: %d, x: %d, i: %d, o: "
+ "%d, index: %d",
+ cy, cx, cx0, ci, co, max_x, y, x, i, o, hwio_to_sm_16b(max_x, y, x, i, o));
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
+ if (copy_data) {
+ blockize_hwc_16b(nhwc_vtcm, nhwc_flat->data, nhwc_flat->shape[1], nhwc_flat->shape[2],
+ nhwc_flat->shape[3]);
+ }
+
+ return SDLTensor<4>(nhwc_vtcm, nhwc_flat->dtype, nhwc_vtcm, {n, h / 8, w / 4, c / 32});
+}
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate one block for filter data. We will need to create our own
+ // pointer table. The reason is that filter chunks cannot be padded
Review Comment:
Why can they not be padded in height or width?
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
Review Comment:
What is the purpose of DEBUG_BLOCK?
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
+ ", pad_top=%d, pad_left=%d\n",
+ filt_height, filt_width, filt_idepth, pad_top, pad_left);
+
+ assert(pad_top < 8 && pad_left < 4);
+
+ DEBUG_BLOCK(int a_height = cr_act.shape[1]; int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0]; int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3]; int b_depth = bias_flat.shape[0];)
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ debug("a: %dx%dx%dx%d, w: %dx%dx%dx%d, o: %dx%dx%dx%d, b: %d, out_shape: %dx%d\n", 1, a_height,
+ a_width, a_depth, w_height, w_width, static_cast<int>(cr_filt.shape[2]),
+ static_cast<int>(cr_filt.shape[3]), 1, o_height, o_width, o_depth, b_depth, out_height,
+ out_width);
+
+ assert(a_depth == cr_filt.shape[2]);
+ assert(o_depth == cr_filt.shape[3]);
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
+ * vector representation of weights, where the elements are represented as
+ * "out_channel-input_channel"
+ *
+ *
+ * Same 2 input channels have to be multiplied across all output channels in the weights.
+ *
+ * Activations vector would thus be:
+ * ----------- ------ ----- ---- --
+ * act_vec = [i0,i1,i0,i1,i0,i1,...,i0,i1] - 2 elements of the input channels broadcasted 32 times
Review Comment:
I think the mapping is `lambda n, h, w, c: [n, h//8, w//4, c//32, h%8, (w%4)//2, c%32, w%2]`
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
+ ", pad_top=%d, pad_left=%d\n",
+ filt_height, filt_width, filt_idepth, pad_top, pad_left);
+
+ assert(pad_top < 8 && pad_left < 4);
+
+ DEBUG_BLOCK(int a_height = cr_act.shape[1]; int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0]; int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3]; int b_depth = bias_flat.shape[0];)
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ debug("a: %dx%dx%dx%d, w: %dx%dx%dx%d, o: %dx%dx%dx%d, b: %d, out_shape: %dx%d\n", 1, a_height,
+ a_width, a_depth, w_height, w_width, static_cast<int>(cr_filt.shape[2]),
+ static_cast<int>(cr_filt.shape[3]), 1, o_height, o_width, o_depth, b_depth, out_height,
+ out_width);
+
+ assert(a_depth == cr_filt.shape[2]);
+ assert(o_depth == cr_filt.shape[3]);
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
Review Comment:
It will be good to document how this packing should be represented in terms of an index map which will be required for integration with TVM scheduling. For example,
`weight_index_map = lambda h, w, i, o: [h, w, i//16, o//32, (i%16)//2, o%32, i%2]`
Not sure if this is the correct mapping for FP16 please correct it if not and if possible include a similar mapping for the activation in the comments.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
Review Comment:
Neat
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
Review Comment:
Agreed with @cconvey here, and hexagon_common.cc implements similar functionality into TVM's runtime logger which we should aim to use for debugging in the runtime. This can be done with `LOG(INFO/WARNING/FATAL) << "message"` which is automatically directed to FARF.
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 10, 14, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 64),
+ "wgt_shape": (3, 3, 64, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 40),
+ "wgt_shape": (3, 3, 40, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 5, 5, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 6, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 7, 7, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 64),
+ "wgt_shape": (2, 2, 64, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+]
+
+
+def gen_id(param):
+ """Utility function to generate useful ids for shape_parameters"""
+
+ dims = lambda vals: "x".join(map(str, vals))
+
+ act_shape = param["act_shape"]
+ wgt_shape = param["wgt_shape"]
+ inp_stride = param["inp_stride"]
+ return f"nhwc{dims(act_shape)}-hwio{dims(wgt_shape)}-stride{dims(inp_stride)}"
+
+
+@tvm.testing.requires_hexagon
+@pytest.mark.parametrize("shapes", shape_parameters, ids=map(gen_id, shape_parameters))
Review Comment:
After using tvm.testing.parameter this decoration can be deleted and test_conv2d can take these params as function arguments directly.
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 10, 14, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 64),
+ "wgt_shape": (3, 3, 64, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 40),
+ "wgt_shape": (3, 3, 40, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 5, 5, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 6, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 7, 7, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 64),
+ "wgt_shape": (2, 2, 64, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
Review Comment:
These are all very small convolutions. Can we add some that are representative of real workloads and mark them with the `slow` decorator, or skip.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+constexpr uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth);
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth);
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth);
Review Comment:
Agreed, I made a comment about this above and after reading further I think even my definitions are not accurate.
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
Review Comment:
Instead using tvm's helpers, you can either do
```
act_shape, wgt_shape, inp_offset, inp_stride = tvm.testing.parameters(((1,8,4,3),(3,3,3,3),(0,0),(1,1)), ((1,10,14,3), (3,3,3,3), (0,0), (1,1)), ...)
```
or
```
act_shape = tvm.testing.parameter((1,8,4,3), (1,10,14,3), ...)
wgt_shape = tvm.testing.parameter((3,3,3,3), (3,3,3,64), ...)
inp_offset = tvm.testing_parameter((0, 0))
inp_stride = tvm.testing_parameter((1,1), (2,2))
```
which will do the cartesian product of the individually set parameters.
```
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
Review Comment:
Please add a `# TODO(author): Add support for relu` and similar for all the params which are not supported but planned to be.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r951796717
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
Review Comment:
I've added comments explaining them, thanks for the suggestion. As to the idea of dangling pointers, I'll try and address them separately as I'm not sure how generic the usage of these are going to be.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949329234
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
Review Comment:
This function's name and argument list suggest that it handles tensors with a variety of element dtypes. But its body and return type assume `uint16_t`.
Is there some reason that's a safe assumption in this context?
If yes, then it might be helpful to add some comments explaining why. If not, then I'd suggest some form of disambiguation, e.g.:
- rename this to `getElementPtr_u16`, or
- convert this to a template that's parameterized by dtype.
Even if this `uint16_t` is being used as a stand-in for qfloat16 or IEEE-754 half-precision floats, it's not (immediately) obvious that this code will only ever be used for _16-bit_ HVX values.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945441699
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+constexpr uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth);
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth);
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth);
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data);
+
+int calculate_num_weight_chunks(int64_t* shape_hwio);
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table);
+
+template <size_t N>
+void release(tvm::runtime::DeviceAPI* device_api, const SDLTensor<N>& tensor) {
+ if (auto* data_space = tensor.GetDataSpace()) {
+ device_api->FreeDataSpace(hexagon_device, data_space);
+ }
Review Comment:
In short no. Like I explained above, `data` is either the same as `data_space`, and so gets freed together, or points to some pointer on the stack, and thus freed during return.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945436583
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
Review Comment:
Done thanks.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945436499
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
Review Comment:
I originally had a different understanding, but I'm not completely sure right now. Since @kparzysz-quic introduced the `SDLTensor` type initially, I'll let him explain this part as I don't want to make an incorrect explanation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r946799525
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
Review Comment:
Thanks @quic-sanirudh , happy it was helpful. Please click on the "Ready for re-review" link next to my name so I can provide feedback on the updated code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949649604
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
I think adding extra comments is probably the best at the moment. The names of these functions are specific to particular layouts, so it's not clear what they should be called if they were to work in other cases. I think that `hwio_at` is somewhat generic, so it can be templatized, or made a member. For `nhwc_at` there is some specificity (like returning 0 for out-of-bounds y-accesses), so to avoid additional changes, I suggest just adding comments to it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949399707
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
These functions should be fast. I'd rather not have any extra code in here. It would actually need to be ICHECK, since TVM is compiled without assertions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949623025
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
IMHO both options are reasonable. When you see a `constexpr` function, it could be for either of two reasons:
- a human noticed that it was possible, and added `constexpr` in case it's useful in the future, or
- there is, or shall be in the future, a callsite that _needs_ the function to be `constxepr`.
I haven't noticed any codebases that clarify which of those reasons applies, so there's ambiguity either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949407183
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
Review Comment:
It's not qfloat16, the storage type is _Float16. I'm opposed to using such typedefs, since they actually obscure things instead of clarifying anything.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949620130
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
Review Comment:
Any reason we couldn't just replace `uint16_t` with `_Float16` then? AFAICT both GCC and LLVM recognize `_Float16`, and IIUC we're currently only planning to ever build this code with LLVM.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r940401262
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
Review Comment:
Just my subjective opinion, but there might be less room for confusion / bugs if we used a longer path here, e.g. `#include <runtime/hexagon/ops/conv2d.h`, rather than adding this directory to the include path via the `CMakeLists.txt` file.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
Review Comment:
IIUC, this header file is only #included by `conv2d_hvx.cc`. Would it make sense to simply put this content into that `.cc` file and delete this header?
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
Review Comment:
Trivial nitpick: If the template parameter name was `RANK` or `NDIM` rather than `N`, I would have need ~5 less seconds to understand its role.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
Review Comment:
What's the relationship between `data_ptr` and `data_space`? I.e.:
- Is `data_ptr` always implicitly an address within the `data_space` allocation?
- Is either of these ever allowed to be `nullptr`?
- Is a single data space ever shared by multiple `SDLTensor` instances?
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+constexpr uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth);
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth);
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth);
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data);
+
+int calculate_num_weight_chunks(int64_t* shape_hwio);
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table);
+
+template <size_t N>
+void release(tvm::runtime::DeviceAPI* device_api, const SDLTensor<N>& tensor) {
+ if (auto* data_space = tensor.GetDataSpace()) {
+ device_api->FreeDataSpace(hexagon_device, data_space);
+ }
Review Comment:
(This validity of this comment hinges on the answer to one of my questions, above.)
Is there a possibility that this function will leave one or mode `SDLTensor` instances with stale `data` and/or `data_space` pointers?
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+constexpr uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth);
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth);
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth);
Review Comment:
It might be helpful to explain what "blockize" vs. "chunkify" mean. Neither term means anything specific to me, and I can't tell (based on the names) if they're two different names for the same concept.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
Review Comment:
Trivial nitpick, feel free to ignore: I think `i < N` is a tiny bit more idiomatic than `i != N`.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
Review Comment:
I don't see any users for this macro. Would it make sense to remove it from the PR?
If we keep this header, I think its content belongs somewhere in/under the `tvm::runtime::hexagon` C++ namespace, rather than `::detail`. (At least, that's the idiom I see in other header files under `src/runtime`.)
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
Review Comment:
`constexpr` implies `inline`: https://stackoverflow.com/questions/14391272/does-constexpr-imply-inline/14391320#14391320
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
Review Comment:
Would it make sense for the comment to elaborate on "owns the shape"?
I'm not sure what that means, and it would also be helpful if the comment gave a little more info/context for why this is being done.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
Review Comment:
Same comment as in the header file: I think this belongs in/under the `tvm::runtime::hexagon` namespace.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
Is there a particular reason these functions are `constexpr`? At least with `nhwc_at`, I didn't notice it being used in a context that would benefit from it being `constexpr`.
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
Review Comment:
Would it make sense to `assert` that the integer parameters have valid values?
AFAIK, [c++14 allows `assert`s inside `constexpr` functions](https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/) , and TVM assumes C++14.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945437195
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
Review Comment:
Removed inline thanks. Although, would it still make sense to have the inline keyword as it doesn't hurt in any way and makes it explicit that this function would be inlined (not just a suggestion to compiler in this case, as not inlining it would lead to ODR violation).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945443344
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
Review Comment:
Yes. I used **"splatted"** as the hexagon instruction to do this is called `vsplat`.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
Review Comment:
Done thanks.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945444165
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945446968
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
Review Comment:
Thanks. I've done it the first way (using `tvm.testing.parameters(...)` and not `parameter(...)` because there are a few restrictions on the compatibility of inputs, weights and stride to make sure that we don't get an incorrect output shape. So, using cartesian products to generate the input/weight/stride combinations was creating problems.
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 10, 14, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 64),
+ "wgt_shape": (3, 3, 64, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 40),
+ "wgt_shape": (3, 3, 40, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 5, 5, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 6, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 7, 7, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 64),
+ "wgt_shape": (2, 2, 64, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+]
+
+
+def gen_id(param):
+ """Utility function to generate useful ids for shape_parameters"""
+
+ dims = lambda vals: "x".join(map(str, vals))
+
+ act_shape = param["act_shape"]
+ wgt_shape = param["wgt_shape"]
+ inp_stride = param["inp_stride"]
+ return f"nhwc{dims(act_shape)}-hwio{dims(wgt_shape)}-stride{dims(inp_stride)}"
+
+
+@tvm.testing.requires_hexagon
+@pytest.mark.parametrize("shapes", shape_parameters, ids=map(gen_id, shape_parameters))
Review Comment:
Done, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945445274
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
Review Comment:
I added this initially to avoid **unused variable** warning, when I was using the other `debug` macro for FARF logging. Now that I've switched to using `LOG_INFO`, I don't think these are necessary, so I'll remove them
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949267658
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
The `DLTensor` struct has two members that don't seem to be considered here: [`strides` and `byte_offset`](https://github.com/dmlc/dlpack/blob/ddeb264880a1fa7e7be238ab3901a810324fbe5f/include/dlpack/dlpack.h#L159-L161).
Is it truly safe to ignore those members here? If so, it might be helpful to (ideally) add an `assert` and/or a comment about why they can be ignored.
If the reason we can safely ignore `strides` and `byte_offset` is that we know they were set to `0` and `nullptr`, repsectively, by `SDLTensor`'s constructor, then would it make sense to have these two functions explicitly operate on `SDLTensor` objects rather than `DLTensor` objects? (And if so, then any reason they shouldn't be class members rather than free-standing functions?)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949419338
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
What value does removing `constexpr` add here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949617801
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
That makes sense. If we consider speed to be a primary concern, would any of the following be a good compromise?
- convert these to members of `SDLTensor`, and (optionally) only provide implementations of the two methods for the template specialization `NDIM=4`, or
- leave these as free-standing functions, but still templatize them and only provide implementations for `NDIM=4`, or
- add a few comments indicating the assumptions these functions make about the rank of `a` / `f`, or
- adjust the names of these two functions to indicate their assumption about the rank of `a` / `f`.
The only way I'd see the templatization approaches be worth the additional code complexity is if:
- you're already planning in converting these to methods of `SDLTensor`, which is already a template class, or
- you have future plans for this code that would involve values of `NDIM` other than 4.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r967374511
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
Review Comment:
@csullivan I've added the unit tests with gtest. Could you please check out the test cases and review the patch again when you get a chance, thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949400871
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+inline uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
Review Comment:
No.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945435944
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
Review Comment:
@cconvey Thanks for the suggestion, I've moved them into the `tvm::runtime::hexagon` namespace and removed the HAP_CALL macro
@csullivan I've modified all the debug messages to `LOG_INFO` calls, thanks for that suggestion. I did not know that this has been adapted to make FARF calls for hexagon already.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945441218
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
Review Comment:
Again, as I commented above, @kparzysz-quic could probably provide the proper answer.
Having said that, I think the difference is that `data_space` is meant to store the pointer returned from `AllocDataSpace`, which means we could call `FreeDataSpace` on the same pointer again. `data_ptr` would be the same as `data_space` in the case of activations.
In case of weights, `data_ptr` stores the pointers the first element of each **"chunk"** in layout (I've explained the difference between "chunkified" and "blockized" below and also added them as comments in the code). In short, since chunks can be of different sizes (due to the restriction that there would be no padding along height and width as part of the layout), the `data_ptr` allows efficient access each of the chunks in the data space.
For the questions you've asked:
1. `data_ptr` could be the same as `data_space` or a completely separate pointer pointing to the start addresses of each chunk of weights
2. I think both have to be valid pointers. In case of weights, the `data_ptr` points to addresses on the stack, so it gets freed automatically, but `data_space` is always freed with a call to `FreeDataSpace`
3. I'm not sure, it is right now used only for the activations and weights of convolution, and is there are always separate `SDLTensor` instances for them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r954527836
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_flat Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
+ *
+ * @param out_ptr Base pointer table to be filled with the list of pointers to the first addresses
+ * of the "chunked" weights
+ * @param out_ptr_size The number of chunks
+ * @param out Pointer to pre-allocated output memory
+ * @param inp Pointer to flat input data
+ * @param height
+ * @param width
+ * @param idepth
+ * @param odepth
+ */
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
+ if (copy_data) {
+ blockize_hwc_16b(nhwc_vtcm, nhwc_flat->data, nhwc_flat->shape[1], nhwc_flat->shape[2],
+ nhwc_flat->shape[3]);
+ }
+
+ return SDLTensor<4>(nhwc_vtcm, nhwc_flat->dtype, nhwc_vtcm, {n, h / 8, w / 4, c / 32});
+}
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate one block for filter data. We will need to create our own
+ // pointer table. The reason is that filter chunks cannot be padded
+ // height- or width-wise, so filter chunks may have different sizes.
+ // A filter chunk is a block of size HxWx32x32, where H, W are at most
+ // height and width of a block respectively.
+ int h = hwio_flat->shape[0];
+ int w = hwio_flat->shape[1];
+ int i = round_up(hwio_flat->shape[2], 32);
+ int o = round_up(hwio_flat->shape[3], 32);
+ int64_t shape_1d[] = {h * w * i * o};
+ void* hwio_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 1, shape_1d, hwio_flat->dtype, vtcm_scope);
Review Comment:
This is actually freed at the calling function for `prepare_hwio()` [here](https://github.com/apache/tvm/pull/12204/files#diff-bba4806e87eee7299362c25ccb9ac67c159cf9f6ab01410cd68e45afe31a0376R484-R486)
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_flat Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
+ *
+ * @param out_ptr Base pointer table to be filled with the list of pointers to the first addresses
+ * of the "chunked" weights
+ * @param out_ptr_size The number of chunks
+ * @param out Pointer to pre-allocated output memory
+ * @param inp Pointer to flat input data
+ * @param height
+ * @param width
+ * @param idepth
+ * @param odepth
+ */
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
Review Comment:
This is actually freed at the calling function for `prepare_nhwc()` [here](https://github.com/apache/tvm/pull/12204/files#diff-bba4806e87eee7299362c25ccb9ac67c159cf9f6ab01410cd68e45afe31a0376R484-R486)
##########
src/runtime/hexagon/ops/conv2d_fp16_hvx.cc:
##########
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed_fp16(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Returns the pointer to the element within the given block
+ * assuming fp16 type and speicific layout as mentioned in blockize_hwc_16b.
+ * All the below params are explained with the same layout assumption
+ *
+ * @param block_out_y y-index of block
+ * @param block_out_x x-index of block
+ * @param block_out_c c-index of block
+ * @param yi height-offset within the block
+ * @param xio outer width offset within the block
+ * @param ci channel offset within the block
+ * @param xii inner width offset within the block
+ * @param block base DLTensor
+ *
+ * @return The pointer to the element within the given block
+ */
+static inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi,
+ int xio, int ci, int xii, const DLTensor& tensor) {
+ auto block_ptr = nhwc_at(tensor, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * TODO (quic-sanirudh) Add support for bias add
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * TODO (quic-sanirudh) Add support for relu activation
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+
+ int pad_top = pad_shape.shape[0];
+ int pad_left = pad_shape.shape[1];
+ LOG_INFO << "filt_height=" << filt_height << ", filt_width=" << filt_width
+ << ", filt_idepth=" << filt_idepth << ", pad_top=" << pad_top
+ << ", pad_left=" << pad_left << "\n";
+
+ ICHECK_LT(pad_top, 8) << "pad_top offset cannot be >= 8";
+ ICHECK_LT(pad_left, 4) << "pad_left offset cannot be >= 4";
+
+ int a_height = cr_act.shape[1];
+ int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0];
+ int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3];
+ int b_depth = bias_flat.shape[0];
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ LOG_INFO << "a: 1x" << a_height << "x" << a_width << "x" << a_depth << ", w: " << w_height << "x"
+ << w_width << "x" << static_cast<int>(cr_filt.shape[2]) << "x"
+ << static_cast<int>(cr_filt.shape[3]) << ", o: 1x" << o_height << "x" << o_width << "x"
+ << o_depth << ", b: " << b_depth << ", out_shape: " << out_height << "x" << out_width
+ << "\n";
+
+ ICHECK_EQ(a_depth, cr_filt.shape[2]) << "input depth should match weights input channels";
+ ICHECK_EQ(o_depth, cr_filt.shape[3]) << "output depth should match the weights output channel";
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
+ * vector representation of weights, where the elements are represented as
+ * "out_channel-input_channel"
+ *
+ *
+ * Same 2 input channels have to be multiplied across all output channels in the weights.
+ *
+ * Activations vector would thus be:
+ * ----------- ------ ----- ---- --
+ * act_vec = [i0,i1,i0,i1,i0,i1,...,i0,i1] - 2 elements of the input channels broadcasted 32 times
+ * to fill 64 elements of the vector
+ *
+ *
+ * Thus the computation is just a vmpy(act_vec,weights_vec) followed by a some rearrangement to
+ * add every pair of 16b lanes in the vector to reduce along the input channels
+ *
+ * This result is added to the result of the next pair of input channels all the way until we
+ * have reduced across the entire input channels.
+ *
+ * Then the same vector is added to the results of the following elements along the width and
+ * height to finally get 32 elements representing 32 output channels.
+ *
+ * Since the output block also has the 8h2w32c2w format, the 32 elements of the next element
+ * along the width is also added into the the same vector such that the first 32 channel elements
+ * occupy the even lanes and the next 32 occupy the odd lanes to form a single 64-element vector
+ * which is then stored
+ */
+ auto computeConv = [filt_height, filt_width, wgt_chunk_thin_width, filt_idepth, stride_h,
+ stride_w, &cr_out, &cr_act, &cr_filt](int out_act_y, int out_act_x, int out_c,
+ int h, int wo, bool skip_wi_1 = false) {
+ auto out_element_ptr = getElementPtr(out_act_y, out_act_x, out_c, h, wo, 0, 0, cr_out);
+
+ LOG_INFO << "out_act_y: " << out_act_y << ", out_act_x: " << out_act_x << ", out_c: " << out_c
+ << ", h: " << h << ", wo: " << wo << " out_element_ptr: " << out_element_ptr;
+
+ HVX_Vector* out_vector = reinterpret_cast<HVX_Vector*>(out_element_ptr);
+ HVX_Vector existing_out_vec = *out_vector;
+
+ for (int fh = 0; fh < filt_height; ++fh) {
+ for (int fw = 0; fw < filt_width; ++fw) {
+ int fch = fh / 8;
+ int fcw = 0;
+ if (fw >= wgt_chunk_thin_width) {
+ fcw = (fw - wgt_chunk_thin_width) / 4 + 1;
+ }
+ int fx = (fw < wgt_chunk_thin_width) ? fw : ((fw - wgt_chunk_thin_width) % 4);
+ int fy = fh % 8;
+ for (int c = 0; c < round_up(filt_idepth, 2); c += 2) {
+ int out_act_cc = c / 32;
+ int ci = c % 32;
+ auto wgt_chunk = hwio_at(cr_filt, fch, fcw, out_act_cc, out_c);
+
+ // Find weight chunk offset ptr
+ int max_x = (fcw == 0) ? wgt_chunk_thin_width : 4;
+
+ int wi = 0;
+
+ int out_width_idx = out_act_x * 4 + wo * 2 + wi;
+ int act_width_access_idx = out_width_idx * stride_w + fw;
+ int true_out_act_x = act_width_access_idx / 4;
+ int true_wo = (act_width_access_idx % 4) / 2;
+ int true_wi = act_width_access_idx % 2;
+
+ int out_height_idx = out_act_y * 8 + h;
+ int act_height_access_idx = out_height_idx * stride_h + fh;
+ int true_out_act_y = act_height_access_idx / 8;
+ int true_h = act_height_access_idx % 8;
+
+ int act_channel_idx = out_act_cc * 32 + ci;
+
+ auto act_element_ptr = getElementPtr(true_out_act_y, true_out_act_x, out_act_cc, true_h,
+ true_wo, ci, true_wi, cr_act);
+ HVX_Vector act_vec = getInputVector(act_element_ptr);
+
+ auto wgt_chunk_offset = hwio_to_sm_16b(max_x, fy, fx, ci, 0);
+ auto base_chunk_ptr = reinterpret_cast<uint16_t*>(wgt_chunk);
+ auto chunk_ptr = base_chunk_ptr + wgt_chunk_offset;
+
+ LOG_INFO << "act: 0x" << act_height_access_idx << "x" << act_width_access_idx << "x"
+ << act_channel_idx << ", wgt: " << fh << "x" << fw << "x" << act_channel_idx
+ << "x" << out_c * 32 << ", out: 0x" << out_height_idx << "x" << out_width_idx
+ << "x" << out_c * 32 << ", wgt_chunk_offset: " << wgt_chunk_offset;
+
+ const HVX_Vector* weights_vec_ptr = reinterpret_cast<const HVX_Vector*>(chunk_ptr);
+ HVX_Vector weights_vec = *weights_vec_ptr;
+
+ HVX_Vector reduced_vec_even_elements = computeOuputVector(act_vec, weights_vec);
+
+ if (!skip_wi_1) {
+ wi = 1;
+
+ out_width_idx = out_act_x * 4 + wo * 2 + wi;
+ act_width_access_idx = out_width_idx * stride_w + fw;
+ true_out_act_x = act_width_access_idx / 4;
+ true_wo = (act_width_access_idx % 4) / 2;
+ true_wi = act_width_access_idx % 2;
+
+ act_element_ptr = getElementPtr(true_out_act_y, true_out_act_x, out_act_cc, true_h,
+ true_wo, ci, true_wi, cr_act);
+ act_vec = getInputVector(act_element_ptr);
+
+ LOG_INFO << "act: 0x" << act_height_access_idx << "x" << act_width_access_idx << "x"
+ << act_channel_idx << ", wgt: " << fh << "x" << fw << "x" << act_channel_idx
+ << "x" << out_c * 32 << ", out: 0x" << out_height_idx << "x" << out_width_idx
+ << "x" << out_c * 32 << ", wgt_chunk_offset: " << wgt_chunk_offset;
+
+ HVX_Vector reduced_vec_odd_elements = computeOuputVector(act_vec, weights_vec);
+ reduced_vec_odd_elements = Q6_V_vror_VR(reduced_vec_odd_elements, -2);
+ HVX_Vector out_final = Q6_V_vor_VV(reduced_vec_even_elements, reduced_vec_odd_elements);
+
+ HVX_Vector out_vec_qf16 = Q6_Vqf16_vadd_VhfVhf(out_final, existing_out_vec);
+ existing_out_vec = Q6_Vhf_equals_Vqf16(out_vec_qf16);
+ } else {
+ HVX_Vector out_vec_qf16 =
+ Q6_Vqf16_vadd_VhfVhf(reduced_vec_even_elements, existing_out_vec);
+ existing_out_vec = Q6_Vhf_equals_Vqf16(out_vec_qf16);
+ }
+ }
+ }
+ }
+ *out_vector = existing_out_vec;
+ };
+
+ auto computeFullWidth = [&computeConv](int out_y, int out_x, int out_c, int h) {
+ for (int wo = 0; wo < 2; ++wo) {
+ computeConv(out_y, out_x, out_c, h, wo);
+ }
+ };
+
+ auto computePartialWidth = [out_width, o_width, &computeConv](int out_y, int out_c, int h) {
+ int out_x = o_width - 1;
+ int wo = 0;
+ for (; wo < (out_width % 4) / 2; ++wo) {
+ computeConv(out_y, out_x, out_c, h, wo);
+ }
+
+ if (out_width % 2) {
+ computeConv(out_y, out_x, out_c, h, wo, true /* skip_wi_1 */);
+ }
+ };
+
+ for (int out_c = 0; out_c < cr_filt.shape[3]; ++out_c) {
+ for (int out_act_y = 0; out_act_y < out_height / 8; ++out_act_y) {
+ int out_y = out_act_y;
+ for (int out_act_x = 0; out_act_x < out_width / 4; ++out_act_x) {
+ int out_x = out_act_x;
+ for (int h = 0; h < 8; ++h) {
+ computeFullWidth(out_y, out_x, out_c, h);
+ }
+ }
+
+ for (int h = 0; h < 8; ++h) {
+ computePartialWidth(out_y, out_c, h);
+ }
+ }
+
+ int out_y = o_height - 1;
+ for (int h = 0; h < out_height % 8; ++h) {
+ for (int out_act_x = 0; out_act_x < out_width / 4; ++out_act_x) {
+ int out_x = out_act_x;
+ computeFullWidth(out_y, out_x, out_c, h);
+ }
+ computePartialWidth(out_y, out_c, h);
+ }
+ }
+}
+} // namespace hexagon
+} // namespace runtime
+} // namespace tvm
+
+int conv2d_packed_fp16(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle) {
+ namespace hexagonrt = tvm::runtime::hexagon;
+ ICHECK_EQ(num_args, 7) << "Unexpected number of arguments";
+ ICHECK_EQ(type_codes[0], kTVMDLTensorHandle)
+ << "First argument is expected to be the input tensor"; // Input activations
+ ICHECK_EQ(type_codes[1], kTVMDLTensorHandle)
+ << "Second argument is expected to be the weights tensor"; // Weights
+ ICHECK_EQ(type_codes[2], kDLInt)
+ << "Third argument is expected to be the pad_top offset"; // pad_top offset
+ ICHECK_EQ(type_codes[3], kDLInt)
+ << "Fourth argument is expected to be the pad_left offset"; // pad_left offset
+ ICHECK_EQ(type_codes[4], kDLInt) << "Fifth argument is expected to be the stride_h"; // stride_h
+ ICHECK_EQ(type_codes[5], kDLInt) << "Sixth argument is expected to be the stride_w"; // stride_w
+ ICHECK_EQ(type_codes[6], kTVMDLTensorHandle)
+ << "Seventh argument is expected to be the output tensor"; // output
+
+ auto* act_flat = static_cast<DLTensor*>(args[0].v_handle);
+ auto* wgt_flat = static_cast<DLTensor*>(args[1].v_handle);
+ auto* out_flat = static_cast<DLTensor*>(args[6].v_handle);
+
+ // Temporary assertion until multiple batches are supported
+ ICHECK_EQ(act_flat->shape[0], 1) << "Input batch size more than 1 is not supported yet";
+
+ // Temporary assertion until multiple batches are supported
+ ICHECK_EQ(out_flat->shape[0], 1) << "Output batch size more than 1 is not supported yet";
+
+ int pad_top = args[2].v_int64;
+ int pad_left = args[3].v_int64;
+ int stride_h = args[4].v_int64;
+ int stride_w = args[5].v_int64;
+
+ LOG_INFO << "act.shape=" << act_flat->shape[0] << "x" << act_flat->shape[1] << "x"
+ << act_flat->shape[2] << "x" << act_flat->shape[3]
+ << ", wgt.shape=" << wgt_flat->shape[0] << "x" << wgt_flat->shape[1] << "x"
+ << wgt_flat->shape[2] << "x" << wgt_flat->shape[3] << ", pad_top=" << pad_top
+ << ", pad_left=" << pad_left;
+
+ auto* device_api = tvm::runtime::DeviceAPI::Get(hexagonrt::hexagon_device, false);
+ ICHECK(device_api != nullptr);
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ auto act_vtcm = hexagonrt::prepare_nhwc(device_api, act_flat, /*copy_data=*/true);
+
+ ICHECK_NE(wgt_flat->shape[0], 0) << "Weights height should not be zero";
+ ICHECK_NE(wgt_flat->shape[1], 0) << "Weights width should not be zero";
+ ICHECK_NE(wgt_flat->shape[2], 0) << "Weights input channels should not be zero";
+ ICHECK_NE(wgt_flat->shape[3], 0) << "Weights output channels should not be zero";
+ int num_wgt_chunks = hexagonrt::calculate_num_weight_chunks(wgt_flat->shape);
+ LOG_INFO << "num_wgt_chunks: " << num_wgt_chunks;
+ auto wgt_ptr_table =
+ reinterpret_cast<void**>(__builtin_alloca(num_wgt_chunks * sizeof(uintptr_t)));
Review Comment:
Since we're just allocating on the stack, it'll be freed on function return as defined in the [`__builtin_alloca` documentation](https://clang.llvm.org/docs/LanguageExtensions.html#builtin-alloca)
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
Review Comment:
Sure, I'll try adding a few unit tests and update the PR, thanks for the comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] tmoreau89 merged pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
tmoreau89 merged PR #12204:
URL: https://github.com/apache/tvm/pull/12204
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945442929
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+constexpr uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth);
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth);
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth);
Review Comment:
@csullivan your comment below was almost correct. **blockized** represents modifying the layout to be non-contiguous blocks of data used in hexagon. **chunkified** means that the data is allocated as a packed contiguous blocks, with the only difference being that it cannot be padded along height and width. This leads chunks of different sizes, which is why I think **packed** might not be the right term for this.
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945446054
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
+ ", pad_top=%d, pad_left=%d\n",
+ filt_height, filt_width, filt_idepth, pad_top, pad_left);
+
+ assert(pad_top < 8 && pad_left < 4);
+
+ DEBUG_BLOCK(int a_height = cr_act.shape[1]; int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0]; int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3]; int b_depth = bias_flat.shape[0];)
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ debug("a: %dx%dx%dx%d, w: %dx%dx%dx%d, o: %dx%dx%dx%d, b: %d, out_shape: %dx%d\n", 1, a_height,
+ a_width, a_depth, w_height, w_width, static_cast<int>(cr_filt.shape[2]),
+ static_cast<int>(cr_filt.shape[3]), 1, o_height, o_width, o_depth, b_depth, out_height,
+ out_width);
+
+ assert(a_depth == cr_filt.shape[2]);
+ assert(o_depth == cr_filt.shape[3]);
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
+ * vector representation of weights, where the elements are represented as
+ * "out_channel-input_channel"
+ *
+ *
+ * Same 2 input channels have to be multiplied across all output channels in the weights.
+ *
+ * Activations vector would thus be:
+ * ----------- ------ ----- ---- --
+ * act_vec = [i0,i1,i0,i1,i0,i1,...,i0,i1] - 2 elements of the input channels broadcasted 32 times
Review Comment:
Done here, and yes you're right. That is the layout
https://github.com/apache/tvm/pull/12204/files#diff-ed59135151f4003f2b49baba77aab5d33a2e4ae0ee36e62cf52159f26ff13f5cR27-R33
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on PR #12204:
URL: https://github.com/apache/tvm/pull/12204#issuecomment-1211072314
Also @quic-sanirudh, if @kparzysz-quic is a co-author I suggest amending the commit message with `Co-authored-by: Krzysztof Parzyszek <kp...@quicinc.com>` which will attribute mutual authorship in Git and GitHub.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r954353031
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "conv2d.h"
+
+namespace detail {
+
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+void chunkify_hwio_16b(void** out_ptr, int out_ptr_size, void* out, void* inp, int height,
+ int width, int idepth, int odepth) {
+ auto inp_data = static_cast<uint16_t*>(inp);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_i = odepth;
+ const int stride_x = stride_i * idepth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ // In the chunkified tensor, the chunks are ordered in increasing
+ // x order, but they start from the thin one.
+ for (int cx = width - round_up(width, 4); cx < width; cx += 4) {
+ int cx0 = std::max(0, cx);
+ for (int ci = 0; ci < idepth; ci += 32) {
+ for (int co = 0; co < odepth; co += 32) {
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, cx + 4 - cx0);
+ int max_i = std::min(32, idepth - ci);
+ int max_o = std::min(32, odepth - co);
+
+ auto chunk = reinterpret_cast<uint16_t*>(out_data);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = max_x - 1; x >= 0; --x) {
+ for (int i = 0; i < max_i; ++i) {
+ for (int o = 0; o < max_o; ++o) {
+ debug(
+ "cy: %d, cx: %d, cx0: %d, ci: %d, co: %d, max_x: %d, y: %d, x: %d, i: %d, o: "
+ "%d, index: %d",
+ cy, cx, cx0, ci, co, max_x, y, x, i, o, hwio_to_sm_16b(max_x, y, x, i, o));
+ chunk[hwio_to_sm_16b(max_x, y, x, i, o)] =
+ inp_data[(cy + y) * stride_y + (cx0 + x) * stride_x + (ci + i) * stride_i +
+ (co + o)];
+ }
+ for (int o = max_o; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ for (int i = max_i; i < 32; ++i)
+ for (int o = 0; o < 32; ++o) chunk[hwio_to_sm_16b(max_x, y, x, i, o)] = 0;
+ }
+ }
+
+ *out_ptr++ = chunk;
+ out_data += max_y * max_x * 32 * 32;
+ out_ptr_size--;
+ assert(out_ptr_size >= 0);
+ }
+ }
+ }
+ }
+}
+
+SDLTensor<4> prepare_nhwc(tvm::runtime::DeviceAPI* device_api, const DLTensor* nhwc_flat,
+ bool copy_data) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate blocks for activations. We will use the block pointers
+ // directly from the allocated area.
+ int n = nhwc_flat->shape[0];
+ int h = round_up(nhwc_flat->shape[1], 8);
+ int w = round_up(nhwc_flat->shape[2], 4);
+ int c = round_up(nhwc_flat->shape[3], 32);
+ int64_t shape_2d[2] = {(n * h * w * c) / (8 * 4 * 32), 8 * 4 * 32};
+ void* nhwc_vtcm =
+ device_api->AllocDataSpace(hexagon_device, 2, shape_2d, nhwc_flat->dtype, vtcm_scope);
+ if (copy_data) {
+ blockize_hwc_16b(nhwc_vtcm, nhwc_flat->data, nhwc_flat->shape[1], nhwc_flat->shape[2],
+ nhwc_flat->shape[3]);
+ }
+
+ return SDLTensor<4>(nhwc_vtcm, nhwc_flat->dtype, nhwc_vtcm, {n, h / 8, w / 4, c / 32});
+}
+
+SDLTensor<4> prepare_hwio(tvm::runtime::DeviceAPI* device_api, const DLTensor* hwio_flat,
+ int num_chunks, void** ptr_table) {
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ // Allocate one block for filter data. We will need to create our own
+ // pointer table. The reason is that filter chunks cannot be padded
Review Comment:
Are the chunks being of different shape akin to sparse approaches?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r954356945
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
Review Comment:
Some unit testing of these utilities would be ideal and is possible pre-commit with [gtest](https://github.com/apache/tvm/pull/11239) on the simulator.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945432422
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
Review Comment:
Actually I was planning to reuse this header for the quantized implementation (which might contain a couple different files as well) that I'm working on, which is why I made it into a separate header
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945436692
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
Review Comment:
Done thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949653345
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
+
+inline uint32_t* bias_at(const DLTensor& b, int d) {
+ auto p = static_cast<uint32_t*>(b.data);
+ return p + d;
+}
Review Comment:
It can be deleted. It will need to be resurrected once we support bias.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949404879
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
Review Comment:
We should rename the files to conv2d_fp16_hvx.cc/.h instead.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] csullivan commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
csullivan commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r954356243
##########
src/runtime/hexagon/ops/conv2d_fp16_hvx.cc:
##########
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed_fp16(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Returns the pointer to the element within the given block
+ * assuming fp16 type and speicific layout as mentioned in blockize_hwc_16b.
+ * All the below params are explained with the same layout assumption
+ *
+ * @param block_out_y y-index of block
+ * @param block_out_x x-index of block
+ * @param block_out_c c-index of block
+ * @param yi height-offset within the block
+ * @param xio outer width offset within the block
+ * @param ci channel offset within the block
+ * @param xii inner width offset within the block
+ * @param block base DLTensor
+ *
+ * @return The pointer to the element within the given block
+ */
+static inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi,
+ int xio, int ci, int xii, const DLTensor& tensor) {
+ auto block_ptr = nhwc_at(tensor, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * TODO (quic-sanirudh) Add support for bias add
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * TODO (quic-sanirudh) Add support for relu activation
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+
+ int pad_top = pad_shape.shape[0];
+ int pad_left = pad_shape.shape[1];
+ LOG_INFO << "filt_height=" << filt_height << ", filt_width=" << filt_width
+ << ", filt_idepth=" << filt_idepth << ", pad_top=" << pad_top
+ << ", pad_left=" << pad_left << "\n";
+
+ ICHECK_LT(pad_top, 8) << "pad_top offset cannot be >= 8";
+ ICHECK_LT(pad_left, 4) << "pad_left offset cannot be >= 4";
+
+ int a_height = cr_act.shape[1];
+ int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0];
+ int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3];
+ int b_depth = bias_flat.shape[0];
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ LOG_INFO << "a: 1x" << a_height << "x" << a_width << "x" << a_depth << ", w: " << w_height << "x"
+ << w_width << "x" << static_cast<int>(cr_filt.shape[2]) << "x"
+ << static_cast<int>(cr_filt.shape[3]) << ", o: 1x" << o_height << "x" << o_width << "x"
+ << o_depth << ", b: " << b_depth << ", out_shape: " << out_height << "x" << out_width
+ << "\n";
+
+ ICHECK_EQ(a_depth, cr_filt.shape[2]) << "input depth should match weights input channels";
+ ICHECK_EQ(o_depth, cr_filt.shape[3]) << "output depth should match the weights output channel";
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
+ * vector representation of weights, where the elements are represented as
+ * "out_channel-input_channel"
+ *
+ *
+ * Same 2 input channels have to be multiplied across all output channels in the weights.
+ *
+ * Activations vector would thus be:
+ * ----------- ------ ----- ---- --
+ * act_vec = [i0,i1,i0,i1,i0,i1,...,i0,i1] - 2 elements of the input channels broadcasted 32 times
+ * to fill 64 elements of the vector
+ *
+ *
+ * Thus the computation is just a vmpy(act_vec,weights_vec) followed by a some rearrangement to
+ * add every pair of 16b lanes in the vector to reduce along the input channels
+ *
+ * This result is added to the result of the next pair of input channels all the way until we
+ * have reduced across the entire input channels.
+ *
+ * Then the same vector is added to the results of the following elements along the width and
+ * height to finally get 32 elements representing 32 output channels.
+ *
+ * Since the output block also has the 8h2w32c2w format, the 32 elements of the next element
+ * along the width is also added into the the same vector such that the first 32 channel elements
+ * occupy the even lanes and the next 32 occupy the odd lanes to form a single 64-element vector
+ * which is then stored
+ */
+ auto computeConv = [filt_height, filt_width, wgt_chunk_thin_width, filt_idepth, stride_h,
+ stride_w, &cr_out, &cr_act, &cr_filt](int out_act_y, int out_act_x, int out_c,
+ int h, int wo, bool skip_wi_1 = false) {
+ auto out_element_ptr = getElementPtr(out_act_y, out_act_x, out_c, h, wo, 0, 0, cr_out);
+
+ LOG_INFO << "out_act_y: " << out_act_y << ", out_act_x: " << out_act_x << ", out_c: " << out_c
+ << ", h: " << h << ", wo: " << wo << " out_element_ptr: " << out_element_ptr;
+
+ HVX_Vector* out_vector = reinterpret_cast<HVX_Vector*>(out_element_ptr);
+ HVX_Vector existing_out_vec = *out_vector;
+
+ for (int fh = 0; fh < filt_height; ++fh) {
+ for (int fw = 0; fw < filt_width; ++fw) {
+ int fch = fh / 8;
+ int fcw = 0;
+ if (fw >= wgt_chunk_thin_width) {
+ fcw = (fw - wgt_chunk_thin_width) / 4 + 1;
+ }
+ int fx = (fw < wgt_chunk_thin_width) ? fw : ((fw - wgt_chunk_thin_width) % 4);
+ int fy = fh % 8;
+ for (int c = 0; c < round_up(filt_idepth, 2); c += 2) {
+ int out_act_cc = c / 32;
+ int ci = c % 32;
+ auto wgt_chunk = hwio_at(cr_filt, fch, fcw, out_act_cc, out_c);
+
+ // Find weight chunk offset ptr
+ int max_x = (fcw == 0) ? wgt_chunk_thin_width : 4;
+
+ int wi = 0;
+
+ int out_width_idx = out_act_x * 4 + wo * 2 + wi;
+ int act_width_access_idx = out_width_idx * stride_w + fw;
+ int true_out_act_x = act_width_access_idx / 4;
+ int true_wo = (act_width_access_idx % 4) / 2;
+ int true_wi = act_width_access_idx % 2;
+
+ int out_height_idx = out_act_y * 8 + h;
+ int act_height_access_idx = out_height_idx * stride_h + fh;
+ int true_out_act_y = act_height_access_idx / 8;
+ int true_h = act_height_access_idx % 8;
+
+ int act_channel_idx = out_act_cc * 32 + ci;
+
+ auto act_element_ptr = getElementPtr(true_out_act_y, true_out_act_x, out_act_cc, true_h,
+ true_wo, ci, true_wi, cr_act);
+ HVX_Vector act_vec = getInputVector(act_element_ptr);
+
+ auto wgt_chunk_offset = hwio_to_sm_16b(max_x, fy, fx, ci, 0);
+ auto base_chunk_ptr = reinterpret_cast<uint16_t*>(wgt_chunk);
+ auto chunk_ptr = base_chunk_ptr + wgt_chunk_offset;
+
+ LOG_INFO << "act: 0x" << act_height_access_idx << "x" << act_width_access_idx << "x"
+ << act_channel_idx << ", wgt: " << fh << "x" << fw << "x" << act_channel_idx
+ << "x" << out_c * 32 << ", out: 0x" << out_height_idx << "x" << out_width_idx
+ << "x" << out_c * 32 << ", wgt_chunk_offset: " << wgt_chunk_offset;
+
+ const HVX_Vector* weights_vec_ptr = reinterpret_cast<const HVX_Vector*>(chunk_ptr);
+ HVX_Vector weights_vec = *weights_vec_ptr;
+
+ HVX_Vector reduced_vec_even_elements = computeOuputVector(act_vec, weights_vec);
+
+ if (!skip_wi_1) {
+ wi = 1;
+
+ out_width_idx = out_act_x * 4 + wo * 2 + wi;
+ act_width_access_idx = out_width_idx * stride_w + fw;
+ true_out_act_x = act_width_access_idx / 4;
+ true_wo = (act_width_access_idx % 4) / 2;
+ true_wi = act_width_access_idx % 2;
+
+ act_element_ptr = getElementPtr(true_out_act_y, true_out_act_x, out_act_cc, true_h,
+ true_wo, ci, true_wi, cr_act);
+ act_vec = getInputVector(act_element_ptr);
+
+ LOG_INFO << "act: 0x" << act_height_access_idx << "x" << act_width_access_idx << "x"
+ << act_channel_idx << ", wgt: " << fh << "x" << fw << "x" << act_channel_idx
+ << "x" << out_c * 32 << ", out: 0x" << out_height_idx << "x" << out_width_idx
+ << "x" << out_c * 32 << ", wgt_chunk_offset: " << wgt_chunk_offset;
+
+ HVX_Vector reduced_vec_odd_elements = computeOuputVector(act_vec, weights_vec);
+ reduced_vec_odd_elements = Q6_V_vror_VR(reduced_vec_odd_elements, -2);
+ HVX_Vector out_final = Q6_V_vor_VV(reduced_vec_even_elements, reduced_vec_odd_elements);
+
+ HVX_Vector out_vec_qf16 = Q6_Vqf16_vadd_VhfVhf(out_final, existing_out_vec);
+ existing_out_vec = Q6_Vhf_equals_Vqf16(out_vec_qf16);
+ } else {
+ HVX_Vector out_vec_qf16 =
+ Q6_Vqf16_vadd_VhfVhf(reduced_vec_even_elements, existing_out_vec);
+ existing_out_vec = Q6_Vhf_equals_Vqf16(out_vec_qf16);
+ }
+ }
+ }
+ }
+ *out_vector = existing_out_vec;
+ };
+
+ auto computeFullWidth = [&computeConv](int out_y, int out_x, int out_c, int h) {
+ for (int wo = 0; wo < 2; ++wo) {
+ computeConv(out_y, out_x, out_c, h, wo);
+ }
+ };
+
+ auto computePartialWidth = [out_width, o_width, &computeConv](int out_y, int out_c, int h) {
+ int out_x = o_width - 1;
+ int wo = 0;
+ for (; wo < (out_width % 4) / 2; ++wo) {
+ computeConv(out_y, out_x, out_c, h, wo);
+ }
+
+ if (out_width % 2) {
+ computeConv(out_y, out_x, out_c, h, wo, true /* skip_wi_1 */);
+ }
+ };
+
+ for (int out_c = 0; out_c < cr_filt.shape[3]; ++out_c) {
+ for (int out_act_y = 0; out_act_y < out_height / 8; ++out_act_y) {
+ int out_y = out_act_y;
+ for (int out_act_x = 0; out_act_x < out_width / 4; ++out_act_x) {
+ int out_x = out_act_x;
+ for (int h = 0; h < 8; ++h) {
+ computeFullWidth(out_y, out_x, out_c, h);
+ }
+ }
+
+ for (int h = 0; h < 8; ++h) {
+ computePartialWidth(out_y, out_c, h);
+ }
+ }
+
+ int out_y = o_height - 1;
+ for (int h = 0; h < out_height % 8; ++h) {
+ for (int out_act_x = 0; out_act_x < out_width / 4; ++out_act_x) {
+ int out_x = out_act_x;
+ computeFullWidth(out_y, out_x, out_c, h);
+ }
+ computePartialWidth(out_y, out_c, h);
+ }
+ }
+}
+} // namespace hexagon
+} // namespace runtime
+} // namespace tvm
+
+int conv2d_packed_fp16(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle) {
+ namespace hexagonrt = tvm::runtime::hexagon;
+ ICHECK_EQ(num_args, 7) << "Unexpected number of arguments";
+ ICHECK_EQ(type_codes[0], kTVMDLTensorHandle)
+ << "First argument is expected to be the input tensor"; // Input activations
+ ICHECK_EQ(type_codes[1], kTVMDLTensorHandle)
+ << "Second argument is expected to be the weights tensor"; // Weights
+ ICHECK_EQ(type_codes[2], kDLInt)
+ << "Third argument is expected to be the pad_top offset"; // pad_top offset
+ ICHECK_EQ(type_codes[3], kDLInt)
+ << "Fourth argument is expected to be the pad_left offset"; // pad_left offset
+ ICHECK_EQ(type_codes[4], kDLInt) << "Fifth argument is expected to be the stride_h"; // stride_h
+ ICHECK_EQ(type_codes[5], kDLInt) << "Sixth argument is expected to be the stride_w"; // stride_w
+ ICHECK_EQ(type_codes[6], kTVMDLTensorHandle)
+ << "Seventh argument is expected to be the output tensor"; // output
+
+ auto* act_flat = static_cast<DLTensor*>(args[0].v_handle);
+ auto* wgt_flat = static_cast<DLTensor*>(args[1].v_handle);
+ auto* out_flat = static_cast<DLTensor*>(args[6].v_handle);
+
+ // Temporary assertion until multiple batches are supported
+ ICHECK_EQ(act_flat->shape[0], 1) << "Input batch size more than 1 is not supported yet";
+
+ // Temporary assertion until multiple batches are supported
+ ICHECK_EQ(out_flat->shape[0], 1) << "Output batch size more than 1 is not supported yet";
+
+ int pad_top = args[2].v_int64;
+ int pad_left = args[3].v_int64;
+ int stride_h = args[4].v_int64;
+ int stride_w = args[5].v_int64;
+
+ LOG_INFO << "act.shape=" << act_flat->shape[0] << "x" << act_flat->shape[1] << "x"
+ << act_flat->shape[2] << "x" << act_flat->shape[3]
+ << ", wgt.shape=" << wgt_flat->shape[0] << "x" << wgt_flat->shape[1] << "x"
+ << wgt_flat->shape[2] << "x" << wgt_flat->shape[3] << ", pad_top=" << pad_top
+ << ", pad_left=" << pad_left;
+
+ auto* device_api = tvm::runtime::DeviceAPI::Get(hexagonrt::hexagon_device, false);
+ ICHECK(device_api != nullptr);
+ tvm::runtime::String vtcm_scope = "global.vtcm";
+
+ auto act_vtcm = hexagonrt::prepare_nhwc(device_api, act_flat, /*copy_data=*/true);
+
+ ICHECK_NE(wgt_flat->shape[0], 0) << "Weights height should not be zero";
+ ICHECK_NE(wgt_flat->shape[1], 0) << "Weights width should not be zero";
+ ICHECK_NE(wgt_flat->shape[2], 0) << "Weights input channels should not be zero";
+ ICHECK_NE(wgt_flat->shape[3], 0) << "Weights output channels should not be zero";
+ int num_wgt_chunks = hexagonrt::calculate_num_weight_chunks(wgt_flat->shape);
+ LOG_INFO << "num_wgt_chunks: " << num_wgt_chunks;
+ auto wgt_ptr_table =
+ reinterpret_cast<void**>(__builtin_alloca(num_wgt_chunks * sizeof(uintptr_t)));
Review Comment:
Not freeing this as far as I can tell. Also probably should use the device api to allocate memory on device.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r951795171
##########
src/runtime/hexagon/ops/conv_utils.cc:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+/**
+ * @brief Function to "blockize" the flat input data
+ * The term "blockize" is used to mention that the data is stored in non-contiguous blocks
+ *
+ * The input is mapped into the below mentioned layout (notation similar to index map used for
+ * transform layout):
+ *
+ * lambda n, h, w, c: n, h//8, w//4, c//32, AXIS_SEPARATOR, h%8, (w%4)//2, c%32, w%2
+ *
+ * where AXIS_SEPARATOR represents split up in the physical layout
+ *
+ * @param out Pre-allocated output memory pointer
+ * @param inp_flat Flat input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void blockize_hwc_16b(void* out, void* inp_flat, int height, int width, int depth) {
+ auto inp_data = static_cast<uint16_t*>(inp_flat);
+ auto out_data = static_cast<uintptr_t*>(out);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*out_data++);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ block[xyc_to_sm_16b(y, x, c)] =
+ inp_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)];
+ }
+ for (int c = max_c; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ for (int x = max_x; x < 4; ++x) {
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ }
+ }
+
+ for (int y = max_y; y < 8; ++y)
+ for (int x = 0; x < 4; ++x)
+ for (int c = 0; c < 32; ++c) block[xyc_to_sm_16b(y, x, c)] = 0;
+ } // cc
+ } // cx
+ } // cy
+}
+
+/**
+ * @brief Convert back from non-contguous layout to a flat layout
+ *
+ * @param out_float Pre-allocated output memory pointer
+ * @param inp Blockized input data pointer
+ * @param height
+ * @param width
+ * @param depth
+ */
+void deblockize_hwc_16b(void* out_flat, void* inp, int height, int width, int depth) {
+ uintptr_t* inp_data = static_cast<uintptr_t*>(inp);
+ uint16_t* out_data = static_cast<uint16_t*>(out_flat);
+ const int stride_x = depth;
+ const int stride_y = stride_x * width;
+
+ for (int cy = 0; cy < height; cy += 8) {
+ for (int cx = 0; cx < width; cx += 4) {
+ for (int cc = 0; cc < depth; cc += 32) {
+ auto block = reinterpret_cast<uint16_t*>(*inp_data);
+ int max_y = std::min(8, height - cy);
+ int max_x = std::min(4, width - cx);
+ int max_c = std::min(32, depth - cc);
+ for (int y = 0; y < max_y; ++y) {
+ for (int x = 0; x < max_x; ++x) {
+ for (int c = 0; c < max_c; ++c) {
+ out_data[(cy + y) * stride_y + (cx + x) * stride_x + (cc + c)] =
+ block[xyc_to_sm_16b(y, x, c)];
+ }
+ }
+ }
+
+ inp_data++;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Convert the layout of weights from flat to "chunked". The term chunked is explained below:
+ *
+ * Weights are packed into the below mentioned layout (notation similar to index map):
+ * Since weights cannot be exactly represented into a index map notation, the
+ * base split up is mentioned below with a few gotchas
+ *
+ * lambda h, w, i, o: h//8, w//4, o//32, i//32, h%8, w%4, (i%32)//2, o%32, i%2
+ *
+ * The gotchas are:
+ * - (w%4) is actually stored in the right to left order, as in 3,2,1,0 instead of 0,1,2,3
+ * - The h%8 and (w%4) dimensions are not padded up, leading to chunks of different sizes
+ * (thereby the name "chunked" instead of packed)
+ * - The thinnest chunk of width is stored first. For example, if a kernel is 5x5, the first
+ * chunk along the width has size 1 (representing index 0) and then next one has size 4
+ * representing indices (1,2,3,4)
Review Comment:
I've moved it to the conv2d.h file as well, and I'll create a separate PR with the documentation, thanks for the suggestions.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r951798525
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
I've added some comments, but not sure if that's enough, but I guess we can always add more comments in follow up PRs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945445909
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace detail {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
+
+/**
+ * @brief Compute 2 vectors with ones in the even and odd lanes
+ *
+ * Output vectors are:
+ * vector 1 = [0xFFFF,0x0000,0xFFFFF,0x0000,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * vector 2 = [0x0000,0xFFFF,0x0000,0xFFFFF,...,0xFFFF,0x0000]
+ * vector lanes = [ 0 , 2 , 3 , 4 ,..., 62 , 63 ]
+ *
+ * @return Return the 2 vectors
+ */
+inline std::pair<HVX_Vector, HVX_Vector> getOddEvenOnes() {
+ HVX_Vector v0 = Q6_V_vzero();
+ HVX_Vector v1 = Q6_Vh_vsplat_R(0xFFFF);
+
+ HVX_Vector v1e = Q6_Vh_vshuffe_VhVh(v0, v1);
+ HVX_Vector v1o = Q6_V_vnot_V(v1e);
+ return {v1e, v1o};
+}
+
+/**
+ * @brief Return the input vector filled with the 2 channel elements(which is the 1st and 3rd
+ * element) from base_ptr filled up 32 times to get 64 elements
+ *
+ * 1. It's generated by first creating 2 vectors "splatted" with the 2 required elements
+ * 2. Then we andd it with vectors containing all ones (0xFFFF) in the even and odd lanes
+ * 3. Finally those 2 vectors are OR'ed together
+ *
+ * @param base_ptr pointer to the first of the 2 channel elements to be filled
+ *
+ * @return input vector
+ */
+inline HVX_Vector getInputVector(uint16_t* base_ptr) {
+ HVX_Vector v1 = Q6_Vh_vsplat_R(base_ptr[0]);
+ HVX_Vector v2 = Q6_Vh_vsplat_R(base_ptr[2]);
+
+ auto oddEvenOnes = getOddEvenOnes();
+ auto v1e = oddEvenOnes.first;
+ auto v1o = oddEvenOnes.second;
+
+ HVX_Vector v_even_vals = Q6_V_vand_VV(v1, v1e);
+ HVX_Vector v_odd_vals = Q6_V_vand_VV(v2, v1o);
+
+ return Q6_V_vor_VV(v_even_vals, v_odd_vals);
+}
+
+/**
+ * @brief Return the Output vector which contains the 32 output channels in the even lanes
+ *
+ * The output vector is commputed as:
+ * 1. vector multiply(vmpy) of input and weights
+ * 2. Rotate the vector right by 1 element and add with the first vector to add the 2 input channels
+ * 3. Then convert the results back from qfloat16 to IEEE half-precision float
+ * 4. The added values are in even lanes, so zero out the odd lanes by anding with ones in even
+ * lanes and return
+ *
+ * @param act_vec Input activations vector
+ * @param wgt_vec Weights vector
+ *
+ * @return output vector with 32 output channels even lanes
+ */
+inline HVX_Vector computeOuputVector(HVX_Vector act_vec, HVX_Vector wgt_vec) {
+ HVX_Vector v_res = Q6_Vqf16_vmpy_VhfVhf(act_vec, wgt_vec); // result is in qf16
+ HVX_Vector v_rot = Q6_V_vror_VR(v_res, 2);
+ HVX_Vector v_reduced = Q6_Vqf16_vadd_Vqf16Vqf16(v_res, v_rot);
+ HVX_Vector v_hf = Q6_Vhf_equals_Vqf16(v_reduced);
+ HVX_Vector v1e = getOddEvenOnes().first;
+ HVX_Vector v_reduced_even_lanes = Q6_V_vand_VV(v_hf, v1e);
+ return v_reduced_even_lanes;
+}
+
+static int round_down(int v, int base) { return v - (v % base); }
+
+/**
+ * @brief Compute the convolution of inputs from cr_act, and weights from
+ * cr_filt to update the output to cr_out. The goal is to have an efficient
+ * HVX implementation
+ *
+ * Assumptions:
+ * -----------
+ * - This implementation right now assumes that the dilation is 1
+ * - there is zero padding or the input was already pre-padded.
+ * - block specific spatial padding is only expected at the end and hence
+ * pad_top and pad_left are not yet used
+ * - Relu activation is not used
+ * - Bias add is not done
+ *
+ * @param cr_out blockized output tensor with zeros already filled in
+ * @param cr_act blockized activations
+ * @param cr_filt Chunkified weights as returned from output of prepare_hwio
+ * @param out_shape Original output shape of the tensor before blockization
+ * @param act_shape Original input shape
+ * @param bias_flat Flat bias values and are not used right now
+ * @param filt_shape Original filter shape
+ * @param pad_shape Pad top and pad left shape
+ * @param relu Whether to apply relu after convolution, not done right now
+ * @param zero_block A block filled with zeros
+ *
+ * @return
+ */
+void conv_layer_fp16_hvx(DLTensor& cr_out, const DLTensor& cr_act, // NOLINT(*)
+ const DLTensor& cr_filt, const DLTensor& out_shape,
+ const DLTensor& act_shape, const DLTensor& bias_flat,
+ const DLTensor& filt_shape, const DLTensor& pad_shape, bool relu,
+ int stride_h, int stride_w, uintptr_t zero_block) {
+ int64_t filt_height = filt_shape.shape[0];
+ int64_t filt_width = filt_shape.shape[1];
+ int64_t filt_idepth = filt_shape.shape[2];
+ (void)filt_idepth;
+
+ DEBUG_BLOCK(int pad_top = pad_shape.shape[0]; int pad_left = pad_shape.shape[1];)
+
+ debug("filt_height=%" PRId64 ", filt_width=%" PRId64 ", filt_idepth=%" PRId64
+ ", pad_top=%d, pad_left=%d\n",
+ filt_height, filt_width, filt_idepth, pad_top, pad_left);
+
+ assert(pad_top < 8 && pad_left < 4);
+
+ DEBUG_BLOCK(int a_height = cr_act.shape[1]; int a_width = cr_act.shape[2];
+ int a_depth = cr_act.shape[3];
+
+ int w_height = cr_filt.shape[0]; int w_width = cr_filt.shape[1];
+
+ int o_depth = cr_out.shape[3]; int b_depth = bias_flat.shape[0];)
+
+ int o_height = cr_out.shape[1];
+ int o_width = cr_out.shape[2];
+
+ int out_height = out_shape.shape[1];
+ int out_width = out_shape.shape[2];
+
+ debug("a: %dx%dx%dx%d, w: %dx%dx%dx%d, o: %dx%dx%dx%d, b: %d, out_shape: %dx%d\n", 1, a_height,
+ a_width, a_depth, w_height, w_width, static_cast<int>(cr_filt.shape[2]),
+ static_cast<int>(cr_filt.shape[3]), 1, o_height, o_width, o_depth, b_depth, out_height,
+ out_width);
+
+ assert(a_depth == cr_filt.shape[2]);
+ assert(o_depth == cr_filt.shape[3]);
+
+ int rd = round_down(filt_width, 4);
+ int wgt_chunk_thin_width = filt_width - rd;
+
+ /*
+ * Compute the output vector of either 1 or 2 elements along the width and max 32 elements along
+ * the depth to constitue a maximum of 64 elements
+ *
+ * The weights are loaded directly in the order they're stored, which results
+ * in 2 input channels and 32 output channels
+ *
+ * Weights vector illustration:
+ * ------- ------ ------------
+ * weights_vec = [0-0,0-1,1-0,1-1,2-0,2-1,3-0,3-1,4-0,4-1,...,31-0,31-1] -> This is the
Review Comment:
I've added the index map representation for the packing as comments in the `conv_utils.cc` file where the `blockize` and `chunkify` functions are called.
https://github.com/apache/tvm/pull/12204/files#diff-ed59135151f4003f2b49baba77aab5d33a2e4ae0ee36e62cf52159f26ff13f5cR27-R33
https://github.com/apache/tvm/pull/12204/files#diff-ed59135151f4003f2b49baba77aab5d33a2e4ae0ee36e62cf52159f26ff13f5cR113-R127
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945437568
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t N>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i != N; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = N;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[N];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+inline constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+inline constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+constexpr uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+constexpr uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
Yeah I removed constexpr calls on this, but then wondered if any of the above functions could benefit from constexpr either, as the files are compiled as part of the runtime, and before the model compilation (which is when the op shapes and other constants are known).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949341528
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
Review Comment:
If `uint16_t` is a stand-in for qfloat16, is there a typename supplied by the Hexagon SDK that might be more descriptive?
This might not be a valid example, but I'm thinking of something _like_ `qhl_q0_t`.
Or if the SDK provides no such type name, could it make sense to just define one here? E.g., `using qfloat16_ptr = uint16_t*;`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949649604
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
Sounds good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949654199
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
Review Comment:
The recognizing of `_Float16` in clang at least is target-specific. For Hexagon, clang recognizes it when the target architecture is >= v68.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r940401262
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "conv2d.h"
Review Comment:
Just my subjective opinion, but there might be less room for confusion / bugs if we used a longer path here, e.g. `#include <runtime/hexagon/ops/conv2d.h>`, rather than adding this directory to the include path via the `CMakeLists.txt` file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] quic-sanirudh commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
quic-sanirudh commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r945487347
##########
tests/python/contrib/test_hexagon/topi/test_conv2d_fp16_intrin.py:
##########
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import numpy as np
+import pytest
+import sys
+
+import tvm
+import tvm.contrib.hexagon
+from tvm.topi.testing import conv2d_nhwc_python
+
+
+def build_conv2d(target):
+ an, ah, aw, ac = (
+ tvm.te.var("an"),
+ tvm.te.var("ah"),
+ tvm.te.var("aw"),
+ tvm.te.var("ac"),
+ )
+ fh, fw, fo = tvm.te.var("fh"), tvm.te.var("fw"), tvm.te.var("fo")
+ off_l, off_t = tvm.te.var("off_l"), tvm.te.var("off_t")
+ stride_h, stride_w = tvm.te.var("stride_h"), tvm.te.var("stride_w")
+
+ act_flat = tvm.te.placeholder(shape=(an, ah, aw, ac), dtype="float16", name="act_flat")
+ wgt_flat = tvm.te.placeholder(shape=(fh, fw, ac, fo), dtype="float16", name="wgt_flat")
+
+ out_flat = tvm.te.extern(
+ shape=(an, (ah - fh) // stride_h + 1, (aw - fw) // stride_w + 1, fo),
+ inputs=[act_flat, wgt_flat],
+ fcompute=lambda ins, outs: tvm.tir.call_cpacked(
+ "conv2d_packed", # Function from TVM runtime
+ ins[0],
+ ins[1],
+ off_t,
+ off_l,
+ stride_h,
+ stride_w,
+ outs[0],
+ tvm.runtime.const(0), # resource_handle (unused)
+ ),
+ dtype="float16",
+ )
+
+ s = tvm.te.create_schedule(out_flat.op)
+
+ func_name = "extern_conv"
+ with tvm.transform.PassContext(opt_level=3):
+ module = tvm.build(
+ s,
+ [act_flat, wgt_flat, off_t, off_l, stride_h, stride_w, out_flat],
+ target=target,
+ name=func_name,
+ )
+
+ return module
+
+
+shape_parameters = [
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 10, 14, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 14, 6, 64),
+ "wgt_shape": (3, 3, 64, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 40),
+ "wgt_shape": (3, 3, 40, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 4, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 5, 5, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 6, 6, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 7, 7, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 8, 64),
+ "wgt_shape": (2, 2, 64, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (1, 1),
+ },
+ {
+ "act_shape": (1, 8, 4, 3),
+ "wgt_shape": (3, 3, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (3, 3, 3, 64),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 14, 6, 3),
+ "wgt_shape": (5, 5, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
+ {
+ "act_shape": (1, 8, 8, 3),
+ "wgt_shape": (2, 2, 3, 3),
+ "inp_offset": (0, 0),
+ "inp_stride": (2, 2),
+ },
Review Comment:
I tried adding larger convolution tests earlier, but realized that it won't work due to VTCM limit. I can still add them and mark them as skip if you think that's a good idea for the future, but as it stands now, the op has to be split into chunks to accommodate VTCM size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949613464
##########
src/runtime/hexagon/ops/conv2d_hvx.cc:
##########
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_compute_res.h>
+#include <hexagon_types.h>
+#include <hvx_hexagon_protos.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cinttypes>
+
+#include "tvm/runtime/hexagon/ops/conv2d.h"
+
+// Current limitations:
+// - N in NHWC must be 1
+// - dilated convolutions are not supported
+// - Bias is not accepted
+// - Optional "relu" is not performed
+
+// Packed arguments:
+// 0: DLTensor activations (NHWC)
+// 1: DLTensor weights (HWIO)
+// 2: int offset_top
+// 3: int offset_left
+// 4: int stride_h
+// 5: int stride_w
+// 6: DLTensor output (NHWC)
+extern "C" int conv2d_packed(TVMValue* args, int* type_codes, int num_args, TVMValue* out_val,
+ int out_code, void* res_handle);
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+
+inline uint16_t* getElementPtr(int block_out_y, int block_out_x, int block_out_c, int yi, int xio,
+ int ci, int xii, const DLTensor& block) {
+ auto block_ptr = nhwc_at(block, 0, block_out_y, block_out_x, block_out_c);
+ auto block_offset = yi * 128 + xio * 64 + ci * 2 + xii;
+ auto first_element_ptr = reinterpret_cast<uint16_t*>(block_ptr);
+ return first_element_ptr + block_offset;
+}
Review Comment:
I like the idea of renaming the files. Even with that, we should consider any symbols that will be visible outside of `convey2d_fp16_hvx.cc`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949624162
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
That sounds plausible. My suggestion could be a case of premature optimization; I guess we'll know either way if this turns out to be a hotspot during profiling.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] cconvey commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
cconvey commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949267658
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
The `DLTensor` struct has two members that don't seem to be considered here: [`strides` and `byte_offset`](https://github.com/dmlc/dlpack/blob/ddeb264880a1fa7e7be238ab3901a810324fbe5f/include/dlpack/dlpack.h#L159-L161).
Is it truly safe to ignore those members here? If so, it might be helpful to (ideally) add an `assert` and/or a comment about why they can be ignored.
If the reason we can safely ignore `strides` and `byte_offset` is that we know they were set to `0` and `nullptr`, repsectively, by `SDLTensor`'s constructor, then would it make sense to have these two functions explicitly operate on `SDLTensor` objects rather than `DLTensor` objects? (And if so, is there any reason they shouldn't be class methods of `SDLTensor` rather than free-standing functions?)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949397877
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
+ return p[y * f.shape[1] * f.shape[2] * f.shape[3] + x * f.shape[2] * f.shape[3] + i * f.shape[3] +
+ o];
+}
Review Comment:
The compiler should be able to take care of it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949401536
##########
include/tvm/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+namespace tvm {
+namespace runtime {
+namespace hexagon {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
+template <size_t NDIM>
+class SDLTensor : public DLTensor {
+ public:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space, const int64_t* data_dims)
+ : SDLTensor(data_ptr, data_type, data_space) {
+ for (size_t i = 0; i < NDIM; ++i) dims[i] = data_dims[i];
+ }
+
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space,
+ std::initializer_list<int64_t> data_dims)
+ : SDLTensor(data_ptr, data_type, data_space, data_dims.begin()) {}
+
+ void* GetDataSpace() const { return data_space; }
+
+ private:
+ SDLTensor(void* data_ptr, DLDataType data_type, void* data_space) : data_space(data_space) {
+ data = data_ptr;
+ device = hexagon_device;
+ ndim = NDIM;
+ dtype = data_type;
+ shape = dims;
+ strides = nullptr;
+ byte_offset = 0;
+ }
+
+ void* data_space = nullptr;
+ int64_t dims[NDIM];
+};
+
+inline void* to_ptr(uintptr_t v) { return reinterpret_cast<void*>(v); }
+
+inline uintptr_t to_uint(void* ptr) { return reinterpret_cast<uintptr_t>(ptr); }
+
+constexpr int xyc_to_sm_16b(int y, int x, int c) {
+ // Map y,x,c coordinates within a block to the offset (in 16-bit elements)
+ // from the beginning of the block in spatial-major layout.
+ // 10-bit spatial mask: yyyxcccccx
+ assert(y >= 0 && x >= 0 && c >= 0);
+ return y << 7 | (x & 2) << 5 | c << 1 | (x & 1);
+}
+
+constexpr int hwio_to_sm_16b(int width, int y, int x, int i, int o) {
+ // Map y,x,i,o coordinates within a chunk (assuming the origin at the
+ // top-left spatial corner) to the offset (in 16-bit elements) from the
+ // beginning of the chunk in spatial-major layout.
+ // Spatial mask: p..piiiioooooi, where p..p are position bits.
+ assert(width >= 1);
+ assert(y >= 0 && x >= 0 && i >= 0 && o >= 0);
+ int p = y * width + (width - 1 - x);
+ return p << 10 | (i & 0x1e) << 5 | o << 1 | (i & 1);
+}
+
+inline constexpr int round_up(int v, int p2) { return (v + p2 - 1) & -p2; }
+
+inline uintptr_t nhwc_at(const DLTensor& a, int n, int y, int x, int c) {
+ if (y < 0 || y >= a.shape[1]) return uintptr_t(0);
+ auto p = static_cast<uintptr_t*>(a.data);
+ assert(n == 0);
+ return p[y * a.shape[2] * a.shape[3] + x * a.shape[3] + c];
+}
+
+inline uintptr_t hwio_at(const DLTensor& f, int y, int x, int i, int o) {
+ auto p = static_cast<uintptr_t*>(f.data);
Review Comment:
Same here, there is no way to guarantee anything like that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [tvm] kparzysz-quic commented on a diff in pull request #12204: [Hexagon] Add Hand written HVX conv2d
Posted by GitBox <gi...@apache.org>.
kparzysz-quic commented on code in PR #12204:
URL: https://github.com/apache/tvm/pull/12204#discussion_r949410521
##########
src/runtime/hexagon/ops/conv2d.h:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <HAP_farf.h>
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/runtime/device_api.h>
+
+#include <cassert>
+
+#ifndef TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+#define TVM_RUNTIME_HEXAGON_OPS_CONV2D_H_
+
+#ifdef DEBUG_CONV
+#define DEBUG_BLOCK(X) \
+ { X }
+#define debug(...) FARF(ALWAYS, ##__VA_ARGS__)
+#else
+#define DEBUG_BLOCK(X)
+#define debug(...)
+#endif
+
+#define HAP_CALL(hap_fn, ...) \
+ { \
+ int rc = hap_fn(__VA_ARGS__); \
+ if (rc != 0) { \
+ debug("%s failed: rc=%x", #hap_fn, rc); \
+ } \
+ }
+
+namespace detail {
+static constexpr auto hexagon_device = DLDevice{static_cast<DLDeviceType>(kDLHexagon), 0};
+
+// Standalone DLTensor: the standalone-ness means that this object owns the shape
+// (as opposed to a DLTensor).
Review Comment:
`DLTensor` has a pointer to an array of dimensions, but doesn't own the storage, someone else has to allocate it somewhere, put the shape in it, and store the pointer in the DLTensor. This is annoying, since DLTensor is not an entity that can live on its own (the strides are also like this, iirc, but we don't use them here). Standalone DLTensor was invented to contain the shape as a part of the tensor object.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org