You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/03/20 22:56:57 UTC
[GitHub] piiswrong closed pull request #9939: [MXNET-40]add multi proposal operator (cpu version) and fix the bug in proposal op (gpu version)
piiswrong closed pull request #9939: [MXNET-40]add multi proposal operator (cpu version) and fix the bug in proposal op (gpu version)
URL: https://github.com/apache/incubator-mxnet/pull/9939
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index c09c559b189..aceec7c09bb 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -158,3 +158,4 @@ List of Contributors
* [Sorokin Evgeniy](https://github.com/TheTweak)
* [dwSun](https://github.com/dwSun/)
* [David Braude](https://github.com/dabraude/)
+* [Kan Wu](https://github.com/wkcn)
diff --git a/src/operator/contrib/multi_proposal.cc b/src/operator/contrib/multi_proposal.cc
index cd00e877a11..0c52b9b7cfc 100644
--- a/src/operator/contrib/multi_proposal.cc
+++ b/src/operator/contrib/multi_proposal.cc
@@ -22,11 +22,262 @@
* Licensed under The Apache-2.0 License [see LICENSE for details]
* \file multi_proposal.cc
* \brief
- * \author Xizhou Zhu
+ * \author Xizhou Zhu, Kan Wu
*/
#include "./multi_proposal-inl.h"
+//============================
+// Bounding Box Transform Utils
+//============================
+namespace mxnet {
+namespace op {
+namespace utils {
+
+// bbox prediction and clip to the image borders
+inline void BBoxTransformInv(const mshadow::Tensor<cpu, 2>& boxes,
+ const mshadow::Tensor<cpu, 3>& deltas,
+ const float im_height,
+ const float im_width,
+ const int real_height,
+ const int real_width,
+ mshadow::Tensor<cpu, 2> *out_pred_boxes) {
+ CHECK_GE(boxes.size(1), 4);
+ CHECK_GE(out_pred_boxes->size(1), 4);
+ int anchors = deltas.size(0) / 4;
+ int heights = deltas.size(1);
+ int widths = deltas.size(2);
+
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int index = 0; index < anchors * heights * widths; ++index) {
+ // index_t index = h * (widths * anchors) + w * (anchors) + a;
+ int a = index % anchors;
+ int w = (index / anchors) % widths;
+ int h = index / (widths * anchors);
+
+ float width = boxes[index][2] - boxes[index][0] + 1.0;
+ float height = boxes[index][3] - boxes[index][1] + 1.0;
+ float ctr_x = boxes[index][0] + 0.5 * (width - 1.0);
+ float ctr_y = boxes[index][1] + 0.5 * (height - 1.0);
+
+ float dx = deltas[a*4 + 0][h][w];
+ float dy = deltas[a*4 + 1][h][w];
+ float dw = deltas[a*4 + 2][h][w];
+ float dh = deltas[a*4 + 3][h][w];
+
+ float pred_ctr_x = dx * width + ctr_x;
+ float pred_ctr_y = dy * height + ctr_y;
+ float pred_w = exp(dw) * width;
+ float pred_h = exp(dh) * height;
+
+ float pred_x1 = pred_ctr_x - 0.5 * (pred_w - 1.0);
+ float pred_y1 = pred_ctr_y - 0.5 * (pred_h - 1.0);
+ float pred_x2 = pred_ctr_x + 0.5 * (pred_w - 1.0);
+ float pred_y2 = pred_ctr_y + 0.5 * (pred_h - 1.0);
+
+ pred_x1 = std::max(std::min(pred_x1, im_width - 1.0f), 0.0f);
+ pred_y1 = std::max(std::min(pred_y1, im_height - 1.0f), 0.0f);
+ pred_x2 = std::max(std::min(pred_x2, im_width - 1.0f), 0.0f);
+ pred_y2 = std::max(std::min(pred_y2, im_height - 1.0f), 0.0f);
+
+ (*out_pred_boxes)[index][0] = pred_x1;
+ (*out_pred_boxes)[index][1] = pred_y1;
+ (*out_pred_boxes)[index][2] = pred_x2;
+ (*out_pred_boxes)[index][3] = pred_y2;
+
+ if (h >= real_height || w >= real_width) {
+ (*out_pred_boxes)[index][4] = -1.0;
+ }
+ }
+}
+
+// iou prediction and clip to the image border
+inline void IoUTransformInv(const mshadow::Tensor<cpu, 2>& boxes,
+ const mshadow::Tensor<cpu, 3>& deltas,
+ const float im_height,
+ const float im_width,
+ const int real_height,
+ const int real_width,
+ mshadow::Tensor<cpu, 2> *out_pred_boxes) {
+ CHECK_GE(boxes.size(1), 4);
+ CHECK_GE(out_pred_boxes->size(1), 4);
+ int anchors = deltas.size(0) / 4;
+ int heights = deltas.size(1);
+ int widths = deltas.size(2);
+
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int index = 0; index < anchors * heights * widths; ++index) {
+ // index_t index = h * (widths * anchors) + w * (anchors) + a;
+ int a = index % anchors;
+ int w = (index / anchors) % widths;
+ int h = index / (widths * anchors);
+
+ float x1 = boxes[index][0];
+ float y1 = boxes[index][1];
+ float x2 = boxes[index][2];
+ float y2 = boxes[index][3];
+
+ float dx1 = deltas[a * 4 + 0][h][w];
+ float dy1 = deltas[a * 4 + 1][h][w];
+ float dx2 = deltas[a * 4 + 2][h][w];
+ float dy2 = deltas[a * 4 + 3][h][w];
+
+ float pred_x1 = x1 + dx1;
+ float pred_y1 = y1 + dy1;
+ float pred_x2 = x2 + dx2;
+ float pred_y2 = y2 + dy2;
+
+ pred_x1 = std::max(std::min(pred_x1, im_width - 1.0f), 0.0f);
+ pred_y1 = std::max(std::min(pred_y1, im_height - 1.0f), 0.0f);
+ pred_x2 = std::max(std::min(pred_x2, im_width - 1.0f), 0.0f);
+ pred_y2 = std::max(std::min(pred_y2, im_height - 1.0f), 0.0f);
+
+ (*out_pred_boxes)[index][0] = pred_x1;
+ (*out_pred_boxes)[index][1] = pred_y1;
+ (*out_pred_boxes)[index][2] = pred_x2;
+ (*out_pred_boxes)[index][3] = pred_y2;
+
+ if (h >= real_height || w >= real_width) {
+ (*out_pred_boxes)[index][4] = -1.0f;
+ }
+ }
+}
+
+// filter box by set confidence to zero
+// * height or width < rpn_min_size
+inline void FilterBox(mshadow::Tensor<cpu, 2> *dets,
+ const float min_size) {
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int i = 0; i < static_cast<int>(dets->size(0)); ++i) {
+ float iw = (*dets)[i][2] - (*dets)[i][0] + 1.0f;
+ float ih = (*dets)[i][3] - (*dets)[i][1] + 1.0f;
+ if (iw < min_size || ih < min_size) {
+ (*dets)[i][0] -= min_size / 2;
+ (*dets)[i][1] -= min_size / 2;
+ (*dets)[i][2] += min_size / 2;
+ (*dets)[i][3] += min_size / 2;
+ (*dets)[i][4] = -1.0f;
+ }
+ }
+}
+
+} // namespace utils
+} // namespace op
+} // namespace mxnet
+
+//=====================
+// NMS Utils
+//=====================
+namespace mxnet {
+namespace op {
+namespace utils {
+
+struct ReverseArgsortCompl {
+ const float *val_;
+ explicit ReverseArgsortCompl(float *val)
+ : val_(val) {}
+ bool operator() (float i, float j) {
+ return (val_[static_cast<index_t>(i)] >
+ val_[static_cast<index_t>(j)]);
+ }
+};
+
+// copy score and init order
+inline void CopyScore(const mshadow::Tensor<cpu, 2>& dets,
+ mshadow::Tensor<cpu, 1> *score,
+ mshadow::Tensor<cpu, 1> *order) {
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int i = 0; i < static_cast<int>(dets.size(0)); ++i) {
+ (*score)[i] = dets[i][4];
+ (*order)[i] = i;
+ }
+}
+
+// sort order array according to score
+inline void ReverseArgsort(const mshadow::Tensor<cpu, 1>& score,
+ mshadow::Tensor<cpu, 1> *order) {
+ ReverseArgsortCompl cmpl(score.dptr_);
+ std::sort(order->dptr_, order->dptr_ + score.size(0), cmpl);
+}
+
+// reorder proposals according to order and keep the pre_nms_top_n proposals
+// dets.size(0) == pre_nms_top_n
+inline void ReorderProposals(const mshadow::Tensor<cpu, 2>& prev_dets,
+ const mshadow::Tensor<cpu, 1>& order,
+ const index_t pre_nms_top_n,
+ mshadow::Tensor<cpu, 2> *dets) {
+ CHECK_EQ(dets->size(0), pre_nms_top_n);
+ const int dets_size0 = static_cast<int>(dets->size(0));
+ const int dets_size1 = static_cast<int>(dets->size(1));
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int k = 0; k < dets_size0 * dets_size1; ++k) {
+ int i = k / dets_size1;
+ int j = k % dets_size1;
+ const index_t index = order[i];
+ (*dets)[i][j] = prev_dets[index][j];
+ }
+}
+
+// greedily keep the max detections (already sorted)
+inline void NonMaximumSuppression(const mshadow::Tensor<cpu, 2>& dets,
+ const float thresh,
+ const index_t post_nms_top_n,
+ mshadow::Tensor<cpu, 1> *area,
+ mshadow::Tensor<cpu, 1> *suppressed,
+ mshadow::Tensor<cpu, 1> *keep,
+ int *out_size) {
+ CHECK_EQ(dets.shape_[1], 5) << "dets: [x1, y1, x2, y2, score]";
+ CHECK_GT(dets.shape_[0], 0);
+ CHECK_EQ(dets.CheckContiguous(), true);
+ CHECK_EQ(area->CheckContiguous(), true);
+ CHECK_EQ(suppressed->CheckContiguous(), true);
+ CHECK_EQ(keep->CheckContiguous(), true);
+ // calculate area
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int i = 0; i < static_cast<int>(dets.size(0)); ++i) {
+ (*area)[i] = (dets[i][2] - dets[i][0] + 1) *
+ (dets[i][3] - dets[i][1] + 1);
+ }
+
+ // calculate nms
+ *out_size = 0;
+ for (index_t i = 0; i < dets.size(0) && (*out_size) < static_cast<int>(post_nms_top_n); ++i) {
+ float ix1 = dets[i][0];
+ float iy1 = dets[i][1];
+ float ix2 = dets[i][2];
+ float iy2 = dets[i][3];
+ float iarea = (*area)[i];
+
+ if ((*suppressed)[i] > 0.0f) {
+ continue;
+ }
+
+ (*keep)[(*out_size)++] = i;
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int j = i + 1; j < static_cast<int>(dets.size(0)); ++j) {
+ if ((*suppressed)[j] > 0.0f) {
+ continue;
+ }
+ float xx1 = std::max(ix1, dets[j][0]);
+ float yy1 = std::max(iy1, dets[j][1]);
+ float xx2 = std::min(ix2, dets[j][2]);
+ float yy2 = std::min(iy2, dets[j][3]);
+ float w = std::max(0.0f, xx2 - xx1 + 1.0f);
+ float h = std::max(0.0f, yy2 - yy1 + 1.0f);
+ float inter = w * h;
+ float ovr = inter / (iarea + (*area)[j] - inter);
+ if (ovr > thresh) {
+ (*suppressed)[j] = 1.0f;
+ }
+ }
+ }
+}
+
+} // namespace utils
+} // namespace op
+} // namespace mxnet
+
+
namespace mxnet {
namespace op {
@@ -43,7 +294,168 @@ class MultiProposalOp : public Operator{
const std::vector<OpReqType> &req,
const std::vector<TBlob> &out_data,
const std::vector<TBlob> &aux_states) {
- LOG(FATAL) << "not implemented";
+ using namespace mshadow;
+ using namespace mshadow::expr;
+ CHECK_EQ(in_data.size(), 3);
+ CHECK_EQ(out_data.size(), 2);
+ CHECK_GT(req.size(), 1);
+ CHECK_EQ(req[proposal::kOut], kWriteTo);
+
+ Stream<xpu> *s = ctx.get_stream<xpu>();
+
+ Tensor<cpu, 4> scores = in_data[proposal::kClsProb].get<cpu, 4, real_t>(s);
+ Tensor<cpu, 4> bbox_deltas = in_data[proposal::kBBoxPred].get<cpu, 4, real_t>(s);
+ Tensor<cpu, 2> im_info = in_data[proposal::kImInfo].get<cpu, 2, real_t>(s);
+
+ Tensor<cpu, 2> out = out_data[proposal::kOut].get<cpu, 2, real_t>(s);
+ Tensor<cpu, 2> out_score = out_data[proposal::kScore].get<cpu, 2, real_t>(s);
+
+ int num_images = scores.size(0);
+ int num_anchors = scores.size(1) / 2;
+ int height = scores.size(2);
+ int width = scores.size(3);
+ int count_anchors = num_anchors * height * width;
+ int rpn_pre_nms_top_n =
+ (param_.rpn_pre_nms_top_n > 0) ? param_.rpn_pre_nms_top_n : count_anchors;
+ rpn_pre_nms_top_n = std::min(rpn_pre_nms_top_n, count_anchors);
+ int rpn_post_nms_top_n = std::min(param_.rpn_post_nms_top_n, rpn_pre_nms_top_n);
+
+ int workspace_size =
+ num_images * (count_anchors * 5 + 2 * count_anchors +
+ rpn_pre_nms_top_n * 5 + 3 * rpn_pre_nms_top_n);
+
+ Tensor<cpu, 1> workspace = ctx.requested[proposal::kTempResource].get_space<cpu>(
+ Shape1(workspace_size), s);
+ int start = 0;
+ Tensor<cpu, 3> workspace_proposals(workspace.dptr_ +
+ start, Shape3(num_images, count_anchors, 5));
+ start += num_images * count_anchors * 5;
+ Tensor<cpu, 3> workspace_pre_nms(workspace.dptr_ + start, Shape3(num_images, 2, count_anchors));
+ start += num_images * 2 * count_anchors;
+ Tensor<cpu, 3> workspace_ordered_proposals(workspace.dptr_ + start,
+ Shape3(num_images, rpn_pre_nms_top_n, 5));
+ start += num_images * rpn_pre_nms_top_n * 5;
+ Tensor<cpu, 3> workspace_nms(workspace.dptr_ + start, Shape3(num_images, 3, rpn_pre_nms_top_n));
+ start += num_images * 3 * rpn_pre_nms_top_n;
+ CHECK_EQ(workspace_size, start) << workspace_size << " " << start << std::endl;
+
+ // Generate anchors
+ std::vector<float> base_anchor(4);
+ base_anchor[0] = 0.0;
+ base_anchor[1] = 0.0;
+ base_anchor[2] = param_.feature_stride - 1.0;
+ base_anchor[3] = param_.feature_stride - 1.0;
+ CHECK_EQ(num_anchors, param_.ratios.ndim() * param_.scales.ndim());
+ std::vector<float> anchors;
+ utils::GenerateAnchors(base_anchor,
+ param_.ratios,
+ param_.scales,
+ &anchors);
+ std::memcpy(workspace_proposals.dptr_, &anchors[0], sizeof(float) * anchors.size());
+
+ Tensor<cpu, 2> workspace_proposals0 = workspace_proposals[0];
+ // Enumerate all shifted anchors
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int index = 0; index < num_anchors * height * width; ++index) {
+ // index_t index = j * (width * num_anchors) + k * (num_anchors) + i;
+ int i = index % num_anchors;
+ int k = (index / num_anchors) % width;
+ int j = index / (width * num_anchors);
+ workspace_proposals0[index][0] =
+ workspace_proposals0[i][0] + k * param_.feature_stride;
+ workspace_proposals0[index][1] =
+ workspace_proposals0[i][1] + j * param_.feature_stride;
+ workspace_proposals0[index][2] =
+ workspace_proposals0[i][2] + k * param_.feature_stride;
+ workspace_proposals0[index][3] =
+ workspace_proposals0[i][3] + j * param_.feature_stride;
+ workspace_proposals0[index][4] = scores[0][i + num_anchors][j][k];
+ }
+
+ // Copy shifted anchors to other images
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int t = count_anchors; t < num_images * count_anchors; ++t) {
+ int b = t / count_anchors;
+ int index = t % count_anchors;
+ int i = index % num_anchors;
+ int k = (index / num_anchors) % width;
+ int j = index / (width * num_anchors);
+ for (int w = 0; w < 4; ++w) {
+ workspace_proposals[b][index][w] = workspace_proposals[0][index][w];
+ }
+ workspace_proposals[b][index][4] = scores[b][i + num_anchors][j][k];
+ }
+
+ // Assign Foreground Scores for each anchor
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int b = 0; b < num_images; ++b) {
+ // prevent padded predictions
+ int real_height = static_cast<int>(im_info[b][0] / param_.feature_stride);
+ int real_width = static_cast<int>(im_info[b][1] / param_.feature_stride);
+ CHECK_GE(height, real_height) << height << " " << real_height << std::endl;
+ CHECK_GE(width, real_width) << width << " " << real_width << std::endl;
+
+ Tensor<cpu, 2> workspace_proposals_i = workspace_proposals[b];
+ Tensor<cpu, 2> workspace_pre_nms_i = workspace_pre_nms[b];
+ Tensor<cpu, 2> workspace_ordered_proposals_i =
+ workspace_ordered_proposals[b];
+ Tensor<cpu, 2> workspace_nms_i = workspace_nms[b];
+
+ if (param_.iou_loss) {
+ utils::IoUTransformInv(workspace_proposals_i, bbox_deltas[b], im_info[b][0], im_info[b][1],
+ real_height, real_width, &(workspace_proposals_i));
+ } else {
+ utils::BBoxTransformInv(workspace_proposals_i, bbox_deltas[b], im_info[b][0], im_info[b][1],
+ real_height, real_width, &(workspace_proposals_i));
+ }
+ utils::FilterBox(&workspace_proposals_i, param_.rpn_min_size * im_info[b][2]);
+
+ Tensor<cpu, 1> score = workspace_pre_nms_i[0];
+ Tensor<cpu, 1> order = workspace_pre_nms_i[1];
+
+ utils::CopyScore(workspace_proposals_i,
+ &score,
+ &order);
+ utils::ReverseArgsort(score,
+ &order);
+ utils::ReorderProposals(workspace_proposals_i,
+ order,
+ rpn_pre_nms_top_n,
+ &workspace_ordered_proposals_i);
+ int out_size = 0;
+ Tensor<cpu, 1> area = workspace_nms_i[0];
+ Tensor<cpu, 1> suppressed = workspace_nms_i[1];
+ Tensor<cpu, 1> keep = workspace_nms_i[2];
+ suppressed = 0; // surprised!
+
+ utils::NonMaximumSuppression(workspace_ordered_proposals_i,
+ param_.threshold,
+ rpn_post_nms_top_n,
+ &area,
+ &suppressed,
+ &keep,
+ &out_size);
+
+ // fill in output rois and output scores
+ #pragma omp parallel for num_threads(engine::OpenMP::Get()->GetRecommendedOMPThreadCount())
+ for (int i = 0; i < param_.rpn_post_nms_top_n; ++i) {
+ int out_index = b * param_.rpn_post_nms_top_n + i;
+ out[out_index][0] = b;
+ if (i < out_size) {
+ index_t index = keep[i];
+ for (index_t j = 0; j < 4; ++j) {
+ out[out_index][j + 1] = workspace_ordered_proposals_i[index][j];
+ }
+ out_score[out_index][0] = workspace_ordered_proposals_i[index][4];
+ } else {
+ index_t index = keep[i % out_size];
+ for (index_t j = 0; j < 4; ++j) {
+ out[out_index][j + 1] = workspace_ordered_proposals_i[index][j];
+ }
+ out_score[out_index][0] = workspace_ordered_proposals_i[index][4];
+ }
+ }
+ }
}
virtual void Backward(const OpContext &ctx,
@@ -53,7 +465,19 @@ class MultiProposalOp : public Operator{
const std::vector<OpReqType> &req,
const std::vector<TBlob> &in_grad,
const std::vector<TBlob> &aux_states) {
- LOG(FATAL) << "not implemented";
+ using namespace mshadow;
+ using namespace mshadow::expr;
+ CHECK_EQ(in_grad.size(), 3);
+
+ Stream<xpu> *s = ctx.get_stream<xpu>();
+ Tensor<xpu, 4> gscores = in_grad[proposal::kClsProb].get<xpu, 4, real_t>(s);
+ Tensor<xpu, 4> gbbox = in_grad[proposal::kBBoxPred].get<xpu, 4, real_t>(s);
+ Tensor<xpu, 2> ginfo = in_grad[proposal::kImInfo].get<xpu, 2, real_t>(s);
+
+ // can not assume the grad would be zero
+ Assign(gscores, req[proposal::kClsProb], 0);
+ Assign(gbbox, req[proposal::kBBoxPred], 0);
+ Assign(ginfo, req[proposal::kImInfo], 0);
}
private:
diff --git a/src/operator/contrib/multi_proposal.cu b/src/operator/contrib/multi_proposal.cu
index a2a17d74908..4552ae49d70 100644
--- a/src/operator/contrib/multi_proposal.cu
+++ b/src/operator/contrib/multi_proposal.cu
@@ -326,6 +326,7 @@ __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh,
void _nms(const mshadow::Tensor<gpu, 2>& boxes,
const float nms_overlap_thresh,
+ const int rpn_post_nms_top_n,
int *keep,
int *num_out) {
const int threadsPerBlock = sizeof(uint64_t) * 8;
@@ -363,6 +364,7 @@ void _nms(const mshadow::Tensor<gpu, 2>& boxes,
if (!(remv[nblock] & (1ULL << inblock))) {
keep[num_to_keep++] = i;
+ if (num_to_keep >= rpn_post_nms_top_n) break;
uint64_t *p = &mask_host[0] + i * col_blocks;
for (int j = nblock; j < col_blocks; j++) {
remv[j] |= p[j];
@@ -555,6 +557,7 @@ class MultiProposalGPUOp : public Operator{
int out_size = 0;
_nms(workspace_ordered_proposals,
param_.threshold,
+ rpn_post_nms_top_n,
&_keep[0],
&out_size);
@@ -563,11 +566,12 @@ class MultiProposalGPUOp : public Operator{
cudaMemcpyHostToDevice));
// copy results after nms
- dimGrid.x = (rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock;
+ dimGrid.x = (param_.rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock;
CheckLaunchParam(dimGrid, dimBlock, "PrepareOutput");
PrepareOutput << <dimGrid, dimBlock >> >(
- rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size, b,
- out.dptr_ + b * rpn_post_nms_top_n * 5, out_score.dptr_ + b * rpn_post_nms_top_n);
+ param_.rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size, b,
+ out.dptr_ + b * param_.rpn_post_nms_top_n * 5,
+ out_score.dptr_ + b * param_.rpn_post_nms_top_n);
FRCNN_CUDA_CHECK(cudaPeekAtLastError());
}
// free temporary memory
diff --git a/src/operator/contrib/proposal.cc b/src/operator/contrib/proposal.cc
index dd6ed5a93da..fa28c26ace6 100644
--- a/src/operator/contrib/proposal.cc
+++ b/src/operator/contrib/proposal.cc
@@ -400,8 +400,8 @@ class ProposalOp : public Operator{
&keep,
&out_size);
- // fill in output rois
- for (index_t i = 0; i < out.size(0); ++i) {
+ // fill in output rois and output score
+ for (index_t i = 0; i < static_cast<index_t>(param_.rpn_post_nms_top_n); ++i) {
// batch index 0
out[i][0] = 0;
if (i < out_size) {
@@ -409,21 +409,12 @@ class ProposalOp : public Operator{
for (index_t j = 0; j < 4; ++j) {
out[i][j + 1] = workspace_ordered_proposals[index][j];
}
+ out_score[i][0] = workspace_ordered_proposals[index][4];
} else {
index_t index = keep[i % out_size];
for (index_t j = 0; j < 4; ++j) {
out[i][j + 1] = workspace_ordered_proposals[index][j];
}
- }
- }
-
- // fill in output score
- for (index_t i = 0; i < out_score.size(0); i++) {
- if (i < out_size) {
- index_t index = keep[i];
- out_score[i][0] = workspace_ordered_proposals[index][4];
- } else {
- index_t index = keep[i % out_size];
out_score[i][0] = workspace_ordered_proposals[index][4];
}
}
diff --git a/src/operator/contrib/proposal.cu b/src/operator/contrib/proposal.cu
index 2d676aca93a..446c92b35ce 100644
--- a/src/operator/contrib/proposal.cu
+++ b/src/operator/contrib/proposal.cu
@@ -307,6 +307,7 @@ __global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh,
void _nms(const mshadow::Tensor<gpu, 2>& boxes,
const float nms_overlap_thresh,
+ const int rpn_post_nms_top_n,
int *keep,
int *num_out) {
const int threadsPerBlock = sizeof(uint64_t) * 8;
@@ -344,6 +345,7 @@ void _nms(const mshadow::Tensor<gpu, 2>& boxes,
if (!(remv[nblock] & (1ULL << inblock))) {
keep[num_to_keep++] = i;
+ if (num_to_keep >= rpn_post_nms_top_n) break;
uint64_t *p = &mask_host[0] + i * col_blocks;
for (int j = nblock; j < col_blocks; j++) {
remv[j] |= p[j];
@@ -543,6 +545,7 @@ class ProposalGPUOp : public Operator{
int out_size = 0;
_nms(workspace_ordered_proposals,
param_.threshold,
+ rpn_post_nms_top_n,
&_keep[0],
&out_size);
@@ -553,10 +556,10 @@ class ProposalGPUOp : public Operator{
cudaMemcpyHostToDevice));
// copy results after nms
- dimGrid.x = (rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock;
+ dimGrid.x = (param_.rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock;
CheckLaunchParam(dimGrid, dimBlock, "PrepareOutput");
PrepareOutput<<<dimGrid, dimBlock>>>(
- rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size,
+ param_.rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size,
out.dptr_, out_score.dptr_);
FRCNN_CUDA_CHECK(cudaPeekAtLastError());
diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py
index 2cef29c03fa..f6ef66fb10b 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -1641,6 +1641,94 @@ def test_cross_device_autograd():
assert_almost_equal(dx, x.grad.asnumpy())
+@with_seed()
+def test_multi_proposal_op():
+ # paramters
+ feature_stride = 16
+ scales = (8, 16, 32)
+ ratios = (0.5, 1, 2)
+ rpn_pre_nms_top_n = 12000
+ rpn_post_nms_top_n = 2000
+ threshold = 0.7
+ rpn_min_size = feature_stride
+
+ feat_len = (1000 + 15) // 16
+ H, W = feat_len, feat_len
+ num_anchors = len(scales) * len(ratios)
+ count_anchors = H * W * num_anchors
+
+ def get_new_data(batch_size, ctx):
+ '''
+ cls_prob: (batch_size, 2 * num_anchors, H, W)
+ bbox_pred: (batch_size, 4 * num_anchors, H, W)
+ im_info: (batch_size, 3)
+ '''
+
+ dtype = np.float32
+ cls_prob = mx.nd.empty((batch_size, 2 * num_anchors, H, W), dtype = dtype, ctx = ctx)
+ bbox_pred = mx.nd.empty((batch_size, 4 * num_anchors, H, W), dtype = dtype, ctx = ctx)
+ im_info = mx.nd.empty((batch_size, 3), dtype = dtype, ctx = ctx)
+
+ cls = [1.0 * (i + 1) / cls_prob.size for i in range(cls_prob.size)]
+ np.random.shuffle(cls)
+ cls_prob = mx.nd.reshape(mx.nd.array(cls, dtype = dtype, ctx = ctx), shape = cls_prob.shape)
+ bbox_pred = mx.nd.array(np.random.randint(-2, 3, size = bbox_pred.shape), dtype = dtype, ctx = ctx)
+
+ for i in range(batch_size):
+ im_size = np.random.randint(600, feat_len * feature_stride, size = (2,))
+ im_scale = np.random.randint(80, 100) / 100.0
+ im_info[i, :] = [im_size[0], im_size[1], im_scale]
+ return cls_prob, bbox_pred, im_info
+
+ def check_proposal_consistency(op, batch_size):
+ '''
+ op is mx.nd.contrib.Proposal or mx.nd.contrib.MultiProposal
+ '''
+ cls_prob, bbox_pred, im_info = get_new_data(batch_size, mx.cpu(0))
+ rois_cpu, score_cpu = op(
+ cls_score = cls_prob,
+ bbox_pred = bbox_pred,
+ im_info = im_info,
+ feature_stride = feature_stride,
+ scales = scales,
+ ratios = ratios,
+ rpn_pre_nms_top_n = rpn_pre_nms_top_n,
+ rpn_post_nms_top_n = rpn_post_nms_top_n,
+ threshold = threshold,
+ rpn_min_size = rpn_min_size, output_score = True)
+
+ gpu_ctx = mx.gpu(0)
+
+ # copy data to gpu from cpu
+ cls_prob_gpu = cls_prob.as_in_context(gpu_ctx)
+ bbox_pred_gpu = bbox_pred.as_in_context(gpu_ctx)
+ im_info_gpu = im_info.as_in_context(gpu_ctx)
+
+ rois_gpu, score_gpu = op(
+ cls_score = cls_prob_gpu,
+ bbox_pred = bbox_pred_gpu,
+ im_info = im_info_gpu,
+ feature_stride = feature_stride,
+ scales = scales,
+ ratios = ratios,
+ rpn_pre_nms_top_n = rpn_pre_nms_top_n,
+ rpn_post_nms_top_n = rpn_post_nms_top_n,
+ threshold = threshold,
+ rpn_min_size = rpn_min_size, output_score = True)
+
+ rois_cpu_np = rois_cpu.asnumpy()
+ rois_gpu_np = rois_gpu.asnumpy()
+
+ score_cpu_np = score_cpu.asnumpy()
+ score_gpu_np = score_gpu.asnumpy()
+
+ assert_almost_equal(score_cpu_np, score_gpu_np, atol = 1e-3, rtol = 1e-3)
+ assert_almost_equal(rois_cpu_np, rois_gpu_np, atol = 1e-3, rtol = 1e-3)
+
+ check_proposal_consistency(mx.nd.contrib.Proposal, 1)
+ check_proposal_consistency(mx.nd.contrib.MultiProposal, 20)
+
+
# The following 2 functions launch 0-thread kernels, an error that should be caught and signaled.
def kernel_error_check_imperative():
os.environ['MXNET_ENGINE_TYPE'] = 'NaiveEngine'
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index 7889e084f74..419f28cfae5 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -5064,6 +5064,126 @@ def check_squeeze_op(shape, axis=None):
test = mx.sym.squeeze(data, axis=(2, 4))
check_numeric_gradient(test, [data_tmp])
+@with_seed()
+def test_multi_proposal_op():
+ # paramters
+ feature_stride = 16
+ scales = (8, 16, 32)
+ ratios = (0.5, 1, 2)
+ rpn_pre_nms_top_n = 12000
+ rpn_post_nms_top_n = 2000
+ threshold = 0.7
+ rpn_min_size = 16
+
+ batch_size = 20
+ feat_len = 14
+ H, W = feat_len, feat_len
+ num_anchors = len(scales) * len(ratios)
+ count_anchors = H * W * num_anchors
+
+ '''
+ cls_prob: (batch_size, 2 * num_anchors, H, W)
+ bbox_pred: (batch_size, 4 * num_anchors, H, W)
+ im_info: (batch_size, 3)
+ '''
+
+ cls_prob = mx.nd.empty((batch_size, 2 * num_anchors, H, W), dtype = np.float32)
+ bbox_pred = mx.nd.empty((batch_size, 4 * num_anchors, H, W), dtype = np.float32)
+ im_info = mx.nd.empty((batch_size, 3), dtype = np.float32)
+
+ cls_prob = mx.nd.array(np.random.random(cls_prob.shape))
+ bbox_pred = mx.nd.array(np.random.random(bbox_pred.shape))
+
+ for i in range(batch_size):
+ im_size = np.random.randint(100, feat_len * feature_stride, size = (2,))
+ im_scale = np.random.randint(70, 100) / 100.0
+ im_info[i, :] = [im_size[0], im_size[1], im_scale]
+
+ def get_sub(arr, i):
+ new_shape = list(arr.shape)
+ new_shape[0] = 1
+ res = arr[i].reshape(new_shape)
+ return res
+
+ def check_forward(rpn_pre_nms_top_n, rpn_post_nms_top_n):
+ single_proposal = []
+ single_score = []
+ for i in range(batch_size):
+ rois, score = mx.nd.contrib.Proposal(
+ cls_score = get_sub(cls_prob, i),
+ bbox_pred = get_sub(bbox_pred, i),
+ im_info = get_sub(im_info, i),
+ feature_stride = feature_stride,
+ scales = scales,
+ ratios = ratios,
+ rpn_pre_nms_top_n = rpn_pre_nms_top_n,
+ rpn_post_nms_top_n = rpn_post_nms_top_n,
+ threshold = threshold,
+ rpn_min_size = rpn_min_size, output_score = True)
+ single_proposal.append(rois)
+ single_score.append(score)
+
+ multi_proposal, multi_score = mx.nd.contrib.MultiProposal(
+ cls_score = cls_prob,
+ bbox_pred = bbox_pred,
+ im_info = im_info,
+ feature_stride = feature_stride,
+ scales = scales,
+ ratios = ratios,
+ rpn_pre_nms_top_n = rpn_pre_nms_top_n,
+ rpn_post_nms_top_n = rpn_post_nms_top_n,
+ threshold = threshold,
+ rpn_min_size = rpn_min_size, output_score = True)
+
+ single_proposal = mx.nd.stack(*single_proposal).reshape(multi_proposal.shape)
+ single_score = mx.nd.stack(*single_score).reshape(multi_score.shape)
+
+ single_proposal_np = single_proposal.asnumpy()
+ multi_proposal_np = multi_proposal.asnumpy()
+
+ single_score_np = single_score.asnumpy()
+ multi_score_np = multi_score.asnumpy()
+
+ # check rois x1,y1,x2,y2
+ assert np.allclose(single_proposal_np[:, 1:], multi_proposal_np[:, 1:])
+ # check rois batch_idx
+ for i in range(batch_size):
+ start = i * rpn_post_nms_top_n
+ end = start + rpn_post_nms_top_n
+ assert (multi_proposal_np[start:end, 0] == i).all()
+ # check score
+ assert np.allclose(single_score_np, multi_score_np)
+
+ def check_backward(rpn_pre_nms_top_n, rpn_post_nms_top_n):
+
+ im_info_sym = mx.sym.Variable('im_info')
+ cls_prob_sym = mx.sym.Variable('cls_prob')
+ bbox_pred_sym = mx.sym.Variable('bbox_pred')
+
+ sym = mx.sym.contrib.MultiProposal(
+ cls_prob = cls_prob_sym,
+ bbox_pred = bbox_pred_sym,
+ im_info = im_info_sym,
+ feature_stride = feature_stride,
+ scales = scales,
+ ratios = ratios,
+ rpn_pre_nms_top_n = rpn_pre_nms_top_n,
+ rpn_post_nms_top_n = rpn_post_nms_top_n,
+ threshold = threshold,
+ rpn_min_size = rpn_min_size, output_score = False)
+
+ location = [cls_prob.asnumpy(), bbox_pred.asnumpy(), im_info.asnumpy()]
+
+ expected = [np.zeros_like(e) for e in location]
+
+ out_grads = [np.ones((rpn_post_nms_top_n, 5))]
+
+ check_symbolic_backward(sym, location, out_grads, expected)
+
+ check_forward(rpn_pre_nms_top_n, rpn_post_nms_top_n)
+ check_forward(rpn_pre_nms_top_n, 1500)
+ check_forward(1000, 500)
+ check_backward(rpn_pre_nms_top_n, rpn_post_nms_top_n)
if __name__ == '__main__':
import nose
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services