You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/06/14 05:31:27 UTC
incubator-singa git commit: SINGA-190 - Add prelu layer and flatten
layer
Repository: incubator-singa
Updated Branches:
refs/heads/dev f488070e3 -> a4d9aab52
SINGA-190 - Add prelu layer and flatten layer
Add test for PReLU using cudagpu device.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/a4d9aab5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/a4d9aab5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/a4d9aab5
Branch: refs/heads/dev
Commit: a4d9aab52db083f18b42f6231dda93e835dba609
Parents: f488070
Author: jixin <ji...@comp.nus.edu.sg>
Authored: Tue Jun 14 13:27:31 2016 +0800
Committer: jixin <ji...@comp.nus.edu.sg>
Committed: Tue Jun 14 13:27:31 2016 +0800
----------------------------------------------------------------------
src/model/layer/prelu.cc | 7 +--
src/model/layer/prelu.h | 1 +
test/singa/test_prelu.cc | 109 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 114 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/src/model/layer/prelu.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/prelu.cc b/src/model/layer/prelu.cc
index b916bed..6766d75 100644
--- a/src/model/layer/prelu.cc
+++ b/src/model/layer/prelu.cc
@@ -41,7 +41,7 @@ const Tensor PReLU::Forward(int flag, const Tensor &input) {
h = temp.shape(2);
w = temp.shape(3);
temp.Reshape(Shape{n * c, h * w});
- Tensor temp_a(Shape{n, c});
+ Tensor temp_a(Shape{n, c}, input.device(), input.data_type());
Uniform(1.f, 1.f, &temp_a);
MultRow(a_, &temp_a);
temp_a.Reshape(Shape{n * c});
@@ -87,7 +87,7 @@ const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag,
h = temp1.shape(2);
w = temp1.shape(3);
temp1.Reshape(Shape{n * c, h * w});
- Tensor temp_a(Shape{n, c});
+ Tensor temp_a(Shape{n, c}, grad.device(), grad.data_type());
Uniform(1.f, 1.f, &temp_a);
MultRow(a_, &temp_a);
temp_a.Reshape(Shape{n * c});
@@ -108,8 +108,9 @@ const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag,
LOG(FATAL) << "Incorrect input format for prelu layer.";
}
input_grad = grad * input * ((input > 0.f) + temp1);
- Tensor temp2 = grad * input * (input <= 0.f), temp3(Shape{n * c});
+ Tensor temp2 = grad * input * (input <= 0.f);
if (format_ == "NCHW") {
+ Tensor temp3(Shape{n * c}, grad.device(), grad.data_type());
temp2.Reshape(Shape{n * c, h * w});
SumColumns(temp2, &temp3);
temp3.Reshape(Shape{n, c});
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/src/model/layer/prelu.h
----------------------------------------------------------------------
diff --git a/src/model/layer/prelu.h b/src/model/layer/prelu.h
index d165fe2..2ee5e9f 100644
--- a/src/model/layer/prelu.h
+++ b/src/model/layer/prelu.h
@@ -21,6 +21,7 @@
#include <string>
#include <vector>
#include "singa/model/layer.h"
+#include "singa_config.h"
namespace singa {
class PReLU : public Layer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4d9aab5/test/singa/test_prelu.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_prelu.cc b/test/singa/test_prelu.cc
index 6fc372b..883935d 100644
--- a/test/singa/test_prelu.cc
+++ b/test/singa/test_prelu.cc
@@ -135,3 +135,112 @@ TEST(PReLU, BackwardCPU) {
for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(dx[i], xptr[i]);
for (size_t i = 0; i < params; i++) EXPECT_FLOAT_EQ(da[i], aptr[i]);
}
+
+#ifdef USE_CUDA
+TEST(PReLU, ForwardGPU) {
+ const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f,
+ -1.f, 2.f, -1.f, -2.f, -2.f, -1.f};
+ size_t n = sizeof(x) / sizeof(float);
+ size_t batchsize = 2, c = 3, h = 2, w = 1;
+ singa::CudaGPU cuda(0, 1);
+ singa::Tensor in(singa::Shape{batchsize, h, w, c}, &cuda);
+ in.CopyDataFromHostPtr<float>(x, n);
+
+ PReLU prelu;
+ singa::LayerConf conf;
+ singa::PReLUConf *preluconf = conf.mutable_prelu_conf();
+ preluconf->set_channel_shared(false);
+ preluconf->set_format("NHWC");
+ prelu.Setup(conf);
+
+ const float neg_slope[] = {0.25f, 0.5f, 0.75f};
+ singa::Tensor a(singa::Shape{c}, &cuda);
+ a.CopyDataFromHostPtr<float>(neg_slope, c);
+ prelu.Set_a(a);
+
+ singa::Tensor out = prelu.Forward(singa::kTrain, in);
+ singa::CppCPU host(0, 1);
+ out.ToDevice(&host);
+ const float *yptr = out.data<const float *>();
+ EXPECT_EQ(n, out.Size());
+
+ float *y = new float[n];
+ size_t div_factor = prelu.Channel_shared() ? c : 1;
+ if (prelu.Format() == "NCHW") {
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i / (h * w) % c / div_factor;
+ y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f);
+ }
+ } else if (prelu.Format() == "NHWC") {
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i % c / div_factor;
+ y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f);
+ }
+ }
+ for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(y[i], yptr[i]);
+}
+
+TEST(PReLU, BackwardGPU) {
+ const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f,
+ -1.f, 2.f, -1.f, -2.f, -2.f, -1.f};
+ size_t n = sizeof(x) / sizeof(float);
+ size_t batchsize = 2, c = 3, h = 2, w = 1;
+ singa::CudaGPU cuda(0, 1);
+ singa::Tensor in(singa::Shape{batchsize, c, h, w}, &cuda);
+ in.CopyDataFromHostPtr<float>(x, n);
+
+ PReLU prelu;
+ singa::LayerConf conf;
+ singa::PReLUConf *preluconf = conf.mutable_prelu_conf();
+ preluconf->set_channel_shared(false);
+ preluconf->set_format("NCHW");
+ prelu.Setup(conf);
+
+ const float neg_slope[] = {0.25f, 0.5f, 0.75f};
+ singa::Tensor a(singa::Shape{c}, &cuda);
+ a.CopyDataFromHostPtr<float>(neg_slope, c);
+ prelu.Set_a(a);
+
+ singa::Tensor out = prelu.Forward(singa::kTrain, in);
+ const float grad[] = {1.f, 2.f, -2.f, -1.f, -1.f, -3.f,
+ 2.f, -2.f, 1.f, 1.f, -2.f, 0.f};
+ singa::Tensor out_diff(singa::Shape{batchsize, c, h, w}, &cuda);
+ out_diff.CopyDataFromHostPtr<float>(grad, n);
+ const auto ret = prelu.Backward(singa::kTrain, out_diff);
+
+ singa::Tensor in_diff = ret.first;
+ singa::CppCPU host(0, 1);
+ in_diff.ToDevice(&host);
+ const float *xptr = in_diff.data<const float *>();
+ singa::Tensor a_diff = ret.second.at(0);
+ a_diff.ToDevice(&host);
+ const float *aptr = a_diff.data<const float *>();
+ float *dx = new float[n];
+ size_t div_factor = prelu.Channel_shared() ? c : 1;
+ size_t params = prelu.Channel_shared() ? 1 : c;
+ float da[] = {0.f, 0.f, 0.f};
+ if (prelu.Format() == "NCHW") {
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i / (h * w) % c / div_factor;
+ dx[i] = grad[i] *
+ (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f));
+ }
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i / (h * w) % c / div_factor;
+ da[pos] += grad[i] * std::min(x[i], 0.f);
+ }
+ } else if (prelu.Format() == "NHWC") {
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i % c / div_factor;
+ dx[i] = grad[i] *
+ (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f));
+ }
+ for (size_t i = 0; i < n; i++) {
+ size_t pos = i % c / div_factor;
+ da[pos] += grad[i] * std::min(x[i], 0.f);
+ }
+ }
+ for (size_t i = 0; i < n; i++) EXPECT_FLOAT_EQ(dx[i], xptr[i]);
+ for (size_t i = 0; i < params; i++) EXPECT_FLOAT_EQ(da[i], aptr[i]);
+}
+#endif