You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/12/16 13:11:33 UTC
[01/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Repository: incubator-singa
Updated Branches:
refs/heads/master 1981874fc -> 8ac511c70
SINGA-100 Implement layers using CUDNN for GPU training
Update dropout layer to run for both cudnn and cpu training mode.
Update the cpu part of math_blob and math_addr to use Context class for
getting cpu random generators.
TODO update math_blob and math_addr for GPU code, e.g., sampling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5d35ef26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5d35ef26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5d35ef26
Branch: refs/heads/master
Commit: 5d35ef267326be519bc0a7ea7d2847a6c5056181
Parents: 49293a6
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri Nov 27 15:27:31 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:01:29 2015 +0800
----------------------------------------------------------------------
include/singa/utils/math_addr.h | 50 ++-
include/singa/utils/math_blob.h | 408 ++++++++++++---------
include/singa/utils/singa_op.h | 28 ++
src/neuralnet/layer.cc | 7 +-
src/neuralnet/loss_layer/cudnn_softmaxloss.cu | 52 +++
src/neuralnet/neuron_layer/activation.cc | 83 +++++
src/neuralnet/neuron_layer/dropout.cc | 4 +-
7 files changed, 445 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index b21ea45..f63ff78 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -64,11 +64,21 @@ void cpu_gemv(const Dtype * A, const Dtype * B, const int m, const int n,
}
template<typename Dtype>
-void cpu_axpy(const Dtype * A, const int n, const Dtype alpha, Dtype * B) {
+void cpu_axpy(const int n, const Dtype alpha, const Dtype * A, Dtype * B) {
cblas_saxpy(n, alpha, A, 1, B, 1);
}
template<typename Dtype>
+void cpu_scale(const int n, const Dtype alpha, Dtype * A) {
+ cblas_sscal(n, alpha, A, 1);
+}
+
+template<typename Dtype>
+void cpu_copy(const int n, const Dtype* A, Dtype *B) {
+ cblas_scopy(n, A, 1, B, 1);
+}
+
+template<typename Dtype>
Dtype cpu_dot(const Dtype * A, const Dtype * B, const int n) {
Dtype sum = 0;
for (int i = 0 ; i < n ; i++)
@@ -122,22 +132,42 @@ void cpu_expand_f(const Dtype * A, const int m, const int n, Dtype * B) {
Op::Map(A[i], n, B+i*n);
}
}
-// expand each element in A into a row of B
+
template<typename Dtype>
-void cpu_sample_uniform(int n, Dtype low, Dtype high, Dtype* A);
+void cpu_softmax(int nb_rows, int nb_cols, const Dtype* A, Dtype* B) {
+ for (int i = 0; i < nb_rows; i++) {
+ const Dtype* dptr = A + i * nb_cols;
+ Dtype mmax = dptr[0];
+ for (int x = 1; x < nb_cols; ++x )
+ if (mmax < dptr[x]) mmax = dptr[x];
+ Dtype sum = 0.0f;
+ for(int x = 0; x < nb_cols; ++x ) {
+ dptr[x] = std::exp(dptr[x] - mmax );
+ sum += dptr[x];
+ }
+ for(int x = 0; x < nb_cols; ++x ) {
+ dptr[x] /= sum;
+ }
+ }
+}
-template<>
-inline void cpu_sample_uniform<float>(int n, float low, float high, float* A) {
-}
-template<typename Dtype>
-void cpu_sample_gaussian(int n, Dtype mean, Dtype std, Dtype* A);
-template<>
-inline void cpu_sample_gaussian<float>(int n, float mean, float std, float* A) {
+template<typename Dtype, typename URNG>
+void cpu_sample_uniform(URNG& g, int n, Dtype low, Dtype high, Dtype* A) {
+ std::uniform_real_distribution<Dtype> distribution(low, high);
+ for (int i = 0; i < n; i++)
+ A[i] = distribution(g);
+}
+template<typename Dtype, typename URNG>
+void cpu_sample_gaussian(URNG& g, int n, Dtype mean, Dtype std, Dtype* A) {
+ std::normal_distribution<Dtype> distribution(mean, std);
+ for (int i = 0; i < n; i++)
+ A[i] = distribution(g);
}
+
#ifdef USE_GPU
template<typename Dtype>
void gpu_gemm(const Dtype * A, const Dtype * B, const int m, const int n,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index ca75205..ce40d4f 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -42,30 +42,34 @@ enum XPU {cpu, gpu, any};
* Use blas scale internally.
*/
template<typename Dtype>
-void Scale(XPU xpu, Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
- CHECK_EQ(A.count(), B->count());
- if (xpu == cpu)
- cpu_scale(A.count(), alpha, A.cpu_data(), B->mutable_cpu_data());
+void Scale(Dtype alpha, Blob<Dtype> * B) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1)
+ cpu_scale(B->count(), alpha, B->mutable_cpu_data());
+ else {
#ifdef USE_GPU
+ // TODO(haibo) check it.
+// gpu_scale(B->count(), alpha, B->mutable_gpu_data());
#endif
+ }
}
/**
* Element-wise operation: Bi = alpha*Ai+Bi. A and B should have the same size
*/
template<typename Dtype>
-void AXPY(XPU xpu, Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
+void AXPY(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count(), B->count());
- if (xpu == cpu) {
- cpu_axpy(A.cpu_data(), A.count(),
- alpha, B->mutable_cpu_data());
- }
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_axpy(A.count(), alpha, A.cpu_data(), B->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
- gpu_axpy(A.gpu_data(), A.count(),
- alpha, B->mutable_gpu_data());
- }
+ gpu_axpy(A.count(), alpha, A.gpu_data(), B->mutable_gpu_data());
#endif // USE_GPU
+ }
}
/************* BLAS level 2 *****************/
@@ -83,7 +87,7 @@ void AXPY(XPU xpu, Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
* @param[in, out] C, vector
*/
template<typename Dtype>
-void GEMV(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
+void GEMV(Dtype alpha, Dtype beta, const Blob<Dtype>& A,
const Blob<Dtype>& B, Blob<Dtype>* C) {
CHECK_EQ(A.shape().size(), 2) << "A must be a matrix";
int a1, a2, m, n;
@@ -95,17 +99,18 @@ void GEMV(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
CHECK_EQ(a1, n) << "# rows of A(.T) must = length of C";
bool TranA = A.transpose();
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_gemv(A.cpu_data(), B.cpu_data(), m, n, alpha, beta, TranA,
C->mutable_cpu_data());
- }
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
// gpu part
gpu_gemv(A.gpu_data(), B.gpu_data(), m, n, alpha, beta, TranA,
C->mutable_gpu_data());
- }
#endif // USE_GPU
+ }
}
/**
* Matrix vector multiplication, C = A(.T) * B, transpose is considered.
@@ -119,9 +124,9 @@ void GEMV(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
* @param[out] C output vector
*/
template <typename Dtype>
-void MVDot(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
+void MVDot(const Blob<Dtype>& A, const Blob<Dtype>& B,
Blob<Dtype>* C) {
- GEMV(xpu, Dtype(1), Dtype(0), A, B, C);
+ GEMV(Dtype(1), Dtype(0), A, B, C);
}
/************* BLAS level 3 *****************/
@@ -140,7 +145,7 @@ void MVDot(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
* @param[in, out] C, matrix
*/
template <typename Dtype>
-void GEMM(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
+void GEMM( Dtype alpha, Dtype beta, const Blob<Dtype>& A,
const Blob<Dtype> & B, Blob<Dtype> * C) {
CHECK_EQ(A.shape().size(), 2);
CHECK_EQ(B.shape().size(), 2);
@@ -160,17 +165,18 @@ void GEMM(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
int k = A.transpose() ? A.shape(0) : A.shape(1);
bool TranA = A.transpose();
bool TranB = B.transpose();
- if (xpu == cpu) {
- cpu_gemm(A.cpu_data(), B.cpu_data(), m, n, k, alpha, beta,
- TranA, TranB, C->mutable_cpu_data());
- }
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_gemm(A.cpu_data(), B.cpu_data(), m, n, k, alpha, beta, TranA, TranB,
+ C->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
// gpu part
gpu_gemm(A.gpu_data(), B.gpu_data(), m, n, k, alpha, beta,
TranA, TranB, C->mutable_gpu_data());
- }
#endif // USE_GPU
+ }
}
/**
* Matrix multiplication, C = A(.T) * B(.T), transpose is considered.
@@ -183,9 +189,9 @@ void GEMM(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype>& A,
* @param[out] C output matrix
*/
template <typename Dtype>
-void MMDot(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
+void MMDot(const Blob<Dtype>& A, const Blob<Dtype>& B,
Blob<Dtype>* C) {
- GEMM(xpu, Dtype(1), Dtype(0), A, B, C);
+ GEMM(Dtype(1), Dtype(0), A, B, C);
}
@@ -199,19 +205,20 @@ void MMDot(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
* @return inner product value.
*/
template <typename Dtype>
-Dtype VVDot(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B) {
+Dtype VVDot(const Blob<Dtype> & A, const Blob<Dtype> & B) {
Dtype res = 0;
CHECK_EQ(A.count(), B.count());
int n = A.count();
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
res = cpu_dot(A.cpu_data(), B.cpu_data(), n);
- }
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
// gpu part
res = gpu_dot(A.gpu_data(), B.gpu_data(), n);
- }
#endif // USE_GPU
+ }
return res;
}
@@ -224,25 +231,24 @@ Dtype VVDot(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B) {
* @param[out] C, output matrix
*/
template <typename Dtype>
-void OuterProduct(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
- Blob<Dtype> * C) {
+void OuterProduct(const Blob<Dtype>& A, const Blob<Dtype>& B, Blob<Dtype> * C) {
CHECK(!C->transpose()); // do not support C.T now.
int m = A.count();
int n = B.count();
CHECK_EQ(C->count(), m * n);
-
- if (xpu == cpu) {
- cpu_gemm(A.cpu_data(), B.cpu_data(), m, n, 1, 1, 0,
- false, false, C->mutable_cpu_data());
- }
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_gemm(A.cpu_data(), B.cpu_data(), m, n, 1, 1, 0, false, false,
+ C->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
// gpu part
gpu_gemm(A.gpu_data(), B.gpu_data(), m, n, 1, 1, 0,
false, false, C->mutable_gpu_data());
- }
#endif // USE_GPU
+ }
}
/*********************** Element-wise functions ***********************/
/**
@@ -251,17 +257,18 @@ void OuterProduct(XPU xpu, const Blob<Dtype>& A, const Blob<Dtype>& B,
* Loose shape checking, A.count() == B.count().
*/
template<typename Op, typename Dtype>
-void Map(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
+void Map(const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count(), B->count()) << "Blobs must have the same size";
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_e_f<Op>(A.count(), A.cpu_data(), B->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
- if (xpu == gpu) {
// gpu part
gpu_e_f<Op>(A.count(), A.gpu_data(), B->mutable_gpu_data());
- }
#endif // SINGA_GPU
+ }
}
/**
@@ -270,19 +277,20 @@ void Map(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
* Loose shape checking, A, B and C are of the same size.
*/
template<typename Op, typename Dtype>
-void Map(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
- Blob<Dtype> * C) {
+void Map(const Blob<Dtype> & A, const Blob<Dtype> & B, Blob<Dtype> * C) {
CHECK_EQ(A.count(), B.count()) << "Blobs must have the same size";
CHECK_EQ(A.count(), C->count()) << "Blobs must have the same size";
- if (xpu == cpu) {
+ cpu_e_f<Op>(A.count(), A.cpu_data(), B.cpu_data(), C->mutable_cpu_data());
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_e_f<Op>(A.count(), A.cpu_data(), B.cpu_data(), C->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
- if (xpu == gpu) {
// gpu part
gpu_e_f<Op>(A.count(), A.gpu_data(), B.gpu_data(), C->mutable_gpu_data());
- }
#endif // SINGA_GPU
+ }
}
/**
@@ -290,28 +298,34 @@ void Map(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
* Loose shape checking, A.count() == B.count().
*/
template<typename Op, typename Dtype>
-void Map(XPU xpu, Dtype alpha, const Blob<Dtype>& A, Blob<Dtype>* B) {
+void Map(Dtype alpha, const Blob<Dtype>& A, Blob<Dtype>* B) {
CHECK_EQ(A.count(), B->count()) << "Blobs must have the same size";
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_e_f<Op>(A.count(), alpha, A.cpu_data(), B->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
#endif // SINGA_GPU
+ }
}
/**
* Ci = Op(alpha, Ai, Bi)
* Loose shape checking, A, B and C are of the same size.
*/
template<typename Op, typename Dtype>
-void Map(XPU xpu, Dtype alpha, const Blob<Dtype>& A, const Blob<Dtype>& B,
+void Map(Dtype alpha, const Blob<Dtype>& A, const Blob<Dtype>& B,
Blob<Dtype>* C) {
CHECK_EQ(A.count(), B->count()) << "Blobs must have the same size";
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_e_f<Op>(A.count(), alpha, A.cpu_data(), B->cpu_data(),
C->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
#endif // SINGA_GPU
+ }
}
/**
@@ -322,24 +336,46 @@ void Map(XPU xpu, Dtype alpha, const Blob<Dtype>& A, const Blob<Dtype>& B,
* Loose shape checking, A.count() == B.count().
*/
template<typename Dtype>
-void Copy(XPU xpu, const Blob<Dtype>& A, Blob<Dtype>* B) {
+void Copy(const Blob<Dtype>& A, Blob<Dtype>* B) {
CHECK_EQ(A.count(), B->count()) << "Blobs must have the same size";
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
std::copy(A.cpu_data(), A.cpu_data() + A.count(), B->mutable_cpu_data());
} else {
- LOG(FATAL) << "Not implemented";
+#ifdef USE_GPU
+#endif
}
}
+
+/**
+ * B = alpha + A
+ * Implemented using Copy and AXPY.
+ */
+template<typename Dtype>
+void Add(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
+ Map<singa::op::Add<Dtype>>(alpha, A, B);
+}
+
/**
* C = A + B
* Implemented using Copy and AXPY.
*/
template<typename Dtype>
-void Add(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
+void Add(const Blob<Dtype> & A, const Blob<Dtype> & B,
Blob<Dtype> * C) {
Copy(A, C);
- AXPY(B, C, 1);
+ AXPY(Dtype(1), B, C);
+}
+
+/**
+ * B = alpha - A
+ * Implemented using Copy and AXPY.
+ */
+template<typename Dtype>
+void Sub(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype>* B) {
+ Map<singa::op::Sub<Dtype>>(alpha, A, B);
}
/**
@@ -347,10 +383,10 @@ void Add(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
* Implemented using Copy and AXPY.
*/
template<typename Dtype>
-void Sub(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
+void Sub(const Blob<Dtype> & A, const Blob<Dtype> & B,
Blob<Dtype> * C) {
- Copy(xpu, A, C);
- AXPY(xpu, B, C, -1);
+ Copy(A, C);
+ AXPY(Dtype(-1), B, C);
}
/**
@@ -360,19 +396,8 @@ void Sub(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
template<typename Dtype>
void Mult(const Blob<Dtype> & A, const Blob<Dtype> & B,
Blob<Dtype> * C) {
- //Map<singa::op::Mult<Dtype>>(xpu, A, B, C);
+ Map<singa::op::Mult<Dtype>>(A, B, C);
// TODO(wangwei) use MKL's vector func
- auto context = Singleton<Context>::Instance();
- int device = context->device_id(std::this_thread::get_id());
- if (device == -1)
- cpu_e_f<op::Mult<Dtype>>(C->count(), A.cpu_data(), B.cpu_data(),
- C->mutable_cpu_data());
- else {
-#ifdef USE_GPU
- gpu_e_f<op::Mult<Dtype>>(C->count(), A.gpu_data(), B.gpu_data(),
- C->mutable_gpu_data());
-#endif
- }
}
/**
@@ -380,11 +405,46 @@ void Mult(const Blob<Dtype> & A, const Blob<Dtype> & B,
* Map(XPU, const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
*/
template<typename Dtype>
-void Div(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
+void Div(const Blob<Dtype> & A, const Blob<Dtype> & B,
Blob<Dtype> * C) {
- Map<singa::op::Div<Dtype>>(xpu, A, B, C);
+ Map<singa::op::Div<Dtype>>(A, B, C);
// TODO(wangwei) use MKL's vector func
}
+/**
+ * B = sqrt(A)
+ */
+template<typename Dtype>
+void Sqrt(const Blob<Dtype> & A, Blob<Dtype>* B) {
+ Map<singa::op::Sqrt<Dtype>, Dtype>(A, B);
+}
+/**
+ * B = square(A)
+ */
+template<typename Dtype>
+void Square(const Blob<Dtype> & A, Blob<Dtype>* B) {
+ Map<singa::op::Square<Dtype>, Dtype>(A, B);
+}
+/**
+ * B = exp(A)
+ */
+template<typename Dtype>
+void Exp(const Blob<Dtype> & A, Blob<Dtype>* B) {
+ Map<singa::op::Exp<Dtype>, Dtype>(A, B);
+}
+/**
+ * B = log(A)
+ */
+template<typename Dtype>
+void Log(const Blob<Dtype>& A, Blob<Dtype>* B) {
+ Map<singa::op::Log<Dtype>, Dtype>(A, B);
+}
+/**
+ * B = tanh(A)
+ */
+template<typename Dtype>
+void Tanh(const Blob<Dtype>& A, Blob<Dtype>* B) {
+ Map<singa::op::Tanh<Dtype>, Dtype>(A, B);
+}
/*************************1D<-->2D op/transform***************************/
/**
* Add A to each column of B, i.e., Bij = alpha*Ai + beta*Bij
@@ -392,28 +452,26 @@ void Div(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
* # columns of B = B.count() / A.count().
*/
template<typename Dtype>
-void MVAddCol(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
- Blob<Dtype> * B) {
+void MVAddCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
if (B->transpose()) {
- Blob<Dtype>* tmp = Transpose(* B);
- MVAddRow(xpu, alpha, beta, A, tmp);
- delete tmp;
+ B->set_transpose(false);
+ MVAddRow(alpha, beta, A, B);
+ B->set_transpose(true);
} else {
CHECK_EQ(B->count() % A.count(), 0) << "#col of B not match length of A";
int m = A.count(), n = B->count() / m;
- if (xpu == cpu) {
- Blob<Dtype> one(n);
- one.SetValue(1);
- cpu_gemm(A.cpu_data(), one.cpu_data(), m, n, 1, alpha, beta,
- false, false, B->mutable_cpu_data());
- }
+ Blob<Dtype> one(n);
+ one.SetValue(1);
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_gemm(A.cpu_data(), one.cpu_data(), m, n, 1, alpha, beta, false, false,
+ B->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
- singa_gpu_add_vec_row(B->gpu_data(),
- A.gpu_data(), A.gpu_data(), m, n, n);
- // gpu part
- }
+ singa_gpu_add_vec_row(B->gpu_data(), A.gpu_data(), A.gpu_data(), m, n, n);
#endif // USE_GPU
+ }
}
}
/**
@@ -422,8 +480,8 @@ void MVAddCol(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
* # columns of B = B.count() / A.count().
*/
template<typename Dtype>
-void MVAddCol(XPU xpu, const Blob<Dtype> & A, Blob<Dtype>* B) {
- MVAddCol(xpu, Dtype(1), Dtype(1), A, B);
+void MVAddCol(const Blob<Dtype> & A, Blob<Dtype>* B) {
+ MVAddCol(Dtype(1), Dtype(1), A, B);
}
/**
@@ -432,28 +490,26 @@ void MVAddCol(XPU xpu, const Blob<Dtype> & A, Blob<Dtype>* B) {
* # rows of B = B.count() / A.count().
*/
template<typename Dtype>
-void MVAddRow(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
- Blob<Dtype> * B) {
+void MVAddRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
if (B->transpose()) {
- Blob<Dtype>* tmp = Transpose(* B);
- MVAddCol(xpu, alpha, beta, A, tmp);
- delete tmp;
+ B->set_transpose(false);
+ MVAddCol(alpha, beta, A, B);
+ B->set_transpose(true);
} else {
CHECK_EQ(B->count() % A.count(), 0) << "#col of B not match length of A";
int m = A.count(), n = B->count() / m;
- if (xpu == cpu) {
- Blob<Dtype> one(n);
- one.SetValue(1);
+ Blob<Dtype> one(n);
+ one.SetValue(1);
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_gemm(one.cpu_data(), A.cpu_data(), n, m, 1, alpha, beta,
false, false, B->mutable_cpu_data());
- }
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
- // gpu part
- singa_gpu_add_vec_row(B->gpu_data(),
- A.gpu_data(), A.gpu_data(), m, n, n);
- }
+ singa_gpu_add_vec_row(B->gpu_data(), A.gpu_data(), A.gpu_data(), m, n, n);
#endif // USE_GPU
+ }
}
}
/**
@@ -462,8 +518,8 @@ void MVAddRow(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
* # rows of B = B.count() / A.count().
*/
template<typename Dtype>
-void MVAddRow(XPU xpu, const Blob<Dtype> & A, Blob<Dtype>* B) {
- MVAddRow(xpu, Dtype(1), Dtype(1), A, B);
+void MVAddRow(const Blob<Dtype> & A, Blob<Dtype>* B) {
+ MVAddRow(Dtype(1), Dtype(1), A, B);
}
/**
@@ -472,8 +528,8 @@ void MVAddRow(XPU xpu, const Blob<Dtype> & A, Blob<Dtype>* B) {
* # columns of B = B.count() / A.count().
*/
template<typename Dtype>
-void RepmatCol(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
- MVAddCol(xpu, Dtype(1), Dtype(0), A, B);
+void RepmatCol(const Blob<Dtype> & A, Blob<Dtype> * B) {
+ MVAddCol(Dtype(1), Dtype(0), A, B);
}
/**
@@ -482,8 +538,8 @@ void RepmatCol(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
* # rows of B = B.count() / A.count().
*/
template<typename Dtype>
-void RepmatRow(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
- MVAddRow(xpu, Dtype(1), Dtype(0), A, B);
+void RepmatRow(const Blob<Dtype> & A, Blob<Dtype> * B) {
+ MVAddRow(Dtype(1), Dtype(0), A, B);
}
/**
@@ -493,18 +549,18 @@ void RepmatRow(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
* # columns of A = A.count() / B.count().
*/
template<typename Dtype>
-void MVSumCol(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
- Blob<Dtype> * B) {
+void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count() % B->count(), 0) << "length of B must = # of cols of A";
int m = B->count(), n = A.count() / m;
- if (xpu == cpu) {
- Blob<Dtype> one(n);
- one.SetValue(1);
+ Blob<Dtype> one(n);
+ one.SetValue(1);
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_gemm(A.cpu_data(), one.cpu_data(), m, 1, n, alpha, beta,
A.transpose(), false, B->mutable_cpu_data());
- }
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
singa_gpu_sum_col(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
}
@@ -518,18 +574,18 @@ void MVSumCol(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
* # rows of A = A.count() / B.count().
*/
template<typename Dtype>
-void MVSumRow(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
- Blob<Dtype> * B) {
+void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count() % B->count(), 0) << "length of B must = # of cols of A";
int m = B->count(), n = A.count() / m;
- if (xpu == cpu) {
- Blob<Dtype> one(n);
- one.SetValue(1);
- cpu_gemm(one.cpu_data(), A.cpu_data(), 1, m, n, alpha, beta,
- A.transpose(), false, B->mutable_cpu_data());
- }
+ Blob<Dtype> one(n);
+ one.SetValue(1);
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_gemm(one.cpu_data(), A.cpu_data(), 1, m, n, alpha, beta, A.transpose(),
+ false, B->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- if (xpu == gpu) {
singa_gpu_sum_row(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
}
@@ -542,18 +598,19 @@ void MVSumRow(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
* # columns of A = A.count() / B.count().
*/
template<typename Op, typename Dtype>
-void Reduce2D(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
+void Reduce2D(const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count() % B->count(), 0) << "Row size not match B length";
int m = B->count(), n = A.count() / m;
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_reduce_f<Op>(A.cpu_data(), m, n, B->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
- if (xpu == gpu) {
// gpu part
gpu_reduce_f<Op>(A.gpu_data(), m, n, B->mutable_gpu_data());
- }
#endif // SINGA_GPU
+ }
}
/**
* Duplicate each element of A into a row of B.
@@ -561,62 +618,67 @@ void Reduce2D(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
* # columns of B = B.count() / A.count().
*/
template<typename Op, typename Dtype>
-void Expand2D(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
+void Expand2D(const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(B->count() % A.count(), 0) << "Row size of B not match length of A";
int m = A.count(), n = B->count() / m;
- if (xpu == cpu) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
cpu_expand_f<Op>(A.cpu_data(), m, n, B->mutable_cpu_data());
- }
+ } else {
#ifdef SINGA_GPU
- if (xpu == gpu) {
- // gpu part
gpu_expand_f<Op>(A.gpu_data(), m, n, B->mutable_gpu_data());
- }
#endif // SINGA_GPU
+ }
}
/**
* Average the absolute values.
*/
-template <typename Dtype>
-Dtype Asum(XPU xpu, const Blob<Dtype>& A) {
+template<typename Dtype>
+Dtype Asum(const Blob<Dtype>& A) {
if (A.count() == 0) return Dtype(0);
- if (xpu == cpu)
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
return cpu_asum(A.count(), A.cpu_data(), 1) / A.count();
- return Dtype(0); // avoid compile warning
+ } else {
#ifdef USE_GPU
+ return 0; // TODO(haibo)
#endif
+ }
}
+
/*************Random Sample***************/
template<typename Dtype>
-void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A);
-
-template<>
-inline void SampleUniform<float>(float low, float high, Blob<float> *A) {
+void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A) {
auto context = Singleton<Context>::Instance();
- int device = context->device_id(std::this_thread::get_id());
- if (device == -1)
- cpu_sample_uniform(A->count(), low, high, A->mutable_cpu_data());
- else {
+ const auto& thread = std::this_thread::get_id();
+ int device = context->device_id(thread);
+ if (device == -1) {
+ cpu_sample_uniform(*context->rand_generator(thread), A->count(), low, high,
+ A->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
+ // TODO(haibo) check
gpu_sample_uniform(A->count(), low, high, A->mutable_gpu_data());
#endif
}
}
template<typename Dtype>
-void SampleGaussian(Dtype mean, Dtype std, Blob<Dtype>* A);
-
-template<>
-inline void SampleGaussian<float>(float low, float high, Blob<float> *A) {
+void SampleGaussian(Dtype mean, Dtype std, Blob<Dtype>* A) {
auto context = Singleton<Context>::Instance();
- int device = context->device_id(std::this_thread::get_id());
- if (device == -1)
- cpu_sample_gaussian(A->count(), low, high, A->mutable_cpu_data());
- else {
+ const auto& thread = std::this_thread::get_id();
+ int device = context->device_id(thread);
+ if (device == -1) {
+ cpu_sample_gaussian(*context->rand_generator(thread), A->count(), mean, std,
+ A->mutable_cpu_data());
+ } else {
#ifdef USE_GPU
- gpu_sample_gaussian(A->count(), low, high, A->mutable_gpu_data());
+ // TODO(haibo) check it.
+ gpu_sample_gaussian(A->count(), mean, std, A->mutable_gpu_data());
#endif
}
}
@@ -627,11 +689,15 @@ void Softmax(int nb_rows, const Blob<Dtype>& A, Blob<Dtype>* B) {
CHECK_GT(nb_rows, 0);
CHECK_EQ(A.count() % nb_rows, 0);
CHECK_EQ(A.count(), B->count());
-
-#ifdef USE_GPU
- cpu_softmax(nb_rows, A.count() / nb_rows, A.cpu_data(),
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1) {
+ cpu_softmax(nb_rows, A.count() / nb_rows, A.cpu_data(),
B->mutable_cpu_data());
+ } else {
+#ifdef USE_GPU
#endif // USE_GPU
+ }
}
} // end of namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/include/singa/utils/singa_op.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/singa_op.h b/include/singa/utils/singa_op.h
index c934050..7499eb1 100644
--- a/include/singa/utils/singa_op.h
+++ b/include/singa/utils/singa_op.h
@@ -211,6 +211,34 @@ struct Pow {
}
#endif // USE_GPU
};
+
+template<typename Dtype>
+struct Add {
+ inline static void Map(const Dtype & a, const Dtype & b, Dtype * c) {
+ *c = a + b;
+ }
+#ifdef USE_GPU
+ inline static void CudaMap(const Dtype * a,
+ const Dtype * b, Dtype * c, int n) {
+// singa::singa_gpu_add(a, b, c, n); // TODO(haibo)
+ }
+#endif // USE_GPU
+};
+
+template<typename Dtype>
+struct Sub {
+ inline static void Map(const Dtype & a, const Dtype & b, Dtype * c) {
+ *c = a - b;
+ }
+#ifdef USE_GPU
+ inline static void CudaMap(const Dtype * a,
+ const Dtype * b, Dtype * c, int n) {
+// singa::singa_gpu_add(a, b, c, n); // TODO(haibo)
+ }
+#endif // USE_GPU
+};
+
+
template<typename Dtype>
struct Mult {
inline static void Map(const Dtype & a, const Dtype & b, Dtype * c) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index be38ac5..9414948 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -48,17 +48,16 @@ const std::string Layer::ToString(bool debug, int flag) {
return "";
string ret = StringPrintf("Layer %10s ", name().c_str());
if ((flag & kForward) == kForward && data_.count() !=0) {
- ret += StringPrintf("data norm1 %13.9f", Asum(cpu, data_));
+ ret += StringPrintf("data norm1 %13.9f", Asum(data_));
} else if ((flag & kBackward) == kBackward) {
if (grad_.count() != 0)
- ret += StringPrintf("grad norm1 %13.9f\n", Asum(cpu, grad_));
+ ret += StringPrintf("grad norm1 %13.9f\n", Asum(grad_));
}
if ((flag & kTrain) == kTrain) {
for (Param* p : GetParams()) {
ret += StringPrintf(
"param id %2d, name %10s, value norm1 %13.9f, grad norm1 %13.9f\n",
- p->id(), p->name().c_str(), Asum(cpu, p->data()),
- Asum(cpu, p->grad()));
+ p->id(), p->name().c_str(), Asum(p->data()), Asum(p->grad()));
}
}
return ret;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu b/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
new file mode 100644
index 0000000..e0af05f
--- /dev/null
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
@@ -0,0 +1,52 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/loss_layer.h"
+
+namespace singa {
+void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ CudnnSoftmaxLayer::Setup(conf, srclayers);
+ topk_ = conf.softmaxloss_conf().topk();
+ loss_ = accuracy_ = 0.0f;
+ counter_ = 0;
+}
+void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ CudnnSoftmaxLayer::ComputeFeature(flag, srclayers);
+ // compute loss
+ counter_++;
+ // add loss and accuracy
+}
+
+void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ // compute gradient
+}
+
+const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
+ string disp = "Loss = " + std::to_string(loss_ / counter_)
+ + ", accuracy = " + std::to_string(accuracy_ / counter_);
+ counter_ = 0;
+ loss_ = accuracy_ = 0;
+ return disp;
+}
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/src/neuralnet/neuron_layer/activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/activation.cc b/src/neuralnet/neuron_layer/activation.cc
new file mode 100644
index 0000000..6f62646
--- /dev/null
+++ b/src/neuralnet/neuron_layer/activation.cc
@@ -0,0 +1,83 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include "singa/neuralnet/neuron_layer.h"
+#include "singa/utils/math_blob.h"
+#include "singa/utils/singa_op.h"
+#include "singa/proto/job.pb.h"
+namespace singa {
+
+void ActivationLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ NeuronLayer::Setup(conf, srclayers);
+ data_.ReshapeLike(srclayers[0]->data(this));
+ grad_.ReshapeLike(*(srclayers[0]->mutable_grad(this)));
+}
+void ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers)
+{
+ switch (layer_conf_.activation_conf().type()) {
+ case RELU:
+ Map<op::Relu<float>, float>(srclayers[0]->data(this), &data_);
+ break;
+ case SIGMOID:
+ Map<op::Sigmoid<float>, float>(srclayers[0]->data(this), &data_);
+ break;
+ case TANH:
+ Map<op::Tanh<float>, float>(srclayers[0]->data(this), &data_);
+ break;
+ /*
+ case ActivationType_STANH:
+ Map<op::STanh<float>, float>(srclayers[0]->data(this), &data_);
+ break;
+ */
+ default:
+ LOG(ERROR) << "Unknow activation type " <<
+ layer_conf_.activation_conf().type();
+ }
+}
+void ActivationLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+{
+ Blob<float> * gsrc = srclayers[0]->mutable_grad(this);
+ switch (layer_conf_.activation_conf().type()) {
+ case RELU:
+ Map<op::Relu<float>, float>(data_, gsrc);
+ Mult(*gsrc, grad_, gsrc);
+ break;
+ case SIGMOID:
+ Map<op::SigmoidGrad<float>, float>(data_, gsrc);
+ Mult(*gsrc, grad_, gsrc);
+ break;
+ case TANH:
+ Map<op::Tanh<float>, float>(data_, gsrc);
+ Mult(*gsrc, grad_, gsrc);
+ break;
+ /*
+ case ActivationType_STANH:
+ Map<op::STanh<float>, float>(data_, gsrc);
+ Mult(*gsrc, grad_, gsrc);
+ break;
+ */
+ default:
+ LOG(ERROR) << "Unknow activation type " <<
+ layer_conf_.activation_conf().type();
+ }
+}
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5d35ef26/src/neuralnet/neuron_layer/dropout.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dropout.cc b/src/neuralnet/neuron_layer/dropout.cc
index ad8c10f..030a663 100644
--- a/src/neuralnet/neuron_layer/dropout.cc
+++ b/src/neuralnet/neuron_layer/dropout.cc
@@ -48,8 +48,8 @@ void DropoutLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
float pkeep = 1 - pdrop_;
Blob<float> rand(data_.count());
SampleUniform(0.0f, 1.0f, &rand);
- // Threashold(pkeep, rand, &mask_);
- // Scale(1.0f / pkeep, &mask_);
+ Map<op::Threshold<float>, float>(pkeep, rand, &mask_);
+ Scale(1.0f / pkeep, &mask_);
Mult(srclayers[0]->data(this), mask_, &data_);
}
[19/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
check with cpplint
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8ac511c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8ac511c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8ac511c7
Branch: refs/heads/master
Commit: 8ac511c70a43c9cb5db354643b186fd4205a7104
Parents: f3b47c7
Author: WANG Sheng <wa...@gmail.com>
Authored: Wed Dec 16 20:10:06 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Wed Dec 16 20:10:06 2015 +0800
----------------------------------------------------------------------
include/singa/neuralnet/loss_layer.h | 3 +++
include/singa/neuralnet/neuron_layer.h | 2 +-
include/singa/neuralnet/output_layer.h | 1 +
include/singa/utils/context.h | 1 -
include/singa/utils/math_blob.h | 4 ++--
src/driver.cc | 8 +++-----
src/neuralnet/input_layer/deprecated.cc | 2 +-
src/neuralnet/loss_layer/euclidean.cc | 2 +-
src/neuralnet/loss_layer/softmax.cc | 2 +-
src/neuralnet/neuron_layer/dummy.cc | 2 +-
src/neuralnet/neuron_layer/rbm.cc | 1 -
src/neuralnet/output_layer/accuracy.cc | 2 +-
src/test/test_connection_layers.cc | 3 ---
src/test/test_math.cc | 10 ++++++----
src/utils/blob.cc | 2 +-
src/utils/math_kernel.cu | 3 ---
16 files changed, 22 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/include/singa/neuralnet/loss_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer.h b/include/singa/neuralnet/loss_layer.h
index 3a45370..53ddc82 100644
--- a/include/singa/neuralnet/loss_layer.h
+++ b/include/singa/neuralnet/loss_layer.h
@@ -22,8 +22,11 @@
#ifndef SINGA_NEURALNET_LOSS_LAYER_H_
#define SINGA_NEURALNET_LOSS_LAYER_H_
+#include <vector>
+#include <string>
#include "singa/neuralnet/layer.h"
#include "singa/neuralnet/neuron_layer.h"
+
namespace singa {
using std::vector;
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 393eeb4..0b936c4 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -23,6 +23,7 @@
#define SINGA_NEURALNET_NEURON_LAYER_H_
#include <vector>
+#include <string>
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
#include "singa/utils/context.h"
@@ -416,7 +417,6 @@ class RBMVisLayer: public RBMLayer, public LossLayer {
Layer* input_layer_;
float error_ = 0.0f;
int counter_ = 0;
-
};
/**
* RBM hidden layer
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/include/singa/neuralnet/output_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer.h b/include/singa/neuralnet/output_layer.h
index 49a3c19..c7e5d6a 100644
--- a/include/singa/neuralnet/output_layer.h
+++ b/include/singa/neuralnet/output_layer.h
@@ -23,6 +23,7 @@
#define SINGA_NEURALNET_OUTPUT_LAYER_H_
#include <vector>
+#include <string>
#include "singa/neuralnet/layer.h"
#include "singa/io/store.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/include/singa/utils/context.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/context.h b/include/singa/utils/context.h
index 5d49dc1..a2bc64f 100644
--- a/include/singa/utils/context.h
+++ b/include/singa/utils/context.h
@@ -263,7 +263,6 @@ class Context {
std::vector<cudnnHandle_t> cudnn_handle_;
#endif
#endif
-
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index 8807bdb..301b72f 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -586,8 +586,8 @@ void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
if (device == -1) {
Blob<Dtype> one(m);
one.SetValue(1);
- cpu_gemm(one.cpu_data(), A.cpu_data(), 1, n, m, alpha, beta, false, A.transpose(),
- B->mutable_cpu_data());
+ cpu_gemm(one.cpu_data(), A.cpu_data(), 1, n, m, alpha, beta, false,
+ A.transpose(), B->mutable_cpu_data());
} else {
#ifdef USE_GPU
singa_gpu_sum_row(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index b86da7c..ed1b6b4 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -142,16 +142,14 @@ void Driver::Init(int argc, char **argv) {
RegisterParamGenerator<UniformSqrtFanInOutGen>(kUniformSqrtFanInOut);
}
-void Driver::InitLog(char* arg)
-{
+void Driver::InitLog(char* arg) {
google::InitGoogleLogging(arg);
}
-void Driver::Train(bool resume, const std::string str){
+void Driver::Train(bool resume, const std::string str) {
JobProto job_conf;
job_conf.ParseFromString(str);
- Train(resume,job_conf);
-
+ Train(resume, job_conf);
}
void Driver::Train(bool resume, const JobProto& job_conf) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
index dfba675..d2901f7 100644
--- a/src/neuralnet/input_layer/deprecated.cc
+++ b/src/neuralnet/input_layer/deprecated.cc
@@ -130,7 +130,7 @@ void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (random_skip_) {
std::uniform_int_distribution<int> distribution(0, random_skip_);
auto generator =
- Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
+ Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
int nskip = distribution(*generator);
int n = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/loss_layer/euclidean.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc
index 49179d6..67c0cd5 100644
--- a/src/neuralnet/loss_layer/euclidean.cc
+++ b/src/neuralnet/loss_layer/euclidean.cc
@@ -51,7 +51,7 @@ void EuclideanLossLayer::ComputeFeature(int flag,
(input_dptr[i] - reconstruct_dptr[i]);
}
loss_ += loss / srclayers[0]->data(this).shape()[0];
- counter_ ++;
+ counter_++;
}
void EuclideanLossLayer::ComputeGradient(int flag,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index 5a3d257..7956470 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -82,7 +82,7 @@ void SoftmaxLossLayer::ComputeFeature(int flag,
CHECK_EQ(probptr, prob.dptr + prob.shape.Size());
loss_ += loss * scale_ / (1.0f * batchsize_);
accuracy_ += precision * scale_ / (1.0f * batchsize_);
- counter_ ++;
+ counter_++;
}
void SoftmaxLossLayer::ComputeGradient(int flag,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/neuron_layer/dummy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc
index 9b352dd..a242f75 100644
--- a/src/neuralnet/neuron_layer/dummy.cc
+++ b/src/neuralnet/neuron_layer/dummy.cc
@@ -19,8 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/neuron_layer/rbm.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/rbm.cc b/src/neuralnet/neuron_layer/rbm.cc
index 69ffa62..92c2371 100644
--- a/src/neuralnet/neuron_layer/rbm.cc
+++ b/src/neuralnet/neuron_layer/rbm.cc
@@ -52,7 +52,6 @@ void RBMLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) {
hdim_ = conf.rbm_conf().hdim();
gaussian_ = conf.rbm_conf().gaussian();
first_gibbs_ = true;
- //pos_data_, neg_data_, neg_sample_, pos_sample_
datavec_.clear();
datavec_.push_back(&pos_data_);
datavec_.push_back(&neg_data_);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/neuralnet/output_layer/accuracy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/accuracy.cc b/src/neuralnet/output_layer/accuracy.cc
index 21107df..53a9406 100644
--- a/src/neuralnet/output_layer/accuracy.cc
+++ b/src/neuralnet/output_layer/accuracy.cc
@@ -46,7 +46,7 @@ void AccuracyLayer::ComputeFeature(int flag,
}
}
accuracy_ += ncorrect * 1.0f / batchsize_;
- counter_ ++;
+ counter_++;
}
const std::string AccuracyLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index a10d3be..28ae341 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -26,9 +26,6 @@
#include "singa/comm/msg.h"
#include "singa/comm/socket.h"
#include "singa/neuralnet/connection_layer.h"
-#include "singa/neuralnet/connection_layer.h"
-#include "singa/neuralnet/connection_layer.h"
-#include "singa/neuralnet/connection_layer.h"
#include "singa/neuralnet/neuron_layer.h"
#include "singa/proto/job.pb.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/test/test_math.cc
----------------------------------------------------------------------
diff --git a/src/test/test_math.cc b/src/test/test_math.cc
index 6583b3d..2e7deec 100644
--- a/src/test/test_math.cc
+++ b/src/test/test_math.cc
@@ -158,7 +158,8 @@ TEST(MathTest, TestGemmGPU) {
cudaMemcpy(B_gpu, B, 3*2*sizeof(float), cudaMemcpyHostToDevice);
auto context = Singleton<Context>::Instance();
context->SetupDevice(std::this_thread::get_id(), 0);
- gpu_gemm<float>(context->cublas_handle(0), A_gpu, B_gpu, 2, 2, 3 , 1, 0, true, false, C_gpu);
+ gpu_gemm<float>(context->cublas_handle(0), A_gpu, B_gpu, 2, 2, 3 , 1, 0, true,
+ false, C_gpu);
cudaMemcpy(C, C_gpu, 2*2*sizeof(float), cudaMemcpyDeviceToHost);
@@ -212,7 +213,8 @@ TEST(MathTest, TestGemvGPU) {
cudaMemcpy(C_gpu, C, 3*sizeof(float), cudaMemcpyHostToDevice);
auto context = Singleton<Context>::Instance();
context->SetupDevice(std::this_thread::get_id(), 0);
- gpu_gemv<float>(context->cublas_handle(0), A_gpu, B_gpu, 4, 3, 1.0f, 1.0f, true, C_gpu);
+ gpu_gemv<float>(context->cublas_handle(0), A_gpu, B_gpu, 4, 3, 1.0f, 1.0f,
+ true, C_gpu);
cudaMemcpy(C, C_gpu, 3*sizeof(float), cudaMemcpyDeviceToHost);
@@ -320,8 +322,8 @@ TEST(MathTest, TestSingaSumRowGPU) {
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 4; j++) {
- //A[i][j] = i + j;
- A[i][j] = 1.0f;
+ // A[i][j] = i + j;
+ A[i][j] = 1.0f;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index 8e30060..a49d169 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -215,7 +215,7 @@ void Blob<Dtype>::CopyFrom(const Blob& source) {
template <typename Dtype>
void Blob<Dtype>::CopyFrom(const Blob& source, bool shape_check) {
LOG(WARNING) << "Better use Copy(const Blob&, Blob*)";
- CHECK_EQ (source.count(), count()) << " cp between blobs of diff size";
+ CHECK_EQ(source.count(), count()) << " cp between blobs of diff size";
if (shape_check &&
!std::equal(shape_.begin(), shape_.end(), source.shape_.begin())) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8ac511c7/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index fe543d7..65d7067 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -131,9 +131,6 @@ void kernel_sum_row(const float *src_mat_data,
__syncthreads();
dst_vec_data[j] = aux[0];
-
-
-
}
__global__
[17/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Minor changes. Fix the :0:9: warning.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/372245f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/372245f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/372245f1
Branch: refs/heads/master
Commit: 372245f12ba0302369e02c6d1b318e3e9dd11264
Parents: f9d4268
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Mon Dec 14 16:57:20 2015 +0800
Committer: xiezl <xi...@comp.nus.edu.sg>
Committed: Mon Dec 14 16:57:20 2015 +0800
----------------------------------------------------------------------
configure.ac | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/372245f1/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 5ae2e8c..07dbdc3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -71,7 +71,7 @@ if test x"$cudaval" = x"yes"; then
LIBS="$LIBS $CUDA_LIBS"
LDFLAGS="$LDFLAGS $CUDA_LDFLAGS"
NVCC="nvcc"
- DEBUG="-DUSE_GPU "
+ DEBUG="-DUSE_GPU"
AC_DEFINE(DCUDA,[1],[Defined if CUDA should be used])
AC_CHECK_LIB([cuda], [main], [], [
AC_MSG_ERROR([unable to find cuda library])
@@ -115,7 +115,7 @@ if test x"$enable_cudnn" == x"yes"; then
CUDNN_LIBS="-lcudnn"
LIBS="$LIBS $CUDNN_LIBS"
LDFLAGS="$LDFLAGS $CUDNN_LDFLAGS"
- DEBUG+=" -DUSE_CUDNN "
+ DEBUG+=" -DUSE_CUDNN"
AC_DEFINE(DCUDNN,[1],[Defined if CUDNN should be used])
AC_CHECK_LIB([cudnn], [main], [], [
AC_MSG_ERROR([unable to find cudnn library])
@@ -175,9 +175,9 @@ AC_ARG_ENABLE(debug,
[enable_debug=yes],[enable_debug=no])
AM_CONDITIONAL(DEBUG, test "$enable_debug" = yes)
if test x"$enable_debug" != x"no"; then
- DEBUG+='-g'
+ DEBUG+=' -g'
else
- DEBUG+='-O2'
+ DEBUG+=' -O2'
fi
AC_ARG_ENABLE(python,
[13/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Support compiling with cudnn related code.
Please run:
./configure --enable-cuda --with-cuda=/CUDA/PATH --enable-cudnn --with-cudnn=/CUDNN/PATH
to generate makefile.
Fix a bug that CPU compilation will detect nvcc existence.
There exists 1 failed test when running "make test", caused by src/test/test_math.cc:349.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/15b23a62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/15b23a62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/15b23a62
Branch: refs/heads/master
Commit: 15b23a62f9bb4fbcf53e0b5db446886d997e6e30
Parents: f8be9af
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Fri Dec 11 15:49:18 2015 +0800
Committer: xiezl <xi...@comp.nus.edu.sg>
Committed: Fri Dec 11 15:49:18 2015 +0800
----------------------------------------------------------------------
Makefile.am | 66 +++++++++++++++++++++++++++++++++++++---------
configure.ac | 51 +++++++++++++++++++++++++++++------
src/test/test_math.cc | 2 +-
3 files changed, 98 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/15b23a62/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 470ea8a..1959fb6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -27,7 +27,7 @@ AUTOMAKE_OPTIONS = foreign subdir-objects
MSHADOW_FLAGS = -DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
DEFAULT_FLAGS = -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
- $(MSHADOW_FLAGS) -DCPU_ONLY=1 -funroll-loops -DTHREADED
+ $(MSHADOW_FLAGS) -funroll-loops -DTHREADED
CFLAGS = $(DEBUG)
CXXFLAGS = $(DEBUG)
@@ -49,8 +49,17 @@ PROTO_PYS := tool/python/pb2/singa_pb2.py \
tool/python/pb2/common_pb2.py
CUDA_SRCS := src/utils/math_kernel.cu
+CUDA_HDRS := include/singa/utils/math_kernel.h
-PY_SRCS := tool/python/singa/driver_wrap.cxx
+CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
+ src/neuralnet/neuron_layer/cudnn_softmax.cc \
+ src/neuralnet/neuron_layer/cudnn_pooling.cc \
+ src/neuralnet/neuron_layer/cudnn_activation.cc \
+ src/neuralnet/neuron_layer/cudnn_lrn.cc \
+ src/neuralnet/neuron_layer/cudnn_convolution.cc
+
+
+PY_SRCS := tool/python/singa/driver_wrap.cxx \
src/driver.cc
SINGA_SRCS := src/driver.cc \
@@ -103,7 +112,9 @@ SINGA_SRCS := src/driver.cc \
src/utils/image_transform.cc
SINGA_HDRS := include/singa.h \
- include/utils/cluster.h \
+ include/singa/utils/math_blob.h \
+ include/singa/utils/math_addr.h \
+ include/singa/utils/cluster.h \
include/utils/cluster_rt.h \
include/utils/param.h \
include/utils/common.h \
@@ -170,14 +181,20 @@ py_LTLIBRARIES = $(PY_PROGS)
#lib_LTLIBRARIES = libsinga.la
libsinga_la_SOURCES = $(PROTO_SRCS) $(SINGA_SRCS)
libsinga_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
+libsinga_la_LDFLAGS =
if LMDB
libsinga_la_CXXFLAGS += -DUSE_LMDB
endif
#libsinga_la_LDFLAGS = -I$(top_srcdir)/include
if DCUDA
-libsinga_la_SOURCES += $(CUDA_SRCS)
+libsinga_la_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
libsinga_la_CXXFLAGS += $(CUDA_CFLAGS)
-libsinga_la_LDFLAGS = $(CUDA_LDFLAGS) $(CUDA_LIBS)
+libsinga_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
+endif
+if DCUDNN
+libsinga_la_SOURCES += $(CUDNN_SRCS)
+libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
+libsinga_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
endif
@@ -197,19 +214,34 @@ singa_LDFLAGS += -llmdb
endif
if DCUDA
-singa_SOURCES += $(CUDA_SRCS)
+singa_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
singa_CXXFLAGS += $(CUDA_CFLAGS)
singa_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+if DCUDNN
+singa_SOURCES += $(CUDNN_SRCS)
+singa_CXXFLAGS += $(CUDNN_CFLAGS)
+singa_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
+endif
#bin_PROGRAMS += singatool
-singatool_SOURCES = src/utils/tool.cc
+singatool_SOURCES = src/utils/tool.cc #$(CUDA_SRCS) $(CUDA_HDRS) $(CUDNN_SRCS)
singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \
- -funroll-loops -DTHREADED -I$(top_srcdir)/include
+ -funroll-loops -DTHREADED -I$(top_srcdir)/include $(DEFAULT_FLAGS)
singatool_LDFLAGS = -lsinga \
-lglog \
-lprotobuf \
- -lzookeeper_mt
+ -lzookeeper_mt
+if DCUDA
+singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
+singatool_CXXFLAGS += $(CUDA_CFLAGS)
+singatool_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
+endif
+if DCUDNN
+singatool_SOURCES += $(CUDNN_SRCS)
+singatool_CXXFLAGS += $(CUDNN_CFLAGS)
+singatool_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
+endif
#lib_LTLIBRARIES += libgtest.la
libgtest_la_SOURCES = $(GTEST_HDRS) $(GTEST_SRCS)
@@ -236,21 +268,31 @@ singatest_LDFLAGS = -lsinga \
if LMDB
singatest_LDFLAGS += -llmdb
endif
-if DCUDA
-singatest_SOURCES += $(CUDA_SRCS)
+if DCUDA
+singatest_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
singatest_CXXFLAGS += $(CUDA_CFLAGS)
singatest_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+if DCUDNN
+singatest_SOURCES += $(CUDNN_SRCS)
+singatest_CXXFLAGS += $(CUDNN_CFLAGS)
+singatest_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
+endif
_driver_la_SOURCES = $(PY_SRCS)
_driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include $(PYFLAGS)
_driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir)
if DCUDA
-_driver_la_CXXFLAGS += $(CUDA_CFLAGS)
+_driver_la_CXXFLAGS += $(CUDA_CFLAGS)
_driver_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+if DCUDNN
+_driver_la_CXXFLAGS += $(CUDNN_CFLAGS)
+_driver_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
+endif
+
clean-local:
rm -rf $(PROTO_SRCS) $(PROTO_HDRS)
rm -rf $(PROTO_PYS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/15b23a62/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 902beab..85cc5fb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -55,16 +55,16 @@ if test "$cuda_prefix" == "yes"; then
fi
fi
-AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
-if test -x "$cuda_prefix/bin/nvcc"; then
- AC_MSG_RESULT([found])
- AC_DEFINE_UNQUOTED([NVCC_PATH], ["$cuda_prefix/bin/nvcc"], [Path to nvcc binary])
-else
- AC_MSG_RESULT([not found!])
- AC_MSG_FAILURE([nvcc was not found in $cuda_prefix/bin])
-fi
if test x"$cudaval" = x"yes"; then
+ AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
+ if test -x "$cuda_prefix/bin/nvcc"; then
+ AC_MSG_RESULT([found])
+ AC_DEFINE_UNQUOTED([NVCC_PATH], ["$cuda_prefix/bin/nvcc"], [Path to nvcc binary])
+ else
+ AC_MSG_RESULT([not found!])
+ AC_MSG_FAILURE([nvcc was not found in $cuda_prefix/bin])
+ fi
CUDA_CFLAGS="-I$cuda_prefix/include"
CUDA_LDFLAGS="-L$cuda_prefix/lib64 -L$cuda_prefix/lib"
CUDA_LIBS="-lcublas -lcudart -lcurand"
@@ -85,12 +85,47 @@ else
CUDA_LDFLAGS=""
CUDA_LIBS=""
NVCC=""
+ DEBUG="-DCPU_ONLY"
fi
AC_SUBST(NVCC)
AC_SUBST(CUDA_LDFLAGS)
AC_SUBST(CUDA_LIBS)
AC_SUBST(CUDA_CFLAGS)
+# Setup custom CUDA paths
+AC_ARG_ENABLE([cudnn],
+ [AS_HELP_STRING(--enable-cudnn,enable CUDNN support)],
+ [enable_cudnn=yes], [enable_cudnn=no])
+AM_CONDITIONAL(DCUDNN, [test "$enable_cudnn" = "yes"])
+
+AC_ARG_WITH([cudnn],
+ [AS_HELP_STRING([--with-cudnn=PATH], [prefix where CUDNN is installed])],
+ [cudnn_prefix=$cudnnwithval], [cudnn_prefix="/usr/local/cuda"])
+if test "$cudnn_prefix" == "yes"; then
+ if test "$cudnnwithval" == "yes"; then
+ cudnn_prefix="/usr/local/cuda"
+ fi
+fi
+
+if test x"$enable_cudnn" == x"yes"; then
+ CUDNN_CFLAGS="-I$cudnn_prefix/include"
+ CUDNN_LDFLAGS="-L$cudnn_prefix/lib64 -L$cudnn_prefix/lib"
+ CUDNN_LIBS="-lcudnn"
+ DEBUG+=" -DUSE_CUDNN "
+ AC_DEFINE(DCUDNN,[1],[Defined if CUDNN should be used])
+ AC_CHECK_LIB([cudnn], [main], [], [
+ AC_MSG_ERROR([unable to find cudnn library])
+ ])
+else
+ CUDNN_CFLAGS=""
+ CUDNN_LDFLAGS=""
+ CUDNN_LIBS=""
+fi
+
+AC_SUBST(CUDNN_CFLAGS)
+AC_SUBST(CUDNN_LDFLAGS)
+AC_SUBST(CUDNN_LIBS)
+
# Checks for libraries.
AC_SEARCH_LIBS([cblas_sgemm], [openblas], [], [
AC_MSG_ERROR([unable to find cblas_sgemm() function])
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/15b23a62/src/test/test_math.cc
----------------------------------------------------------------------
diff --git a/src/test/test_math.cc b/src/test/test_math.cc
index c2730a4..0b9f0ff 100644
--- a/src/test/test_math.cc
+++ b/src/test/test_math.cc
@@ -335,7 +335,7 @@ TEST(MathTest, TestSingaSumColGPU) {
cudaMalloc(reinterpret_cast<void**>(&A_gpu), 12*sizeof(float));
cudaMalloc(reinterpret_cast<void**>(&B_gpu), 4*sizeof(float));
cudaMemcpy(A_gpu, A, 12*sizeof(float), cudaMemcpyHostToDevice);
- singa_gpu_sum_by_col(A_gpu, B_gpu, 3, 4, 4);
+ //singa_gpu_sum_row(A_gpu, B_gpu, 3, 4, 4);
cudaMemcpy(B, B_gpu, 4*sizeof(float), cudaMemcpyDeviceToHost);
[07/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
fix compilation errors.
pass all gtest files.
tmp commit;
1. CudnnSoftmaxloss class
ComputeFeature(): computer loss
ComputeGradient(): compute gradient
2. Add cuda math kernel functions
1) singa_gpu_softmax_loss(): computer loss kernel function
2) singa_gpu_softmax_gradient(): compute gradient kernel function
tmp commit; debug accuracy
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f31ba645
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f31ba645
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f31ba645
Branch: refs/heads/master
Commit: f31ba64589fde0d2b095804366c613988f89fc27
Parents: 8cacd83
Author: seaokcs <se...@gmail.com>
Authored: Tue Dec 1 17:54:38 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:23 2015 +0800
----------------------------------------------------------------------
examples/cifar10/job.conf | 7 +-
include/singa/neuralnet/loss_layer.h | 3 +-
include/singa/neuralnet/neuron_layer.h | 15 +-
include/singa/utils/math_blob.h | 37 ++--
include/singa/utils/math_kernel.h | 6 +
src/driver.cc | 2 +-
src/neuralnet/layer.cc | 4 +-
src/neuralnet/loss_layer/cudnn_softmaxloss.cu | 32 ++-
src/neuralnet/neuron_layer/cudnn_activation.cc | 100 +++++++++
src/neuralnet/neuron_layer/cudnn_activation.cu | 100 ---------
src/neuralnet/neuron_layer/cudnn_convolution.cc | 207 +++++++++++++++++++
src/neuralnet/neuron_layer/cudnn_convolution.cu | 205 ------------------
src/neuralnet/neuron_layer/cudnn_lrn.cc | 87 ++++++++
src/neuralnet/neuron_layer/cudnn_lrn.cu | 87 --------
src/neuralnet/neuron_layer/cudnn_pooling.cc | 96 +++++++++
src/neuralnet/neuron_layer/cudnn_pooling.cu | 96 ---------
src/neuralnet/neuron_layer/cudnn_softmax.cc | 75 +++++++
src/neuralnet/neuron_layer/cudnn_softmax.cu | 75 -------
src/neuralnet/neuron_layer/inner_product.cc | 6 +-
src/stub.cc | 6 +-
src/test/test_csv_input_layer.cc | 2 +-
src/test/test_math.cc | 27 ++-
src/test/test_record_input_layer.cc | 2 +-
src/utils/math_kernel.cu | 36 ++++
src/worker.cc | 4 +-
25 files changed, 702 insertions(+), 615 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index a7540a2..7e42ed8 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -1,10 +1,11 @@
name: "cifar10-convnet"
-train_steps: 1000
+train_steps: 30
test_steps: 100
-test_freq: 300
+test_freq: 0
#validate_steps: 100
#validate_freq: 300
-disp_freq: 30
+disp_freq: 10
+debug: true
#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
train_one_batch {
alg: kBP
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/include/singa/neuralnet/loss_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer.h b/include/singa/neuralnet/loss_layer.h
index 50e6c24..f78d7e0 100644
--- a/include/singa/neuralnet/loss_layer.h
+++ b/include/singa/neuralnet/loss_layer.h
@@ -61,7 +61,8 @@ class CudnnSoftmaxLossLayer : public LossLayer, public CudnnSoftmaxLayer {
const std::string ToString(bool debug, int flag) override;
private:
- int topk_;
+ float scale_;
+ int topk_, dim_;
int counter_;
float loss_, accuracy_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 830f731..9ae2738 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -259,19 +259,24 @@ class STanhLayer : public NeuronLayer {
class CudnnLayer : virtual public NeuronLayer {
public:
~CudnnLayer() {
- CHECK_CUDNN(cudnnDestroyTensorDescriptor(src_desc_));
- CHECK_CUDNN(cudnnDestroyTensorDescriptor(my_desc_));
- CHECK_CUDNN(cudnnDestroy(handle_));
+ if (handle_ != nullptr)
+ CHECK_CUDNN(cudnnDestroy(handle_));
+ if (src_desc_ != nullptr)
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(src_desc_));
+ if (my_desc_ != nullptr)
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(my_desc_));
}
void virtual InitCudnn() {
CHECK(!has_init_cudnn_);
CHECK_CUDNN(cudnnCreate(&handle_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&src_desc_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&my_desc_));
has_init_cudnn_ = true;
}
protected:
bool has_init_cudnn_ = false;
- cudnnHandle_t handle_;
- cudnnTensorDescriptor_t src_desc_, my_desc_;
+ cudnnHandle_t handle_ = nullptr;
+ cudnnTensorDescriptor_t src_desc_ = nullptr, my_desc_ = nullptr;
};
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index 586554e..629839a 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -76,9 +76,10 @@ void AXPY(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
/************* BLAS level 2 *****************/
/**
* Matrix vector multiplication, C = alpha A(.T) * B + beta C.
- * Strict shape checking:
- * - dim of A ==2
- * columsn of A(.T) == B.count()
+ * Loose shape checking:
+ * - dim of A >=2
+ * - row of A is shape(0) (no transpose)
+ * - column of A(.T) == B.count()
* - rows of A(.T) == C.count()
*
* @param[in] alpha
@@ -90,10 +91,10 @@ void AXPY(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
template<typename Dtype>
void GEMV(Dtype alpha, Dtype beta, const Blob<Dtype>& A,
const Blob<Dtype>& B, Blob<Dtype>* C) {
- CHECK_EQ(A.shape().size(), 2) << "A must be a matrix";
+ CHECK_EQ(A.shape().size(), 2);
int a1, a2, m, n;
- a1 = A.transpose() ? A.shape(1) : A.shape(0);
- a2 = A.transpose() ? A.shape(0) : A.shape(1);
+ a1 = A.transpose() ? A.count() / A.shape(0) : A.shape(0);
+ a2 = A.transpose() ? A.shape(0) : A.count() / A.shape(0);
m = B.count();
n = C->count();
CHECK_EQ(a2, m) << "# columns of A(.T) must = length of B";
@@ -134,8 +135,8 @@ void MVDot(const Blob<Dtype>& A, const Blob<Dtype>& B,
* Matrix multiplication, C = alpha A*B + beta C, A, B and C are matrix.
*
* Tranpose is considered for A and B.
- * Strict shape checking:
- * - all are matrix
+ * Loose shape checking:
+ * - the first dimension is row (no transpose) or col (with transpose) size
* - shapes match for matrix multiplication
*
* @param[in] alpha
@@ -147,17 +148,17 @@ void MVDot(const Blob<Dtype>& A, const Blob<Dtype>& B,
template <typename Dtype>
void GEMM( Dtype alpha, Dtype beta, const Blob<Dtype>& A,
const Blob<Dtype> & B, Blob<Dtype> * C) {
- CHECK_EQ(A.shape().size(), 2);
- CHECK_EQ(B.shape().size(), 2);
- CHECK_EQ(C->shape().size(), 2);
+ CHECK_GE(A.shape().size(), 2);
+ CHECK_GE(B.shape().size(), 2);
+ CHECK_GE(C->shape().size(), 2);
int a1, a2, b1, b2, m, n;
CHECK(!C->transpose());
- a1 = A.transpose() ? A.shape(1) : A.shape(0);
- a2 = A.transpose() ? A.shape(0) : A.shape(1);
- b1 = B.transpose() ? B.shape(1) : B.shape(0);
- b2 = B.transpose() ? B.shape(0) : B.shape(1);
+ a1 = A.transpose() ? A.count() / A.shape(0) : A.shape(0);
+ a2 = A.transpose() ? A.shape(0) : A.count() / A.shape(0);
+ b1 = B.transpose() ? B.count() /B.shape(0) : B.shape(0);
+ b2 = B.transpose() ? B.shape(0) : B.count() / B.shape(0);
m = C->shape(0);
- n = C->shape(1);
+ n = C->count() / C->shape(0);
CHECK_EQ(a2, b1);
CHECK_EQ(a1, m);
CHECK_EQ(b2, n);
@@ -560,7 +561,7 @@ void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
A.transpose(), false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_col(A.gpu_data(), B->gpu_data(), m, n, n);
+ singa_gpu_sum_by_col(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
@@ -585,7 +586,7 @@ void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_vec(A.gpu_data(), B->gpu_data(), m, n, n);
+ singa_gpu_sum_by_row(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/include/singa/utils/math_kernel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_kernel.h b/include/singa/utils/math_kernel.h
index 37a9356..8cfa562 100644
--- a/include/singa/utils/math_kernel.h
+++ b/include/singa/utils/math_kernel.h
@@ -24,6 +24,12 @@
namespace singa {
extern "C" {
+ void singa_gpu_softmax_loss(const float *prob, const int *label,
+ float *loss, int n, int dim);
+
+ void singa_gpu_softmax_gradient(float *grad, const int *label ,
+ int n, int dim, float scale);
+
void singa_gpu_sum_vec(float *data, float *sum , int n);
void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 1ae6d9f..b963912 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -216,7 +216,7 @@ void Driver::Train(const JobProto& job_conf) {
threads.push_back(std::thread(&Server::Run, server));
int gpu = 0;
auto context = Singleton<Context>::Instance();
- CHECK_LE(workers.size(), job_conf.gpu_size());
+ // CHECK_LE(workers.size(), job_conf.gpu_size());
for (auto worker : workers) {
threads.push_back(std::thread(&Worker::Run, worker));
if (gpu < job_conf.gpu_size()) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index 9414948..df77239 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -49,8 +49,8 @@ const std::string Layer::ToString(bool debug, int flag) {
string ret = StringPrintf("Layer %10s ", name().c_str());
if ((flag & kForward) == kForward && data_.count() !=0) {
ret += StringPrintf("data norm1 %13.9f", Asum(data_));
- } else if ((flag & kBackward) == kBackward) {
- if (grad_.count() != 0)
+ }
+ if ((flag & kBackward) == kBackward && grad_.count() != 0) {
ret += StringPrintf("grad norm1 %13.9f\n", Asum(grad_));
}
if ((flag & kTrain) == kTrain) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu b/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
index e0af05f..53420d3 100644
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
@@ -20,6 +20,8 @@
*************************************************************/
#include "singa/neuralnet/loss_layer.h"
+#include "singa/utils/blob.h"
+#include "singa/utils/math_kernel.h"
namespace singa {
void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
@@ -33,13 +35,41 @@ void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
CudnnSoftmaxLayer::ComputeFeature(flag, srclayers);
// compute loss
+ float *prob = data_.mutable_gpu_data();
+ Blob<int> label(batchsize_);
+ int *labelptr = label.mutable_cpu_data();
+
+ //aux_data: vector<int>, convert vector to int array.
+ for(int i = 0; i < batchsize_; ++i) {
+ labelptr[i] = srclayers[1]->aux_data(this)[i];
+ }
+
+ Blob<float> loss(batchsize_);
+
+ singa_gpu_softmax_loss(prob , label.mutable_gpu_data() , loss.mutable_gpu_data(),
+ batchsize_, dim_);
+
counter_++;
- // add loss and accuracy
+ // TODO add loss and accuracy
}
void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
// compute gradient
+ Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
+ gsrcblob->CopyFrom(data_);
+ float* gsrcptr = gsrcblob->mutable_gpu_data();
+
+ Blob<int> label(batchsize_);
+ int *labelptr = label.mutable_cpu_data();
+
+ //aux_data: vector<int>, convert vector to int array.
+ for(int i = 0; i < batchsize_; ++i) {
+ labelptr[i] = srclayers[1]->aux_data(this)[i];
+ }
+
+ singa_gpu_softmax_gradient(gsrcptr, label.mutable_gpu_data(), batchsize_, dim_, scale_);
+
}
const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cc b/src/neuralnet/neuron_layer/cudnn_activation.cc
new file mode 100644
index 0000000..e8a7b41
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_activation.cc
@@ -0,0 +1,100 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+void CudnnActivationLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+
+ // TODO(wangwei) make the mode case insensitive
+ if (layer_conf_.activation_conf().type() == SIGMOID)
+ mode_ = CUDNN_ACTIVATION_SIGMOID;
+ else if (layer_conf_.activation_conf().type() == TANH)
+ mode_ = CUDNN_ACTIVATION_TANH;
+ else if (layer_conf_.activation_conf().type() == RELU)
+ mode_ = CUDNN_ACTIVATION_RELU;
+ else {
+ LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().type();
+ }
+
+ const auto& shape = data_.shape();
+ CHECK_GT(shape.size(), 0);
+ // size of each dimension
+ int* sdim= new int[shape.size()];
+ int* stride = new int[shape.size()];
+ stride[shape.size() -1] = 1;
+ int i = shape.size() - 1;
+ sdim[i] = shape[i];
+ stride[i] = 1;
+ for (--i; i >= 0; i--) {
+ sdim[i] = shape[i];
+ stride[i] = shape[i + 1] * stride[i + 1];
+ }
+ CHECK_CUDNN(cudnnSetTensorNdDescriptor(src_desc_,
+ CUDNN_DATA_FLOAT,
+ shape.size(),
+ sdim,
+ stride));
+ CHECK_CUDNN(cudnnSetTensorNdDescriptor(my_desc_,
+ CUDNN_DATA_FLOAT,
+ shape.size(),
+ sdim,
+ stride));
+ delete[] sdim;
+ delete[] stride;
+}
+
+void CudnnActivationLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
+ // currently only consider single src layer
+ CHECK_EQ(srclayers.size(), 1);
+ CHECK_CUDNN(cudnnActivationForward(handle_,
+ mode_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CudnnActivationLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnActivationBackward(handle_,
+ mode_,
+ &alpha,
+ my_desc_,
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+}
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_activation.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cu b/src/neuralnet/neuron_layer/cudnn_activation.cu
deleted file mode 100644
index e8a7b41..0000000
--- a/src/neuralnet/neuron_layer/cudnn_activation.cu
+++ /dev/null
@@ -1,100 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/neuron_layer.h"
-
-namespace singa {
-
-void CudnnActivationLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
-
- // TODO(wangwei) make the mode case insensitive
- if (layer_conf_.activation_conf().type() == SIGMOID)
- mode_ = CUDNN_ACTIVATION_SIGMOID;
- else if (layer_conf_.activation_conf().type() == TANH)
- mode_ = CUDNN_ACTIVATION_TANH;
- else if (layer_conf_.activation_conf().type() == RELU)
- mode_ = CUDNN_ACTIVATION_RELU;
- else {
- LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().type();
- }
-
- const auto& shape = data_.shape();
- CHECK_GT(shape.size(), 0);
- // size of each dimension
- int* sdim= new int[shape.size()];
- int* stride = new int[shape.size()];
- stride[shape.size() -1] = 1;
- int i = shape.size() - 1;
- sdim[i] = shape[i];
- stride[i] = 1;
- for (--i; i >= 0; i--) {
- sdim[i] = shape[i];
- stride[i] = shape[i + 1] * stride[i + 1];
- }
- CHECK_CUDNN(cudnnSetTensorNdDescriptor(src_desc_,
- CUDNN_DATA_FLOAT,
- shape.size(),
- sdim,
- stride));
- CHECK_CUDNN(cudnnSetTensorNdDescriptor(my_desc_,
- CUDNN_DATA_FLOAT,
- shape.size(),
- sdim,
- stride));
- delete[] sdim;
- delete[] stride;
-}
-
-void CudnnActivationLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- if (!has_init_cudnn_)
- InitCudnn();
- float alpha = 1.0f, beta = 0.0f;
- // currently only consider single src layer
- CHECK_EQ(srclayers.size(), 1);
- CHECK_CUDNN(cudnnActivationForward(handle_,
- mode_,
- &alpha,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta,
- my_desc_,
- data_.mutable_gpu_data()));
-}
-
-void CudnnActivationLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- float alpha = 1.0f, beta = 0.0f;
- CHECK_CUDNN(cudnnActivationBackward(handle_,
- mode_,
- &alpha,
- my_desc_,
- data_.gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta,
- src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cc b/src/neuralnet/neuron_layer/cudnn_convolution.cc
new file mode 100644
index 0000000..e08b57a
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_convolution.cc
@@ -0,0 +1,207 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+CudnnConvLayer::~CudnnConvLayer() {
+ if (has_init_cudnn_) {
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(bias_desc_));
+ CHECK_CUDNN(cudnnDestroyFilterDescriptor(filter_desc_));
+ CHECK_CUDNN(cudnnDestroyConvolutionDescriptor(conv_desc_));
+ }
+}
+
+void CudnnConvLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ // convert MB to bytes
+ workspace_byte_limit_ = layer_conf_.convolution_conf().workspace_byte_limit() << 20;
+
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&bias_desc_));
+ CHECK_CUDNN(cudnnCreateFilterDescriptor(&filter_desc_));
+ CHECK_CUDNN(cudnnCreateConvolutionDescriptor(&conv_desc_));
+
+ CHECK_CUDNN(cudnnSetConvolution2dDescriptor(conv_desc_,
+ pad_y_,
+ pad_x_,
+ stride_y_,
+ stride_x_,
+ 1,
+ 1,
+ CUDNN_CROSS_CORRELATION));
+ CHECK_CUDNN(cudnnSetFilter4dDescriptor(filter_desc_,
+ CUDNN_DATA_FLOAT,
+ num_filters_,
+ channels_,
+ kernel_y_,
+ kernel_x_));
+ if (bias_) {
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(bias_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ 1,
+ num_filters_,
+ 1,
+ 1));
+ }
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_filters_,
+ conv_height_,
+ conv_width_));
+
+ CHECK_CUDNN(cudnnGetConvolutionForwardAlgorithm(handle_,
+ src_desc_,
+ filter_desc_,
+ conv_desc_,
+ my_desc_,
+ CUDNN_CONVOLUTION_FWD_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &fp_alg_));
+
+ CHECK_CUDNN(cudnnGetConvolutionBackwardFilterAlgorithm(handle_,
+ src_desc_,
+ my_desc_,
+ conv_desc_,
+ filter_desc_,
+ CUDNN_CONVOLUTION_BWD_FILTER_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &bp_filter_alg_));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardDataAlgorithm(handle_,
+ filter_desc_,
+ my_desc_,
+ conv_desc_,
+ src_desc_,
+ CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &bp_data_alg_));
+
+ size_t fp_byte, bp_data_byte, bp_filter_byte;
+ CHECK_CUDNN(cudnnGetConvolutionForwardWorkspaceSize(handle_,
+ src_desc_,
+ filter_desc_,
+ conv_desc_,
+ my_desc_,
+ fp_alg_,
+ &fp_byte));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(handle_,
+ filter_desc_,
+ my_desc_,
+ conv_desc_,
+ src_desc_,
+ bp_data_alg_,
+ &bp_data_byte));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardFilterWorkspaceSize(handle_,
+ src_desc_,
+ my_desc_,
+ conv_desc_,
+ filter_desc_,
+ bp_filter_alg_,
+ &bp_filter_byte));
+ workspace_count_ = std::max(std::max(fp_byte, bp_data_byte), bp_filter_byte)
+ / sizeof(float) + 1;
+}
+
+void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.f, beta = 0.f;
+ Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
+ CHECK_CUDNN(cudnnConvolutionForward(handle_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ filter_desc_,
+ weight_->data().gpu_data(),
+ conv_desc_,
+ fp_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+
+ if (bias_) {
+ beta = 1.f;
+ CHECK_CUDNN(cudnnAddTensor(handle_,
+ CUDNN_ADD_SAME_C,
+ &alpha,
+ bias_desc_,
+ bias_->data().gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+ }
+}
+
+void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+{
+ float alpha = 1.f, beta = 0.f;
+ Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
+ if (bias_) {
+ CHECK_CUDNN(cudnnConvolutionBackwardBias(handle_,
+ &alpha,
+ my_desc_,
+ grad_.gpu_data(),
+ &beta,
+ bias_desc_,
+ bias_->mutable_grad()->mutable_gpu_data()));
+ }
+ CHECK_CUDNN(cudnnConvolutionBackwardFilter_v3(handle_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ conv_desc_,
+ bp_filter_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ filter_desc_,
+ weight_->mutable_grad()->mutable_gpu_data()));
+ if (srclayers[0]->mutable_grad(this) != nullptr) {
+ CHECK_CUDNN(cudnnConvolutionBackwardData_v3(handle_,
+ &alpha,
+ filter_desc_,
+ weight_->data().gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ conv_desc_,
+ bp_data_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+ }
+}
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_convolution.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cu b/src/neuralnet/neuron_layer/cudnn_convolution.cu
deleted file mode 100644
index 13e9f65..0000000
--- a/src/neuralnet/neuron_layer/cudnn_convolution.cu
+++ /dev/null
@@ -1,205 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/neuron_layer.h"
-
-namespace singa {
-
-CudnnConvLayer::~CudnnConvLayer() {
- if (has_init_cudnn_) {
- CHECK_CUDNN(cudnnDestroyTensorDescriptor(bias_desc_));
- CHECK_CUDNN(cudnnDestroyFilterDescriptor(filter_desc_));
- CHECK_CUDNN(cudnnDestroyConvolutionDescriptor(conv_desc_));
- }
-}
-
-void CudnnConvLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
- // convert MB to bytes
- workspace_byte_limit_ = layer_conf_.convolution_conf().workspace_byte_limit() << 20;
-
- CHECK_CUDNN(cudnnCreateTensorDescriptor(&bias_desc_));
- CHECK_CUDNN(cudnnCreateFilterDescriptor(&filter_desc_));
- CHECK_CUDNN(cudnnCreateConvolutionDescriptor(&conv_desc_));
-
- CHECK_CUDNN(cudnnSetConvolution2dDescriptor(conv_desc_,
- pad_y_,
- pad_x_,
- stride_y_,
- stride_x_,
- 1,
- 1,
- CUDNN_CROSS_CORRELATION));
- CHECK_CUDNN(cudnnSetFilter4dDescriptor(filter_desc_,
- CUDNN_DATA_FLOAT,
- num_filters_,
- channels_,
- kernel_y_,
- kernel_x_));
- if (bias_) {
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(bias_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- 1,
- num_filters_,
- 1,
- 1));
- }
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- channels_,
- height_,
- width_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- num_filters_,
- conv_height_,
- conv_width_));
-
- CHECK_CUDNN(cudnnGetConvolutionForwardAlgorithm(handle_,
- src_desc_,
- filter_desc_,
- conv_desc_,
- my_desc_,
- CUDNN_CONVOLUTION_FWD_PREFER_FASTEST,
- workspace_byte_limit_,
- &fp_alg_));
-
- CHECK_CUDNN(cudnnGetConvolutionBackwardFilterAlgorithm(handle_,
- src_desc_,
- my_desc_,
- conv_desc_,
- filter_desc_,
- CUDNN_CONVOLUTION_BWD_FILTER_PREFER_FASTEST,
- workspace_byte_limit_,
- &bp_filter_alg_));
- CHECK_CUDNN(cudnnGetConvolutionBackwardDataAlgorithm(handle_,
- filter_desc_,
- my_desc_,
- conv_desc_,
- src_desc_,
- CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST,
- workspace_byte_limit_,
- &bp_data_alg_));
-
- size_t fp_byte, bp_data_byte, bp_filter_byte;
- CHECK_CUDNN(cudnnGetConvolutionForwardWorkspaceSize(handle_,
- src_desc_,
- filter_desc_,
- conv_desc_,
- my_desc_,
- fp_alg_,
- &fp_byte));
- CHECK_CUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(handle_,
- filter_desc_,
- my_desc_,
- conv_desc_,
- src_desc_,
- bp_data_alg_,
- &bp_data_byte));
- CHECK_CUDNN(cudnnGetConvolutionBackwardFilterWorkspaceSize(handle_,
- src_desc_,
- my_desc_,
- conv_desc_,
- filter_desc_,
- bp_filter_alg_,
- &bp_filter_byte));
- workspace_count_ = std::max(std::max(fp_byte, bp_data_byte), bp_filter_byte)
- / sizeof(float) + 1;
-}
-
-void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (!has_init_cudnn_)
- InitCudnn();
- float alpha = 1.f, beta = 0.f;
- Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
- CHECK_CUDNN(cudnnConvolutionForward(handle_,
- &alpha,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- filter_desc_,
- weight_->data().gpu_data(),
- conv_desc_,
- fp_alg_,
- workspace.mutable_gpu_data(),
- workspace_count_ * sizeof(float),
- &beta,
- my_desc_,
- data_.mutable_gpu_data()));
-
- if (bias_) {
- beta = 1.f;
- CHECK_CUDNN(cudnnAddTensor(handle_,
- CUDNN_ADD_SAME_C,
- &alpha,
- bias_desc_,
- bias_->data().gpu_data(),
- &beta,
- my_desc_,
- data_.mutable_gpu_data()));
- }
-}
-
-void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
-{
- float alpha = 1.f, beta = 0.f;
- Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
- if (bias_) {
- CHECK_CUDNN(cudnnConvolutionBackwardBias(handle_,
- &alpha,
- my_desc_,
- grad_.gpu_data(),
- &beta,
- bias_desc_,
- bias_->mutable_grad()->mutable_gpu_data()));
- }
- CHECK_CUDNN(cudnnConvolutionBackwardFilter_v3(handle_,
- &alpha,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- conv_desc_,
- bp_filter_alg_,
- workspace.mutable_gpu_data(),
- workspace_count_ * sizeof(float),
- &beta,
- filter_desc_,
- weight_->mutable_grad()->mutable_gpu_data()));
- CHECK_CUDNN(cudnnConvolutionBackwardData_v3(handle_,
- &alpha,
- filter_desc_,
- weight_->data().gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- conv_desc_,
- bp_data_alg_,
- workspace.mutable_gpu_data(),
- workspace_count_ * sizeof(float),
- &beta,
- src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
-}
-} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_lrn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cc b/src/neuralnet/neuron_layer/cudnn_lrn.cc
new file mode 100644
index 0000000..4a2b695
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_lrn.cc
@@ -0,0 +1,87 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+CudnnLRNLayer::~CudnnLRNLayer() {
+ if (has_init_cudnn_) {
+ cudnnDestroyLRNDescriptor(norm_desc_);
+ }
+}
+
+void CudnnLRNLayer::InitCudnn() {
+ mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnCreateLRNDescriptor(&norm_desc_));
+ CHECK_CUDNN(cudnnSetLRNDescriptor(norm_desc_,
+ lsize_,
+ alpha_,
+ beta_,
+ knorm_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&src_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&my_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+}
+void CudnnLRNLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ CHECK_CUDNN(cudnnLRNCrossChannelForward(handle_,
+ norm_desc_,
+ mode_,
+ &alpha_,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta_,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+void CudnnLRNLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
+ CHECK_CUDNN(cudnnLRNCrossChannelBackward(handle_,
+ norm_desc_,
+ mode_,
+ &alpha_,
+ my_desc_, // ???
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta_,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+}
+
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_lrn.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cu b/src/neuralnet/neuron_layer/cudnn_lrn.cu
deleted file mode 100644
index 4a2b695..0000000
--- a/src/neuralnet/neuron_layer/cudnn_lrn.cu
+++ /dev/null
@@ -1,87 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/neuron_layer.h"
-
-namespace singa {
-CudnnLRNLayer::~CudnnLRNLayer() {
- if (has_init_cudnn_) {
- cudnnDestroyLRNDescriptor(norm_desc_);
- }
-}
-
-void CudnnLRNLayer::InitCudnn() {
- mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
- CudnnLayer::InitCudnn();
- CHECK_CUDNN(cudnnCreateLRNDescriptor(&norm_desc_));
- CHECK_CUDNN(cudnnSetLRNDescriptor(norm_desc_,
- lsize_,
- alpha_,
- beta_,
- knorm_));
- CHECK_CUDNN(cudnnCreateTensorDescriptor(&src_desc_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- channels_,
- height_,
- width_));
- CHECK_CUDNN(cudnnCreateTensorDescriptor(&my_desc_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- channels_,
- height_,
- width_));
-}
-void CudnnLRNLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (!has_init_cudnn_)
- InitCudnn();
- CHECK_CUDNN(cudnnLRNCrossChannelForward(handle_,
- norm_desc_,
- mode_,
- &alpha_,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta_,
- my_desc_,
- data_.mutable_gpu_data()));
-}
-void CudnnLRNLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- CHECK_CUDNN(cudnnLRNCrossChannelBackward(handle_,
- norm_desc_,
- mode_,
- &alpha_,
- my_desc_, // ???
- data_.gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta_,
- src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
-}
-
-
-} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cc b/src/neuralnet/neuron_layer/cudnn_pooling.cc
new file mode 100644
index 0000000..ffdfb3b
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_pooling.cc
@@ -0,0 +1,96 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+CudnnPoolLayer::~CudnnPoolLayer() {
+ if (has_init_cudnn_) {
+ CHECK_CUDNN(cudnnDestroyPoolingDescriptor(pool_desc_));
+ }
+}
+
+void CudnnPoolLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnCreatePoolingDescriptor(&pool_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ pooled_height_,
+ pooled_width_));
+ auto pool_method = CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING;
+ if (pool_ == PoolingProto_PoolMethod_MAX)
+ pool_method = CUDNN_POOLING_MAX;
+ CHECK_CUDNN(cudnnSetPooling2dDescriptor(pool_desc_,
+ pool_method,
+ kernel_y_,
+ kernel_x_,
+ pad_y_,
+ pad_x_,
+ stride_y_,
+ stride_x_));
+
+}
+
+void CudnnPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
+ // currently only consider single src layer
+ CHECK_EQ(srclayers.size(), 1);
+ CHECK_CUDNN(cudnnPoolingForward(handle_,
+ pool_desc_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+{
+ float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnPoolingBackward(handle_,
+ pool_desc_,
+ &alpha,
+ my_desc_,
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+}
+} /* singa */
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_pooling.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cu b/src/neuralnet/neuron_layer/cudnn_pooling.cu
deleted file mode 100644
index ffdfb3b..0000000
--- a/src/neuralnet/neuron_layer/cudnn_pooling.cu
+++ /dev/null
@@ -1,96 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/neuron_layer.h"
-
-namespace singa {
-
-CudnnPoolLayer::~CudnnPoolLayer() {
- if (has_init_cudnn_) {
- CHECK_CUDNN(cudnnDestroyPoolingDescriptor(pool_desc_));
- }
-}
-
-void CudnnPoolLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
- CHECK_CUDNN(cudnnCreatePoolingDescriptor(&pool_desc_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- channels_,
- height_,
- width_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- channels_,
- pooled_height_,
- pooled_width_));
- auto pool_method = CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING;
- if (pool_ == PoolingProto_PoolMethod_MAX)
- pool_method = CUDNN_POOLING_MAX;
- CHECK_CUDNN(cudnnSetPooling2dDescriptor(pool_desc_,
- pool_method,
- kernel_y_,
- kernel_x_,
- pad_y_,
- pad_x_,
- stride_y_,
- stride_x_));
-
-}
-
-void CudnnPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (!has_init_cudnn_)
- InitCudnn();
- float alpha = 1.0f, beta = 0.0f;
- // currently only consider single src layer
- CHECK_EQ(srclayers.size(), 1);
- CHECK_CUDNN(cudnnPoolingForward(handle_,
- pool_desc_,
- &alpha,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta,
- my_desc_,
- data_.mutable_gpu_data()));
-}
-
-void CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
-{
- float alpha = 1.0f, beta = 0.0f;
- CHECK_CUDNN(cudnnPoolingBackward(handle_,
- pool_desc_,
- &alpha,
- my_desc_,
- data_.gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta,
- src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
-}
-} /* singa */
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cc b/src/neuralnet/neuron_layer/cudnn_softmax.cc
new file mode 100644
index 0000000..7fade3e
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cc
@@ -0,0 +1,75 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+void CudnnSoftmaxLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_softmax_per_instance_,
+ count_per_softmax_,
+ 1));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_softmax_per_instance_,
+ count_per_softmax_,
+ 1));
+}
+
+void CudnnSoftmaxLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ const float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnSoftmaxForward(handle_,
+ CUDNN_SOFTMAX_ACCURATE,
+ CUDNN_SOFTMAX_MODE_CHANNEL,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CudnnSoftmaxLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ const float alpha = 1.f, beta = 0.f;
+ CHECK_CUDNN(cudnnSoftmaxBackward(handle_,
+ CUDNN_SOFTMAX_ACCURATE,
+ CUDNN_SOFTMAX_MODE_CHANNEL,
+ &alpha,
+ my_desc_,
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+}
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/cudnn_softmax.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cu b/src/neuralnet/neuron_layer/cudnn_softmax.cu
deleted file mode 100644
index 7fade3e..0000000
--- a/src/neuralnet/neuron_layer/cudnn_softmax.cu
+++ /dev/null
@@ -1,75 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/neuron_layer.h"
-
-namespace singa {
-
-void CudnnSoftmaxLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- num_softmax_per_instance_,
- count_per_softmax_,
- 1));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
- CUDNN_TENSOR_NCHW,
- CUDNN_DATA_FLOAT,
- batchsize_,
- num_softmax_per_instance_,
- count_per_softmax_,
- 1));
-}
-
-void CudnnSoftmaxLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- if (!has_init_cudnn_)
- InitCudnn();
- const float alpha = 1.0f, beta = 0.0f;
- CHECK_CUDNN(cudnnSoftmaxForward(handle_,
- CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_MODE_CHANNEL,
- &alpha,
- src_desc_,
- srclayers[0]->data(this).gpu_data(),
- &beta,
- my_desc_,
- data_.mutable_gpu_data()));
-}
-
-void CudnnSoftmaxLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- const float alpha = 1.f, beta = 0.f;
- CHECK_CUDNN(cudnnSoftmaxBackward(handle_,
- CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_MODE_CHANNEL,
- &alpha,
- my_desc_,
- data_.gpu_data(),
- my_desc_,
- grad_.gpu_data(),
- &beta,
- src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
-}
-} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/neuralnet/neuron_layer/inner_product.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/inner_product.cc b/src/neuralnet/neuron_layer/inner_product.cc
index 3b18cd7..6b5ec36 100644
--- a/src/neuralnet/neuron_layer/inner_product.cc
+++ b/src/neuralnet/neuron_layer/inner_product.cc
@@ -58,7 +58,7 @@ void InnerProductLayer::Setup(const LayerProto& conf,
void InnerProductLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
- MMDot(srclayers[0]->data(this), weight_->data(), &data_);
+ MMDot(srclayers[0]->data(this), weight_->data().T(), &data_);
MVAddRow(bias_->data(), &data_);
}
@@ -66,9 +66,9 @@ void InnerProductLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
MVSumRow(1.0f, 0.0f, grad_, bias_->mutable_grad());
- MVDot(grad_.T(), srclayers[0]->data(this), weight_->mutable_grad());
+ MMDot(grad_.T(), srclayers[0]->data(this), weight_->mutable_grad());
if (srclayers[0]->mutable_grad(this) != nullptr) {
- MVDot(grad_, weight_->data(), srclayers[0]->mutable_grad(this));
+ MMDot(grad_, weight_->data(), srclayers[0]->mutable_grad(this));
}
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/stub.cc
----------------------------------------------------------------------
diff --git a/src/stub.cc b/src/stub.cc
index 3f7c59d..a3605f7 100644
--- a/src/stub.cc
+++ b/src/stub.cc
@@ -207,12 +207,12 @@ void Stub::GenMsgs(int type, int version, ParamEntry* entry, Msg* msg,
Msg* new_msg = nullptr;
if (type == kPut) {
CHECK_GT(entry->num_total, 0);
- new_msg = param->GenPutMsg(dst_procs != procs_id, idx);
+ new_msg = param->GenPutMsg(dst_procs == procs_id, idx);
new_msg->AddFormatFrame("i", entry->num_total);
} else if (type == kGet) {
- new_msg = param->GenGetMsg(dst_procs != procs_id, idx);
+ new_msg = param->GenGetMsg(dst_procs == procs_id, idx);
} else if (type == kUpdate) {
- new_msg = param->GenUpdateMsg(dst_procs != procs_id, idx);
+ new_msg = param->GenUpdateMsg(dst_procs == procs_id, idx);
new_msg->AddFormatFrame("i", entry->num_local);
} else {
LOG(FATAL) << "Wrong type";
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/test/test_csv_input_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_csv_input_layer.cc b/src/test/test_csv_input_layer.cc
index ce5847d..6613d87 100644
--- a/src/test/test_csv_input_layer.cc
+++ b/src/test/test_csv_input_layer.cc
@@ -23,7 +23,7 @@
#include <fstream>
#include "gtest/gtest.h"
-#include "singa/neuralnet/input_layer/csv.h"
+#include "singa/neuralnet/input_layer.h"
#include "singa/proto/job.pb.h"
class CSVInputLayerTest : public ::testing::Test {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/test/test_math.cc
----------------------------------------------------------------------
diff --git a/src/test/test_math.cc b/src/test/test_math.cc
index 39ec2a0..c2730a4 100644
--- a/src/test/test_math.cc
+++ b/src/test/test_math.cc
@@ -7,9 +7,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
-*
+*
* http://www.apache.org/licenses/LICENSE-2.0
-*
+*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,11 +18,13 @@
* under the License.
*
*************************************************************/
-
+#include <thread>
#include "gtest/gtest.h"
#include "singa/utils/math_addr.h"
#include "singa/utils/math_kernel.h"
#include "singa/utils/singa_op.h"
+#include "singa/utils/context.h"
+#include "singa/utils/singleton.h"
#ifdef USE_GPU
#include <cuda_runtime.h>
@@ -82,6 +84,7 @@ TEST(MathTest, TestGemvCPU) {
}
+/*
TEST(MathTest, TestAxpyCPU) {
float A[4][3] = {};
float C[4][3] = {};
@@ -109,7 +112,6 @@ TEST(MathTest, TestAxpyCPU) {
}
}
-/*
TEST(MathTest, TestEopCPU) {
float A[10] = {};
@@ -154,8 +156,9 @@ TEST(MathTest, TestGemmGPU) {
cudaMemcpy(A_gpu, A, 3*2*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(B_gpu, B, 3*2*sizeof(float), cudaMemcpyHostToDevice);
-
- gpu_gemm<float>(A_gpu, B_gpu, 2, 2, 3 , 1, 0, true, false, C_gpu);
+ auto context = Singleton<Context>::Instance();
+ context->SetupDevice(std::this_thread::get_id(), 0);
+ gpu_gemm<float>(context->cublas_handle(0), A_gpu, B_gpu, 2, 2, 3 , 1, 0, true, false, C_gpu);
cudaMemcpy(C, C_gpu, 2*2*sizeof(float), cudaMemcpyDeviceToHost);
@@ -207,8 +210,9 @@ TEST(MathTest, TestGemvGPU) {
cudaMemcpy(A_gpu, A, 4*3*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(B_gpu, B, 4*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(C_gpu, C, 3*sizeof(float), cudaMemcpyHostToDevice);
-
- gpu_gemv<float>(A_gpu, B_gpu, 4, 3, 1, 1, true, C_gpu);
+ auto context = Singleton<Context>::Instance();
+ context->SetupDevice(std::this_thread::get_id(), 0);
+ gpu_gemv<float>(context->cublas_handle(0), A_gpu, B_gpu, 4, 3, 1.0f, 1.0f, true, C_gpu);
cudaMemcpy(C, C_gpu, 3*sizeof(float), cudaMemcpyDeviceToHost);
@@ -294,7 +298,9 @@ TEST(MathTest, TestDotGPU) {
cudaMemcpy(A_gpu, A, 12*sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(B_gpu, B, 12*sizeof(float), cudaMemcpyHostToDevice);
- float gpu_ret = gpu_dot<float>(A_gpu, B_gpu, 12);
+ auto context = Singleton<Context>::Instance();
+ context->SetupDevice(std::this_thread::get_id(), 0);
+ float gpu_ret = gpu_dot<float>(context->cublas_handle(0), A_gpu, B_gpu, 12);
float cpu_ret = 0.0f;
for (int i = 0; i < 12; i++) {
@@ -329,8 +335,7 @@ TEST(MathTest, TestSingaSumColGPU) {
cudaMalloc(reinterpret_cast<void**>(&A_gpu), 12*sizeof(float));
cudaMalloc(reinterpret_cast<void**>(&B_gpu), 4*sizeof(float));
cudaMemcpy(A_gpu, A, 12*sizeof(float), cudaMemcpyHostToDevice);
-
- singa_gpu_sum_col(A_gpu, B_gpu, 3, 4, 4);
+ singa_gpu_sum_by_col(A_gpu, B_gpu, 3, 4, 4);
cudaMemcpy(B, B_gpu, 4*sizeof(float), cudaMemcpyDeviceToHost);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/test/test_record_input_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_record_input_layer.cc b/src/test/test_record_input_layer.cc
index 78e6047..9c953c1 100644
--- a/src/test/test_record_input_layer.cc
+++ b/src/test/test_record_input_layer.cc
@@ -22,7 +22,7 @@
#include <vector>
#include "gtest/gtest.h"
-#include "singa/neuralnet/input_layer/record.h"
+#include "singa/neuralnet/input_layer.h"
#include "singa/proto/job.pb.h"
#include "singa/proto/common.pb.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index 12501fd..a4cd513 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -21,6 +21,7 @@
#include <cmath>
#include <algorithm>
#include "singa/utils/math_kernel.h"
+#include "mshadow/tensor.h" //FLT_MIN?
#define CU2DBLOCK_X 32
#define CU2DBLOCK_Y 32
@@ -31,6 +32,29 @@
// Cuda Kernel Functions
__global__
+void kernel_softmax_loss(const float *prob, const int *label ,
+ float *loss, int n, int dim) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ int num_threads = blockDim.x * gridDim.x;
+ for (; index < n; index += num_threads) {
+ const int label_value = static_cast<int>(label[index]);
+ float prob_of_truth = prob[index * dim + label_value];
+ loss[index] -= log(max(prob_of_truth, FLT_MIN));
+ }
+}
+
+__global__
+void kernel_softmax_gradient(float *grad, const int *label ,
+ int n, int dim, float scale) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ int num_threads = blockDim.x * gridDim.x;
+ for (; index < n; index += num_threads) {
+ int pos = index * dim + static_cast<int>(label[index]);
+ grad[pos] = (grad[pos] - 1.0f) * scale / (1.0 * n);
+ }
+}
+
+__global__
void kernel_sum_vec(float *data, float *sum , int n) {
int THREADS = blockDim.x;
@@ -293,6 +317,18 @@ void kernel_threshold(const float *src_data, float *des_data,
//
namespace singa {
+void singa_gpu_softmax_loss(const float *prob, const int *label,
+ float *loss, int n, int dim) {
+ kernel_softmax_loss<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>
+ (prob, label, loss, n, dim);
+}
+
+void singa_gpu_softmax_gradient(float *grad, const int *label,
+ int n, int dim, float scale) {
+ kernel_softmax_gradient<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>
+ (grad, label, n, dim, scale);
+}
+
void singa_gpu_sum_vec(float *data, float *sum , int n) {
int threads_per_block = n > CU1DBLOCK ? CU1DBLOCK : n;
// here, we only need one block
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f31ba645/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index 8c1d950..d9380b6 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -91,7 +91,7 @@ void Worker::Run() {
}
TrainOneBatch(step_, train_net_);
if (DisplayNow(step_) && grp_id_ == 0 && id_ == 0)
- Display(kTrain, "Train @ step " + std::to_string(step_), train_net_);
+ Display(kTrain | kForward | kBackward, "Train @ step " + std::to_string(step_), train_net_);
step_++;
}
@@ -297,7 +297,7 @@ void Worker::Display(int flag, const std::string& prefix, NeuralNet* net) {
if (job_conf_.debug()) {
const string& info = layer->ToString(true, flag);
if (info.length()) {
- LOG(INFO) << prefix << info;
+ LOG(INFO) << prefix << info;
}
}
}
[03/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/output_layer/csv.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer/csv.h b/include/singa/neuralnet/output_layer/csv.h
deleted file mode 100644
index 439db8c..0000000
--- a/include/singa/neuralnet/output_layer/csv.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_OUTPUT_LAYER_CSV_H_
-#define SINGA_NEURALNET_OUTPUT_LAYER_CSV_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/io/store.h"
-
-namespace singa {
-/**
- * Output data (and label) for its source layer.
- */
-class CSVOutputLayer : public OutputLayer {
- public:
- ~CSVOutputLayer() { delete store_; }
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- int inst_ = 0;
- io::Store* store_ = nullptr;
-};
-} // namespace singa
-#endif // SINGA_NEURALNET_OUTPUT_LAYER_CSV_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/output_layer/record.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer/record.h b/include/singa/neuralnet/output_layer/record.h
deleted file mode 100644
index cd652d5..0000000
--- a/include/singa/neuralnet/output_layer/record.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_OUTPUT_LAYER_RECORD_H_
-#define SINGA_NEURALNET_OUTPUT_LAYER_RECORD_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/io/store.h"
-
-namespace singa {
-
-class RecordOutputLayer : public OutputLayer {
- public:
- ~RecordOutputLayer() { delete store_; }
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- int inst_ = 0; //!< instance No.
- io::Store* store_ = nullptr;
-};
-} // namespace singa
-#endif // SINGA_NEURALNET_OUTPUT_LAYER_RECORD_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/connection_layer/bridge.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/bridge.cc b/src/neuralnet/connection_layer/bridge.cc
index 1ad4b0c..200a3f9 100644
--- a/src/neuralnet/connection_layer/bridge.cc
+++ b/src/neuralnet/connection_layer/bridge.cc
@@ -19,6 +19,7 @@
*
*************************************************************/
+<<<<<<< HEAD
#include "singa/neuralnet/connection_layer/bridge.h"
#include "singa/comm/msg.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index f9d6416..13a7ea7 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -19,12 +19,10 @@
*
*************************************************************/
-#include "singa/neuralnet/connection_layer/concate.h"
+#include "singa/neuralnet/connection_layer.h"
namespace singa {
-using std::vector;
-
void ConcateLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index c69f797..a607a95 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/connection_layer/slice.h"
+#include "singa/neuralnet/connection_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index 36b391c..7ee24fa 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/connection_layer/split.h"
+#include "singa/neuralnet/connection_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/csv.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/csv.cc b/src/neuralnet/input_layer/csv.cc
index 297d05f..53cabff 100644
--- a/src/neuralnet/input_layer/csv.cc
+++ b/src/neuralnet/input_layer/csv.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/csv.h"
+#include "singa/neuralnet/input_layer.h"
#include "singa/utils/tokenizer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
index b0baf1d..1760d4b 100644
--- a/src/neuralnet/input_layer/deprecated.cc
+++ b/src/neuralnet/input_layer/deprecated.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/deprecated.h"
+#include "singa/neuralnet/input_layer.h"
#include "mshadow/tensor.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/image_preprocess.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/image_preprocess.cc b/src/neuralnet/input_layer/image_preprocess.cc
index 110605b..576b096 100644
--- a/src/neuralnet/input_layer/image_preprocess.cc
+++ b/src/neuralnet/input_layer/image_preprocess.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/image_preprocess.h"
+#include "singa/neuralnet/input_layer.h"
#include "singa/utils/image_transform.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/prefetch.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/prefetch.cc b/src/neuralnet/input_layer/prefetch.cc
index 1348a37..9c7f2d9 100644
--- a/src/neuralnet/input_layer/prefetch.cc
+++ b/src/neuralnet/input_layer/prefetch.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/prefetch.h"
+#include "singa/neuralnet/input_layer.h"
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/record.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/record.cc b/src/neuralnet/input_layer/record.cc
index 1983e7b..2fb71c6 100644
--- a/src/neuralnet/input_layer/record.cc
+++ b/src/neuralnet/input_layer/record.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/record.h"
+#include "singa/neuralnet/input_layer.h"
namespace singa {
using std::string;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/input_layer/store.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/store.cc b/src/neuralnet/input_layer/store.cc
index 0355c13..b1b296e 100644
--- a/src/neuralnet/input_layer/store.cc
+++ b/src/neuralnet/input_layer/store.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/input_layer/store.h"
+#include "singa/neuralnet/input_layer.h"
namespace singa {
StoreInputLayer::~StoreInputLayer() {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/loss_layer/euclidean.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc
index daab15c..71e5bae 100644
--- a/src/neuralnet/loss_layer/euclidean.cc
+++ b/src/neuralnet/loss_layer/euclidean.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/loss_layer/euclidean.h"
+#include "singa/neuralnet/loss_layer.h"
#include <glog/logging.h>
#include "mshadow/tensor.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index 497c287..8c100ef 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/loss_layer/softmax.h"
+#include "singa/neuralnet/loss_layer.h"
#include <algorithm>
#include <glog/logging.h>
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/argsort.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/argsort.cc b/src/neuralnet/neuron_layer/argsort.cc
index d0e7f36..d1775c0 100644
--- a/src/neuralnet/neuron_layer/argsort.cc
+++ b/src/neuralnet/neuron_layer/argsort.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/argsort.h"
+#include "singa/neuralnet/output_layer.h"
#include <algorithm>
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/convolution.cc b/src/neuralnet/neuron_layer/convolution.cc
index fa74dcf..4247bee 100644
--- a/src/neuralnet/neuron_layer/convolution.cc
+++ b/src/neuralnet/neuron_layer/convolution.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/convolution.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
@@ -37,14 +37,40 @@ void ConvolutionLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
ConvolutionProto conv_conf = conf.convolution_conf();
- kernel_ = conv_conf.kernel();
- CHECK_GT(kernel_, 0) << "Filter size cannot be zero.";
- pad_ = conv_conf.pad();
- stride_ = conv_conf.stride();
+ if (conv_conf.has_kernel()) {
+ kernel_x_ = kernel_y_ = conv_conf.kernel();
+ } else {
+ CHECK(conv_conf.has_kernel_x());
+ CHECK(conv_conf.has_kernel_y());
+ kernel_x_ = conv_conf.kernel_x();
+ kernel_y_ = conv_conf.kernel_y();
+ }
+
+ if (conv_conf.has_pad()) {
+ pad_x_ = pad_y_ = conv_conf.pad();
+ } else {
+ CHECK(conv_conf.has_pad_x());
+ CHECK(conv_conf.has_pad_y());
+ pad_x_ = conv_conf.pad_x();
+ pad_y_ = conv_conf.pad_y();
+ }
+
+ if (conv_conf.has_stride()) {
+ stride_x_ = stride_y_ = conv_conf.stride();
+ } else {
+ CHECK(conv_conf.has_stride_x());
+ CHECK(conv_conf.has_stride_y());
+ stride_x_ = conv_conf.stride_x();
+ stride_y_ = conv_conf.stride_y();
+ }
+
num_filters_ = conv_conf.num_filters();
+ // partition filters
if (partition_dim() > 0)
num_filters_ /= srclayers.at(0)->num_partitions();
+
const vector<int>& srcshape = srclayers[0]->data(this).shape();
+ batchsize_ = srcshape[0];
int dim = srcshape.size();
CHECK_GT(dim, 2);
width_ = srcshape[dim - 1];
@@ -53,10 +79,10 @@ void ConvolutionLayer::Setup(const LayerProto& conf,
channels_ = srcshape[dim - 3];
else if (dim > 2)
channels_ = 1;
- batchsize_ = srcshape[0];
- conv_height_ = (height_ + 2 * pad_ - kernel_) / stride_ + 1;
- conv_width_ = (width_ + 2 * pad_ - kernel_) / stride_ + 1;
- col_height_ = channels_ * kernel_ * kernel_;
+
+ conv_height_ = (height_ + 2 * pad_y_ - kernel_y_) / stride_y_ + 1;
+ conv_width_ = (width_ + 2 * pad_x_ - kernel_x_) / stride_x_ + 1;
+ col_height_ = channels_ * kernel_x_ * kernel_y_;
col_width_ = conv_height_ * conv_width_;
vector<int> shape{batchsize_, num_filters_, conv_height_, conv_width_};
data_.Reshape(shape);
@@ -64,11 +90,14 @@ void ConvolutionLayer::Setup(const LayerProto& conf,
col_data_.Reshape(vector<int>{col_height_, col_width_});
col_grad_.Reshape(vector<int>{col_height_, col_width_});
weight_ = Param::Create(conf.param(0));
- bias_ = Param::Create(conf.param(1));
weight_->Setup(vector<int>{num_filters_, col_height_});
- bias_->Setup(vector<int>{num_filters_});
+ if (conf.param_size() > 1) {
+ bias_ = Param::Create(conf.param(1));
+ bias_->Setup(vector<int>{num_filters_});
+ }
}
+// TODO(wangwei) remove mshadow's functions
void ConvolutionLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
auto src = Tensor4(srclayers[0]->mutable_data(this));
@@ -78,9 +107,9 @@ void ConvolutionLayer::ComputeFeature(int flag,
auto bias = Tensor1(bias_->mutable_data());
for (int n = 0; n < batchsize_; n++) {
if (pad_ > 0)
- col = expr::unpack_patch2col(pad(src[n], pad_), kernel_, stride_);
+ col = expr::unpack_patch2col(pad(src[n], pad_x_), kernel_x_, stride_x_);
else
- col = expr::unpack_patch2col(src[n], kernel_, stride_);
+ col = expr::unpack_patch2col(src[n], kernel_x_, stride_x_);
data[n] = dot(weight, col);
}
data += expr::broadcast<1>(bias, data.shape);
@@ -107,13 +136,13 @@ void ConvolutionLayer::ComputeGradient(int flag,
Shape<2> imgshp = Shape2(height_, width_);
for (int n = 0; n < batchsize_; n++) {
if (pad_ > 0)
- col = expr::unpack_patch2col(pad(src[n], pad_), kernel_, stride_);
+ col = expr::unpack_patch2col(pad(src[n], pad_x_), kernel_x_, stride_x_);
else
- col = expr::unpack_patch2col(src[n], kernel_, stride_);
+ col = expr::unpack_patch2col(src[n], kernel_x_, stride_x_);
gweight += dot(grad[n], col.T());
if (gsrcblob != nullptr) {
gcol = dot(weight.T(), grad[n]);
- gsrc[n] = crop(expr::pack_col2patch(gcol, padshp, kernel_, stride_),
+ gsrc[n] = crop(expr::pack_col2patch(gcol, padshp, kernel_x_, stride_x_),
imgshp);
}
}
@@ -130,7 +159,7 @@ void CConvolutionLayer::ComputeFeature(int flag,
for (int n = 0; n < batchsize_; n++) {
Im2col(src[n].dptr, channels_, height_, width_,
- kernel_, kernel_, pad_, pad_, stride_, stride_, col.dptr);
+ kernel_x_, kernel_y_, pad_x_, pad_y_, stride_x_, stride_y_, col.dptr);
data[n] = dot(weight, col);
}
data += expr::broadcast<1>(bias, data.shape);
@@ -154,12 +183,13 @@ void CConvolutionLayer::ComputeGradient(int flag,
gbias = expr::sumall_except_dim<1>(grad);
for (int n = 0; n < batchsize_; n++) {
Im2col(src[n].dptr, channels_, height_, width_,
- kernel_, kernel_, pad_, pad_, stride_, stride_, col.dptr);
+ kernel_x_, kernel_y_, pad_x_, pad_y_, stride_x_, stride_y_, col.dptr);
gweight += dot(grad[n], col.T());
if (gsrcblob != nullptr) {
gcol = dot(weight.T(), grad[n]);
Col2im(gcol.dptr, channels_, height_, width_,
- kernel_, kernel_, pad_, pad_, stride_, stride_, gsrc[n].dptr);
+ kernel_x_, kernel_y_, pad_x_, pad_y_, stride_x_, stride_y_,
+ gsrc[n].dptr);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/cudnn_activation.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cu b/src/neuralnet/neuron_layer/cudnn_activation.cu
new file mode 100644
index 0000000..f77a8a8
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_activation.cu
@@ -0,0 +1,100 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+void CudnnActivationLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+
+ // TODO(wangwei) make the mode case insensitive
+ if (layer_conf_.activation_conf().mode() == "sigmoid")
+ mode_ = CUDNN_ACTIVATION_SIGMOID;
+ else if (layer_conf_.activation_conf().mode() == "tanh")
+ mode_ = CUDNN_ACTIVATION_TANH;
+ else if (layer_conf_.activation_conf().mode() == "relu")
+ mode_ = CUDNN_ACTIVATION_RELU;
+ else {
+ LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().mode();
+ }
+
+ const auto& shape = data_.shape();
+ CHECK_GT(shape.size(), 0);
+ // size of each dimension
+ int* sdim= new int[shape.size()];
+ int* stride = new int[shape.size()];
+ stride[shape.size() -1] = 1;
+ int i = shape.size() - 1;
+ sdim[i] = shape[i];
+ stride[i] = 1;
+ for (--i; i >= 0; i--) {
+ sdim[i] = shape[i];
+ stride[i] = shape[i + 1] * stride[i + 1]
+ }
+ CHECK_CUDNN(cudnnSetTensorNdDescriptor(src_desc_,
+ CUDNN_DATA_FLOAT,
+ shape.size(),
+ sdim,
+ stride));
+ CHECK_CUDNN(cudnnSetTensorNdDescriptor(my_desc_,
+ CUDNN_DATA_FLOAT,
+ shape.size(),
+ sdim,
+ stride));
+ delete[] sdim;
+ delete[] stride;
+}
+
+void CudnnActivationLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
+ // currently only consider single src layer
+ CHECK_EQ(srclayers.size(), 1);
+ CHECK_CUDNN(cudnnActivationForward(handle_,
+ mode_,
+ &alpha,
+ src_desc_,
+ srclayers[0].data(this)->gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CudnnActivationLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnActivationBackward(handle_,
+ mode_,
+ &alpha,
+ my_desc_,
+ data_.gpu_data()
+ my_desc_,
+ grad_.gpu_data(),
+ src_desc_,
+ srclayers[0].data(this)->gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0].mutable_grad(this)->mutable_gpu_data()));
+}
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/cudnn_convolution.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cu b/src/neuralnet/neuron_layer/cudnn_convolution.cu
new file mode 100644
index 0000000..debe4c3
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_convolution.cu
@@ -0,0 +1,205 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+CudnnConvLayer::~CudnnConvLayer() {
+ if (has_init_cudnn_) {
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(bias_desc_));
+ CHECK_CUDNN(cudnnDestroyFilterDescriptor(filter_desc_));
+ CHECK_CUDNN(cudnnDestroyConvolutionDescriptor(conv_desc_));
+ }
+}
+
+void CudnnConvLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ // convert MB to bytes
+ workspace_byte_limit_ = layer_conf_.convolution_conf().workspace_byte_limit() << 20;
+
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&bias_desc_));
+ CHECK_CUDNN(cudnnCreateFilterDescriptor(&filter_desc_));
+ CHECK_CUDNN(cudnnCreateConvolutionDescriptor(&conv_desc_));
+
+ CHECK_CUDNN(cudnnSetConvolution2dDescriptor(conv_desc_,
+ pad_y_,
+ pad_x_,
+ stride_y_,
+ stride_x_,
+ 1,
+ 1,
+ CUDNN_CROSS_CORRELATION));
+ CHECK_CUDNN(cudnnSetFilter4dDescriptor(filter_desc_,
+ CUDNN_DATA_FLOAT,
+ num_filters_,
+ channels_,
+ kernel_y_,
+ kernel_x_));
+ if (bias_) {
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(bias_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ 1,
+ num_filters_,
+ 1,
+ 1));
+ }
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(data_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_filters_,
+ conv_height_,
+ conv_width_));
+
+ CHECK_CUDNN(cudnnGetConvolutionForwardAlgorithm(handle_,
+ src_desc_,
+ filter_desc_,
+ conv_desc_,
+ my_desc_,
+ CUDNN_CONVOLUTION_FWD_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &fp_alg_));
+
+ CHECK_CUDNN(cudnnGetConvolutionBackwardFilterAlgorithm(handle_,
+ src_desc_,
+ my_desc_,
+ conv_desc_,
+ filter_desc_,
+ CUDNN_CONVOLUTION_BWD_FILTER_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &bp_filter_alg_));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardDataAlgorithm(handle_,
+ filter_desc_,
+ my_desc_,
+ conv_desc_,
+ src_desc_,
+ CUDNN_CONVOLUTION_BWD_DATA_PREFER_FASTEST,
+ workspace_byte_limit_,
+ &bp_data_alg_));
+
+ size_t fp_byte, bp_data_byte, bp_filter_byte;
+ CHECK_CUDNN(cudnnGetConvolutionForwardWorkspaceSize(handle_,
+ src_desc_,
+ filter_desc_,
+ conv_desc_,
+ my_desc_,
+ fp_alg_,
+ &fp_byte));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardDataWorkspaceSize(handle_,
+ filter_desc_,
+ my_desc_,
+ conv_desc_,
+ src_desc_,
+ bp_data_alg_,
+ &bp_data_byte));
+ CHECK_CUDNN(cudnnGetConvolutionBackwardFilterWorkspaceSize(handle_,
+ src_desc_,
+ my_desc_,
+ conv_desc_,
+ filter_desc_,
+ bp_filter_alg_,
+ &bp_filter_byte));
+ workspace_count_ = std::max(std::max(fp_bypte, bp_data_byte), bp_filter_byte)
+ / sizeof(float) + 1;
+}
+
+void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.f, beta = 0.f;
+ Blob<float> workspace(vector<int>{workspace_count_});
+ CUDNN_CHECK(cudnnConvolutionForward(handle_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ filter_desc_,
+ weight_->data().gpu_data(),
+ conv_descs_,
+ fp_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ data_desc_,
+ data_.mutable_gpu_data()));
+
+ if (bias_) {
+ beta = 1.f;
+ CUDNN_CHECK(cudnnAddTensor(handle_,
+ CUDNN_ADD_SAME_C,
+ &alpha,
+ bias_desc_,
+ bias_->data().gpu_data(),
+ &beta,
+ data_desc_,
+ data_.mutable_gpu_data()));
+ }
+}
+
+void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+{
+ float alpha = 1.f, beta = 0.f;
+ Blob<float> workspace(vector<int>{workspace_count_});
+ if (bias_) {
+ CHECK_CUDNN(cudnnConvolutionBackwardBias(handle_,
+ &alpha,
+ data_desc_,
+ grad_.gpu_data(),
+ &beta,
+ bias_desc_,
+ bias_->mutable_grad()->mutable_gpu_data()));
+ }
+ CHECK_CUDNN(cudnnConvolutionBackwardFilter_v3(handle_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this)->gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ conv_desc_,
+ bp_filter_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ filter_desc_,
+ weight_->mutable_grad()->mutable_gpu_data()));
+ CHECK_CUDNN(cudnnConvolutionBackwardData_v3(handle_,
+ &alpha,
+ filter_desc_,
+ weight_->data()->gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ conv_desc_,
+ bp_data_alg_,
+ workspace.mutable_gpu_data(),
+ workspace_count_ * sizeof(float),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+}
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/cudnn_lrn.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cu b/src/neuralnet/neuron_layer/cudnn_lrn.cu
new file mode 100644
index 0000000..7cb111a
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_lrn.cu
@@ -0,0 +1,95 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+CudnnLRNLayer::~CudnnLRNLayer() {
+ if (!init_cudnn_) {
+ cudnnDestroyLRNDescriptor(norm_desc_);
+ }
+}
+
+void CudnnLRNLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ LRNLayer::Setup(proto, srclayers);
+ mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
+}
+
+void CudnnLRNLayer::InitCudnn() {
+ CudnnLayer::InitCudnn(srclayers);
+ CHECK_EQ(cudnnCreateLRNDescriptor(&norm_desc_), CUDNN_STATUS_SUCCESS);
+ CHECK_EQ(cudnnSetLRNDescriptor(norm_desc_,
+ lsize_,
+ alpha_,
+ beta_,
+ knorm_), CUDNN_STATUS_SUCCESS);
+ CHECK_EQ(cudnnCreateTensorDescriptor(&src_desc_), CUDNN_STATUS_SUCCESS);
+ CHECK_EQ(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_), CUDNN_STATUS_SUCCESS);
+ CHECK_EQ(cudnnCreateTensorDescriptor(&my_desc_), CUDNN_STATUS_SUCCESS);
+ CHECK_EQ(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_), CUDNN_STATUS_SUCCESS);
+}
+void ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (init_cudnn_) {
+ InitCudnn();
+ init_cudnn_ = false;
+ }
+ CHECK_EQ(cudnnLRNCrossChannelForward(handle_,
+ norm_desc_,
+ mode_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()), CUDNN_STATUS_SUCCESS);
+}
+void ComputeGradient(int flag, const vector<Layer*>& srclayers) {
+ CHECK_EQ(cudnnLRNCrossChannelBackward(handle_,
+ norm_desc_,
+ mode_,
+ &alpha,
+ my_desc_, // ???
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data()
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()),
+ CUDNN_STATUS_SUCCESS);
+}
+
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/cudnn_pooling.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cu b/src/neuralnet/neuron_layer/cudnn_pooling.cu
new file mode 100644
index 0000000..619998f
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_pooling.cu
@@ -0,0 +1,95 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+CuDNNPoolLayer::~CuDNNPoolLayer() {
+ if (has_init_cudnn_) {
+ CHECK_EQ(cudnnDestroyPoolingDescriptor(pool_desc_), CUDNN_STATUS_SUCCESS);
+ }
+}
+
+void CuDNNPoolLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnCreatePoolingDescriptor(&pool_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ height_,
+ width_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ channels_,
+ pooled_height_,
+ pooled_width_));
+ auto pool_method = CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING;
+ if (pool_ == PoolingProto_PoolMethod_MAX)
+ pool_method = CUDNN_POOLING_MAX;
+ CHECK_CUDNN(cudnnSetPooling2dDescriptor(pool_desc_,
+ pool_method,
+ kernel_y_,
+ kernel_x_,
+ pad_y_,
+ pad_x_,
+ stride_y_,
+ stride_x_));
+}
+
+void CuDNNPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
+ // currently only consider single src layer
+ CHECK_EQ(srclayers.size(), 1);
+ CHECK_CUDNN(cudnnPoolingForward(handle_,
+ pool_desc_,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ data_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CuDNNPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+{
+ float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnPoolingBackward(handle_,
+ pool_desc_,
+ &alpha,
+ my_desc_,
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this).mutable_gpu_data()));
+}
+} /* singa */
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/cudnn_softmax.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cu b/src/neuralnet/neuron_layer/cudnn_softmax.cu
new file mode 100644
index 0000000..f7e8abe
--- /dev/null
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cu
@@ -0,0 +1,75 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer.h"
+
+namespace singa {
+
+void CudnnSoftmaxLayer::InitCudnn() {
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_softmax_per_instance_,
+ count_per_softmax_,
+ 1));
+ CHECK_EQ(cudnnSetTensor4dDescriptor(my_desc_,
+ CUDNN_TENSOR_NCHW,
+ CUDNN_DATA_FLOAT,
+ batchsize_,
+ num_softmax_per_instance_,
+ count_per_softmax_,
+ 1));
+}
+
+void CudnnSoftmaxLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
+ InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(CudnnSoftmaxForward(handle_,
+ CUDNN_SOFTMAX_ACCURATE,
+ CUDNN_SOFTMAX_CHANNEL,
+ &alpha,
+ src_desc_,
+ srclayers[0]->data(this).gpu_data(),
+ &beta,
+ my_desc_,
+ data_.mutable_gpu_data()));
+}
+
+void CudnnSoftmaxLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ float alpha = 1.f, beta = 0.f;
+ CHECK_CUDNN(CudnnSoftmaxForward(handle_,
+ CUDNN_SOFTMAX_ACCURATE,
+ CUDNN_SOFTMAX_CHANNEL,
+ &alpha,
+ my_desc_,
+ data_.gpu_data(),
+ my_desc_,
+ grad_.gpu_data(),
+ &beta,
+ src_desc_,
+ srclayers[0]->mutable_grad(this).mutable_gpu_data()));
+}
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/dropout.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dropout.cc b/src/neuralnet/neuron_layer/dropout.cc
index 59d9bc5..6158a6c 100644
--- a/src/neuralnet/neuron_layer/dropout.cc
+++ b/src/neuralnet/neuron_layer/dropout.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/dropout.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/inner_product.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/inner_product.cc b/src/neuralnet/neuron_layer/inner_product.cc
index 3f38d9c..25b71df 100644
--- a/src/neuralnet/neuron_layer/inner_product.cc
+++ b/src/neuralnet/neuron_layer/inner_product.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/inner_product.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/lrn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/lrn.cc b/src/neuralnet/neuron_layer/lrn.cc
index f1c302f..178b2bf 100644
--- a/src/neuralnet/neuron_layer/lrn.cc
+++ b/src/neuralnet/neuron_layer/lrn.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/lrn.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/pooling.cc b/src/neuralnet/neuron_layer/pooling.cc
index a3c5e6f..fd475d8 100644
--- a/src/neuralnet/neuron_layer/pooling.cc
+++ b/src/neuralnet/neuron_layer/pooling.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/pooling.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/rbm.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/rbm.cc b/src/neuralnet/neuron_layer/rbm.cc
index fc15e79..53a1bd9 100644
--- a/src/neuralnet/neuron_layer/rbm.cc
+++ b/src/neuralnet/neuron_layer/rbm.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/rbm.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/relu.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/relu.cc b/src/neuralnet/neuron_layer/relu.cc
index 247d077..1e030a0 100644
--- a/src/neuralnet/neuron_layer/relu.cc
+++ b/src/neuralnet/neuron_layer/relu.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/relu.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/sigmoid.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/sigmoid.cc b/src/neuralnet/neuron_layer/sigmoid.cc
index 3ddefd8..c449b36 100644
--- a/src/neuralnet/neuron_layer/sigmoid.cc
+++ b/src/neuralnet/neuron_layer/sigmoid.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/sigmoid.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
#include "singa/utils/singleton.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/softmax.cc b/src/neuralnet/neuron_layer/softmax.cc
index be2f337..83bbc5a 100644
--- a/src/neuralnet/neuron_layer/softmax.cc
+++ b/src/neuralnet/neuron_layer/softmax.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/softmax.h"
+#include "singa/neuralnet/neuron_layer.h"
namespace singa {
@@ -37,12 +37,17 @@ void SoftmaxLayer::Setup(const LayerProto& proto,
const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
NeuronLayer::Setup(proto, srclayers);
- data_.Reshape(srclayers[0]->data(this).shape());
+ const auto& srcdata = srclayers[0]->data(this);
+ batchsize_ = data_.shape()[0];
+ num_softmax_per_instance_ = proto.softmax_conf().softmax_dim();
+ count_per_softmax_ = .count() / batchsize_ / num_softmax_per_instance_;
+ data_.Reshape(vector<int>{batchsize_, num_softmax_per_instance_,
+ count_per_softmax_});
+ grad_.ReshapeLike(data_);
}
void SoftmaxLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
- int batchsize = data_.shape()[0];
int dim = data_.count() / batchsize;
Shape<2> s = Shape2(batchsize, dim);
Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/neuron_layer/stanh.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/stanh.cc b/src/neuralnet/neuron_layer/stanh.cc
index 3036850..70b9cd1 100644
--- a/src/neuralnet/neuron_layer/stanh.cc
+++ b/src/neuralnet/neuron_layer/stanh.cc
@@ -19,10 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/stanh.h"
-
-#include <glog/logging.h>
-#include "singa/utils/singleton.h"
+#include "singa/neuralnet/neuron_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/output_layer/csv.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/csv.cc b/src/neuralnet/output_layer/csv.cc
index 856b95f..d2512da 100644
--- a/src/neuralnet/output_layer/csv.cc
+++ b/src/neuralnet/output_layer/csv.cc
@@ -18,7 +18,7 @@
* under the License.
*
*************************************************************/
-#include "singa/neuralnet/output_layer/csv.h"
+#include "singa/neuralnet/output_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/neuralnet/output_layer/record.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/record.cc b/src/neuralnet/output_layer/record.cc
index 727f539..f7b3e01 100644
--- a/src/neuralnet/output_layer/record.cc
+++ b/src/neuralnet/output_layer/record.cc
@@ -18,7 +18,7 @@
* under the License.
*
*************************************************************/
-#include "singa/neuralnet/output_layer/record.h"
+#include "singa/neuralnet/output_layer.h"
#include "singa/proto/common.pb.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index ae461f8..ca5e546 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -349,8 +349,9 @@ message ConvolutionProto {
optional int32 pad = 30 [default = 0];
// the stride
optional int32 stride = 31 [default = 1];
- // whether to have bias terms
- optional bool bias_term = 32 [default = true];
+
+ // cudnn workspace size in MB
+ optional int32 workspace_byte_limit = 33 [default = 512];
}
message DataProto {
@@ -443,6 +444,16 @@ message ReLUProto {
optional float negative_slope = 1 [default = 0];
}
+message SliceProto {
+ required int32 slice_dim = 1;
+}
+
+message SoftmaxProto {
+ // Can be used to do softmax over each channel of one image by setting it to
+ // be the size of the second dimension (the first dimension is batchsize).
+ optional int32 num_softmax_per_instance = 1 [default = 1];
+}
+
message RMSPropProto {
// history=history*rho_+(1-rho_)*(grad*grad_scale);
required float rho = 1;
[04/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
* Add cudnn layers, including convoltuion, pooling, lrn, and activation (relu, sigmoid, tanh)
* move declarations of layers from single file into <category>_layer.h under include/singa/neuralnet/
TODO compile the code
with '#' will be ignored, and an empty message aborts the commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/af1bf509
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/af1bf509
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/af1bf509
Branch: refs/heads/master
Commit: af1bf509d7c7cb5b3d597fc0ff59fc940187e3a4
Parents: 1981874
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Wed Nov 4 21:02:15 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:01:29 2015 +0800
----------------------------------------------------------------------
include/singa/neuralnet/connection_layer.h | 127 ++++++
.../singa/neuralnet/connection_layer/bridge.h | 78 ----
.../singa/neuralnet/connection_layer/concate.h | 44 --
include/singa/neuralnet/input_layer.h | 305 ++++++++++++++
include/singa/neuralnet/input_layer/csv.h | 55 ---
.../singa/neuralnet/input_layer/deprecated.h | 171 --------
.../neuralnet/input_layer/image_preprocess.h | 47 ---
include/singa/neuralnet/input_layer/prefetch.h | 50 ---
include/singa/neuralnet/input_layer/record.h | 56 ---
include/singa/neuralnet/input_layer/store.h | 89 ----
include/singa/neuralnet/layer.h | 22 +-
include/singa/neuralnet/loss_layer.h | 56 +++
include/singa/neuralnet/loss_layer/euclidean.h | 43 --
include/singa/neuralnet/loss_layer/softmax.h | 59 ---
include/singa/neuralnet/neuron_layer.h | 410 +++++++++++++++++++
include/singa/neuralnet/neuron_layer/argsort.h | 52 ---
.../singa/neuralnet/neuron_layer/convolution.h | 68 ---
include/singa/neuralnet/neuron_layer/dropout.h | 46 ---
.../neuralnet/neuron_layer/inner_product.h | 49 ---
include/singa/neuralnet/neuron_layer/lrn.h | 56 ---
include/singa/neuralnet/neuron_layer/pooling.h | 56 ---
include/singa/neuralnet/neuron_layer/relu.h | 39 --
include/singa/neuralnet/neuron_layer/softmax.h | 40 --
include/singa/neuralnet/neuron_layer/stanh.h | 43 --
include/singa/neuralnet/output_layer.h | 72 ++++
include/singa/neuralnet/output_layer/csv.h | 44 --
include/singa/neuralnet/output_layer/record.h | 42 --
src/neuralnet/connection_layer/bridge.cc | 1 +
src/neuralnet/connection_layer/concate.cc | 4 +-
src/neuralnet/connection_layer/slice.cc | 2 +-
src/neuralnet/connection_layer/split.cc | 2 +-
src/neuralnet/input_layer/csv.cc | 2 +-
src/neuralnet/input_layer/deprecated.cc | 2 +-
src/neuralnet/input_layer/image_preprocess.cc | 2 +-
src/neuralnet/input_layer/prefetch.cc | 2 +-
src/neuralnet/input_layer/record.cc | 2 +-
src/neuralnet/input_layer/store.cc | 2 +-
src/neuralnet/loss_layer/euclidean.cc | 2 +-
src/neuralnet/loss_layer/softmax.cc | 2 +-
src/neuralnet/neuron_layer/argsort.cc | 2 +-
src/neuralnet/neuron_layer/convolution.cc | 68 ++-
src/neuralnet/neuron_layer/cudnn_activation.cu | 100 +++++
src/neuralnet/neuron_layer/cudnn_convolution.cu | 205 ++++++++++
src/neuralnet/neuron_layer/cudnn_lrn.cu | 95 +++++
src/neuralnet/neuron_layer/cudnn_pooling.cu | 95 +++++
src/neuralnet/neuron_layer/cudnn_softmax.cu | 75 ++++
src/neuralnet/neuron_layer/dropout.cc | 2 +-
src/neuralnet/neuron_layer/inner_product.cc | 2 +-
src/neuralnet/neuron_layer/lrn.cc | 2 +-
src/neuralnet/neuron_layer/pooling.cc | 2 +-
src/neuralnet/neuron_layer/rbm.cc | 2 +-
src/neuralnet/neuron_layer/relu.cc | 2 +-
src/neuralnet/neuron_layer/sigmoid.cc | 2 +-
src/neuralnet/neuron_layer/softmax.cc | 11 +-
src/neuralnet/neuron_layer/stanh.cc | 5 +-
src/neuralnet/output_layer/csv.cc | 2 +-
src/neuralnet/output_layer/record.cc | 2 +-
src/proto/job.proto | 15 +-
58 files changed, 1644 insertions(+), 1289 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/connection_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h
new file mode 100644
index 0000000..0b14a94
--- /dev/null
+++ b/include/singa/neuralnet/connection_layer.h
@@ -0,0 +1,127 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_CONNECTION_LAYER_H_
+#define SINGA_NEURALNET_CONNECTION_LAYER_H_
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "singa/comm/socket.h"
+#include "singa/neuralnet/layer.h"
+
+namespace singa {
+
+class BridgeLayer : public ConnectionLayer {
+ public:
+ void set_ready(bool a) { ready_ = a; }
+ bool ready() const { return ready_; }
+ // Bind the layer with dealer instance by worker at runtime
+ void MakePaired(Layer* pair, int grp_id, Dealer* dealer,
+ std::unordered_map<std::string, Layer*>* name2bridge);
+ // Send blobs to other workers due to model partitions
+ void SendBlobs(bool handle_data);
+ // Receive blobs from other workers due to model partitions;
+ void ReceiveBlobs(bool handle_data);
+
+ protected:
+ //!< true if received grad from BridgeDstLayer
+ bool ready_ = false;
+ int group_id_ = 0;
+ Layer* pair_ = nullptr;
+ Dealer* dealer_ = nullptr;
+ std::unordered_map<std::string, Layer*>* name2bridge_ = nullptr;
+};
+
+/**
+ * For sending data to layer on other threads which may resident on other nodes
+ * due to layer/data partition.
+ */
+class BridgeSrcLayer : public BridgeLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/**
+ * For recv data from layer on other threads which may resident on other nodes
+ * due to layer/data partiton
+ */
+class BridgeDstLayer : public BridgeLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+/**
+ * Connect multiple (src) layers with a single (dst) layer.
+ *
+ * It concates feature Blobs (i.e., matrix) of src layers on one dimension.
+ * The concated feature Blob will be fed into the dst layer.
+ */
+class ConcateLayer : public ConnectionLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/**
+ * Connect a single (src) layer with multiple (dst) layers.
+ *
+ * It slices the feature Blob (i.e., matrix) of the src layer on one dimension.
+ * The sliced feature Blobs will be fed into dst layers.
+ */
+class SliceLayer : public ConnectionLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ std::vector<Blob<float>> datavec_;
+ std::vector<Blob<float>> gradvec_;
+ int slice_dim_;
+ int slice_num_;
+};
+
+/**
+ * Connect a single (src) layer with multiple dst layers.
+ *
+ * It replicates the feature Blob of the src layer.
+ * Each replicated feature Blob will be fed into one dst layer.
+ * It aggregates gradients set by all dst layers and set it to the src layer.
+ */
+class SplitLayer : public ConnectionLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ Blob<float> grads_;
+};
+
+
+} // namespace singa
+
+#endif // SINGA_NEURALNET_CONNECTION_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/connection_layer/bridge.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/bridge.h b/include/singa/neuralnet/connection_layer/bridge.h
deleted file mode 100644
index b27693d..0000000
--- a/include/singa/neuralnet/connection_layer/bridge.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_CONNECTION_LAYER_BRIDGE_H_
-#define SINGA_NEURALNET_CONNECTION_LAYER_BRIDGE_H_
-
-#include <string>
-#include <unordered_map>
-#include <vector>
-#include "singa/comm/socket.h"
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-
-class BridgeLayer : public ConnectionLayer {
- public:
- void set_ready(bool a) { ready_ = a; }
- bool ready() const { return ready_; }
- // Bind the layer with dealer instance by worker at runtime
- void MakePaired(Layer* pair, int grp_id, Dealer* dealer,
- std::unordered_map<std::string, Layer*>* name2bridge);
- // Send blobs to other workers due to model partitions
- void SendBlobs(bool handle_data);
- // Receive blobs from other workers due to model partitions;
- void ReceiveBlobs(bool handle_data);
-
- protected:
- //!< true if received grad from BridgeDstLayer
- bool ready_ = false;
- int group_id_ = 0;
- Layer* pair_ = nullptr;
- Dealer* dealer_ = nullptr;
- std::unordered_map<std::string, Layer*>* name2bridge_ = nullptr;
-};
-
-/**
- * For sending data to layer on other threads which may resident on other nodes
- * due to layer/data partition.
- */
-class BridgeSrcLayer : public BridgeLayer {
- public:
- void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-/**
- * For recv data from layer on other threads which may resident on other nodes
- * due to layer/data partiton
- */
-class BridgeDstLayer : public BridgeLayer {
- public:
- void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_CONNECTION_LAYER_BRIDGE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/connection_layer/concate.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h
deleted file mode 100644
index 5875835..0000000
--- a/include/singa/neuralnet/connection_layer/concate.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_CONNECTION_LAYER_CONCATE_H_
-#define SINGA_NEURALNET_CONNECTION_LAYER_CONCATE_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-/**
- * Connect multiple (src) layers with a single (dst) layer.
- *
- * It concates feature Blobs (i.e., matrix) of src layers on one dimension.
- * The concated feature Blob will be fed into the dst layer.
- */
-class ConcateLayer : public ConnectionLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_CONNECTION_LAYER_CONCATE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer.h b/include/singa/neuralnet/input_layer.h
new file mode 100644
index 0000000..72593d5
--- /dev/null
+++ b/include/singa/neuralnet/input_layer.h
@@ -0,0 +1,305 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_INPUT_LAYER_H_
+#define SINGA_NEURALNET_INPUT_LAYER_H_
+
+#include <string>
+#include <vector>
+#include <thread>
+#include "singa/io/store.h"
+#include "singa/neuralnet/layer.h"
+
+namespace singa {
+
+/**
+ * Base class for loading data from Store.
+ */
+class StoreInputLayer : virtual public InputLayer {
+ public:
+ ~StoreInputLayer();
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+
+ protected:
+ /**
+ * Parsing the (key, val) tuple to get feature (and label).
+ * Subclasses must implment this function.
+ * @param[in] k parse this tuple as the k-th instance of one mini-batch.
+ * @param[in] flag used to guide the parsing, e.g., kDeploy phase should not
+ * parse labels from the tuple.
+ * @param[in] key
+ * @param[in] val
+ */
+ virtual bool Parse(int k, int flag, const string& key, const string& val) = 0;
+
+ protected:
+ int batchsize_ = 1;
+ int random_skip_ = 0;
+ io::Store* store_ = nullptr;
+};
+
+/**
+ * Base layer for parsing a key-value tuple as a feature vector with fixed
+ * length. The feature shape is indicated by users in the configuration.
+ * Each tuple may has a label.
+ */
+class SingleLabelRecordLayer : public StoreInputLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ /**
+ * Load a single record (tuple), e.g., the mean or standard variance vector.
+ */
+ virtual void LoadRecord(const string& backend, const string& path,
+ Blob<float>* to) = 0;
+
+ protected:
+ /**
+ * Feature standardization by processing each feature dimension via
+ * @f$ y = (x - mu)/ std @f$
+ * <a href= "http://ufldl.stanford.edu/wiki/index.php/Data_Preprocessing">
+ * UFLDL</a>
+ */
+ Blob<float> mean_, std_;
+};
+/**
+ * Specific layer that parses the value string loaded by Store as a line from
+ * a CSV file.
+ *
+ * It assumes the first column is the label except that has_label_ is configured
+ * to false. Or the data is used in deploy mode.
+ */
+class CSVInputLayer : public SingleLabelRecordLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+
+ protected:
+ bool Parse(int k, int flag, const string& key, const string& val) override;
+ void LoadRecord(const string& backend,
+ const string& path,
+ Blob<float>* to) override;
+
+ private:
+ std::string sep_;
+ bool has_label_;
+};
+
+
+/**
+ * Specific layer that parses the value string loaded by Store into a
+ * RecordProto.
+ */
+class RecordInputLayer : public SingleLabelRecordLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+
+ protected:
+ /**
+ * Parse key as instance ID and val into RecordProto.
+ * @copydetails StoreInputLayer::Parse()
+ */
+ bool Parse(int k, int flag, const string& key, const string& val) override;
+ void LoadRecord(const string& backend,
+ const string& path,
+ Blob<float>* to) override;
+
+ private:
+ // TODO(wangwei) decode the image
+ bool encoded_;
+};
+
+/**
+ * Do preprocessing for images, including cropping, mirroring, resizing.
+ */
+class ImagePreprocessLayer : public InputLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers);
+
+ private:
+ bool mirror_ = false;
+ int cropsize_ = 0;
+ int resize_ = 0;
+ float scale_ = 1;
+};
+
+/**
+ * TODO(wangwei) Layer for prefetching data records and parsing them.
+ *
+ * This layer controls the prefetching thread, i.e.,
+ * creating and joining the prefetching thread.
+ */
+class PrefetchLayer : public Layer {
+ public:
+ ~PrefetchLayer();
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
+
+ protected:
+ std::thread thread_;
+};
+
+/****************Deprecated layers******************/
+/**
+ * @deprecated please use the StoreInputLayer.
+ *
+ * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc.
+ */
+class DataLayer: virtual public InputLayer {
+ public:
+ Blob<float>* mutable_data(const Layer* layer) override { return nullptr; }
+ ConnectionType dst_layer_connection() const override {
+ return kOneToMany;
+ }
+
+ inline int batchsize() const { return batchsize_; }
+ virtual const Record& sample() const {
+ return sample_;
+ }
+ /**
+ * @return the loaded records
+ */
+ virtual const std::vector<Record>& records() const {
+ return records_;
+ }
+
+ protected:
+ int random_skip_;
+ int batchsize_;
+ Record sample_;
+ std::vector<Record> records_;
+};
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer.
+ *
+ * Layer for loading Record from DataShard.
+ */
+class ShardDataLayer : public DataLayer {
+ public:
+ ~ShardDataLayer();
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ DataShard* shard_;
+};
+/**
+ * @deprecated please use the subclasses of StoreInputLayer.
+ *
+ * Layer for loading Record from LMDB.
+ */
+#ifdef USE_LMDB
+#include <lmdb.h>
+class LMDBDataLayer : public DataLayer {
+ public:
+ ~LMDBDataLayer();
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void OpenLMDB(const std::string& path);
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
+ SingleLabelImageRecord* record);
+
+ private:
+ MDB_env* mdb_env_;
+ MDB_dbi mdb_dbi_;
+ MDB_txn* mdb_txn_;
+ MDB_cursor* mdb_cursor_;
+ MDB_val mdb_key_, mdb_value_;
+};
+#endif
+
+/******************Parser layers***************/
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer which load and parse
+ * data in a single layer.
+ *
+ * Base layer for parsing the input records into Blobs.
+ */
+class ParserLayer : public InputLayer {
+ public:
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
+ ConnectionType dst_layer_connection() const override {
+ return kOneToMany;
+ }
+ /**
+ * Parse records from DataLayer into blob.
+ */
+ virtual void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) = 0;
+};
+/**
+ *
+ * @deprecated Please use the SingleLabelRecordLayer which parses both feature
+ * and label for each record. Its aux_data() function returns the parsed labels.
+ *
+ * Derived from ParserLayer to parse label in SingaleLabelImageRecord loaded by
+ * ShardDataLayer.
+ */
+class LabelLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+};
+
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer.
+ *
+ * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord.
+ */
+class MnistLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+
+ protected:
+ float norm_a_, norm_b_;
+};
+/**
+ * @deprecated please use the ImagePreprocessLayer which preprocess image
+ * feature from data Blob of source layers.
+ *
+ * Derived from ParserLayer to parse RGB image feature from
+ * SingaleLabelImageRecord.
+ */
+class RGBImageLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+
+ private:
+ float scale_;
+ int cropsize_;
+ bool mirror_;
+ Blob<float> mean_;
+};
+} // namespace singa
+
+#endif // SINGA_NEURALNET_INPUT_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/csv.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/csv.h b/include/singa/neuralnet/input_layer/csv.h
deleted file mode 100644
index 32c0f76..0000000
--- a/include/singa/neuralnet/input_layer/csv.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_CSV_H_
-#define SINGA_NEURALNET_INPUT_LAYER_CSV_H_
-
-#include <string>
-#include <vector>
-#include "singa/neuralnet/input_layer/store.h"
-
-namespace singa {
-
-/**
- * Specific layer that parses the value string loaded by Store as a line from
- * a CSV file.
- *
- * It assumes the first column is the label except that has_label_ is configured
- * to false. Or the data is used in deploy mode.
- */
-class CSVInputLayer : public SingleLabelRecordLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
-
- protected:
- bool Parse(int k, int flag, const string& key, const string& val) override;
- void LoadRecord(const string& backend,
- const string& path,
- Blob<float>* to) override;
-
- private:
- std::string sep_;
- bool has_label_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_CSV_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/deprecated.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/deprecated.h b/include/singa/neuralnet/input_layer/deprecated.h
deleted file mode 100644
index 2da590b..0000000
--- a/include/singa/neuralnet/input_layer/deprecated.h
+++ /dev/null
@@ -1,171 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
-#define SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
-
-#include <string>
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/io/kvfile.h"
-
-namespace singa {
-/**
- * @deprecated please use the StoreInputLayer.
- *
- * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc.
- */
-class DataLayer: virtual public InputLayer {
- public:
- Blob<float>* mutable_data(const Layer* layer) override { return nullptr; }
- ConnectionType dst_layer_connection() const override {
- return kOneToMany;
- }
-
- inline int batchsize() const { return batchsize_; }
- virtual const Record& sample() const {
- return sample_;
- }
- /**
- * @return the loaded records
- */
- virtual const std::vector<Record>& records() const {
- return records_;
- }
-
- protected:
- int random_skip_;
- int batchsize_;
- Record sample_;
- std::vector<Record> records_;
-};
-/**
- * @deprecated Please use the subclasses of StoreInputLayer.
- *
- * Layer for loading Record from DataShard.
- */
-class ShardDataLayer : public DataLayer {
- public:
- ~ShardDataLayer();
-
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- DataShard* shard_;
-};
-/**
- * @deprecated please use the subclasses of StoreInputLayer.
- *
- * Layer for loading Record from LMDB.
- */
-#ifdef USE_LMDB
-#include <lmdb.h>
-class LMDBDataLayer : public DataLayer {
- public:
- ~LMDBDataLayer();
-
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void OpenLMDB(const std::string& path);
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record);
-
- private:
- MDB_env* mdb_env_;
- MDB_dbi mdb_dbi_;
- MDB_txn* mdb_txn_;
- MDB_cursor* mdb_cursor_;
- MDB_val mdb_key_, mdb_value_;
-};
-#endif
-
-/******************Parser layers***************/
-/**
- * @deprecated Please use the subclasses of StoreInputLayer which load and parse
- * data in a single layer.
- *
- * Base layer for parsing the input records into Blobs.
- */
-class ParserLayer : public InputLayer {
- public:
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
- ConnectionType dst_layer_connection() const override {
- return kOneToMany;
- }
- /**
- * Parse records from DataLayer into blob.
- */
- virtual void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) = 0;
-};
-/**
- *
- * @deprecated Please use the SingleLabelRecordLayer which parses both feature
- * and label for each record. Its aux_data() function returns the parsed labels.
- *
- * Derived from ParserLayer to parse label in SingaleLabelImageRecord loaded by
- * ShardDataLayer.
- */
-class LabelLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-};
-
-/**
- * @deprecated Please use the subclasses of StoreInputLayer.
- *
- * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord.
- */
-class MnistLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-
- protected:
- float norm_a_, norm_b_;
-};
-/**
- * @deprecated please use the ImagePreprocessLayer which preprocess image
- * feature from data Blob of source layers.
- *
- * Derived from ParserLayer to parse RGB image feature from
- * SingaleLabelImageRecord.
- */
-class RGBImageLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-
- private:
- float scale_;
- int cropsize_;
- bool mirror_;
- Blob<float> mean_;
-};
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/image_preprocess.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/image_preprocess.h b/include/singa/neuralnet/input_layer/image_preprocess.h
deleted file mode 100644
index 7f8a3a8..0000000
--- a/include/singa/neuralnet/input_layer/image_preprocess.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_
-#define SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-using std::vector;
-/**
- * Do preprocessing for images, including cropping, mirroring, resizing.
- */
-class ImagePreprocessLayer : public InputLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers);
-
- private:
- bool mirror_ = false;
- int cropsize_ = 0;
- int resize_ = 0;
- float scale_ = 1;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/prefetch.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/prefetch.h b/include/singa/neuralnet/input_layer/prefetch.h
deleted file mode 100644
index d480618..0000000
--- a/include/singa/neuralnet/input_layer/prefetch.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_PREFETCH_H_
-#define SINGA_NEURALNET_INPUT_LAYER_PREFETCH_H_
-
-#include <string>
-#include <vector>
-#include <thread>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-using std::vector;
-/**
- * TODO(wangwei) Layer for prefetching data records and parsing them.
- *
- * This layer controls the prefetching thread, i.e.,
- * creating and joining the prefetching thread.
- */
-class PrefetchLayer : public Layer {
- public:
- ~PrefetchLayer();
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
-
- protected:
- std::thread thread_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_PREFETCH_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/record.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/record.h b/include/singa/neuralnet/input_layer/record.h
deleted file mode 100644
index f4571ca..0000000
--- a/include/singa/neuralnet/input_layer/record.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_RECORD_H_
-#define SINGA_NEURALNET_INPUT_LAYER_RECORD_H_
-
-#include <string>
-#include <vector>
-#include "singa/neuralnet/input_layer/store.h"
-
-namespace singa {
-
-/**
- * Specific layer that parses the value string loaded by Store into a
- * RecordProto.
- */
-class RecordInputLayer : public SingleLabelRecordLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
-
- protected:
- /**
- * Parse key as instance ID and val into RecordProto.
- * @copydetails StoreInputLayer::Parse()
- */
- bool Parse(int k, int flag, const string& key, const string& val) override;
- void LoadRecord(const string& backend,
- const string& path,
- Blob<float>* to) override;
-
- private:
- // TODO(wangwei) decode the image
- bool encoded_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_RECORD_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/input_layer/store.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/store.h b/include/singa/neuralnet/input_layer/store.h
deleted file mode 100644
index 88e6ca4..0000000
--- a/include/singa/neuralnet/input_layer/store.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_STORE_H_
-#define SINGA_NEURALNET_INPUT_LAYER_STORE_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-
-/**
- * Base class for loading data from Store.
- */
-class StoreInputLayer : virtual public InputLayer {
- public:
- ~StoreInputLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
-
- protected:
- /**
- * Parsing the (key, val) tuple to get feature (and label).
- * Subclasses must implment this function.
- * @param[in] k parse this tuple as the k-th instance of one mini-batch.
- * @param[in] flag used to guide the parsing, e.g., kDeploy phase should not
- * parse labels from the tuple.
- * @param[in] key
- * @param[in] val
- */
- virtual bool Parse(int k, int flag, const string& key, const string& val) = 0;
-
- protected:
- int batchsize_ = 1;
- int random_skip_ = 0;
- io::Store* store_ = nullptr;
-};
-
-/**
- * Base layer for parsing a key-value tuple as a feature vector with fixed
- * length. The feature shape is indicated by users in the configuration.
- * Each tuple may has a label.
- */
-class SingleLabelRecordLayer : public StoreInputLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
- protected:
- /**
- * Load a single record (tuple), e.g., the mean or standard variance vector.
- */
- virtual void LoadRecord(const string& backend, const string& path,
- Blob<float>* to) = 0;
-
- protected:
- /**
- * Feature standardization by processing each feature dimension via
- * @f$ y = (x - mu)/ std @f$
- * <a href= "http://ufldl.stanford.edu/wiki/index.php/Data_Preprocessing">
- * UFLDL</a>
- */
- Blob<float> mean_, std_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/layer.h b/include/singa/neuralnet/layer.h
index 7a92ced..5e2692e 100644
--- a/include/singa/neuralnet/layer.h
+++ b/include/singa/neuralnet/layer.h
@@ -22,9 +22,7 @@
#ifndef SINGA_NEURALNET_LAYER_H_
#define SINGA_NEURALNET_LAYER_H_
-#include <map>
#include <string>
-#include <thread>
#include <vector>
#include "singa/proto/common.pb.h"
#include "singa/proto/job.pb.h"
@@ -267,7 +265,7 @@ class Layer {
vector<AuxType> aux_data_;
vector<Blob<float>*> datavec_, gradvec_;
};
-
+/**************** Layer categories *****************/
/**
* Base layer for connecting layers when neural net is partitioned.
*/
@@ -275,6 +273,7 @@ class ConnectionLayer : virtual public Layer {
// defined as a layer category
};
+
/**
* Base layer for getting input data. May include layers for loading records,
* parsing records.
@@ -295,6 +294,14 @@ class InputLayer : virtual public Layer {
using SingleLabelImageRecord = RecordProto;
+/**
+ * Base layer for feature transformation, e.g., ConvolutionLayer, PoolingLayer,
+ * etc.
+ */
+class NeuronLayer : virtual public Layer {
+ // defined as a layer category
+};
+
/**
* Base layer for calculating loss and doing BackPropagation.
@@ -315,14 +322,6 @@ class LossLayer : virtual public Layer {
};
/**
- * Base layer for feature transformation, e.g., ConvolutionLayer, PoolingLayer,
- * etc.
- */
-class NeuronLayer : virtual public Layer {
- // defined as a layer category
-};
-
-/**
* Base layer for collecting features into disk file, HTTP stream, etc.
*/
class OutputLayer : virtual public Layer {
@@ -338,5 +337,6 @@ class OutputLayer : virtual public Layer {
}
};
+
} // namespace singa
#endif // SINGA_NEURALNET_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/loss_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer.h b/include/singa/neuralnet/loss_layer.h
new file mode 100644
index 0000000..e7fcd74
--- /dev/null
+++ b/include/singa/neuralnet/loss_layer.h
@@ -0,0 +1,56 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_LOSS_LAYER_H_
+#define SINGA_NEURALNET_LOSS_LAYER_H_
+
+#include "singa/neuralnet/layer.h"
+#include "singa/neuralnet/neuron_layer.h"
+namespace singa {
+using std::vector;
+/**
+ * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is prediction
+ * result, t is the ground truth.
+ */
+class EuclideanLossLayer : public LossLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+/**
+ * Cross-entropy loss applied to the probabilities computed from Softmax.
+ * @f$ L_i = -log P_{t_i}, t_i\in [0, C] @f$ is the label for the i-th object,
+ * C is the total number of classes.
+ */
+class SoftmaxLossLayer : public LossLayer, public SoftmaxLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ float scale_;
+ int topk_;
+};
+} // namespace singa
+
+#endif // SINGA_NEURALNET_LOSS_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/loss_layer/euclidean.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer/euclidean.h b/include/singa/neuralnet/loss_layer/euclidean.h
deleted file mode 100644
index 4d50dc5..0000000
--- a/include/singa/neuralnet/loss_layer/euclidean.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_LOSS_LAYER_EUCLIDEAN_H_
-#define SINGA_NEURALNET_LOSS_LAYER_EUCLIDEAN_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-using std::vector;
-/**
- * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is prediction
- * result, t is the ground truth.
- */
-class EuclideanLossLayer : public LossLayer {
- public:
- void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_LOSS_LAYER_EUCLIDEAN_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/loss_layer/softmax.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer/softmax.h b/include/singa/neuralnet/loss_layer/softmax.h
deleted file mode 100644
index c497377..0000000
--- a/include/singa/neuralnet/loss_layer/softmax.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_LOSS_LAYER_SOFTMAX_H_
-#define SINGA_NEURALNET_LOSS_LAYER_SOFTMAX_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-using std::vector;
-/**
- * Cross-entropy loss applied to the probabilities computed from Softmax.
- * @f$ L_i = -log P_{t_i}, t_i\in [0, C] @f$ is the label for the i-th object,
- * C is the total number of classes.
- */
-class SoftmaxLossLayer : public LossLayer {
- public:
- void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- /**
- * This layer is not recommendeded for partition because it requires the whole
- * src layer for normalization.
- */
- ConnectionType src_neuron_connection(int k) const override {
- // CHECK_LT(k, srclayers_.size());
- return kOneToAll;
- }
-
- private:
- int batchsize_;
- int dim_;
- float scale_;
- int topk_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_LOSS_LAYER_SOFTMAX_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
new file mode 100644
index 0000000..2bf9682
--- /dev/null
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -0,0 +1,410 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_NEURON_LAYER_H_
+#define SINGA_NEURALNET_NEURON_LAYER_H_
+
+#include <vector>
+#include "singa/neuralnet/layer.h"
+#include "singa/proto/job.pb.h"
+namespace singa {
+
+/* Activation layer applies following activations,
+ * - "relu", @f$ f(x) = max(0, x)@f$
+ * - "sigmoid", @f$ f(x)=1/(1+exp(-x)) @f$
+ * - "tanh", @f$ f(x) = tanh(x) @f$
+ * - "stanh", scaled tanh @f$f(x)=1.7159047 * tanh(0.66666667 * x)@f$, valid
+ * only for CPU training.
+ * It may share data and grad with its (single) source layer depending on
+ * the share_srclayer_blob configuration field.
+ */
+class ActivationLayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ bool share_with_srclayer = false;
+ std::string method_;
+};
+
+/**
+ * Convolution layer.
+ * Currently using Mshadow to do convolution operations. TODO(wangwei) remove
+ * dependency on Mshadow and using im2col from Caffe to implement this for CPU
+ * version. For GPU version, there is class CudnnConvLayer.
+ */
+class ConvolutionLayer : public NeuronLayer {
+ public:
+ ~ConvolutionLayer();
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::vector<Param*> GetParams() const override {
+ std::vector<Param*> params{weight_, bias_};
+ return params;
+ }
+ ConnectionType src_neuron_connection(int k) const override {
+ // CHECK_LT(k, srclayers_.size());
+ return kOneToAll;
+ }
+
+ protected:
+ int kernel_x_, pad_x_, stride_x_;
+ int kernel_y_, pad_y_, stride_y_;
+ int batchsize_, channels_, height_, width_;
+ int col_height_, col_width_, conv_height_, conv_width_, num_filters_;
+ Param* weight_ = nullptr, *bias_ = nullptr;
+ Blob<float> col_data_, col_grad_;
+};
+
+/**
+ * Implement convolution operations using im2col from Caffe.
+ */
+class CConvolutionLayer : public ConvolutionLayer {
+ public:
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/**
+ * Layer that drops out some neurons randomly according to a user defined drop
+ * ratio (default is 0.5). It helps reduce overfitting.
+ */
+class DropoutLayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ protected:
+ // drop probability
+ float pdrop_;
+ /* record which neuron is dropped, required for back propagating gradients,
+ * if mask[i]=0, then the i-th neuron is dropped.
+ */
+ Blob<float> mask_;
+};
+
+/**
+ * Layer that applys linear transformations as
+ * @f$ h = v*W+b @f$, where W and b are weight matrix and bias vector.
+ */
+class InnerProductLayer : public NeuronLayer {
+ public:
+ ~InnerProductLayer();
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::vector<Param*> GetParams() const override {
+ std::vector<Param*> params{weight_, bias_};
+ return params;
+ }
+
+ private:
+ int batchsize_;
+ int vdim_, hdim_;
+ bool transpose_;
+ Param *weight_, *bias_;
+};
+
+/**
+ * Local Response Normalization edge
+ *
+ * @f$ b_i=a_i/x_i^beta @f$
+ * @f$x_i=knorm+alpha*\sum_{j=max(0,i-n/2)}^{min(N,i+n/2)}(a_j)^2 @f$
+ * n is size of local response area.
+ * @f$a_i@f$, the activation (after ReLU) of a neuron convolved with the i-th kernel.
+ * @f$b_i@f$, the neuron after normalization, N is the total num of kernels
+ */
+class LRNLayer : public NeuronLayer {
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ //!< shape of the feature blob of the src layer
+ int batchsize_, channels_, height_, width_;
+ //!< size local response (neighbor) area
+ int lsize_;
+ //!< hyper-parameter
+ float alpha_, beta_, knorm_;
+ Blob<float> norm_;
+};
+
+/**
+ * Layer that applies the pooling operation.
+ * TODO(wangwei) remove dependenices on mshadow
+ */
+class PoolingLayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ int kernel_x_, pad_x_, stride_x_;
+ int kernel_y_, pad_y_, stride_y_;
+ int batchsize_, channels_, height_, width_, pooled_height_, pooled_width_;
+ PoolingProto_PoolMethod pool_;
+};
+/**
+ * Use book-keeping for BP following Caffe's pooling implementation
+ */
+class CPoolingLayer : public PoolingLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers);
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ Blob<float> mask_;
+};
+
+/**
+ * @deprecated {please use ActivationLayer}
+ */
+class ReLULayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/**
+ * Softmax layer applies softmax transformation to features from source layers.
+ * The feature blob of this layer is of shape (batchsize,
+ * num_softmax_per_instance, count_per_softmax), where num_softmax_per_instance
+ * is controled by users (default is 1),
+ * @f$ count_per_softmax = count / batchsize / num_softmax_per_instance @f$.
+ * The softmax is conducted over count_per_softmax elements each time.
+ */
+class SoftmaxLayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ /**
+ * This layer is not recommendeded for partition because it requires the whole
+ * src layer for normalization.
+ */
+ ConnectionType src_neuron_connection(int k) const override {
+ // CHECK_LT(k, srclayers_.size());
+ return kOneToAll;
+ }
+ protected:
+ int batchsize_;
+ //!< set by users (default is 1)
+ int num_softmax_per_instance_;
+ //!< size of the softmax area/length
+ int count_per_softmax_;
+};
+/**
+ * @deprecated {please use ActivationLayer}
+ *
+ * This layer apply Sigmoid function to neuron activations.
+ * f(x)=1/(1+exp(-x))
+ * f'(x)=f(x)*(1-f(x))
+ */
+class SigmoidLayer: public Layer {
+ public:
+ using Layer::ComputeFeature;
+ using Layer::ComputeGradient;
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/**
+ * @deprecated {please use ActivationLayer}
+ * This layer apply scaled Tanh function to neuron activations.
+ * f(x)=1.7159047 tanh(0.66666667 x)
+ */
+class STanhLayer : public NeuronLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+
+/*************** Layers implemented using cudnn v3 ***************/
+#ifdef USE_CUDNN
+#define CHECK_CUDNN(x) CHECK_EQ(x, CUDNN_STATUS_SUCCESS)
+
+class CudnnLayer : virtual public NeuronLayer {
+ public:
+ ~CudnnLayer() {
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(src_desc_));
+ CHECK_CUDNN(cudnnDestroyTensorDescriptor(my_desc_));
+ CHECK_CUDNN(cudnnDestroy(handle_));
+ }
+ void virtual InitCudnn() {
+ CHECK(!has_init_cudnn_);
+ CHECK_CUDNN(cudnnCreate(&handle_));
+ has_init_cudnn_ = true;
+ }
+ protected:
+ bool has_init_cudnn_ = false;
+ cudnnHandle_t handle_;
+ cudnnTensorDescriptor_t src_desc_, my_desc_;
+};
+
+/**
+ * Activation layer implemented using cudnn v3.
+ * Activation methods including
+ * - "sigmoid"
+ * - "tanh"
+ * - "relu"
+ */
+class CudnnActivationLayer : public ActivationLayer, public CudnnLayer {
+ public:
+ void InitCudnn() override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ cudnnActivationMode_t mode_;
+};
+
+/**
+ * Convolution layer implemeneted using cudnn (v3 version backward functions).
+ */
+class CuDNNConvLayer : public ConvolutionLayer, public CudnnLayer {
+ public:
+ ~CuDNNConvLayer();
+ void InitCudnn() override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ cudnnTensorDescriptor_t bias_desc_;
+ cudnnFilterDescriptor_t filter_desc_;
+ cudnnConvolutionDescriptor_t conv_desc_;
+ cudnnConvolutionFwdAlgo_t fp_alg_;
+ cudnnConvolutionBwdFilterAlgo_t bp_filter_alg_;
+ cudnnConvolutionBwdDataAlgo_t bp_data_alg_;
+ size_t workspace_byte_limit_, workspace_count_;
+};
+
+class CudnnLRNLayer : public LRNLayer {
+ public:
+ ~CudnnLRNLayer();
+ void InitCudnn() override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ cudnnLRNMode_t mode_;
+ cudnnLRNDescriptor_t norm_desc_;
+};
+/**
+ * Pooling layer implemented using cudnn.
+ */
+class CuDNNPoolLayer : public PoolingLayer, public CudnnLayer {
+ public:
+ ~CuDNNPoolLayer();
+ void InitCudnn() override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ cudnnPoolingDescriptor_t pool_desc_;
+};
+
+/**
+ * Cudnn Softmax layer.
+ */
+class CudnnSoftmaxLayer : public SoftmaxLayer, public CudnnLayer {
+ public:
+ void InitCudnn() override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+#endif // USE_CUDNN
+
+/******************** RBM layers *****************/
+/**
+ * Base layer for RBM models.
+ */
+class RBMLayer: virtual public NeuronLayer {
+ public:
+ virtual ~RBMLayer() {}
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ const Blob<float>& neg_data(const Layer* layer) {
+ return neg_data_;
+ }
+ Blob<float>* mutable_neg_data(const Layer* layer) {
+ return &neg_data_;
+ }
+ const std::vector<Param*> GetParams() const override {
+ std::vector<Param*> params{weight_, bias_};
+ return params;
+ }
+ virtual Blob<float>* Sample(int flat);
+
+ protected:
+ //! if ture, sampling according to guassian distribution
+ bool gaussian_;
+ //! dimension of the hidden layer
+ int hdim_;
+ //! dimension of the visible layer
+ int vdim_;
+ int batchsize_;
+ bool first_gibbs_;
+ Param* weight_, *bias_;
+
+ Blob<float> neg_data_;
+ Blob<float> neg_sample_;
+ Blob<float> sample_;
+};
+
+/**
+ * RBM visible layer
+ */
+class RBMVisLayer: public RBMLayer, public LossLayer {
+ public:
+ ~RBMVisLayer();
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ RBMLayer* hid_layer_;
+ Layer* input_layer_;
+};
+/**
+ * RBM hidden layer
+ */
+class RBMHidLayer: public RBMLayer {
+ public:
+ ~RBMHidLayer();
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ RBMLayer *vis_layer_;
+};
+
+} // namespace singa
+#define SINGA_NEURALNET_NEURON_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/argsort.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/argsort.h b/include/singa/neuralnet/neuron_layer/argsort.h
deleted file mode 100644
index 1ab1d4b..0000000
--- a/include/singa/neuralnet/neuron_layer/argsort.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_
-#define SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_
-
-#include <glog/logging.h>
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-namespace singa {
-
-/**
- * ArgSort layer used to get topk prediction labels.
- *
- * It sort the labels based on its score (e.g., probability) from large to
- * small. Topk labels will be kepted in the data field. It should not be called
- * during training because this layer does not implement ComputeGradient()
- * function.
- */
-class ArgSortLayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not Implemented";
- }
-
- private:
- int batchsize_, dim_;
- int topk_;
-};
-} // namespace singa
-#endif // SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/convolution.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/convolution.h b/include/singa/neuralnet/neuron_layer/convolution.h
deleted file mode 100644
index 2ffbf46..0000000
--- a/include/singa/neuralnet/neuron_layer/convolution.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_CONVOLUTION_H_
-#define SINGA_NEURALNET_NEURON_LAYER_CONVOLUTION_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * Convolution layer.
- */
-class ConvolutionLayer : public NeuronLayer {
- public:
- ~ConvolutionLayer();
-
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- const std::vector<Param*> GetParams() const override {
- std::vector<Param*> params{weight_, bias_};
- return params;
- }
- ConnectionType src_neuron_connection(int k) const override {
- // CHECK_LT(k, srclayers_.size());
- return kOneToAll;
- }
-
- protected:
- int kernel_, pad_, stride_;
- int batchsize_, channels_, height_, width_;
- int col_height_, col_width_, conv_height_, conv_width_, num_filters_;
- Param* weight_, *bias_;
- Blob<float> col_data_, col_grad_;
-};
-
-/**
- * Use im2col from Caffe
- */
-class CConvolutionLayer : public ConvolutionLayer {
- public:
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_CONVOLUTION_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/dropout.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/dropout.h b/include/singa/neuralnet/neuron_layer/dropout.h
deleted file mode 100644
index 8c1883c..0000000
--- a/include/singa/neuralnet/neuron_layer/dropout.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_DROPOUT_H_
-#define SINGA_NEURALNET_NEURON_LAYER_DROPOUT_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-
-class DropoutLayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- protected:
- // drop probability
- float pdrop_;
- /* record which neuron is dropped, required for back propagating gradients,
- * if mask[i]=0, then the i-th neuron is dropped.
- */
- Blob<float> mask_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_DROPOUT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/inner_product.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/inner_product.h b/include/singa/neuralnet/neuron_layer/inner_product.h
deleted file mode 100644
index bf3fdee..0000000
--- a/include/singa/neuralnet/neuron_layer/inner_product.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_INNER_PRODUCT_H_
-#define SINGA_NEURALNET_NEURON_LAYER_INNER_PRODUCT_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-namespace singa {
-
-class InnerProductLayer : public NeuronLayer {
- public:
- ~InnerProductLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- const std::vector<Param*> GetParams() const override {
- std::vector<Param*> params{weight_, bias_};
- return params;
- }
-
- private:
- int batchsize_;
- int vdim_, hdim_;
- bool transpose_;
- Param *weight_, *bias_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_INNER_PRODUCT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/lrn.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/lrn.h b/include/singa/neuralnet/neuron_layer/lrn.h
deleted file mode 100644
index 42b10ed..0000000
--- a/include/singa/neuralnet/neuron_layer/lrn.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_LRN_H_
-#define SINGA_NEURALNET_NEURON_LAYER_LRN_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * Local Response Normalization edge
- *
- * b_i=a_i/x_i^beta
- * x_i=knorm+alpha*\sum_{j=max(0,i-n/2}^{min(N,i+n/2}(a_j)^2
- * n is size of local response area.
- * a_i, the activation (after ReLU) of a neuron convolved with the i-th kernel.
- * b_i, the neuron after normalization, N is the total num of kernels
- */
-class LRNLayer : public NeuronLayer {
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- protected:
- //! shape of the bottom layer feature
- int batchsize_, channels_, height_, width_;
- //! size local response (neighbor) area
- int lsize_;
- //! hyper-parameter
- float alpha_, beta_, knorm_;
- Blob<float> norm_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_LRN_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/pooling.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/pooling.h b/include/singa/neuralnet/neuron_layer/pooling.h
deleted file mode 100644
index a66a4ef..0000000
--- a/include/singa/neuralnet/neuron_layer/pooling.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_POOLING_H_
-#define SINGA_NEURALNET_NEURON_LAYER_POOLING_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-
-class PoolingLayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- protected:
- int kernel_, pad_, stride_;
- int batchsize_, channels_, height_, width_, pooled_height_, pooled_width_;
- PoolingProto_PoolMethod pool_;
-};
-/**
- * Use book-keeping for BP following Caffe's pooling implementation
- */
-class CPoolingLayer : public PoolingLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers);
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- private:
- Blob<float> mask_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_POOLING_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/relu.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/relu.h b/include/singa/neuralnet/neuron_layer/relu.h
deleted file mode 100644
index 48a40a8..0000000
--- a/include/singa/neuralnet/neuron_layer/relu.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_RELU_H_
-#define SINGA_NEURALNET_NEURON_LAYER_RELU_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-namespace singa {
-
-class ReLULayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_RELU_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/softmax.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/softmax.h b/include/singa/neuralnet/neuron_layer/softmax.h
deleted file mode 100644
index cffae21..0000000
--- a/include/singa/neuralnet/neuron_layer/softmax.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_
-#define SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-namespace singa {
-
-/**
- * Softmax layer.
- */
-class SoftmaxLayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-} // namespace singa
-#endif // SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/neuron_layer/stanh.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/stanh.h b/include/singa/neuralnet/neuron_layer/stanh.h
deleted file mode 100644
index 08edb7d..0000000
--- a/include/singa/neuralnet/neuron_layer/stanh.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_STANH_H_
-#define SINGA_NEURALNET_NEURON_LAYER_STANH_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * This layer apply scaled Tanh function to neuron activations.
- * f(x)=1.7159047 tanh(0.66666667 x)
- */
-class STanhLayer : public NeuronLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_STANH_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af1bf509/include/singa/neuralnet/output_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer.h b/include/singa/neuralnet/output_layer.h
new file mode 100644
index 0000000..a7d92d7
--- /dev/null
+++ b/include/singa/neuralnet/output_layer.h
@@ -0,0 +1,72 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_OUTPUT_LAYER_H_
+#define SINGA_NEURALNET_OUTPUT_LAYER_H_
+
+#include <vector>
+#include "singa/neuralnet/layer.h"
+#include "singa/io/store.h"
+
+namespace singa {
+/**
+ * ArgSort layer used to get topk prediction labels.
+ *
+ * It sort the labels based on its score (e.g., probability) from large to
+ * small. Topk labels will be kepted in the data field. It should not be called
+ * during training because this layer does not implement ComputeGradient()
+ * function.
+ */
+class ArgSortLayer : public OutputLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ int batchsize_, dim_;
+ int topk_;
+};
+/**
+ * Output data (and label) for its source layer.
+ */
+class CSVOutputLayer : public OutputLayer {
+ public:
+ ~CSVOutputLayer() { delete store_; }
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ int inst_ = 0;
+ io::Store* store_ = nullptr;
+};
+
+class RecordOutputLayer : public OutputLayer {
+ public:
+ ~RecordOutputLayer() { delete store_; }
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ int inst_ = 0; //!< instance No.
+ io::Store* store_ = nullptr;
+};
+} // namespace singa
+#endif // SINGA_NEURALNET_OUTPUT_LAYER_H_
[16/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Automatically add library path when using `--with-cuda' and `--with-cudnn'.
Users can pass the lib check even if they do not set cuda/cudnn library path in environment variables.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f9d42687
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f9d42687
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f9d42687
Branch: refs/heads/master
Commit: f9d42687a623e6f010aedeca14692baaadc8241d
Parents: 6f81adb
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Sun Dec 13 11:25:39 2015 +0800
Committer: xiezl <xi...@comp.nus.edu.sg>
Committed: Sun Dec 13 11:25:39 2015 +0800
----------------------------------------------------------------------
Makefile.am | 4 ++--
configure.ac | 4 ++++
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f9d42687/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 90807be..bfbd409 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -29,8 +29,8 @@ MSHADOW_FLAGS = -DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
DEFAULT_FLAGS = -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
$(MSHADOW_FLAGS) -funroll-loops -DTHREADED
-CFLAGS = $(DEBUG)
-CXXFLAGS = $(DEBUG)
+CFLAGS += $(DEBUG)
+CXXFLAGS += $(DEBUG)
AC_CXXFLAGS = $(DEBUG)
INCLUDES = -I$(top_srcdir)/include
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f9d42687/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 1127994..5ae2e8c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -68,6 +68,8 @@ if test x"$cudaval" = x"yes"; then
CUDA_CFLAGS="-I$cuda_prefix/include"
CUDA_LDFLAGS="-L$cuda_prefix/lib64 -L$cuda_prefix/lib"
CUDA_LIBS="-lcublas -lcudart -lcurand"
+ LIBS="$LIBS $CUDA_LIBS"
+ LDFLAGS="$LDFLAGS $CUDA_LDFLAGS"
NVCC="nvcc"
DEBUG="-DUSE_GPU "
AC_DEFINE(DCUDA,[1],[Defined if CUDA should be used])
@@ -111,6 +113,8 @@ if test x"$enable_cudnn" == x"yes"; then
CUDNN_CFLAGS="-I$cudnn_prefix/include"
CUDNN_LDFLAGS="-L$cudnn_prefix/lib64 -L$cudnn_prefix/lib"
CUDNN_LIBS="-lcudnn"
+ LIBS="$LIBS $CUDNN_LIBS"
+ LDFLAGS="$LDFLAGS $CUDNN_LDFLAGS"
DEBUG+=" -DUSE_CUDNN "
AC_DEFINE(DCUDNN,[1],[Defined if CUDNN should be used])
AC_CHECK_LIB([cudnn], [main], [], [
[14/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Fix a bug:
a failed test when running "make test",caused by src/test/test_math.cc:349.
modified file:
math_kernel.cu: singa_gpu_sum_row() and singa_gpu_sum_col() set
threads_per_block and num_blocks wrong value
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/05680dd1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/05680dd1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/05680dd1
Branch: refs/heads/master
Commit: 05680dd174297cd7be8447aacf334299724c6575
Parents: 15b23a6
Author: seaok <se...@gmail.com>
Authored: Fri Dec 11 17:21:07 2015 +0800
Committer: seaok <se...@gmail.com>
Committed: Fri Dec 11 17:21:07 2015 +0800
----------------------------------------------------------------------
src/test/test_math.cc | 7 ++++---
src/utils/math_kernel.cu | 8 ++++----
2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/05680dd1/src/test/test_math.cc
----------------------------------------------------------------------
diff --git a/src/test/test_math.cc b/src/test/test_math.cc
index 0b9f0ff..6583b3d 100644
--- a/src/test/test_math.cc
+++ b/src/test/test_math.cc
@@ -313,14 +313,15 @@ TEST(MathTest, TestDotGPU) {
cudaFree(B_gpu);
}
-TEST(MathTest, TestSingaSumColGPU) {
+TEST(MathTest, TestSingaSumRowGPU) {
float A[3][4];
float B[4];
float C[4];
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 4; j++) {
- A[i][j] = i + j;
+ //A[i][j] = i + j;
+ A[i][j] = 1.0f;
}
}
@@ -335,7 +336,7 @@ TEST(MathTest, TestSingaSumColGPU) {
cudaMalloc(reinterpret_cast<void**>(&A_gpu), 12*sizeof(float));
cudaMalloc(reinterpret_cast<void**>(&B_gpu), 4*sizeof(float));
cudaMemcpy(A_gpu, A, 12*sizeof(float), cudaMemcpyHostToDevice);
- //singa_gpu_sum_row(A_gpu, B_gpu, 3, 4, 4);
+ singa_gpu_sum_row(A_gpu, B_gpu, 3, 4, 4);
cudaMemcpy(B, B_gpu, 4*sizeof(float), cudaMemcpyDeviceToHost);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/05680dd1/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index 3650c09..fe543d7 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -339,21 +339,21 @@ void singa_gpu_sum_vec(float *data, float *sum , int n) {
kernel_sum_vec<<<num_blocks, threads_per_block>>>(data, sum, n);
}
-void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
+void singa_gpu_sum_row(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride) {
int threads_per_block = rows > CU1DBLOCK ? CU1DBLOCK : rows;
int num_blocks = cols;
- kernel_sum_col<<<num_blocks, threads_per_block>>>(src_mat_data,
+ kernel_sum_row<<<num_blocks, threads_per_block>>>(src_mat_data,
dst_vec_data, rows, cols, stride);
}
-void singa_gpu_sum_row(const float *src_mat_data, float *dst_vec_data,
+void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride) {
int threads_per_block = cols > CU1DBLOCK ? CU1DBLOCK : cols;
int num_blocks = rows;
- kernel_sum_row<<<num_blocks, threads_per_block>>>(src_mat_data,
+ kernel_sum_col<<<num_blocks, threads_per_block>>>(src_mat_data,
dst_vec_data, rows, cols, stride);
}
[06/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Fix the error from wrong accuracy caused by inner-product layer.
The MMDot accpets two dimension Blob, but inner-product passes 4-d blob.
The error is fixed by assuming the Blob is of shape (shape[0], count()/shape[0]).
The cifar10 example can reach the same accuracy for GPU and CPU training (running 1000 iterations).
GPU using CUDNN is 10x faster than CPU.
TODO test multiple GPU cards and cifar10 for 70000 steps.
tmp commit; there is bug in cudnn_softmaxloss.cc->segmentfault
code clean with cpplint.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/eb97097a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/eb97097a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/eb97097a
Branch: refs/heads/master
Commit: eb97097aad2ce6aee19aecea243074f5c67c1cc4
Parents: f31ba64
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri Dec 4 14:07:05 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:23 2015 +0800
----------------------------------------------------------------------
examples/cifar10/cudnn.conf | 287 +++++++++++++++++++
include/singa/utils/blob.h | 2 +-
include/singa/utils/math_addr.h | 16 +-
include/singa/utils/math_blob.h | 46 +--
include/singa/utils/math_kernel.h | 8 +-
src/neuralnet/input_layer/deprecated.cc | 41 ++-
src/neuralnet/input_layer/image_preprocess.cc | 16 +-
src/neuralnet/input_layer/store.cc | 12 +-
src/neuralnet/loss_layer/cudnn_softmaxloss.cc | 80 ++++++
src/neuralnet/loss_layer/cudnn_softmaxloss.cu | 82 ------
src/neuralnet/loss_layer/euclidean.cc | 3 +-
src/neuralnet/loss_layer/softmax.cc | 4 +-
src/neuralnet/neuron_layer/activation.cc | 8 +-
src/neuralnet/neuron_layer/convolution.cc | 3 +-
src/neuralnet/neuron_layer/cudnn_activation.cc | 7 +-
src/neuralnet/neuron_layer/cudnn_convolution.cc | 9 +-
src/neuralnet/neuron_layer/cudnn_lrn.cc | 14 +-
src/neuralnet/neuron_layer/cudnn_pooling.cc | 7 +-
src/neuralnet/neuron_layer/cudnn_softmax.cc | 2 +-
src/neuralnet/neuron_layer/dropout.cc | 3 +-
src/neuralnet/neuron_layer/inner_product.cc | 3 +-
src/neuralnet/neuron_layer/lrn.cc | 4 +-
src/neuralnet/neuron_layer/pooling.cc | 4 +-
src/neuralnet/neuron_layer/rbm.cc | 3 +-
src/neuralnet/neuron_layer/relu.cc | 4 +-
src/neuralnet/neuron_layer/sigmoid.cc | 4 +-
src/neuralnet/output_layer/argsort.cc | 2 +-
src/utils/math_kernel.cu | 48 ++--
src/worker.cc | 11 +-
29 files changed, 524 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/examples/cifar10/cudnn.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/cudnn.conf b/examples/cifar10/cudnn.conf
new file mode 100644
index 0000000..4a29b7f
--- /dev/null
+++ b/examples/cifar10/cudnn.conf
@@ -0,0 +1,287 @@
+name: "cifar10-convnet"
+train_steps: 1000
+test_steps: 0
+test_freq: 0
+#validate_steps: 100
+#validate_freq: 300
+disp_freq: 30
+gpu: 0
+gpu: 1
+#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ weight_decay:0.004
+ momentum:0.9
+ learning_rate {
+ type: kFixedStep
+ fixedstep_conf:{
+ step:0
+ step:60000
+ step:65000
+ step_lr:0.001
+ step_lr:0.0001
+ step_lr:0.00001
+ }
+ }
+}
+neuralnet {
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/train_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ #random_skip: 5000
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTrain
+ }
+# layer{
+# name: "data"
+# type: kRecordInput
+# store_conf {
+# backend: "kvfile"
+# path: "examples/cifar10/val_data.bin"
+# mean_file: "examples/cifar10/image_mean.bin"
+# batchsize: 64
+# random_skip: 5000
+# shape: 3
+# shape: 32
+# shape: 32
+# }
+# include: kVal
+# }
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/test_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTest
+ }
+
+ layer {
+ name: "conv1"
+ type: kCudnnConv
+ srclayers: "data"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w1"
+ init {
+ type:kGaussian
+ std:0.0001
+ }
+ }
+ param {
+ name: "b1"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+
+ layer {
+ name: "pool1"
+ type: kCudnnPool
+ srclayers: "conv1"
+ pooling_conf {
+ pool: MAX
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "relu1"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"pool1"
+ }
+ layer {
+ name: "norm1"
+ type: kCudnnLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"relu1"
+ }
+ layer {
+ name: "conv2"
+ type: kCudnnConv
+ srclayers: "norm1"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w2"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b2"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ name: "relu2"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"conv2"
+ }
+ layer {
+ name: "pool2"
+ type: kCudnnPool
+ srclayers: "relu2"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "norm2"
+ type: kCudnnLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"pool2"
+ }
+ layer {
+ name: "conv3"
+ type: kCudnnConv
+ srclayers: "norm2"
+ convolution_conf {
+ num_filters: 64
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w3"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b3"
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ name: "relu3"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"conv3"
+ }
+ layer {
+ name: "pool3"
+ type: kCudnnPool
+ srclayers: "relu3"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "ip1"
+ type: kInnerProduct
+ srclayers:"pool3"
+ innerproduct_conf {
+ num_output: 10
+ }
+ param {
+ name: "w4"
+ wd_scale:250
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b4"
+ lr_scale:2.0
+ wd_scale:0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+# layer {
+# name : "softmax"
+# type: kSoftmax
+# srclayers: "ip1"
+# }
+#
+# layer {
+# name : "argsort"
+# type: kArgSort
+# srclayers: "softmax"
+# }
+ layer{
+ name: "loss"
+ type: kSoftmaxLoss
+ srclayers:"ip1"
+ srclayers: "data"
+ }
+# uncomment "softmax", "argsort", "output" layer and comment "loss" layer
+# to extract features from argsort
+# layer {
+# name : "output"
+# type: kCSVOutput
+# srclayers: "argsort"
+# store_conf {
+# path: "examples/cifar10/out.csv"
+# }
+# }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ nworkers_per_group: 2
+ nworkers_per_procs: 2
+ workspace: "examples/cifar10"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index bb63208..97b59e0 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -1,4 +1,4 @@
-/************************************************************
+/**************************************************************
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index 7ab884e..d641251 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -21,18 +21,16 @@
#ifndef SINGA_UTILS_MATH_ADDR_H_
#define SINGA_UTILS_MATH_ADDR_H_
+
extern "C" {
- #include <cblas.h>
+#include <cblas.h>
}
#ifdef USE_GPU
#include <cuda_runtime.h>
-#endif
-#include "singa/utils/singa_op.h"
-#ifdef USE_GPU
-#include "cuda_utils.h"
#include <cublas_v2.h>
#endif
+#include "singa/utils/singa_op.h"
namespace singa {
template<typename Dtype>
@@ -140,14 +138,14 @@ void cpu_softmax(int nb_rows, int nb_cols, const Dtype* A, Dtype* B) {
for (int i = 0; i < nb_rows; i++) {
const Dtype* dptr = A + i * nb_cols;
Dtype mmax = dptr[0];
- for (int x = 1; x < nb_cols; ++x )
+ for (int x = 1; x < nb_cols; ++x)
if (mmax < dptr[x]) mmax = dptr[x];
Dtype sum = 0.0f;
- for(int x = 0; x < nb_cols; ++x ) {
- dptr[x] = std::exp(dptr[x] - mmax );
+ for (int x = 0; x < nb_cols; ++x) {
+ dptr[x] = std::exp(dptr[x] - mmax);
sum += dptr[x];
}
- for(int x = 0; x < nb_cols; ++x ) {
+ for (int x = 0; x < nb_cols; ++x) {
dptr[x] /= sum;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index 629839a..cf989fa 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -31,10 +31,11 @@
#include "singa/utils/singleton.h"
#include "singa/utils/context.h"
-
namespace singa {
-enum XPU {cpu, gpu, any};
-
+/**
+ * \file math_blob.h is not tested thorough.
+ * Only GEMM() and MMDot() MVSumRow() andMVAddRow() are used now.
+ */
/************* BLAS level 1 *****************/
/**
* Scale each element of A with alpha, and put the result into B.
@@ -45,9 +46,9 @@ template<typename Dtype>
void Scale(Dtype alpha, Blob<Dtype> * B) {
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
- if (device == -1)
+ if (device == -1) {
cpu_scale(B->count(), alpha, B->mutable_cpu_data());
- else {
+ } else {
#ifdef USE_GPU
gpu_scale(context->cublas_handle(device), B->count(), alpha,
B->mutable_gpu_data());
@@ -146,24 +147,24 @@ void MVDot(const Blob<Dtype>& A, const Blob<Dtype>& B,
* @param[in, out] C, matrix
*/
template <typename Dtype>
-void GEMM( Dtype alpha, Dtype beta, const Blob<Dtype>& A,
- const Blob<Dtype> & B, Blob<Dtype> * C) {
+void GEMM(Dtype alpha, Dtype beta, const Blob<Dtype>& A, const Blob<Dtype>& B,
+ Blob<Dtype> * C) {
CHECK_GE(A.shape().size(), 2);
CHECK_GE(B.shape().size(), 2);
CHECK_GE(C->shape().size(), 2);
int a1, a2, b1, b2, m, n;
CHECK(!C->transpose());
a1 = A.transpose() ? A.count() / A.shape(0) : A.shape(0);
- a2 = A.transpose() ? A.shape(0) : A.count() / A.shape(0);
+ a2 = A.count() / a1;
b1 = B.transpose() ? B.count() /B.shape(0) : B.shape(0);
- b2 = B.transpose() ? B.shape(0) : B.count() / B.shape(0);
+ b2 = B.count() / b1;
m = C->shape(0);
- n = C->count() / C->shape(0);
+ n = C->count() / m;
CHECK_EQ(a2, b1);
CHECK_EQ(a1, m);
CHECK_EQ(b2, n);
- int k = A.transpose() ? A.shape(0) : A.shape(1);
+ int k = a2;
bool TranA = A.transpose();
bool TranB = B.transpose();
auto context = Singleton<Context>::Instance();
@@ -173,8 +174,8 @@ void GEMM( Dtype alpha, Dtype beta, const Blob<Dtype>& A,
C->mutable_cpu_data());
} else {
#ifdef USE_GPU
- gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), m, n, k,
- alpha, beta, TranA, TranB, C->mutable_gpu_data());
+ gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(),
+ m, n, k, alpha, beta, TranA, TranB, C->mutable_gpu_data());
#endif // USE_GPU
}
}
@@ -216,7 +217,8 @@ Dtype VVDot(const Blob<Dtype> & A, const Blob<Dtype> & B) {
} else {
#ifdef USE_GPU
// gpu part
- res = gpu_dot(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), n);
+ res = gpu_dot(context->cublas_handle(device), A.gpu_data(), B.gpu_data(),
+ n);
#endif // USE_GPU
}
return res;
@@ -244,8 +246,8 @@ void OuterProduct(const Blob<Dtype>& A, const Blob<Dtype>& B, Blob<Dtype> * C) {
C->mutable_cpu_data());
} else {
#ifdef USE_GPU
- gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), m, n, 1, 1, 0,
- false, false, C->mutable_gpu_data());
+ gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(),
+ m, n, 1, 1, 0, false, false, C->mutable_gpu_data());
#endif // USE_GPU
}
}
@@ -389,7 +391,7 @@ void Sub(const Blob<Dtype> & A, const Blob<Dtype> & B,
/**
* C = A * B, implemented using
- * Map(XPU, const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
+ * Map(const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
*/
template<typename Dtype>
void Mult(const Blob<Dtype> & A, const Blob<Dtype> & B,
@@ -400,7 +402,7 @@ void Mult(const Blob<Dtype> & A, const Blob<Dtype> & B,
/**
* C = A / B, implemented using
- * Map(XPU, const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
+ * Map(const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
*/
template<typename Dtype>
void Div(const Blob<Dtype> & A, const Blob<Dtype> & B,
@@ -561,7 +563,7 @@ void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
A.transpose(), false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_by_col(A.gpu_data(), B->gpu_data(), m, n, n);
+ singa_gpu_sum_by_col(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
@@ -586,7 +588,7 @@ void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_by_row(A.gpu_data(), B->gpu_data(), m, n, n);
+ singa_gpu_sum_by_row(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
@@ -645,7 +647,7 @@ Dtype Asum(const Blob<Dtype>& A) {
ret = cpu_asum(A.count(), A.cpu_data(), 1) / A.count();
} else {
#ifdef USE_GPU
- ret = gpu_asum(context->cublas_handle(device), A.count(), A.cpu_data(), 1)
+ ret = gpu_asum(context->cublas_handle(device), A.count(), A.gpu_data(), 1)
/ A.count();
#endif
}
@@ -665,7 +667,7 @@ void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A) {
} else {
#ifdef USE_GPU
gpu_sample_uniform(context->curand_generator(thread), A->count(), low, high,
- A->mutable_gpu_data());
+ A->mutable_gpu_data());
#endif
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/include/singa/utils/math_kernel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_kernel.h b/include/singa/utils/math_kernel.h
index 8cfa562..59bc3bf 100644
--- a/include/singa/utils/math_kernel.h
+++ b/include/singa/utils/math_kernel.h
@@ -24,11 +24,11 @@
namespace singa {
extern "C" {
- void singa_gpu_softmax_loss(const float *prob, const int *label,
- float *loss, int n, int dim);
+ void singa_gpu_softmaxloss_forward(int n, int dim, const float *prob,
+ const int *label, float *loss);
- void singa_gpu_softmax_gradient(float *grad, const int *label ,
- int n, int dim, float scale);
+ void singa_gpu_softmaxloss_backward(int n, int dim, float scale,
+ const int *label, float *grad);
void singa_gpu_sum_vec(float *data, float *sum , int n);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
index 1760d4b..0f98279 100644
--- a/src/neuralnet/input_layer/deprecated.cc
+++ b/src/neuralnet/input_layer/deprecated.cc
@@ -20,6 +20,8 @@
*************************************************************/
#include "singa/neuralnet/input_layer.h"
+#include "singa/utils/context.h"
+#include "singa/utils/singleton.h"
#include "mshadow/tensor.h"
namespace singa {
@@ -57,7 +59,10 @@ void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
shard_ = new DataShard(layer_conf_.sharddata_conf().path(),
DataShard::kRead);
if (random_skip_) {
- int nskip = rand() % random_skip_;
+ std::uniform_int_distribution<int> distribution(0, random_skip_);
+ auto generator =
+ Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+ int nskip = distribution(generator);
LOG(INFO) << "Random Skip " << nskip << " records, there are "
<< shard_->Count() << " records in total";
string key;
@@ -123,25 +128,29 @@ void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (mdb_cursor_ == nullptr)
OpenLMDB(layer_conf_.lmdbdata_conf().path());
if (random_skip_) {
- int nskip = rand() % random_skip_;
+ std::uniform_int_distribution<int> distribution(0, random_skip_);
+ auto generator =
+ Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+ int nskip = distribution(generator);
+
int n = 0;
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
+ &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
n++;
LOG(INFO) << "Random Skip " << nskip << " records of total "
- << n << "records";
+ << n << "records";
// We have reached the end. Restart from the first.
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
for (int i = 0; i < nskip; i++) {
if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
+ &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
// We have reached the end. Restart from the first.
DLOG(INFO) << "Restarting data prefetching from start.";
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
}
}
random_skip_ = 0;
@@ -254,10 +263,16 @@ void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
AllocSpace(croped_image);
int rid = 0;
const float* meandptr = mean_.cpu_data();
+
+ std::uniform_int_distribution<int> distribution(0, r.shape(0) - cropsize_);
+ auto generator =
+ Singleton<Context>::Instance()->generator(std::this_thread::get_id());
for (const Record& record : records) {
auto image = images[rid];
bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain);
- bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
+ bool do_mirror = mirror_
+ && (distribution(generator) % 2)
+ && ((flag & kTrain) == kTrain);
float* dptr = nullptr;
if (do_crop || do_mirror)
dptr = raw_image.dptr;
@@ -274,8 +289,8 @@ void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
for (int i = 0; i < mean_.count(); i++)
dptr[i] -= meandptr[i];
if (do_crop) {
- int hoff = rand() % (r.shape(1) - cropsize_);
- int woff = rand() % (r.shape(2) - cropsize_);
+ int hoff = distribution(generator);
+ int woff = distribution(generator);
Shape<2> cropshape = Shape2(cropsize_, cropsize_);
if (do_mirror) {
croped_image = expr::crop(raw_image, cropshape, hoff, woff);
@@ -355,6 +370,4 @@ void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
}
CHECK_EQ(rid, blob->shape()[0]);
}
-
-
-} // namespace singa
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/input_layer/image_preprocess.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/image_preprocess.cc b/src/neuralnet/input_layer/image_preprocess.cc
index 576b096..251f456 100644
--- a/src/neuralnet/input_layer/image_preprocess.cc
+++ b/src/neuralnet/input_layer/image_preprocess.cc
@@ -21,6 +21,9 @@
#include "singa/neuralnet/input_layer.h"
#include "singa/utils/image_transform.h"
+#include "singa/utils/context.h"
+#include "singa/utils/singleton.h"
+
namespace singa {
using std::vector;
@@ -52,13 +55,20 @@ void ImagePreprocessLayer::ComputeFeature(int flag,
float* dptr = data_.mutable_cpu_data();
int srcimage_size = channel * height * width;
int image_size = channel * data_.shape()[2] * data_.shape()[3];
+ std::uniform_int_distribution<int> rand1(0, srcdata.shape()[1] - cropsize_);
+ std::uniform_int_distribution<int> rand2(0, srcdata.shape()[2] - cropsize_);
+ auto generator =
+ Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+
for (int k = 0; k < batchsize; k++) {
int h_offset = 0, w_offset = 0;
if (cropsize_> 0 && ((flag & kTrain) == kTrain)) {
- h_offset = rand() % (srcdata.shape()[1] - cropsize_);
- w_offset = rand() % (srcdata.shape()[2] - cropsize_);
+ h_offset = rand1(generator);
+ w_offset = rand2(generator);
}
- bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
+ bool do_mirror = mirror_
+ && (rand1(generator) % 2)
+ && ((flag & kTrain) == kTrain);
ImageTransform(srcdptr + k * srcimage_size, nullptr, do_mirror, cropsize_,
cropsize_, h_offset, w_offset, srcdata.shape()[1], height, width,
scale_, dptr + image_size);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/input_layer/store.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/store.cc b/src/neuralnet/input_layer/store.cc
index b1b296e..dbb1874 100644
--- a/src/neuralnet/input_layer/store.cc
+++ b/src/neuralnet/input_layer/store.cc
@@ -20,6 +20,9 @@
*************************************************************/
#include "singa/neuralnet/input_layer.h"
+#include "singa/utils/context.h"
+#include "singa/utils/singleton.h"
+
namespace singa {
StoreInputLayer::~StoreInputLayer() {
@@ -35,8 +38,13 @@ void StoreInputLayer::Setup(const LayerProto& conf,
if (conf.partition_dim() == 0) {
batchsize_ /= conf.num_partitions();
}
- if (conf.store_conf().random_skip() > 0)
- random_skip_ = rand() % conf.store_conf().random_skip();
+ if (conf.store_conf().random_skip() > 0) {
+ std::uniform_int_distribution<int>
+ distribution(0, conf.store_conf().random_skip());
+ auto generator =
+ Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+ random_skip_ = distribution(generator);
+ }
}
void StoreInputLayer::ComputeFeature(int flag,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
new file mode 100644
index 0000000..b18a751
--- /dev/null
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
@@ -0,0 +1,80 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/loss_layer.h"
+#include "singa/utils/blob.h"
+#include "singa/utils/math_kernel.h"
+
+namespace singa {
+void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ CudnnSoftmaxLayer::Setup(conf, vector<Layer*> {srclayers.at(0)});
+ topk_ = conf.softmaxloss_conf().topk();
+ loss_ = accuracy_ = 0.0f;
+ counter_ = 0;
+}
+void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ CudnnSoftmaxLayer::ComputeFeature(flag, srclayers);
+ // compute loss
+ float *prob = data_.mutable_gpu_data();
+ Blob<int> label(batchsize_);
+ int *labelptr = label.mutable_cpu_data();
+
+ // aux_data: vector<int>, convert vector to int array.
+ for (int i = 0; i < batchsize_; ++i) {
+ labelptr[i] = srclayers[1]->aux_data(this)[i];
+ }
+
+ Blob<float> loss(batchsize_);
+
+ singa_gpu_softmaxloss_forward(batchsize_, dim_, prob, label.gpu_data(),
+ loss.mutable_gpu_data());
+
+ counter_++;
+}
+
+void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
+ const vector<Layer*>& srclayers) {
+ Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
+ gsrcblob->CopyFrom(data_);
+ float* gsrcptr = gsrcblob->mutable_gpu_data();
+
+ Blob<int> label(batchsize_);
+ int *labelptr = label.mutable_cpu_data();
+
+ // aux_data: vector<int>, convert vector to int array.
+ for (int i = 0; i < batchsize_; ++i) {
+ labelptr[i] = srclayers[1]->aux_data(this)[i];
+ }
+
+ singa_gpu_softmaxloss_backward(batchsize_, dim_, scale_, label.gpu_data(),
+ gsrcptr);
+}
+
+const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
+ string disp = "Loss = " + std::to_string(loss_ / counter_)
+ + ", accuracy = " + std::to_string(accuracy_ / counter_);
+ counter_ = 0;
+ loss_ = accuracy_ = 0;
+ return disp;
+}
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu b/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
deleted file mode 100644
index 53420d3..0000000
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cu
+++ /dev/null
@@ -1,82 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/loss_layer.h"
-#include "singa/utils/blob.h"
-#include "singa/utils/math_kernel.h"
-
-namespace singa {
-void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- CudnnSoftmaxLayer::Setup(conf, srclayers);
- topk_ = conf.softmaxloss_conf().topk();
- loss_ = accuracy_ = 0.0f;
- counter_ = 0;
-}
-void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- CudnnSoftmaxLayer::ComputeFeature(flag, srclayers);
- // compute loss
- float *prob = data_.mutable_gpu_data();
- Blob<int> label(batchsize_);
- int *labelptr = label.mutable_cpu_data();
-
- //aux_data: vector<int>, convert vector to int array.
- for(int i = 0; i < batchsize_; ++i) {
- labelptr[i] = srclayers[1]->aux_data(this)[i];
- }
-
- Blob<float> loss(batchsize_);
-
- singa_gpu_softmax_loss(prob , label.mutable_gpu_data() , loss.mutable_gpu_data(),
- batchsize_, dim_);
-
- counter_++;
- // TODO add loss and accuracy
-}
-
-void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- // compute gradient
- Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- gsrcblob->CopyFrom(data_);
- float* gsrcptr = gsrcblob->mutable_gpu_data();
-
- Blob<int> label(batchsize_);
- int *labelptr = label.mutable_cpu_data();
-
- //aux_data: vector<int>, convert vector to int array.
- for(int i = 0; i < batchsize_; ++i) {
- labelptr[i] = srclayers[1]->aux_data(this)[i];
- }
-
- singa_gpu_softmax_gradient(gsrcptr, label.mutable_gpu_data(), batchsize_, dim_, scale_);
-
-}
-
-const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
- string disp = "Loss = " + std::to_string(loss_ / counter_)
- + ", accuracy = " + std::to_string(accuracy_ / counter_);
- counter_ = 0;
- loss_ = accuracy_ = 0;
- return disp;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/loss_layer/euclidean.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc
index 71e5bae..b6aa12a 100644
--- a/src/neuralnet/loss_layer/euclidean.cc
+++ b/src/neuralnet/loss_layer/euclidean.cc
@@ -19,9 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/loss_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/loss_layer.h"
#include "mshadow/tensor.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index 8c100ef..0f3d5bf 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -19,10 +19,10 @@
*
*************************************************************/
-#include "singa/neuralnet/loss_layer.h"
-#include <algorithm>
#include <glog/logging.h>
+#include <algorithm>
+#include "singa/neuralnet/loss_layer.h"
#include "mshadow/tensor.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/activation.cc b/src/neuralnet/neuron_layer/activation.cc
index 6f62646..492e453 100644
--- a/src/neuralnet/neuron_layer/activation.cc
+++ b/src/neuralnet/neuron_layer/activation.cc
@@ -30,8 +30,8 @@ void ActivationLayer::Setup(const LayerProto& conf,
data_.ReshapeLike(srclayers[0]->data(this));
grad_.ReshapeLike(*(srclayers[0]->mutable_grad(this)));
}
-void ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers)
-{
+void
+ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
switch (layer_conf_.activation_conf().type()) {
case RELU:
Map<op::Relu<float>, float>(srclayers[0]->data(this), &data_);
@@ -52,8 +52,8 @@ void ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers)
layer_conf_.activation_conf().type();
}
}
-void ActivationLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
-{
+void
+ActivationLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
Blob<float> * gsrc = srclayers[0]->mutable_grad(this);
switch (layer_conf_.activation_conf().type()) {
case RELU:
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/convolution.cc b/src/neuralnet/neuron_layer/convolution.cc
index edfabb6..bde2f08 100644
--- a/src/neuralnet/neuron_layer/convolution.cc
+++ b/src/neuralnet/neuron_layer/convolution.cc
@@ -19,9 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/cudnn_activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cc b/src/neuralnet/neuron_layer/cudnn_activation.cc
index e8a7b41..9088d84 100644
--- a/src/neuralnet/neuron_layer/cudnn_activation.cc
+++ b/src/neuralnet/neuron_layer/cudnn_activation.cc
@@ -33,14 +33,13 @@ void CudnnActivationLayer::InitCudnn() {
mode_ = CUDNN_ACTIVATION_TANH;
else if (layer_conf_.activation_conf().type() == RELU)
mode_ = CUDNN_ACTIVATION_RELU;
- else {
+ else
LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().type();
- }
const auto& shape = data_.shape();
CHECK_GT(shape.size(), 0);
// size of each dimension
- int* sdim= new int[shape.size()];
+ int* sdim = new int[shape.size()];
int* stride = new int[shape.size()];
stride[shape.size() -1] = 1;
int i = shape.size() - 1;
@@ -97,4 +96,4 @@ void CudnnActivationLayer::ComputeGradient(int flag,
src_desc_,
srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
-} // namespace singa
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/cudnn_convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cc b/src/neuralnet/neuron_layer/cudnn_convolution.cc
index e08b57a..2cf376c 100644
--- a/src/neuralnet/neuron_layer/cudnn_convolution.cc
+++ b/src/neuralnet/neuron_layer/cudnn_convolution.cc
@@ -34,7 +34,8 @@ CudnnConvLayer::~CudnnConvLayer() {
void CudnnConvLayer::InitCudnn() {
CudnnLayer::InitCudnn();
// convert MB to bytes
- workspace_byte_limit_ = layer_conf_.convolution_conf().workspace_byte_limit() << 20;
+ workspace_byte_limit_
+ = layer_conf_.convolution_conf().workspace_byte_limit() << 20;
CHECK_CUDNN(cudnnCreateTensorDescriptor(&bias_desc_));
CHECK_CUDNN(cudnnCreateFilterDescriptor(&filter_desc_));
@@ -162,8 +163,8 @@ void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
}
}
-void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
-{
+void
+CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
float alpha = 1.f, beta = 0.f;
Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
if (bias_) {
@@ -204,4 +205,4 @@ void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
}
-} /* singa */
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/cudnn_lrn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cc b/src/neuralnet/neuron_layer/cudnn_lrn.cc
index 4a2b695..f976b16 100644
--- a/src/neuralnet/neuron_layer/cudnn_lrn.cc
+++ b/src/neuralnet/neuron_layer/cudnn_lrn.cc
@@ -57,31 +57,33 @@ void CudnnLRNLayer::InitCudnn() {
void CudnnLRNLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (!has_init_cudnn_)
InitCudnn();
+ float alpha = 1.0f, beta = 0.0f;
CHECK_CUDNN(cudnnLRNCrossChannelForward(handle_,
norm_desc_,
mode_,
- &alpha_,
+ &alpha,
src_desc_,
srclayers[0]->data(this).gpu_data(),
- &beta_,
+ &beta,
my_desc_,
data_.mutable_gpu_data()));
}
void CudnnLRNLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
+ float alpha = 1.0f, beta = 0.0f;
CHECK_CUDNN(cudnnLRNCrossChannelBackward(handle_,
norm_desc_,
mode_,
- &alpha_,
- my_desc_, // ???
+ &alpha,
+ my_desc_,
data_.gpu_data(),
my_desc_,
grad_.gpu_data(),
src_desc_,
srclayers[0]->data(this).gpu_data(),
- &beta_,
+ &beta,
src_desc_,
srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
-} /* singa */
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/cudnn_pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cc b/src/neuralnet/neuron_layer/cudnn_pooling.cc
index ffdfb3b..217df51 100644
--- a/src/neuralnet/neuron_layer/cudnn_pooling.cc
+++ b/src/neuralnet/neuron_layer/cudnn_pooling.cc
@@ -57,7 +57,6 @@ void CudnnPoolLayer::InitCudnn() {
pad_x_,
stride_y_,
stride_x_));
-
}
void CudnnPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
@@ -76,8 +75,8 @@ void CudnnPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
data_.mutable_gpu_data()));
}
-void CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
-{
+void
+CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
float alpha = 1.0f, beta = 0.0f;
CHECK_CUDNN(cudnnPoolingBackward(handle_,
pool_desc_,
@@ -92,5 +91,5 @@ void CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
src_desc_,
srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
-} /* singa */
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/cudnn_softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cc b/src/neuralnet/neuron_layer/cudnn_softmax.cc
index 7fade3e..21d17c4 100644
--- a/src/neuralnet/neuron_layer/cudnn_softmax.cc
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cc
@@ -72,4 +72,4 @@ void CudnnSoftmaxLayer::ComputeGradient(int flag,
src_desc_,
srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
-} /* singa */
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/dropout.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dropout.cc b/src/neuralnet/neuron_layer/dropout.cc
index 030a663..6e0db76 100644
--- a/src/neuralnet/neuron_layer/dropout.cc
+++ b/src/neuralnet/neuron_layer/dropout.cc
@@ -19,9 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
#include "singa/utils/singa_op.h"
#include "singa/utils/math_blob.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/inner_product.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/inner_product.cc b/src/neuralnet/neuron_layer/inner_product.cc
index 6b5ec36..7e6318d 100644
--- a/src/neuralnet/neuron_layer/inner_product.cc
+++ b/src/neuralnet/neuron_layer/inner_product.cc
@@ -19,9 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
#include "singa/utils/math_blob.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/lrn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/lrn.cc b/src/neuralnet/neuron_layer/lrn.cc
index 178b2bf..9a5ba37 100644
--- a/src/neuralnet/neuron_layer/lrn.cc
+++ b/src/neuralnet/neuron_layer/lrn.cc
@@ -19,11 +19,11 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
+
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/pooling.cc b/src/neuralnet/neuron_layer/pooling.cc
index 5b408ba..4eda2e4 100644
--- a/src/neuralnet/neuron_layer/pooling.cc
+++ b/src/neuralnet/neuron_layer/pooling.cc
@@ -19,11 +19,11 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
+
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/rbm.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/rbm.cc b/src/neuralnet/neuron_layer/rbm.cc
index 53a1bd9..fadd1df 100644
--- a/src/neuralnet/neuron_layer/rbm.cc
+++ b/src/neuralnet/neuron_layer/rbm.cc
@@ -19,9 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/relu.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/relu.cc b/src/neuralnet/neuron_layer/relu.cc
index 1e030a0..5d4d954 100644
--- a/src/neuralnet/neuron_layer/relu.cc
+++ b/src/neuralnet/neuron_layer/relu.cc
@@ -19,11 +19,11 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
+
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/neuron_layer/sigmoid.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/sigmoid.cc b/src/neuralnet/neuron_layer/sigmoid.cc
index c449b36..9348011 100644
--- a/src/neuralnet/neuron_layer/sigmoid.cc
+++ b/src/neuralnet/neuron_layer/sigmoid.cc
@@ -19,11 +19,11 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-
#include <glog/logging.h>
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/utils/singleton.h"
+
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/neuralnet/output_layer/argsort.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/argsort.cc b/src/neuralnet/output_layer/argsort.cc
index 63969b3..869bc65 100644
--- a/src/neuralnet/output_layer/argsort.cc
+++ b/src/neuralnet/output_layer/argsort.cc
@@ -19,8 +19,8 @@
*
*************************************************************/
-#include "singa/neuralnet/output_layer.h"
#include <algorithm>
+#include "singa/neuralnet/output_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index a4cd513..9c54520 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -21,7 +21,7 @@
#include <cmath>
#include <algorithm>
#include "singa/utils/math_kernel.h"
-#include "mshadow/tensor.h" //FLT_MIN?
+#include "mshadow/tensor.h" // FLT_MIN?
#define CU2DBLOCK_X 32
#define CU2DBLOCK_Y 32
@@ -32,25 +32,25 @@
// Cuda Kernel Functions
__global__
-void kernel_softmax_loss(const float *prob, const int *label ,
- float *loss, int n, int dim) {
+void kernel_softmax_loss(const float *prob, const int *label , float *loss,
+ int n, int dim) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
int num_threads = blockDim.x * gridDim.x;
for (; index < n; index += num_threads) {
- const int label_value = static_cast<int>(label[index]);
- float prob_of_truth = prob[index * dim + label_value];
- loss[index] -= log(max(prob_of_truth, FLT_MIN));
+ const int label_value = static_cast<int>(label[index]);
+ float prob_of_truth = prob[index * dim + label_value];
+ loss[index] -= log(max(prob_of_truth, FLT_MIN));
}
}
__global__
void kernel_softmax_gradient(float *grad, const int *label ,
- int n, int dim, float scale) {
+ int n, int dim, float scale) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
int num_threads = blockDim.x * gridDim.x;
for (; index < n; index += num_threads) {
- int pos = index * dim + static_cast<int>(label[index]);
- grad[pos] = (grad[pos] - 1.0f) * scale / (1.0 * n);
+ int pos = index * dim + static_cast<int>(label[index]);
+ grad[pos] = (grad[pos] - 1.0f) * scale / (1.0 * n);
}
}
@@ -128,9 +128,9 @@ void kernel_sum_by_row(const float *src_mat_data,
int num_threads = blockDim.x * gridDim.x;
for (; index < rows; index += num_threads) {
dst_vec_data[index] = 0.0f;
- for (int k = 0; k < cols; k++) {
- dst_vec_data[index] += src_mat_data[index * stride + k];
- }
+ for (int k = 0; k < cols; k++) {
+ dst_vec_data[index] += src_mat_data[index * stride + k];
+ }
}
}
@@ -317,16 +317,16 @@ void kernel_threshold(const float *src_data, float *des_data,
//
namespace singa {
-void singa_gpu_softmax_loss(const float *prob, const int *label,
- float *loss, int n, int dim) {
- kernel_softmax_loss<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>
- (prob, label, loss, n, dim);
+void singa_gpu_softmaxloss_forward(int n, int dim, const float *prob,
+ const int *label, float *loss) {
+ kernel_softmax_loss<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>(prob, label, loss, n,
+ dim);
}
-void singa_gpu_softmax_gradient(float *grad, const int *label,
- int n, int dim, float scale) {
- kernel_softmax_gradient<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>
- (grad, label, n, dim, scale);
+void singa_gpu_softmaxloss_backward(int n, int dim, float scale,
+ const int *label, float *grad) {
+ kernel_softmax_gradient<<<ceil(n/CU1DBLOCKF), CU1DBLOCKF>>>(grad, label, n,
+ dim, scale);
}
void singa_gpu_sum_vec(float *data, float *sum , int n) {
@@ -342,8 +342,8 @@ void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
int threads_per_block = rows > CU1DBLOCK ? CU1DBLOCK : rows;
int num_blocks = cols;
- kernel_sum_by_col<<<num_blocks, threads_per_block>>>(src_mat_data, dst_vec_data,
- rows, cols, stride);
+ kernel_sum_by_col<<<num_blocks, threads_per_block>>>(src_mat_data,
+ dst_vec_data, rows, cols, stride);
}
void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
@@ -351,8 +351,8 @@ void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
int threads_per_block = cols > CU1DBLOCK ? CU1DBLOCK : cols;
int num_blocks = rows;
- kernel_sum_by_row<<<num_blocks, threads_per_block>>>(src_mat_data, dst_vec_data,
- rows, cols, stride);
+ kernel_sum_by_row<<<num_blocks, threads_per_block>>>(src_mat_data,
+ dst_vec_data, rows, cols, stride);
}
void singa_gpu_add_vec_row(const float *src_vec_data, const float *src_mat_data,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/eb97097a/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index d9380b6..a41e3a8 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -61,11 +61,12 @@ Worker::~Worker() {
}
void Worker::Run() {
- LOG(ERROR) << "Worker (group = " << grp_id_ <<", id = " << id_ << ") start";
// setup gpu device
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
- if (device > 0)
+ LOG(ERROR) << "Worker (group = " << grp_id_ <<", id = " << id_
+ << ") start on device " << device;
+ if (device >= 0)
context->ActivateDevice(device);
auto cluster = Cluster::Get();
int svr_grp = grp_id_ / cluster->nworker_groups_per_server_group();
@@ -90,8 +91,10 @@ void Worker::Run() {
job_conf_.set_step(step_);
}
TrainOneBatch(step_, train_net_);
- if (DisplayNow(step_) && grp_id_ == 0 && id_ == 0)
- Display(kTrain | kForward | kBackward, "Train @ step " + std::to_string(step_), train_net_);
+ if (DisplayNow(step_) && grp_id_ == 0 && id_ == 0) {
+ Display(kTrain | kForward | kBackward,
+ "Train @ step " + std::to_string(step_), train_net_);
+ }
step_++;
}
[09/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
tmp commit; setup device in driver.cc, register cudnn layers;
tmp commit;
1.Add kernel_sum_by_row() cuda kernel functions
2.Fixed neuron_layer.h bug:
cudnn.h should be placed outside the namespace
3.Register CudnnSoftmaxLoss class (job.proto)
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8cacd83c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8cacd83c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8cacd83c
Branch: refs/heads/master
Commit: 8cacd83ceb5b22df473fb96fcf2e371e470e6d43
Parents: 6e56334
Author: seaokcs <se...@gmail.com>
Authored: Mon Nov 30 14:48:15 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:23 2015 +0800
----------------------------------------------------------------------
include/singa/neuralnet/neuron_layer.h | 12 ++++++---
include/singa/utils/blob.h | 5 ++++
include/singa/utils/cuda_utils.h | 14 -----------
include/singa/utils/math_addr.h | 2 +-
include/singa/utils/math_blob.h | 8 +++---
include/singa/utils/math_kernel.h | 5 +++-
src/driver.cc | 23 +++++++++++++++--
src/neuralnet/neuron_layer/inner_product.cc | 32 +++++-------------------
src/proto/job.proto | 9 +++++++
src/utils/math_kernel.cu | 30 +++++++++++++++++++---
src/utils/param.cc | 4 +++
src/worker.cc | 6 +++++
12 files changed, 95 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 097b209..830f731 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -25,6 +25,11 @@
#include <vector>
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
+
+#ifdef USE_CUDNN
+#include <cudnn.h>
+#endif
+
namespace singa {
/* Activation layer applies following activations,
@@ -249,7 +254,6 @@ class STanhLayer : public NeuronLayer {
/*************** Layers implemented using cudnn v3 ***************/
#ifdef USE_CUDNN
-#include <cudnn.h>
#define CHECK_CUDNN(x) CHECK_EQ(x, CUDNN_STATUS_SUCCESS)
class CudnnLayer : virtual public NeuronLayer {
@@ -273,9 +277,9 @@ class CudnnLayer : virtual public NeuronLayer {
/**
* Activation layer implemented using cudnn v3.
* Activation methods including
- * - "sigmoid"
- * - "tanh"
- * - "relu"
+ * - SIGMOID
+ * - TANH
+ * - RELU
*/
class CudnnActivationLayer : public ActivationLayer, public CudnnLayer {
public:
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index e7bf995..bb63208 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -294,6 +294,11 @@ class Blob {
inline bool transpose() const {
return transpose_;
}
+ inline const Blob<Dtype> T() const {
+ Blob<Dtype> ret(*this);
+ ret.transpose_ = !transpose_;
+ return ret;
+ }
protected:
std::shared_ptr<SyncedMemory> data_ = nullptr;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/utils/cuda_utils.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/cuda_utils.h b/include/singa/utils/cuda_utils.h
index b27a6bb..0b3e0a5 100644
--- a/include/singa/utils/cuda_utils.h
+++ b/include/singa/utils/cuda_utils.h
@@ -74,18 +74,4 @@
CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \
} while (0)
-#define CUBLAS_CHECK(condition) \
- do { \
- cublasStatus_t status = condition; \
- CHECK_EQ(status, CUBLAS_STATUS_SUCCESS) << " " \
- << caffe::cublasGetErrorString(status); \
- } while (0)
-
-#define CURAND_CHECK(condition) \
- do { \
- curandStatus_t status = condition; \
- CHECK_EQ(status, CURAND_STATUS_SUCCESS) << " " \
- << caffe::curandGetErrorString(status); \
- } while (0)
-
#endif // SINGA_UTILS_CUDA_UTILS_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index 3b0eefd..7ab884e 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -231,7 +231,7 @@ void gpu_e_f(const int n, const Dtype * A, Dtype * B) {
}
template<typename Op, typename Dtype>
-void gpu_e_f(const int n, const Dtype * A, const Dtype * B, const Dtype * C) {
+void gpu_e_f(const int n, const Dtype * A, const Dtype * B, Dtype * C) {
Op::CudaMap(A, B, C, n);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index 97d5cf7..586554e 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -466,7 +466,8 @@ void MVAddCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_add_vec_row(B->gpu_data(), A.gpu_data(), A.gpu_data(), m, n, n);
+ singa_gpu_add_vec_row(A.gpu_data(), B->gpu_data(), B->mutable_gpu_data(),
+ m, n, n);
#endif // USE_GPU
}
}
@@ -504,7 +505,8 @@ void MVAddRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
false, false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_add_vec_row(B->gpu_data(), A.gpu_data(), A.gpu_data(), m, n, n);
+ singa_gpu_add_vec_row(A.gpu_data(), B->gpu_data(), B->mutable_gpu_data(),
+ m, n, n);
#endif // USE_GPU
}
}
@@ -583,7 +585,7 @@ void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_row(A.gpu_data(), B->gpu_data(), m, n, n);
+ singa_gpu_sum_vec(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/include/singa/utils/math_kernel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_kernel.h b/include/singa/utils/math_kernel.h
index d763283..37a9356 100644
--- a/include/singa/utils/math_kernel.h
+++ b/include/singa/utils/math_kernel.h
@@ -26,7 +26,10 @@ namespace singa {
extern "C" {
void singa_gpu_sum_vec(float *data, float *sum , int n);
- void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
+ void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
+ int rows, int cols, int stride);
+
+ void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride);
void singa_gpu_add_vec_row(const float *src_vec_data,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 8a48c30..1ae6d9f 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -29,6 +29,8 @@
#include "singa/utils/common.h"
#include "singa/utils/tinydir.h"
#include "singa/utils/cluster.h"
+#include "singa/utils/context.h"
+#include "singa/proto/job.pb.h"
#include "singa/server.h"
#include "singa/stub.h"
#include "singa/worker.h"
@@ -75,7 +77,16 @@ void Driver::Init(int argc, char **argv) {
RegisterLayer<CConvolutionLayer, int>(kCConvolution);
RegisterLayer<CPoolingLayer, int>(kCPooling);
RegisterLayer<ConcateLayer, int>(kConcate);
- RegisterLayer<DropoutLayer, int>(kDropout);
+
+#ifdef USE_CUDNN
+ RegisterLayer<CudnnActivationLayer, int>(kCudnnActivation);
+ RegisterLayer<CudnnConvLayer, int>(kCudnnConv);
+ RegisterLayer<CudnnPoolLayer, int>(kCudnnPool);
+ RegisterLayer<CudnnLRNLayer, int>(kCudnnLRN);
+ RegisterLayer<CudnnSoftmaxLayer, int>(kCudnnSoftmax);
+ RegisterLayer<CudnnSoftmaxLossLayer, int>(kCudnnSoftmaxLoss);
+#endif
+
RegisterLayer<EuclideanLossLayer, int>(kEuclideanLoss);
RegisterLayer<InnerProductLayer, int>(kInnerProduct);
RegisterLayer<LabelLayer, int>(kLabel);
@@ -203,8 +214,16 @@ void Driver::Train(const JobProto& job_conf) {
vector<std::thread> threads;
for (auto server : servers)
threads.push_back(std::thread(&Server::Run, server));
- for (auto worker : workers)
+ int gpu = 0;
+ auto context = Singleton<Context>::Instance();
+ CHECK_LE(workers.size(), job_conf.gpu_size());
+ for (auto worker : workers) {
threads.push_back(std::thread(&Worker::Run, worker));
+ if (gpu < job_conf.gpu_size()) {
+ int device_id = job_conf.gpu(gpu++);
+ context->SetupDevice(threads.back().get_id(), device_id);
+ }
+ }
if (grp_size > 1 || nserver_grps > 0) {
int nservers_per_grp = cluster->nservers_per_group();
int lcm = LeastCommonMultiple(nservers_per_grp, nserver_grps);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/neuralnet/neuron_layer/inner_product.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/inner_product.cc b/src/neuralnet/neuron_layer/inner_product.cc
index 25b71df..3b18cd7 100644
--- a/src/neuralnet/neuron_layer/inner_product.cc
+++ b/src/neuralnet/neuron_layer/inner_product.cc
@@ -23,6 +23,7 @@
#include <glog/logging.h>
#include "singa/utils/singleton.h"
+#include "singa/utils/math_blob.h"
namespace singa {
@@ -57,38 +58,17 @@ void InnerProductLayer::Setup(const LayerProto& conf,
void InnerProductLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
- auto data = Tensor2(&data_);
- auto src = Tensor2(srclayers[0]->mutable_data(this));
- auto weight = Tensor2(weight_->mutable_data());
- auto bias = Tensor1(bias_->mutable_data());
- if (transpose_)
- data = dot(src, weight);
- else
- data = dot(src, weight.T());
- // repmat: repeat bias vector into batchsize rows
- data += expr::repmat(bias, batchsize_);
+ MMDot(srclayers[0]->data(this), weight_->data(), &data_);
+ MVAddRow(bias_->data(), &data_);
}
void InnerProductLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
- auto src = Tensor2(srclayers[0]->mutable_data(this));
- auto grad = Tensor2(&grad_);
- auto weight = Tensor2(weight_->mutable_data());
- auto gweight = Tensor2(weight_->mutable_grad());
- auto gbias = Tensor1(bias_->mutable_grad());
- gbias = expr::sum_rows(grad);
- if (transpose_)
- gweight = dot(src.T(), grad);
- else
- gweight = dot(grad.T(), src);
+ MVSumRow(1.0f, 0.0f, grad_, bias_->mutable_grad());
+ MVDot(grad_.T(), srclayers[0]->data(this), weight_->mutable_grad());
if (srclayers[0]->mutable_grad(this) != nullptr) {
- auto gsrc = Tensor2(srclayers[0]->mutable_grad(this));
- if (transpose_)
- gsrc = dot(grad, weight.T());
- else
- gsrc = dot(grad, weight);
+ MVDot(grad_, weight_->data(), srclayers[0]->mutable_grad(this));
}
}
-
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 12f6929..88caa44 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -52,6 +52,9 @@ message JobProto {
required int32 train_steps = 16;
// frequency of displaying training info
optional int32 disp_freq = 17 [default = 0];
+ // GPU device IDs for use, if fewer than workers per procs, some workers run
+ // on GPU and the rest run on CPU.
+ repeated int32 gpu = 18;
// frequency of test, e.g., do test every 100 training steps
optional int32 test_freq = 20 [default = 0];
@@ -603,6 +606,12 @@ enum LayerType {
kConvolution = 1;
kCConvolution = 27;
kCPooling = 28;
+ kCudnnConv = 50;
+ kCudnnPool = 51;
+ kCudnnLRN = 52;
+ kCudnnSoftmax = 53;
+ kCudnnActivation = 54;
+ kCudnnSoftmaxLoss = 55;
kDropout = 4;
kInnerProduct = 5;
kLRN = 6;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index 4dd91e0..12501fd 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -63,7 +63,7 @@ void kernel_sum_vec(float *data, float *sum , int n) {
}
__global__
-void kernel_sum_col(const float *src_mat_data,
+void kernel_sum_by_col(const float *src_mat_data,
float *dst_vec_data, int rows, int cols, int stride) {
int j = blockIdx.x;
int THREADS = blockDim.x;
@@ -98,6 +98,19 @@ void kernel_sum_col(const float *src_mat_data,
}
__global__
+void kernel_sum_by_row(const float *src_mat_data,
+ float *dst_vec_data, int rows, int cols, int stride) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ int num_threads = blockDim.x * gridDim.x;
+ for (; index < rows; index += num_threads) {
+ dst_vec_data[index] = 0.0f;
+ for (int k = 0; k < cols; k++) {
+ dst_vec_data[index] += src_mat_data[index * stride + k];
+ }
+ }
+}
+
+__global__
void kernel_add_vec_row(const float *src_vec_data, const float *src_mat_data,
float* des_mat_data, int rows, int cols, int stride) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
@@ -143,7 +156,7 @@ void kernel_sigmoid_grad(const float *src_data, float *des_data, int n) {
int index = blockIdx.x * blockDim.x + threadIdx.x;
int num_threads = blockDim.x * gridDim.x;
for (; index < n; index += num_threads) {
- des_data[index] = src_data[index] * (1.0f - src_data[index]);
+ des_data[index] = src_data[index] * (1.0f - src_data[index]);
}
}
@@ -288,12 +301,21 @@ void singa_gpu_sum_vec(float *data, float *sum , int n) {
kernel_sum_vec<<<num_blocks, threads_per_block>>>(data, sum, n);
}
-void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
+void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride) {
int threads_per_block = rows > CU1DBLOCK ? CU1DBLOCK : rows;
int num_blocks = cols;
- kernel_sum_col<<<num_blocks, threads_per_block>>>(src_mat_data, dst_vec_data,
+ kernel_sum_by_col<<<num_blocks, threads_per_block>>>(src_mat_data, dst_vec_data,
+ rows, cols, stride);
+}
+
+void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
+ int rows, int cols, int stride) {
+ int threads_per_block = cols > CU1DBLOCK ? CU1DBLOCK : cols;
+ int num_blocks = rows;
+
+ kernel_sum_by_row<<<num_blocks, threads_per_block>>>(src_mat_data, dst_vec_data,
rows, cols, stride);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/utils/param.cc
----------------------------------------------------------------------
diff --git a/src/utils/param.cc b/src/utils/param.cc
index 2ccc5a8..54fe2aa 100644
--- a/src/utils/param.cc
+++ b/src/utils/param.cc
@@ -237,11 +237,15 @@ Msg* Param::GenUpdateMsg(bool copy, int idx) {
msg->set_type(kUpdate);
msg->AddFormatFrame("i", copy);
void* ptr = grad_.mutable_cpu_data() + slice_offset_[idx];
+ // to change the head of SyncMem to cpu; otherwise, the updated parameter
+ // // values would not be synced to gpu (since the head is at gpu).
+ data_->mutable_cpu_data();
if (copy) {
msg->AddFrame(ptr, slice_size_[idx]*sizeof(float));
} else {
msg->AddFormatFrame("p", ptr); // to share values of grad blob
}
+
pending_update_[idx] = true;
num_pending_requests_++;
return msg;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8cacd83c/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index 4756514..8c1d950 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -28,6 +28,7 @@
#include "singa/utils/cluster.h"
#include "singa/utils/factory.h"
#include "singa/utils/singleton.h"
+#include "singa/utils/context.h"
namespace singa {
@@ -61,6 +62,11 @@ Worker::~Worker() {
void Worker::Run() {
LOG(ERROR) << "Worker (group = " << grp_id_ <<", id = " << id_ << ") start";
+ // setup gpu device
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device > 0)
+ context->ActivateDevice(device);
auto cluster = Cluster::Get();
int svr_grp = grp_id_ / cluster->nworker_groups_per_server_group();
CHECK(cluster->runtime()->JoinSGroup(grp_id_, id_, svr_grp));
[15/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Fixs include path problems on cudnn.
A reminder:
Users should configure their library path(LD_LIBRARY_PATH & LIBRARY_PATH) after they install cudnn libs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/6f81adba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/6f81adba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/6f81adba
Branch: refs/heads/master
Commit: 6f81adba402bcd217dd020faa4188968b3200bb7
Parents: 05680dd
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Sat Dec 12 10:40:36 2015 +0800
Committer: xiezl <xi...@comp.nus.edu.sg>
Committed: Sat Dec 12 10:44:21 2015 +0800
----------------------------------------------------------------------
Makefile.am | 8 +++++++-
configure.ac | 4 ++--
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6f81adba/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 1959fb6..90807be 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -181,7 +181,7 @@ py_LTLIBRARIES = $(PY_PROGS)
#lib_LTLIBRARIES = libsinga.la
libsinga_la_SOURCES = $(PROTO_SRCS) $(SINGA_SRCS)
libsinga_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
-libsinga_la_LDFLAGS =
+libsinga_la_LDFLAGS =
if LMDB
libsinga_la_CXXFLAGS += -DUSE_LMDB
endif
@@ -191,6 +191,7 @@ libsinga_la_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
libsinga_la_CXXFLAGS += $(CUDA_CFLAGS)
libsinga_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+
if DCUDNN
libsinga_la_SOURCES += $(CUDNN_SRCS)
libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
@@ -218,6 +219,7 @@ singa_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
singa_CXXFLAGS += $(CUDA_CFLAGS)
singa_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+
if DCUDNN
singa_SOURCES += $(CUDNN_SRCS)
singa_CXXFLAGS += $(CUDNN_CFLAGS)
@@ -232,11 +234,13 @@ singatool_LDFLAGS = -lsinga \
-lglog \
-lprotobuf \
-lzookeeper_mt
+
if DCUDA
singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
singatool_CXXFLAGS += $(CUDA_CFLAGS)
singatool_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+
if DCUDNN
singatool_SOURCES += $(CUDNN_SRCS)
singatool_CXXFLAGS += $(CUDNN_CFLAGS)
@@ -268,11 +272,13 @@ singatest_LDFLAGS = -lsinga \
if LMDB
singatest_LDFLAGS += -llmdb
endif
+
if DCUDA
singatest_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
singatest_CXXFLAGS += $(CUDA_CFLAGS)
singatest_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+
if DCUDNN
singatest_SOURCES += $(CUDNN_SRCS)
singatest_CXXFLAGS += $(CUDNN_CFLAGS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6f81adba/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 85cc5fb..1127994 100644
--- a/configure.ac
+++ b/configure.ac
@@ -100,9 +100,9 @@ AM_CONDITIONAL(DCUDNN, [test "$enable_cudnn" = "yes"])
AC_ARG_WITH([cudnn],
[AS_HELP_STRING([--with-cudnn=PATH], [prefix where CUDNN is installed])],
- [cudnn_prefix=$cudnnwithval], [cudnn_prefix="/usr/local/cuda"])
+ [cudnn_prefix=$withval], [cudnn_prefix="/usr/local/cuda"])
if test "$cudnn_prefix" == "yes"; then
- if test "$cudnnwithval" == "yes"; then
+ if test "$withval" == "yes"; then
cudnn_prefix="/usr/local/cuda"
fi
fi
[08/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Pass cublas handle from math_blob to math_addr.
Test configure-make for cpu code.
Compile success for Makefile.gpu.
Todo set up Context when creating worker threads.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/6e563344
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/6e563344
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/6e563344
Branch: refs/heads/master
Commit: 6e5633441da13625b28971a42afda416ee05e1c6
Parents: 2ed18a5
Author: seaokcs <se...@gmail.com>
Authored: Fri Nov 27 21:25:00 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:23 2015 +0800
----------------------------------------------------------------------
include/singa/utils/context.h | 2 +
include/singa/utils/math_addr.h | 45 ++++++------------
include/singa/utils/math_blob.h | 64 +++++++++++++-------------
src/neuralnet/neuron_layer/convolution.cc | 8 +---
src/neuralnet/neuron_layer/pooling.cc | 10 ++--
src/proto/job.proto | 20 ++++----
6 files changed, 61 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/include/singa/utils/context.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/context.h b/include/singa/utils/context.h
index 905b810..1d1802c 100644
--- a/include/singa/utils/context.h
+++ b/include/singa/utils/context.h
@@ -77,8 +77,10 @@ class Context {
*/
Context() {
for (int i = 0; i < kMaxNumGPU; i++) {
+#ifdef USE_GPU
cublas_handle_.push_back(nullptr);
curand_generator_.push_back(nullptr);
+#endif
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index f548606..3b0eefd 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -171,65 +171,51 @@ void cpu_sample_gaussian(URNG& g, int n, Dtype mean, Dtype std, Dtype* A) {
#ifdef USE_GPU
template<typename Dtype>
-Dtype gpu_asum(int n, const Dtype* A, int inc) {
+Dtype gpu_asum(cublasHandle_t handle, int n, const Dtype* A, int inc) {
Dtype result = 0.0;
- cublasHandle_t handle;
- cublasCreate(&handle);
cublasSasum(handle, n, A, inc, &result);
- cublasDestroy(handle);
return result;
}
template<typename Dtype>
-void gpu_gemm(const Dtype * A, const Dtype * B, const int m, const int n,
- const int k, const Dtype alpha, const Dtype beta, const bool TranA,
- const bool TranB, Dtype * C) {
+void gpu_gemm(cublasHandle_t handle, const Dtype * A, const Dtype * B,
+ const int m, const int n, const int k, const Dtype alpha, const Dtype beta,
+ const bool TranA, const bool TranB, Dtype * C) {
int lda = TranA ? m : k;
int ldb = TranB ? k : n;
int ldc = n;
cublasOperation_t tA = (TranA == false) ? CUBLAS_OP_N : CUBLAS_OP_T;
cublasOperation_t tB = (TranB == false) ? CUBLAS_OP_N : CUBLAS_OP_T;
- cublasHandle_t handle;
- cublasCreate(&handle);
cublasSgemm(handle, tB, tA, n, m, k, &alpha, B, ldb,
A, lda, &beta, C, ldc);
- cublasDestroy(handle);
}
template<typename Dtype>
-void gpu_gemv(const Dtype * A, const Dtype * B, const int m, const int n,
- const Dtype alpha, const Dtype beta, const bool TranA, Dtype * C) {
+void gpu_gemv(cublasHandle_t handle, const Dtype * A, const Dtype * B,
+ const int m, const int n, const Dtype alpha, const Dtype beta,
+ const bool TranA, Dtype * C) {
int lda = n;
cublasOperation_t tA = (TranA == true) ? CUBLAS_OP_N : CUBLAS_OP_T;
- cublasHandle_t handle;
- cublasCreate(&handle);
cublasSgemv(handle, tA, n, m, &alpha , A, lda, B, 1, &beta, C, 1);
- cublasDestroy(handle);
}
template<typename Dtype>
-void gpu_axpy(const Dtype * A, const int n, const Dtype alpha, Dtype * B) {
- cublasHandle_t handle;
- cublasCreate(&handle);
+void gpu_axpy(cublasHandle_t handle, const Dtype * A, const int n,
+ const Dtype alpha, Dtype * B) {
cublasSaxpy(handle, n, &alpha, A, 1, B, 1);
- cublasDestroy(handle);
}
template<typename Dtype>
-void gpu_scale(const int n, const Dtype alpha, Dtype * A) {
- cublasHandle_t handle;
- cublasCreate(&handle);
+void gpu_scale(cublasHandle_t handle, const int n, const Dtype alpha,
+ Dtype * A) {
cublasSscal(handle, n, &alpha, A, 1);
- cublasDestroy(handle);
}
template<typename Dtype>
-Dtype gpu_dot(const Dtype * A, const Dtype * B, const int n) {
- cublasHandle_t handle;
- cublasCreate(&handle);
+Dtype gpu_dot(cublasHandle_t handle, const Dtype * A, const Dtype * B,
+ const int n) {
Dtype result = 0.0;
cublasSdot(handle, n, A, 1, B, 1, &result);
- cublasDestroy(handle);
return result;
}
@@ -280,16 +266,11 @@ void gpu_expand_f(const Dtype * A, const int m, const int n, Dtype * B) {
template<typename Dtype, typename URNG>
void gpu_sample_uniform(URNG g, int n, Dtype low, Dtype high, Dtype* A) {
- //curandGenerator_t gen;
- //curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT);
- curandSetPseudoRandomGeneratorSeed(g, time(NULL));
curandGenerateUniform(g, A, n);
- //curandDestroyGenerator(gen);
}
template<typename Dtype, typename URNG>
void gpu_sample_gaussian(URNG g, int n, Dtype mean, Dtype std, Dtype* A) {
- curandSetPseudoRandomGeneratorSeed(g, time(NULL));
curandGenerateNormal(g, A, n, mean, std);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index bbf7cc0..97d5cf7 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -49,8 +49,8 @@ void Scale(Dtype alpha, Blob<Dtype> * B) {
cpu_scale(B->count(), alpha, B->mutable_cpu_data());
else {
#ifdef USE_GPU
- // TODO(haibo) check it.
- gpu_scale(B->count(), alpha, B->mutable_gpu_data());
+ gpu_scale(context->cublas_handle(device), B->count(), alpha,
+ B->mutable_gpu_data());
#endif
}
}
@@ -67,7 +67,8 @@ void AXPY(Dtype alpha, const Blob<Dtype> & A, Blob<Dtype> * B) {
cpu_axpy(A.count(), alpha, A.cpu_data(), B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- gpu_axpy(A.count(), alpha, A.gpu_data(), B->mutable_gpu_data());
+ gpu_axpy(context->cublas_handle(device), A.count(), alpha, A.gpu_data(),
+ B->mutable_gpu_data());
#endif // USE_GPU
}
}
@@ -106,9 +107,8 @@ void GEMV(Dtype alpha, Dtype beta, const Blob<Dtype>& A,
C->mutable_cpu_data());
} else {
#ifdef USE_GPU
- // gpu part
- gpu_gemv(A.gpu_data(), B.gpu_data(), m, n, alpha, beta, TranA,
- C->mutable_gpu_data());
+ gpu_gemv(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), m, n,
+ alpha, beta, TranA, C->mutable_gpu_data());
#endif // USE_GPU
}
}
@@ -172,9 +172,8 @@ void GEMM( Dtype alpha, Dtype beta, const Blob<Dtype>& A,
C->mutable_cpu_data());
} else {
#ifdef USE_GPU
- // gpu part
- gpu_gemm(A.gpu_data(), B.gpu_data(), m, n, k, alpha, beta,
- TranA, TranB, C->mutable_gpu_data());
+ gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), m, n, k,
+ alpha, beta, TranA, TranB, C->mutable_gpu_data());
#endif // USE_GPU
}
}
@@ -216,7 +215,7 @@ Dtype VVDot(const Blob<Dtype> & A, const Blob<Dtype> & B) {
} else {
#ifdef USE_GPU
// gpu part
- res = gpu_dot(A.gpu_data(), B.gpu_data(), n);
+ res = gpu_dot(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), n);
#endif // USE_GPU
}
return res;
@@ -244,8 +243,7 @@ void OuterProduct(const Blob<Dtype>& A, const Blob<Dtype>& B, Blob<Dtype> * C) {
C->mutable_cpu_data());
} else {
#ifdef USE_GPU
- // gpu part
- gpu_gemm(A.gpu_data(), B.gpu_data(), m, n, 1, 1, 0,
+ gpu_gemm(context->cublas_handle(device), A.gpu_data(), B.gpu_data(), m, n, 1, 1, 0,
false, false, C->mutable_gpu_data());
#endif // USE_GPU
}
@@ -264,10 +262,9 @@ void Map(const Blob<Dtype> & A, Blob<Dtype> * B) {
if (device == -1) {
cpu_e_f<Op>(A.count(), A.cpu_data(), B->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
- // gpu part
+#ifdef USE_GPU
gpu_e_f<Op>(A.count(), A.gpu_data(), B->mutable_gpu_data());
-#endif // SINGA_GPU
+#endif // USE_GPU
}
}
@@ -286,10 +283,10 @@ void Map(const Blob<Dtype> & A, const Blob<Dtype> & B, Blob<Dtype> * C) {
if (device == -1) {
cpu_e_f<Op>(A.count(), A.cpu_data(), B.cpu_data(), C->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
+#ifdef USE_GPU
// gpu part
gpu_e_f<Op>(A.count(), A.gpu_data(), B.gpu_data(), C->mutable_gpu_data());
-#endif // SINGA_GPU
+#endif // USE_GPU
}
}
@@ -305,8 +302,8 @@ void Map(Dtype alpha, const Blob<Dtype>& A, Blob<Dtype>* B) {
if (device == -1) {
cpu_e_f<Op>(A.count(), alpha, A.cpu_data(), B->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
-#endif // SINGA_GPU
+#ifdef USE_GPU
+#endif // USE_GPU
}
}
/**
@@ -323,8 +320,8 @@ void Map(Dtype alpha, const Blob<Dtype>& A, const Blob<Dtype>& B,
cpu_e_f<Op>(A.count(), alpha, A.cpu_data(), B->cpu_data(),
C->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
-#endif // SINGA_GPU
+#ifdef USE_GPU
+#endif // USE_GPU
}
}
@@ -563,8 +560,8 @@ void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
#ifdef USE_GPU
singa_gpu_sum_col(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
- }
#endif // USE_GPU
+ }
}
/**
@@ -588,8 +585,8 @@ void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
#ifdef USE_GPU
singa_gpu_sum_row(A.gpu_data(), B->gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
- }
#endif // USE_GPU
+ }
}
/**
@@ -606,10 +603,10 @@ void Reduce2D(const Blob<Dtype> & A, Blob<Dtype> * B) {
if (device == -1) {
cpu_reduce_f<Op>(A.cpu_data(), m, n, B->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
+#ifdef USE_GPU
// gpu part
gpu_reduce_f<Op>(A.gpu_data(), m, n, B->mutable_gpu_data());
-#endif // SINGA_GPU
+#endif // USE_GPU
}
}
/**
@@ -626,9 +623,9 @@ void Expand2D(const Blob<Dtype> & A, Blob<Dtype> * B) {
if (device == -1) {
cpu_expand_f<Op>(A.cpu_data(), m, n, B->mutable_cpu_data());
} else {
-#ifdef SINGA_GPU
+#ifdef USE_GPU
gpu_expand_f<Op>(A.gpu_data(), m, n, B->mutable_gpu_data());
-#endif // SINGA_GPU
+#endif // USE_GPU
}
}
@@ -640,13 +637,16 @@ Dtype Asum(const Blob<Dtype>& A) {
if (A.count() == 0) return Dtype(0);
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
+ Dtype ret = Dtype(0);
if (device == -1) {
- return cpu_asum(A.count(), A.cpu_data(), 1) / A.count();
+ ret = cpu_asum(A.count(), A.cpu_data(), 1) / A.count();
} else {
#ifdef USE_GPU
- return gpu_asum(A.count(), A.cpu_data(), 1) / A.count(); // TODO(haibo)
+ ret = gpu_asum(context->cublas_handle(device), A.count(), A.cpu_data(), 1)
+ / A.count();
#endif
}
+ return ret;
}
@@ -661,7 +661,6 @@ void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A) {
A->mutable_cpu_data());
} else {
#ifdef USE_GPU
- // TODO(haibo) check
gpu_sample_uniform(context->curand_generator(thread), A->count(), low, high,
A->mutable_gpu_data());
#endif
@@ -678,9 +677,8 @@ void SampleGaussian(Dtype mean, Dtype std, Blob<Dtype>* A) {
A->mutable_cpu_data());
} else {
#ifdef USE_GPU
- // TODO(haibo) check it.
- gpu_sample_gaussian(context->curand_generator(thread), A->count(), mean, std,
- A->mutable_gpu_data());
+ gpu_sample_gaussian(context->curand_generator(thread), A->count(),
+ mean, std, A->mutable_gpu_data());
#endif
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/src/neuralnet/neuron_layer/convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/convolution.cc b/src/neuralnet/neuron_layer/convolution.cc
index 2b4c6d9..edfabb6 100644
--- a/src/neuralnet/neuron_layer/convolution.cc
+++ b/src/neuralnet/neuron_layer/convolution.cc
@@ -40,17 +40,15 @@ void ConvolutionLayer::Setup(const LayerProto& conf,
if (conv_conf.has_kernel()) {
kernel_x_ = kernel_y_ = conv_conf.kernel();
} else {
- CHECK(conv_conf.has_kernel_x());
- CHECK(conv_conf.has_kernel_y());
kernel_x_ = conv_conf.kernel_x();
kernel_y_ = conv_conf.kernel_y();
}
+ CHECK_NE(kernel_x_, 0);
+ CHECK_NE(kernel_y_, 0);
if (conv_conf.has_pad()) {
pad_x_ = pad_y_ = conv_conf.pad();
} else {
- CHECK(conv_conf.has_pad_x());
- CHECK(conv_conf.has_pad_y());
pad_x_ = conv_conf.pad_x();
pad_y_ = conv_conf.pad_y();
}
@@ -58,8 +56,6 @@ void ConvolutionLayer::Setup(const LayerProto& conf,
if (conv_conf.has_stride()) {
stride_x_ = stride_y_ = conv_conf.stride();
} else {
- CHECK(conv_conf.has_stride_x());
- CHECK(conv_conf.has_stride_y());
stride_x_ = conv_conf.stride_x();
stride_y_ = conv_conf.stride_y();
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/src/neuralnet/neuron_layer/pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/pooling.cc b/src/neuralnet/neuron_layer/pooling.cc
index 2e246fc..5b408ba 100644
--- a/src/neuralnet/neuron_layer/pooling.cc
+++ b/src/neuralnet/neuron_layer/pooling.cc
@@ -37,24 +37,22 @@ void PoolingLayer::Setup(const LayerProto& conf,
if (pool_conf.has_kernel()) {
kernel_x_ = kernel_y_ = pool_conf.kernel();
} else {
- CHECK(pool_conf.has_kernel_x());
- CHECK(pool_conf.has_kernel_y());
kernel_x_ = pool_conf.kernel_x();
kernel_y_ = pool_conf.kernel_y();
}
+ CHECK_NE(kernel_x_, 0);
+ CHECK_NE(kernel_y_, 0);
+
if (pool_conf.has_pad()) {
pad_x_ = pad_y_ = pool_conf.pad();
} else {
- CHECK(pool_conf.has_pad_x());
- CHECK(pool_conf.has_pad_y());
pad_x_ = pool_conf.pad_x();
pad_y_ = pool_conf.pad_y();
}
+
if (pool_conf.has_stride()) {
stride_x_ = stride_y_ = pool_conf.stride();
} else {
- CHECK(pool_conf.has_stride_x());
- CHECK(pool_conf.has_stride_y());
stride_x_ = pool_conf.stride_x();
stride_y_ = pool_conf.stride_y();
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/6e563344/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 9d64c4b..12f6929 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -362,15 +362,14 @@ message ConvolutionProto {
// The number of outputs for the layer
optional int32 num_filters = 1;
// the kernel height/width
- optional int32 kernel = 2;
-
+ optional int32 kernel = 2 [default = 3];
// The padding height/width
optional int32 pad = 30 [default = 0];
// the stride
optional int32 stride = 31 [default = 1];
- optional int32 kernel_x = 41;
- optional int32 kernel_y = 42;
+ optional int32 kernel_x = 41 [default = 3];
+ optional int32 kernel_y = 42 [default = 3];
optional int32 pad_x = 44 [default = 0];
optional int32 pad_y = 45 [default = 0];
@@ -452,7 +451,7 @@ message LRNProto {
message PoolingProto {
// The kernel size (square)
- optional int32 kernel= 1;
+ optional int32 kernel= 1 [default = 3];
enum PoolMethod {
MAX = 0;
AVG = 1;
@@ -462,17 +461,16 @@ message PoolingProto {
// The padding size
optional uint32 pad = 31 [default = 0];
// The stride
- optional uint32 stride = 32 [default = 1];
-
+ optional uint32 stride = 32 [default = 2];
- optional int32 kernel_x = 41;
- optional int32 kernel_y = 42;
+ optional int32 kernel_x = 41 [default = 3];
+ optional int32 kernel_y = 42 [default = 3];
optional int32 pad_x = 44 [default = 0];
optional int32 pad_y = 45 [default = 0];
- optional int32 stride_x = 47 [default = 1];
- optional int32 stride_y = 48 [default = 1];
+ optional int32 stride_x = 47 [default = 2];
+ optional int32 stride_y = 48 [default = 2];
}
[18/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Replace CopyFrom with Copy in softmax loss layers. CopyFrom copies both
cpu and gpu memory, while Copy copies either cpu or gpu memory.
Rename CudnnLayer to CudnnBase, which was changed but missed during rebase..
Fix bugs from getting stuck using GPU (CUDA) operations.
Possible causes include,
1. different threads access the same GPU memory addr. The non-owner thread would get stuck.
2. cudnnCreate gets stuck if the GPU device is used by other programs.
3. CudnnSoftmaxLossLayer::Setup forgets to call LossLayer::Setup which leaves layer_conf_ un-initialized.
TODO
1. move cudnn headers and libs before cuda in compiling, otherwise there would be
link conflicts if there are multiple versions of cudnn in the system.
2. replace malloc with cudaMalloc in blob.cc(h) which pins the CPU
memory for efficient and stable memory transferring between CPU and GPU.
Tested with multiple GPUs and single GPU.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f3b47c70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f3b47c70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f3b47c70
Branch: refs/heads/master
Commit: f3b47c706e09315f2eb8eb1d599597093c980962
Parents: 372245f
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Mon Dec 14 22:27:12 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Wed Dec 16 19:20:44 2015 +0800
----------------------------------------------------------------------
Makefile.gpu | 16 ++++----
include/singa/neuralnet/neuron_layer.h | 20 ++++-----
include/singa/utils/blob.h | 3 ++
include/singa/utils/context.h | 43 ++++++++++++++++++++
include/singa/utils/math_addr.h | 4 +-
include/singa/utils/math_blob.h | 2 +
src/neuralnet/loss_layer/cudnn_softmaxloss.cc | 13 +++---
src/neuralnet/loss_layer/softmax.cc | 4 +-
src/neuralnet/neuron_layer/cudnn_activation.cc | 2 +-
src/neuralnet/neuron_layer/cudnn_convolution.cc | 6 ++-
src/neuralnet/neuron_layer/cudnn_lrn.cc | 2 +-
src/neuralnet/neuron_layer/cudnn_pooling.cc | 2 +-
src/neuralnet/neuron_layer/cudnn_softmax.cc | 2 +-
src/server.cc | 1 +
src/stub.cc | 8 ++--
src/utils/blob.cc | 12 +++---
src/utils/param.cc | 12 +++---
src/worker.cc | 25 +++++++++++-
18 files changed, 123 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/Makefile.gpu
----------------------------------------------------------------------
diff --git a/Makefile.gpu b/Makefile.gpu
index 9cbbc42..c35a445 100644
--- a/Makefile.gpu
+++ b/Makefile.gpu
@@ -20,17 +20,17 @@
###################User Config Varaibles #############################
# third-party library installation folder
-HOME_DIR := /home/wangwei/local
-
-CUDA_DIR := /usr/local/cuda
+HOME_DIR := /media/hd1/home/wangwei/local
# must config the cudnn folder if using cudnn
-CUDNN_DIR := /home/wangwei/local/cuda
+CUDNN_DIR := $(HOME_DIR)/cudnn
+
+CUDA_DIR := /usr/local/cuda
# Lib folder for system and external libs. You may need to change it.
-LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(HOME_DIR)/local/lib $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib $(CUDNN_DIR)/lib64
+LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(CUDNN_DIR)/lib64 $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib
# Header folder for system and external libs. You may need to change it.
-INCLUDE_DIRS := $(HOME_DIR)/include ./include $(HOME_DIR)/local/include/zookeeper $(CUDA_DIR)/include $(CUDNN_DIR)/include
+INCLUDE_DIRS := ./include $(HOME_DIR)/include $(CUDNN_DIR)/include $(CUDA_DIR)/include
# g++ location, should support c++11, tested with 4.8.1
CXX := g++
CUCXX := nvcc
@@ -49,7 +49,7 @@ BUILD_DIR := .libs
MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
ZK_FLAGS :=-DTHREADED -fpermissive
CXXFLAGS := -O2 -msse3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
- $(MSHADOW_FLAGS) $(ZK_FLAGS)\
+ $(MSHADOW_FLAGS) -DUSE_CUDNN $(ZK_FLAGS)\
-funroll-loops $(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
CUCXXFLAGS := -DUSE_CUDNN $(MSHADOW_FLAGS) -std=c++11 $(CUDA_ARCH) \
$(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
@@ -103,7 +103,7 @@ singa: $(PROTO_OBJS) $(SINGA_OBJS) $(SINGA_CUDA_OBJS)
$(CXX) -shared -o $(BUILD_DIR)/libsinga.so $(SINGA_OBJS)
$(CXX) $(SINGA_OBJS) $(SINGA_CUDA_OBJS) src/main.cc -o singa $(CXXFLAGS) $(LDFLAGS)
@echo
- $(CXX) $(SINGA_OBJS) $(SINGA_CUDA_OBJS) src/utils/tool.cc -o singatool $(CXXFLAGS) $(LDFLAGS)
+ $(CXX) $(BUILD_DIR)/libsinga.so src/utils/tool.cc -o singatool $(CXXFLAGS) $(LDFLAGS) -Wl,-unresolved-symbols=ignore-in-shared-libs
@echo
loader: proto $(LOADER_OBJS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 17891e1..393eeb4 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -25,6 +25,8 @@
#include <vector>
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
+#include "singa/utils/context.h"
+#include "singa/utils/singleton.h"
#ifdef USE_CUDNN
#include <cudnn.h>
@@ -274,11 +276,9 @@ class STanhLayer : public NeuronLayer {
#ifdef USE_CUDNN
#define CHECK_CUDNN(x) CHECK_EQ(x, CUDNN_STATUS_SUCCESS)
-class CudnnLayer : virtual public NeuronLayer {
+class CudnnBase : virtual public NeuronLayer {
public:
- ~CudnnLayer() {
- if (handle_ != nullptr)
- CHECK_CUDNN(cudnnDestroy(handle_));
+ ~CudnnBase() {
if (src_desc_ != nullptr)
CHECK_CUDNN(cudnnDestroyTensorDescriptor(src_desc_));
if (my_desc_ != nullptr)
@@ -286,9 +286,9 @@ class CudnnLayer : virtual public NeuronLayer {
}
void virtual InitCudnn() {
CHECK(!has_init_cudnn_);
- CHECK_CUDNN(cudnnCreate(&handle_));
CHECK_CUDNN(cudnnCreateTensorDescriptor(&src_desc_));
CHECK_CUDNN(cudnnCreateTensorDescriptor(&my_desc_));
+ handle_ = Singleton<Context>::Instance()->cudnn_handle();
has_init_cudnn_ = true;
}
protected:
@@ -304,7 +304,7 @@ class CudnnLayer : virtual public NeuronLayer {
* - TANH
* - RELU
*/
-class CudnnActivationLayer : public ActivationLayer, public CudnnLayer {
+class CudnnActivationLayer : public ActivationLayer, public CudnnBase {
public:
void InitCudnn() override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
@@ -317,7 +317,7 @@ class CudnnActivationLayer : public ActivationLayer, public CudnnLayer {
/**
* Convolution layer implemeneted using cudnn (v3 version backward functions).
*/
-class CudnnConvLayer : public ConvolutionLayer, public CudnnLayer {
+class CudnnConvLayer : public ConvolutionLayer, public CudnnBase {
public:
~CudnnConvLayer();
void InitCudnn() override;
@@ -334,7 +334,7 @@ class CudnnConvLayer : public ConvolutionLayer, public CudnnLayer {
size_t workspace_byte_limit_, workspace_count_;
};
-class CudnnLRNLayer : public LRNLayer, public CudnnLayer {
+class CudnnLRNLayer : public LRNLayer, public CudnnBase {
public:
~CudnnLRNLayer();
void InitCudnn() override;
@@ -348,7 +348,7 @@ class CudnnLRNLayer : public LRNLayer, public CudnnLayer {
/**
* Pooling layer implemented using cudnn.
*/
-class CudnnPoolLayer : public PoolingLayer, public CudnnLayer {
+class CudnnPoolLayer : public PoolingLayer, public CudnnBase {
public:
~CudnnPoolLayer();
void InitCudnn() override;
@@ -362,7 +362,7 @@ class CudnnPoolLayer : public PoolingLayer, public CudnnLayer {
/**
* Cudnn Softmax layer.
*/
-class CudnnSoftmaxLayer : public SoftmaxLayer, public CudnnLayer {
+class CudnnSoftmaxLayer : public SoftmaxLayer, public CudnnBase {
public:
void InitCudnn() override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index 7e1e516..256400e 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -71,12 +71,15 @@
namespace singa {
+// TODO(wangwei) use cudaMallocHost depending on Context::device.
inline void MallocHost(void** ptr, size_t size) {
*ptr = malloc(size);
+ // cudaMallocHost(ptr, size);
}
inline void FreeHost(void* ptr) {
free(ptr);
+ // cudaFreeHost(ptr);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/include/singa/utils/context.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/context.h b/include/singa/utils/context.h
index c23b338..5d49dc1 100644
--- a/include/singa/utils/context.h
+++ b/include/singa/utils/context.h
@@ -31,6 +31,11 @@
#ifdef USE_GPU
#include "singa/utils/cuda_utils.h"
+
+#ifdef USE_CUDNN
+#include <cudnn.h>
+#endif
+
#endif
namespace singa {
@@ -64,6 +69,13 @@ class Context {
}
}
}
+#ifdef USE_CUDNN
+ for (auto& handle : cudnn_handle_) {
+ if (handle != nullptr)
+ CHECK_EQ(cudnnDestroy(handle), CUDNN_STATUS_SUCCESS);
+ handle = nullptr;
+ }
+#endif
#endif
for (auto& entry : rand_generator_) {
if (entry.second != nullptr) {
@@ -80,6 +92,9 @@ class Context {
#ifdef USE_GPU
cublas_handle_.push_back(nullptr);
curand_generator_.push_back(nullptr);
+#ifdef USE_CUDNN
+ cudnn_handle_.push_back(nullptr);
+#endif
#endif
}
}
@@ -191,6 +206,7 @@ class Context {
*/
curandGenerator_t curand_generator(const int device_id) {
CHECK_GE(device_id, 0);
+ CHECK_LT(device_id, cudnn_handle_.size());
if (curand_generator_.at(device_id) == nullptr) {
// TODO(wangwei) handle user set seed
/*
@@ -204,6 +220,28 @@ class Context {
return curand_generator_[device_id];
}
+#ifdef USE_CUDNN
+ cudnnHandle_t cudnn_handle() {
+ return cudnn_handle(std::this_thread::get_id());
+ }
+
+ cudnnHandle_t cudnn_handle(const std::thread::id thread_id) {
+ return cudnn_handle(device_id(thread_id));
+ }
+
+ cudnnHandle_t cudnn_handle(const int device_id) {
+ CHECK_GE(device_id, 0);
+ CHECK_LT(device_id, cudnn_handle_.size());
+ if (cudnn_handle_.at(device_id) == nullptr) {
+ ActivateDevice(device_id);
+ // LOG(ERROR) << "create cudnn handle for device " << device_id;
+ CHECK_EQ(cudnnCreate(&cudnn_handle_[device_id]), CUDNN_STATUS_SUCCESS);
+ }
+ // LOG(ERROR) << "use cudnn handle from device " << device_id;
+ return cudnn_handle_[device_id];
+ }
+#endif
+
#endif
protected:
@@ -220,7 +258,12 @@ class Context {
std::vector<cublasHandle_t> cublas_handle_;
//!< cublas rand generator indexed by GPU device ID
std::vector<curandGenerator_t> curand_generator_;
+
+#ifdef USE_CUDNN
+ std::vector<cudnnHandle_t> cudnn_handle_;
#endif
+#endif
+
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index d641251..524e13e 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -198,8 +198,8 @@ void gpu_gemv(cublasHandle_t handle, const Dtype * A, const Dtype * B,
}
template<typename Dtype>
-void gpu_axpy(cublasHandle_t handle, const Dtype * A, const int n,
- const Dtype alpha, Dtype * B) {
+void gpu_axpy(cublasHandle_t handle, const int n, const Dtype alpha,
+ const Dtype * A, Dtype * B) {
cublasSaxpy(handle, n, &alpha, A, 1, B, 1);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index bc38bd4..8807bdb 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -344,6 +344,8 @@ void Copy(const Blob<Dtype>& A, Blob<Dtype>* B) {
std::copy(A.cpu_data(), A.cpu_data() + A.count(), B->mutable_cpu_data());
} else {
#ifdef USE_GPU
+ CUDA_CHECK(cudaMemcpy(static_cast<Dtype*>(B->mutable_gpu_data()),
+ A.gpu_data(), sizeof(Dtype) * A.count(), cudaMemcpyDefault));
#endif
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
index 1fe228b..0d4ba45 100644
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
@@ -27,30 +27,26 @@
namespace singa {
void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
+ LossLayer::Setup(conf, srclayers);
softmax_.Setup(conf, vector<Layer*> {srclayers.at(0)});
data_.Reshape(softmax_.data(this).shape());
data_.ShareData(softmax_.mutable_data(this), false);
batchsize_ = data_.shape(0);
dim_ = data_.count() / batchsize_;
- LOG(ERROR) << batchsize_ << " " << dim_;
}
void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
softmax_.ComputeFeature(flag, srclayers);
- // compute loss
Blob<int> label(batchsize_);
int *labelptr = label.mutable_cpu_data();
-
// aux_data: vector<int>, convert vector to int array.
for (int i = 0; i < batchsize_; ++i) {
labelptr[i] = srclayers[1]->aux_data(this)[i];
}
Blob<float> loss(batchsize_);
-
- const float *prob = data_.gpu_data();
- singa_gpu_softmaxloss_forward(batchsize_, dim_, prob, label.gpu_data(),
- loss.mutable_gpu_data());
+ singa_gpu_softmaxloss_forward(batchsize_, dim_, data_.gpu_data(),
+ label.gpu_data(), loss.mutable_gpu_data());
loss_ += Asum(loss);
counter_++;
}
@@ -58,7 +54,8 @@ void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- gsrcblob->CopyFrom(data_);
+ Copy(data_, gsrcblob);
+ // gsrcblob->CopyFrom(data_);
float* gsrcptr = gsrcblob->mutable_gpu_data();
Blob<int> label(batchsize_);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index ce858ac..5a3d257 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -24,6 +24,7 @@
#include <algorithm>
#include "singa/neuralnet/loss_layer.h"
#include "mshadow/tensor.h"
+#include "singa/utils/math_blob.h"
namespace singa {
@@ -88,7 +89,8 @@ void SoftmaxLossLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
const auto& label = srclayers[1]->aux_data();
Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- gsrcblob->CopyFrom(data_);
+ Copy(data_, gsrcblob);
+// gsrcblob->CopyFrom(data_);
float* gsrcptr = gsrcblob->mutable_cpu_data();
for (int n = 0; n < batchsize_; n++) {
gsrcptr[n*dim_ + static_cast<int>(label[n])] -= 1.0f;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/neuron_layer/cudnn_activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cc b/src/neuralnet/neuron_layer/cudnn_activation.cc
index 9088d84..5405b53 100644
--- a/src/neuralnet/neuron_layer/cudnn_activation.cc
+++ b/src/neuralnet/neuron_layer/cudnn_activation.cc
@@ -24,7 +24,7 @@
namespace singa {
void CudnnActivationLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
+ CudnnBase::InitCudnn();
// TODO(wangwei) make the mode case insensitive
if (layer_conf_.activation_conf().type() == SIGMOID)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/neuron_layer/cudnn_convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cc b/src/neuralnet/neuron_layer/cudnn_convolution.cc
index 2cf376c..ca67a7d 100644
--- a/src/neuralnet/neuron_layer/cudnn_convolution.cc
+++ b/src/neuralnet/neuron_layer/cudnn_convolution.cc
@@ -32,7 +32,7 @@ CudnnConvLayer::~CudnnConvLayer() {
}
void CudnnConvLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
+ CudnnBase::InitCudnn();
// convert MB to bytes
workspace_byte_limit_
= layer_conf_.convolution_conf().workspace_byte_limit() << 20;
@@ -149,7 +149,6 @@ void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
&beta,
my_desc_,
data_.mutable_gpu_data()));
-
if (bias_) {
beta = 1.f;
CHECK_CUDNN(cudnnAddTensor(handle_,
@@ -167,6 +166,7 @@ void
CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
float alpha = 1.f, beta = 0.f;
Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
+ // LOG(ERROR) << "backward bias";
if (bias_) {
CHECK_CUDNN(cudnnConvolutionBackwardBias(handle_,
&alpha,
@@ -176,6 +176,7 @@ CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
bias_desc_,
bias_->mutable_grad()->mutable_gpu_data()));
}
+ // LOG(ERROR) << "backward w";
CHECK_CUDNN(cudnnConvolutionBackwardFilter_v3(handle_,
&alpha,
src_desc_,
@@ -189,6 +190,7 @@ CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
&beta,
filter_desc_,
weight_->mutable_grad()->mutable_gpu_data()));
+ // LOG(ERROR) << "backward src";
if (srclayers[0]->mutable_grad(this) != nullptr) {
CHECK_CUDNN(cudnnConvolutionBackwardData_v3(handle_,
&alpha,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/neuron_layer/cudnn_lrn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cc b/src/neuralnet/neuron_layer/cudnn_lrn.cc
index f976b16..8237b13 100644
--- a/src/neuralnet/neuron_layer/cudnn_lrn.cc
+++ b/src/neuralnet/neuron_layer/cudnn_lrn.cc
@@ -30,7 +30,7 @@ CudnnLRNLayer::~CudnnLRNLayer() {
void CudnnLRNLayer::InitCudnn() {
mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
- CudnnLayer::InitCudnn();
+ CudnnBase::InitCudnn();
CHECK_CUDNN(cudnnCreateLRNDescriptor(&norm_desc_));
CHECK_CUDNN(cudnnSetLRNDescriptor(norm_desc_,
lsize_,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/neuron_layer/cudnn_pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cc b/src/neuralnet/neuron_layer/cudnn_pooling.cc
index 217df51..4c4c038 100644
--- a/src/neuralnet/neuron_layer/cudnn_pooling.cc
+++ b/src/neuralnet/neuron_layer/cudnn_pooling.cc
@@ -30,7 +30,7 @@ CudnnPoolLayer::~CudnnPoolLayer() {
}
void CudnnPoolLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
+ CudnnBase::InitCudnn();
CHECK_CUDNN(cudnnCreatePoolingDescriptor(&pool_desc_));
CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
CUDNN_TENSOR_NCHW,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/neuralnet/neuron_layer/cudnn_softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cc b/src/neuralnet/neuron_layer/cudnn_softmax.cc
index a1a492e..bf5a8d3 100644
--- a/src/neuralnet/neuron_layer/cudnn_softmax.cc
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cc
@@ -24,7 +24,7 @@
namespace singa {
void CudnnSoftmaxLayer::InitCudnn() {
- CudnnLayer::InitCudnn();
+ CudnnBase::InitCudnn();
CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/server.cc
----------------------------------------------------------------------
diff --git a/src/server.cc b/src/server.cc
index 003d4e3..bd7b5f8 100644
--- a/src/server.cc
+++ b/src/server.cc
@@ -200,6 +200,7 @@ const vector<Msg*> Server::HandleUpdate(Msg **msg) {
auto param = entry->shares.at(0);
// extract and aggregate gradients
param->ParseUpdateMsgs(request);
+ // DLOG(ERROR) << "update param " << param->id() << " @ step " << step;
updater_->Update(step, param, 1.0f / entry->num_total);
param->set_version(param->version() + 1);
// response to all shares of this param
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/stub.cc
----------------------------------------------------------------------
diff --git a/src/stub.cc b/src/stub.cc
index 184a1f3..82526de 100644
--- a/src/stub.cc
+++ b/src/stub.cc
@@ -26,11 +26,11 @@
#include <map>
#include <thread>
#include <set>
-#include "mshadow/tensor.h"
#include "singa/proto/common.pb.h"
#include "singa/utils/cluster.h"
#include "singa/utils/common.h"
#include "singa/utils/tinydir.h"
+#include "singa/utils/math_blob.h"
namespace singa {
@@ -242,11 +242,9 @@ const vector<Msg*> Stub::HandleUpdateRequest(ParamEntry *entry, Msg** msg) {
// average local gradient
if (entry->num_local > 1) {
auto it = entry->shares.begin();
- auto shape = mshadow::Shape1((*it)->size());
- mshadow::Tensor<mshadow::cpu, 1> sum((*it)->mutable_cpu_grad(), shape);
+ auto sum = it;
for (++it; it != entry->shares.end(); it++) {
- mshadow::Tensor<mshadow::cpu, 1> grad((*it)->mutable_cpu_grad(), shape);
- sum += grad;
+ AXPY(1.0f, (*it)->grad(), (*sum)->mutable_grad());
}
}
int step = (*msg)->trgt_version();
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index 735226d..8e30060 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -213,13 +213,13 @@ void Blob<Dtype>::CopyFrom(const Blob& source) {
}
template <typename Dtype>
-void Blob<Dtype>::CopyFrom(const Blob& source, bool reshape) {
- if (!std::equal(shape_.begin(), shape_.end(), source.shape_.begin())) {
- if (reshape) {
- Reshape(source.shape_);
- } else {
+void Blob<Dtype>::CopyFrom(const Blob& source, bool shape_check) {
+ LOG(WARNING) << "Better use Copy(const Blob&, Blob*)";
+ CHECK_EQ (source.count(), count()) << " cp between blobs of diff size";
+
+ if (shape_check &&
+ !std::equal(shape_.begin(), shape_.end(), source.shape_.begin())) {
LOG(FATAL) << "Trying to copy blobs of different sizes.";
- }
}
#ifndef CPU_ONLY
CUDA_CHECK(cudaMemcpy(static_cast<Dtype*>(data_->mutable_gpu_data()),
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/utils/param.cc
----------------------------------------------------------------------
diff --git a/src/utils/param.cc b/src/utils/param.cc
index 097fa61..70a969f 100644
--- a/src/utils/param.cc
+++ b/src/utils/param.cc
@@ -212,8 +212,8 @@ Msg* Param::GenPutMsg(bool copy, int idx) {
CHECK_LT(idx, num_slices_);
Msg* msg = new Msg();
msg->set_type(kPut);
- void* ptr = mutable_cpu_data() + slice_offset_[idx];
- void* p = ptr;
+ const void* ptr = data_.cpu_data() + slice_offset_[idx];
+ const void* p = ptr;
if (copy) p = nullptr;
msg->AddFormatFrame("iffp", slice_size_[idx], lr_scale(), wd_scale(), p);
if (copy) {
@@ -226,7 +226,8 @@ Msg* Param::GenGetMsg(bool copy, int idx) {
CHECK_LT(idx, num_slices_);
Msg* msg = new Msg();
msg->set_type(kGet);
- msg->AddFormatFrame("ip", copy, data_.cpu_data() + slice_offset_[idx]);
+ msg->AddFormatFrame("ip", copy, data_.mutable_cpu_data()
+ + slice_offset_[idx]);
pending_get_[idx] = true;
num_pending_requests_++;
return msg;
@@ -237,10 +238,7 @@ Msg* Param::GenUpdateMsg(bool copy, int idx) {
Msg* msg = new Msg();
msg->set_type(kUpdate);
msg->AddFormatFrame("i", copy);
- void* ptr = grad_.mutable_cpu_data() + slice_offset_[idx];
- // to change the head of SyncMem to cpu; otherwise, the updated parameter
- // // values would not be synced to gpu (since the head is at gpu).
- data_.mutable_cpu_data();
+ const void* ptr = grad_.cpu_data() + slice_offset_[idx];
if (copy) {
msg->AddFrame(ptr, slice_size_[idx]*sizeof(float));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f3b47c70/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index 333408d..01872d0 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -241,6 +241,9 @@ int Worker::Put(int step, Param* param) {
LOG(WARNING) << "Null dealer in worker (" << grp_id_ << ", " << id_ << ")";
return 1;
}
+ // set Blob head to cpu to avoid calling cudaMemcpy by the stub thread, which
+ // would hang on some machines.
+ param->data().cpu_data();
Msg* msg = new Msg(Addr(grp_id_, id_, kWorkerParam), Addr(-1, -1, kStub));
msg->set_trgt(ParamTrgt(param->owner(), 0), step);
msg->set_type(kPut);
@@ -255,6 +258,10 @@ int Worker::Get(int step, Param* param) {
LOG(WARNING) << "Null dealer in worker (" << grp_id_ << ", " << id_ << ")";
return 1;
}
+ // set Blob head to cpu to avoid calling cudaMemcpy by the stub thread, which
+ // would hang on some machines.
+ param->mutable_data()->mutable_cpu_data();
+
Msg* msg = new Msg(Addr(grp_id_, id_, kWorkerParam), Addr(-1, -1, kStub));
msg->set_trgt(ParamTrgt(param->owner(), 0), step);
msg->set_type(kGet);
@@ -268,6 +275,18 @@ int Worker::Update(int step, Param* param) {
LOG(WARNING) << "Null dealer in worker (" << grp_id_ << ", " << id_ << ")";
return 1;
}
+ // head of data Blob (SyncMem) to cpu, because the stub thread may use
+ // cudaMemcpy copy gradients into msgs. cudaMemcpy hangs when called by the
+ // stub thread on some GPU machines.
+ // TODO(wangwei) fix this issue and remove the following line.
+ // optimize for training with single worker by removing stub and server, and
+ // updating parameters locally inside the worker GPU. Then we do not need to
+ // transfer gradients and parameter values between GPU-CPU.
+ param->grad().cpu_data();
+ // change the head of SyncMem to cpu; otherwise, the updated parameter
+ // values would not be synced to gpu (since the head is at gpu).
+ param->mutable_data()->mutable_cpu_data();
+
Msg* msg = new Msg(Addr(grp_id_, id_, kWorkerParam), Addr(-1, -1, kStub));
msg->set_trgt(ParamTrgt(param->owner(), 0), step);
msg->set_type(kUpdate);
@@ -288,8 +307,10 @@ int Worker::CollectAll(int step, NeuralNet* net) {
}
int Worker::Collect(int step, Param* param) {
- while (param->version() <= param->last_version())
+ while (param->version() <= param->last_version()) {
std::this_thread::sleep_for(std::chrono::milliseconds(kCollectSleepTime));
+ // LOG(ERROR) << "wait "<< param->id() << " at " << step << " by " <<id_;
+ }
return 1;
}
@@ -332,6 +353,7 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) {
Collect(step, p);
}
}
+ // LOG(ERROR) << layer->name() << " forward";
layer->ComputeFeature(phase | kForward, net->srclayers(layer));
// TODO(wangwei): enable this for model partition
// send data to other workers
@@ -350,6 +372,7 @@ void BPWorker::Backward(int step, NeuralNet* net) {
// send data to other workers
// if (typeid(layer) == typeid(BridgeSrcLayer))
// ReceiveBlobs(false, true, layer, net);
+ // LOG(ERROR) << layer->name() << " backward";
layer->ComputeGradient(kTrain | kBackward, net->srclayers(layer));
for (Param* p : layer->GetParams())
Update(step, p);
[05/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Add some gpu functions:
gpu_scale();
gpu_asum();
gpu_sample_gaussian();
gpu_sample_uniform();
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/2ed18a54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/2ed18a54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/2ed18a54
Branch: refs/heads/master
Commit: 2ed18a547243b0ff160405f969d243cca31672bc
Parents: 5d35ef2
Author: seaok <se...@gmail.com>
Authored: Fri Nov 27 18:51:05 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:47:57 2015 +0800
----------------------------------------------------------------------
include/singa/utils/math_addr.h | 36 ++++++++++++++++++++++++++++++------
include/singa/utils/math_blob.h | 10 ++++++----
2 files changed, 36 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2ed18a54/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index f63ff78..f548606 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -29,6 +29,7 @@ extern "C" {
#endif
#include "singa/utils/singa_op.h"
#ifdef USE_GPU
+#include "cuda_utils.h"
#include <cublas_v2.h>
#endif
@@ -170,6 +171,16 @@ void cpu_sample_gaussian(URNG& g, int n, Dtype mean, Dtype std, Dtype* A) {
#ifdef USE_GPU
template<typename Dtype>
+Dtype gpu_asum(int n, const Dtype* A, int inc) {
+ Dtype result = 0.0;
+ cublasHandle_t handle;
+ cublasCreate(&handle);
+ cublasSasum(handle, n, A, inc, &result);
+ cublasDestroy(handle);
+ return result;
+}
+
+template<typename Dtype>
void gpu_gemm(const Dtype * A, const Dtype * B, const int m, const int n,
const int k, const Dtype alpha, const Dtype beta, const bool TranA,
const bool TranB, Dtype * C) {
@@ -205,6 +216,14 @@ void gpu_axpy(const Dtype * A, const int n, const Dtype alpha, Dtype * B) {
}
template<typename Dtype>
+void gpu_scale(const int n, const Dtype alpha, Dtype * A) {
+ cublasHandle_t handle;
+ cublasCreate(&handle);
+ cublasSscal(handle, n, &alpha, A, 1);
+ cublasDestroy(handle);
+}
+
+template<typename Dtype>
Dtype gpu_dot(const Dtype * A, const Dtype * B, const int n) {
cublasHandle_t handle;
cublasCreate(&handle);
@@ -259,14 +278,19 @@ void gpu_expand_f(const Dtype * A, const int m, const int n, Dtype * B) {
}
-template<typename Dtype>
-void gpu_sample_uniform(int n, Dtype low, Dtype high, Dtype* A) {
-
+template<typename Dtype, typename URNG>
+void gpu_sample_uniform(URNG g, int n, Dtype low, Dtype high, Dtype* A) {
+ //curandGenerator_t gen;
+ //curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT);
+ curandSetPseudoRandomGeneratorSeed(g, time(NULL));
+ curandGenerateUniform(g, A, n);
+ //curandDestroyGenerator(gen);
}
-template<typename Dtype>
-void gpu_sample_gaussian(int n, Dtype mean, Dtype std, Dtype* A) {
-
+template<typename Dtype, typename URNG>
+void gpu_sample_gaussian(URNG g, int n, Dtype mean, Dtype std, Dtype* A) {
+ curandSetPseudoRandomGeneratorSeed(g, time(NULL));
+ curandGenerateNormal(g, A, n, mean, std);
}
// expand each element in A into a row of B
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2ed18a54/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index ce40d4f..bbf7cc0 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -50,7 +50,7 @@ void Scale(Dtype alpha, Blob<Dtype> * B) {
else {
#ifdef USE_GPU
// TODO(haibo) check it.
-// gpu_scale(B->count(), alpha, B->mutable_gpu_data());
+ gpu_scale(B->count(), alpha, B->mutable_gpu_data());
#endif
}
}
@@ -644,7 +644,7 @@ Dtype Asum(const Blob<Dtype>& A) {
return cpu_asum(A.count(), A.cpu_data(), 1) / A.count();
} else {
#ifdef USE_GPU
- return 0; // TODO(haibo)
+ return gpu_asum(A.count(), A.cpu_data(), 1) / A.count(); // TODO(haibo)
#endif
}
}
@@ -662,7 +662,8 @@ void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A) {
} else {
#ifdef USE_GPU
// TODO(haibo) check
- gpu_sample_uniform(A->count(), low, high, A->mutable_gpu_data());
+ gpu_sample_uniform(context->curand_generator(thread), A->count(), low, high,
+ A->mutable_gpu_data());
#endif
}
}
@@ -678,7 +679,8 @@ void SampleGaussian(Dtype mean, Dtype std, Blob<Dtype>* A) {
} else {
#ifdef USE_GPU
// TODO(haibo) check it.
- gpu_sample_gaussian(A->count(), mean, std, A->mutable_gpu_data());
+ gpu_sample_gaussian(context->curand_generator(thread), A->count(), mean, std,
+ A->mutable_gpu_data());
#endif
}
}
[10/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
cudnn for cifar10 reaches accuracy 0.814
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5728c248
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5728c248
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5728c248
Branch: refs/heads/master
Commit: 5728c24832f8ce542325410c033d9c59dadab712
Parents: 8174760
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Thu Dec 10 21:37:43 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:24 2015 +0800
----------------------------------------------------------------------
examples/cifar10/cudnn.conf | 18 ++++--------------
1 file changed, 4 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5728c248/examples/cifar10/cudnn.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/cudnn.conf b/examples/cifar10/cudnn.conf
index 6a5e15f..49132d8 100644
--- a/examples/cifar10/cudnn.conf
+++ b/examples/cifar10/cudnn.conf
@@ -1,21 +1,11 @@
name: "cifar10-convnet"
-<<<<<<< HEAD
-train_steps: 1000
-test_steps: 0
-test_freq: 0
-#validate_steps: 100
-#validate_freq: 300
-disp_freq: 30
-gpu: 0
-=======
-train_steps: 13000
+train_steps: 70000
test_steps: 100
test_freq: 1000
#validate_steps: 100
#validate_freq: 300
disp_freq: 200
->>>>>>> bfb913f... fix bug from cudnnsoftmax and cudnnsoftmaxloss; todo debug accuracy problem. the accuracy improves slower than that from caffe and cannot reach 0.8 finally.
-gpu: 1
+gpu: 0
#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
train_one_batch {
alg: kBP
@@ -298,7 +288,7 @@ neuralnet {
cluster {
nworker_groups: 1
nserver_groups: 1
- nworkers_per_group: 2
- nworkers_per_procs: 2
+ nworkers_per_group: 1
+ nworkers_per_procs: 1
workspace: "examples/cifar10"
}
[11/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
test with multi-gpus on cifar10; setting batchszie=500, it takes one iteration 3s on single gpu, and 2s on 2 gpus
fix bug from cudnnsoftmax and cudnnsoftmaxloss; todo debug accuracy problem. the accuracy improves slower than that from caffe and cannot reach 0.8 finally.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/81747603
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/81747603
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/81747603
Branch: refs/heads/master
Commit: 81747603149afeadd8b2b93ce48a170040982acf
Parents: eb97097
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Sat Dec 5 23:40:37 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:24 2015 +0800
----------------------------------------------------------------------
examples/cifar10/cudnn.conf | 39 ++++++++++----
examples/cifar10/job.conf | 8 +--
include/singa/neuralnet/layer.h | 3 --
include/singa/neuralnet/loss_layer.h | 22 +++++---
include/singa/neuralnet/neuralnet.h | 12 +++--
include/singa/neuralnet/neuron_layer.h | 12 +++--
include/singa/neuralnet/output_layer.h | 13 ++++-
include/singa/utils/blob.h | 6 ++-
include/singa/utils/context.h | 14 +++++
include/singa/utils/math_blob.h | 26 ++++-----
include/singa/utils/math_kernel.h | 4 +-
include/singa/utils/param.h | 47 +++++++++--------
src/driver.cc | 12 +++--
src/neuralnet/input_layer/deprecated.cc | 22 ++++----
src/neuralnet/input_layer/image_preprocess.cc | 8 +--
src/neuralnet/input_layer/store.cc | 19 +++----
src/neuralnet/layer.cc | 11 ----
src/neuralnet/loss_layer/cudnn_softmaxloss.cc | 28 ++++++----
src/neuralnet/loss_layer/euclidean.cc | 11 +++-
src/neuralnet/loss_layer/softmax.cc | 14 ++++-
src/neuralnet/neuralnet.cc | 8 +--
src/neuralnet/neuron_layer/cudnn_softmax.cc | 17 +++---
src/neuralnet/neuron_layer/rbm.cc | 12 ++++-
src/neuralnet/neuron_layer/softmax.cc | 11 ++--
src/neuralnet/output_layer/accuracy.cc | 61 ++++++++++++++++++++++
src/proto/job.proto | 1 +
src/server.cc | 10 ++--
src/stub.cc | 6 ++-
src/utils/blob.cc | 10 +++-
src/utils/math_kernel.cu | 44 ++++++++--------
src/utils/param.cc | 35 +++++++------
src/worker.cc | 18 ++++---
32 files changed, 366 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/examples/cifar10/cudnn.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/cudnn.conf b/examples/cifar10/cudnn.conf
index 4a29b7f..6a5e15f 100644
--- a/examples/cifar10/cudnn.conf
+++ b/examples/cifar10/cudnn.conf
@@ -1,4 +1,5 @@
name: "cifar10-convnet"
+<<<<<<< HEAD
train_steps: 1000
test_steps: 0
test_freq: 0
@@ -6,6 +7,14 @@ test_freq: 0
#validate_freq: 300
disp_freq: 30
gpu: 0
+=======
+train_steps: 13000
+test_steps: 100
+test_freq: 1000
+#validate_steps: 100
+#validate_freq: 300
+disp_freq: 200
+>>>>>>> bfb913f... fix bug from cudnnsoftmax and cudnnsoftmaxloss; todo debug accuracy problem. the accuracy improves slower than that from caffe and cannot reach 0.8 finally.
gpu: 1
#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
train_one_batch {
@@ -93,6 +102,7 @@ neuralnet {
param {
name: "b1"
lr_scale:2.0
+ wd_scale: 0
init {
type: kConstant
value:0
@@ -148,6 +158,7 @@ neuralnet {
param {
name: "b2"
lr_scale:2.0
+ wd_scale: 0
init {
type: kConstant
value:0
@@ -201,6 +212,8 @@ neuralnet {
}
param {
name: "b3"
+ lr_scale: 2
+ wd_scale: 0
init {
type: kConstant
value:0
@@ -250,22 +263,26 @@ neuralnet {
}
}
}
-# layer {
-# name : "softmax"
-# type: kSoftmax
-# srclayers: "ip1"
-# }
-#
-# layer {
-# name : "argsort"
-# type: kArgSort
-# srclayers: "softmax"
-# }
+ layer {
+ name : "softmax"
+ type: kCudnnSoftmax
+ srclayers: "ip1"
+ include: kTest
+ }
+
+ layer {
+ name : "accuracy"
+ type: kAccuracy
+ srclayers: "softmax"
+ srclayers: "data"
+ include: kTest
+ }
layer{
name: "loss"
type: kSoftmaxLoss
srclayers:"ip1"
srclayers: "data"
+ include : kTrain
}
# uncomment "softmax", "argsort", "output" layer and comment "loss" layer
# to extract features from argsort
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 7e42ed8..1dad0f7 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -1,10 +1,10 @@
name: "cifar10-convnet"
-train_steps: 30
+train_steps: 5
test_steps: 100
test_freq: 0
#validate_steps: 100
#validate_freq: 300
-disp_freq: 10
+disp_freq: 1
debug: true
#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
train_one_batch {
@@ -34,8 +34,8 @@ neuralnet {
backend: "kvfile"
path: "examples/cifar10/train_data.bin"
mean_file: "examples/cifar10/image_mean.bin"
- batchsize: 64
- random_skip: 5000
+ batchsize: 100
+ #random_skip: 5000
shape: 3
shape: 32
shape: 32
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/layer.h b/include/singa/neuralnet/layer.h
index 5e2692e..599203f 100644
--- a/include/singa/neuralnet/layer.h
+++ b/include/singa/neuralnet/layer.h
@@ -308,7 +308,6 @@ class NeuronLayer : virtual public Layer {
*/
class LossLayer : virtual public Layer {
public:
- const std::string ToString(bool debug, int flag) override;
Blob<float>* mutable_grad(const Layer* layer) override {
LOG(FATAL) << "Loss layer has no gradient blob";
return nullptr;
@@ -317,8 +316,6 @@ class LossLayer : virtual public Layer {
LOG(FATAL) << "Loss layer has no gradient blob";
return grad_;
}
- protected:
- Metric metric_;
};
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/neuralnet/loss_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer.h b/include/singa/neuralnet/loss_layer.h
index f78d7e0..3a45370 100644
--- a/include/singa/neuralnet/loss_layer.h
+++ b/include/singa/neuralnet/loss_layer.h
@@ -35,25 +35,32 @@ class EuclideanLossLayer : public LossLayer {
void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::string ToString(bool debug, int flag) override;
+
+ private:
+ int counter_ = 0;
+ float loss_ = 0.0f;
};
/**
* Cross-entropy loss applied to the probabilities computed from Softmax.
* @f$ L_i = -log P_{t_i}, t_i\in [0, C] @f$ is the label for the i-th object,
* C is the total number of classes.
*/
-class SoftmaxLossLayer : public LossLayer, public SoftmaxLayer {
+class SoftmaxLossLayer : public LossLayer {
public:
void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::string ToString(bool debug, int flag) override;
private:
+ int batchsize_, topk_, dim_, counter_ = 0;
float scale_;
- int topk_, dim_;
+ float loss_ = 0.0f, accuracy_ = 0.0f;
};
#ifdef USE_CUDNN
-class CudnnSoftmaxLossLayer : public LossLayer, public CudnnSoftmaxLayer {
+class CudnnSoftmaxLossLayer : public LossLayer{
public:
void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
@@ -61,12 +68,11 @@ class CudnnSoftmaxLossLayer : public LossLayer, public CudnnSoftmaxLayer {
const std::string ToString(bool debug, int flag) override;
private:
- float scale_;
- int topk_, dim_;
- int counter_;
- float loss_, accuracy_;
+ int batchsize_, dim_;
+ int counter_ = 0;
+ float loss_ = 0.0f;
- CudnnSoftmaxLayer *softmax_;
+ CudnnSoftmaxLayer softmax_;
};
#endif
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/neuralnet/neuralnet.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuralnet.h b/include/singa/neuralnet/neuralnet.h
index b9a58fe..7dd4edc 100644
--- a/include/singa/neuralnet/neuralnet.h
+++ b/include/singa/neuralnet/neuralnet.h
@@ -90,14 +90,18 @@ class NeuralNet {
*/
/**
* Share memory of parameter values from other neuralnet
+ * @param[in] other the neural net from which to share the Params
+ * @param[in] cpu_only if true only share cpu memory; else, share both cpu
+ * and gpu memory.
*/
- void ShareParamsFrom(NeuralNet* other);
+ void ShareParamsFrom(NeuralNet* other, bool cpu_only);
inline const std::vector<Layer*>& layers() const { return layers_; }
inline const std::vector<Param*>& params() const { return params_; }
inline Layer* name2layer(std::string name) const {
- CHECK(name2layer_.find(name) != name2layer_.end())
- << "No layer with name " << name;
- return name2layer_.at(name);
+ if (name2layer_.find(name) == name2layer_.end())
+ return nullptr;
+ else
+ return name2layer_.at(name);
}
inline const std::vector<Layer*>& srclayers(const Layer* layer) const {
CHECK(src_map_.find(layer) != src_map_.end())
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 9ae2738..39f4d69 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -217,11 +217,11 @@ class SoftmaxLayer : public NeuronLayer {
return kOneToAll;
}
protected:
- int batchsize_;
+ int batchsize_, dim_;
//!< set by users (default is 1)
- int num_softmax_per_instance_;
+ // int num_softmax_per_instance_;
//!< size of the softmax area/length
- int count_per_softmax_;
+ // int count_per_softmax_;
};
/**
* @deprecated {please use ActivationLayer}
@@ -391,16 +391,20 @@ class RBMLayer: virtual public NeuronLayer {
/**
* RBM visible layer
*/
-class RBMVisLayer: public RBMLayer, public LossLayer {
+class RBMVisLayer: public RBMLayer {
public:
~RBMVisLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::string ToString(bool debug, int flag) override;
private:
RBMLayer* hid_layer_;
Layer* input_layer_;
+
+ float error_ = 0.0f;
+ int counter_ = 0;
};
/**
* RBM hidden layer
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/neuralnet/output_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer.h b/include/singa/neuralnet/output_layer.h
index a7d92d7..49a3c19 100644
--- a/include/singa/neuralnet/output_layer.h
+++ b/include/singa/neuralnet/output_layer.h
@@ -40,10 +40,21 @@ class ArgSortLayer : public OutputLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- private:
+ protected:
int batchsize_, dim_;
int topk_;
};
+
+class AccuracyLayer : public ArgSortLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ const std::string ToString(bool debug, int flag) override;
+
+ private:
+ int counter_ = 0;
+ float accuracy_ = 0.0f;
+};
/**
* Output data (and label) for its source layer.
*/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index 97b59e0..b260862 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -242,8 +242,12 @@ class Blob {
*
* It may deallocate the SyncedMemory holding this Blob's data_, as
* shared_ptr calls its destructor when reset with the "=" operator.
+ * @param other the Blob who owns the data
+ * @param cpu_only if true, only share the cpu data; if false, share the whole
+ * data_ field. For training with multi-gpu cards, cpu_only must be true,
+ * becuase gpu memory cannot be shared among different devices.
*/
- void ShareData(const Blob& other);
+ void ShareData(const Blob& other, bool cpu_only = true);
/*
void Swap(Blob& other);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/utils/context.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/context.h b/include/singa/utils/context.h
index 1d1802c..c23b338 100644
--- a/include/singa/utils/context.h
+++ b/include/singa/utils/context.h
@@ -124,6 +124,13 @@ class Context {
}
/**
+ * \copybreif rand_generator(const std::thread::id&);
+ * @return the CPU random generator for the calling thread.
+ */
+ std::mt19937* rand_generator() {
+ return rand_generator(std::this_thread::get_id());
+ }
+ /**
* Get the CPU random generator.
* If the generator does not exist, then create it now.
* If the seed is not set, i.e., seed=-1, then get a seed from system time.
@@ -142,6 +149,13 @@ class Context {
}
#ifdef USE_GPU
/**
+ * \copybreif cublas_handle_(const std::thread::id&);
+ * @return cublas handle for the calling thread.
+ */
+ cublasHandle_t cublas_handle() {
+ return cublas_handle(std::this_thread::get_id());
+ }
+ /**
* Get the handler of the GPU which is assigned to the given thread.
* Calls cublas_handle(const int);
*/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index cf989fa..bc38bd4 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -498,13 +498,13 @@ void MVAddRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
B->set_transpose(true);
} else {
CHECK_EQ(B->count() % A.count(), 0) << "#col of B not match length of A";
- int m = A.count(), n = B->count() / m;
- Blob<Dtype> one(n);
- one.SetValue(1);
+ int n = A.count(), m = B->count() / n;
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
if (device == -1) {
- cpu_gemm(one.cpu_data(), A.cpu_data(), n, m, 1, alpha, beta,
+ Blob<Dtype> one(m);
+ one.SetValue(1);
+ cpu_gemm(one.cpu_data(), A.cpu_data(), m, n, 1, alpha, beta,
false, false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
@@ -554,16 +554,16 @@ template<typename Dtype>
void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count() % B->count(), 0) << "length of B must = # of cols of A";
int m = B->count(), n = A.count() / m;
- Blob<Dtype> one(n);
- one.SetValue(1);
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
if (device == -1) {
+ Blob<Dtype> one(n);
+ one.SetValue(1);
cpu_gemm(A.cpu_data(), one.cpu_data(), m, 1, n, alpha, beta,
A.transpose(), false, B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_by_col(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
+ singa_gpu_sum_col(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
@@ -578,17 +578,17 @@ void MVSumCol(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
template<typename Dtype>
void MVSumRow(Dtype alpha, Dtype beta, const Blob<Dtype> & A, Blob<Dtype> * B) {
CHECK_EQ(A.count() % B->count(), 0) << "length of B must = # of cols of A";
- int m = B->count(), n = A.count() / m;
- Blob<Dtype> one(n);
- one.SetValue(1);
+ int n = B->count(), m = A.count() / n;
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
if (device == -1) {
- cpu_gemm(one.cpu_data(), A.cpu_data(), 1, m, n, alpha, beta, A.transpose(),
- false, B->mutable_cpu_data());
+ Blob<Dtype> one(m);
+ one.SetValue(1);
+ cpu_gemm(one.cpu_data(), A.cpu_data(), 1, n, m, alpha, beta, false, A.transpose(),
+ B->mutable_cpu_data());
} else {
#ifdef USE_GPU
- singa_gpu_sum_by_row(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
+ singa_gpu_sum_row(A.gpu_data(), B->mutable_gpu_data(), m, n, n);
// gpu part (TODO check transpose case)
#endif // USE_GPU
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/utils/math_kernel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_kernel.h b/include/singa/utils/math_kernel.h
index 59bc3bf..8eb7cf5 100644
--- a/include/singa/utils/math_kernel.h
+++ b/include/singa/utils/math_kernel.h
@@ -32,10 +32,10 @@ extern "C" {
void singa_gpu_sum_vec(float *data, float *sum , int n);
- void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
+ void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride);
- void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
+ void singa_gpu_sum_row(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride);
void singa_gpu_add_vec_row(const float *src_vec_data,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/include/singa/utils/param.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/param.h b/include/singa/utils/param.h
index bcfc3f9..33b61ff 100644
--- a/include/singa/utils/param.h
+++ b/include/singa/utils/param.h
@@ -143,8 +143,10 @@ class Param {
* Share the data blob from other Param objects.
*
* @param other the Param object whose owner owns the data blob
+ * @param cpu_only if true, share only cpu memory (used for training with
+ * multi-gpu cards); else, share both cpu and gpu memory.
*/
- void ShareFrom(const Param& other);
+ void ShareFrom(const Param& other, bool cpu_only);
/**
* Init param values from checkpoint blob.
*/
@@ -190,28 +192,27 @@ class Param {
proto_.set_id(id);
proto_.set_owner(id);
}
+ inline int version() const { return version_; }
+ inline void set_version(int v) { version_ = v; }
/**
- * Param version is stored inside the data blob to enable all Param objs
- * sharing the same values have the same version.
- * @return the param version
+ * @return the version of the Param when the last Update request was issued.
*/
- inline int version() const { return data_->version(); }
- inline void set_version(int v) { data_->set_version(v); }
+ inline int last_version() const { return last_version_; }
+ inline void set_last_version(int v) { last_version_ = v; }
+
/**
- * @return the version of the parameter value local to a worker
+ * @return the sharing Param name which is configured by users in conf file.
*/
- inline int local_version() const { return local_version_; }
- inline void set_local_version(int v) { local_version_ = v; }
inline const std::string& share_from() const { return proto_.share_from(); }
/**
- * @return num of floats.
+ * @return num of parameters in this Param obj.
*/
- inline int size() const { return data_->count(); }
- inline const Blob<float>& data() const { return *data_; }
- inline Blob<float>* mutable_data() { return data_.get(); }
+ inline int size() const { return data_.count(); }
+ inline const Blob<float>& data() const { return data_; }
+ inline Blob<float>* mutable_data() { return &data_; }
inline const Blob<float> &grad() const { return grad_; }
inline Blob<float> *mutable_grad() { return &grad_; }
- inline float* mutable_cpu_data() { return data_->mutable_cpu_data(); }
+ inline float* mutable_cpu_data() { return data_.mutable_cpu_data(); }
inline float* mutable_cpu_grad() { return grad_.mutable_cpu_data(); }
inline float* mutable_cpu_history() { return history_.mutable_cpu_data(); }
/**
@@ -333,22 +334,24 @@ class Param {
void ParseResponseMsg(Msg* msg, int slice_idx);
protected:
- int local_version_ = -1;
- // the ID of the first slice
+ //!< param version updated by the Update/Sync/Get response
+ //!< only the owner param is initialized.
+ int version_ = -1;
+ //!< param version before last Update/Sync/Get request, set from version_
+ int last_version_ = -1;
+ //!< the global ID of the first slice
int slice_start_ = 0;
+ //!< total num of slices for this Parm obj
int num_slices_ = 0;
// offset and size of each slice
std::vector<int> slice_offset_;
std::vector<int> slice_size_;
- // for debug checking
- // since put request has no feedback, we do not track its pending status
+ // for debug. Put request has no feedback, we do not track its pending status
std::vector<bool> pending_get_;
std::vector<bool> pending_update_;
int num_pending_requests_ = 0;
- // data field
- std::shared_ptr<Blob<float>> data_ = nullptr;
- // gradient, history gradient of this parameter
- Blob<float> grad_, history_;
+ // data, gradient, history gradient of this parameter
+ Blob<float> data_, grad_, history_;
ParamProto proto_;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index b963912..b86da7c 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -72,6 +72,7 @@ void Driver::Init(int argc, char **argv) {
RegisterLayer<BridgeDstLayer, int>(kBridgeDst);
RegisterLayer<BridgeSrcLayer, int>(kBridgeSrc);
+ RegisterLayer<AccuracyLayer, int>(kAccuracy);
RegisterLayer<ArgSortLayer, int>(kArgSort);
RegisterLayer<ConvolutionLayer, int>(kConvolution);
RegisterLayer<CConvolutionLayer, int>(kCConvolution);
@@ -219,10 +220,11 @@ void Driver::Train(const JobProto& job_conf) {
// CHECK_LE(workers.size(), job_conf.gpu_size());
for (auto worker : workers) {
threads.push_back(std::thread(&Worker::Run, worker));
+ int device_id = -1;
if (gpu < job_conf.gpu_size()) {
- int device_id = job_conf.gpu(gpu++);
- context->SetupDevice(threads.back().get_id(), device_id);
+ device_id = job_conf.gpu(gpu++);
}
+ context->SetupDevice(threads.back().get_id(), device_id);
}
if (grp_size > 1 || nserver_grps > 0) {
int nservers_per_grp = cluster->nservers_per_group();
@@ -307,16 +309,16 @@ const vector<Worker*> Driver::CreateWorkers(const JobProto& job_conf,
// test and validation are performed by the 1st group.
if (gid == 0 && job_conf.test_steps() > 0) {
test_net = NeuralNet::Create(job_conf.neuralnet(), kTest, 1);
- test_net->ShareParamsFrom(train_net);
+ test_net->ShareParamsFrom(train_net, false);
}
if (gid == 0 && job_conf.validate_steps() > 0) {
val_net = NeuralNet::Create(job_conf.neuralnet(), kVal, 1);
- val_net->ShareParamsFrom(train_net);
+ val_net->ShareParamsFrom(train_net, false);
}
} else {
train_net = NeuralNet::Create(job_conf.neuralnet(), kTrain, wgrp_size);
if (cluster->share_memory()) {
- train_net->ShareParamsFrom(net);
+ train_net->ShareParamsFrom(net, true);
} else {
Param::SliceParams(lcm, train_net->params());
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
index 0f98279..dfba675 100644
--- a/src/neuralnet/input_layer/deprecated.cc
+++ b/src/neuralnet/input_layer/deprecated.cc
@@ -19,6 +19,7 @@
*
*************************************************************/
+#include <random>
#include "singa/neuralnet/input_layer.h"
#include "singa/utils/context.h"
#include "singa/utils/singleton.h"
@@ -59,12 +60,11 @@ void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
shard_ = new DataShard(layer_conf_.sharddata_conf().path(),
DataShard::kRead);
if (random_skip_) {
- std::uniform_int_distribution<int> distribution(0, random_skip_);
- auto generator =
- Singleton<Context>::Instance()->generator(std::this_thread::get_id());
- int nskip = distribution(generator);
+ std::uniform_int_distribution<int> distribution(0, random_skip_);
+ auto generator = Singleton<Context>::Instance()->rand_generator();
+ int nskip = distribution(*generator);
LOG(INFO) << "Random Skip " << nskip << " records, there are "
- << shard_->Count() << " records in total";
+ << shard_->Count() << " records in total";
string key;
for (int i = 0; i < nskip; i++) {
shard_->Next(&key, &sample_);
@@ -130,8 +130,8 @@ void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (random_skip_) {
std::uniform_int_distribution<int> distribution(0, random_skip_);
auto generator =
- Singleton<Context>::Instance()->generator(std::this_thread::get_id());
- int nskip = distribution(generator);
+ Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
+ int nskip = distribution(*generator);
int n = 0;
CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
@@ -266,12 +266,12 @@ void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
std::uniform_int_distribution<int> distribution(0, r.shape(0) - cropsize_);
auto generator =
- Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+ Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
for (const Record& record : records) {
auto image = images[rid];
bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain);
bool do_mirror = mirror_
- && (distribution(generator) % 2)
+ && (distribution(*generator) % 2)
&& ((flag & kTrain) == kTrain);
float* dptr = nullptr;
if (do_crop || do_mirror)
@@ -289,8 +289,8 @@ void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
for (int i = 0; i < mean_.count(); i++)
dptr[i] -= meandptr[i];
if (do_crop) {
- int hoff = distribution(generator);
- int woff = distribution(generator);
+ int hoff = distribution(*generator);
+ int woff = distribution(*generator);
Shape<2> cropshape = Shape2(cropsize_, cropsize_);
if (do_mirror) {
croped_image = expr::crop(raw_image, cropshape, hoff, woff);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/input_layer/image_preprocess.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/image_preprocess.cc b/src/neuralnet/input_layer/image_preprocess.cc
index 251f456..c63c957 100644
--- a/src/neuralnet/input_layer/image_preprocess.cc
+++ b/src/neuralnet/input_layer/image_preprocess.cc
@@ -58,16 +58,16 @@ void ImagePreprocessLayer::ComputeFeature(int flag,
std::uniform_int_distribution<int> rand1(0, srcdata.shape()[1] - cropsize_);
std::uniform_int_distribution<int> rand2(0, srcdata.shape()[2] - cropsize_);
auto generator =
- Singleton<Context>::Instance()->generator(std::this_thread::get_id());
+ Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
for (int k = 0; k < batchsize; k++) {
int h_offset = 0, w_offset = 0;
if (cropsize_> 0 && ((flag & kTrain) == kTrain)) {
- h_offset = rand1(generator);
- w_offset = rand2(generator);
+ h_offset = rand1(*generator);
+ w_offset = rand2(*generator);
}
bool do_mirror = mirror_
- && (rand1(generator) % 2)
+ && (rand1(*generator) % 2)
&& ((flag & kTrain) == kTrain);
ImageTransform(srcdptr + k * srcimage_size, nullptr, do_mirror, cropsize_,
cropsize_, h_offset, w_offset, srcdata.shape()[1], height, width,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/input_layer/store.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/store.cc b/src/neuralnet/input_layer/store.cc
index dbb1874..283b0c7 100644
--- a/src/neuralnet/input_layer/store.cc
+++ b/src/neuralnet/input_layer/store.cc
@@ -38,13 +38,6 @@ void StoreInputLayer::Setup(const LayerProto& conf,
if (conf.partition_dim() == 0) {
batchsize_ /= conf.num_partitions();
}
- if (conf.store_conf().random_skip() > 0) {
- std::uniform_int_distribution<int>
- distribution(0, conf.store_conf().random_skip());
- auto generator =
- Singleton<Context>::Instance()->generator(std::this_thread::get_id());
- random_skip_ = distribution(generator);
- }
}
void StoreInputLayer::ComputeFeature(int flag,
@@ -52,8 +45,16 @@ void StoreInputLayer::ComputeFeature(int flag,
string key, val;
if (store_ == nullptr) {
store_ = io::OpenStore(layer_conf_.store_conf().backend(),
- layer_conf_.store_conf().path(),
- io::kRead);
+ layer_conf_.store_conf().path(),
+ io::kRead);
+ if (layer_conf_.store_conf().random_skip() > 0) {
+ std::uniform_int_distribution<int>
+ distribution(0, layer_conf_.store_conf().random_skip());
+ auto generator = Singleton<Context>::Instance()->rand_generator(
+ std::this_thread::get_id());
+ random_skip_ = distribution(*generator);
+ }
+
while (random_skip_ > 0) {
if (!store_->Read(&key, &val)) {
store_->SeekToFirst();
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index df77239..1953d6c 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -62,15 +62,4 @@ const std::string Layer::ToString(bool debug, int flag) {
}
return ret;
}
-
-const std::string LossLayer::ToString(bool debug, int flag) {
- std::string disp;
- if (debug) {
- disp = Layer::ToString(debug, flag);
- } else {
- disp = metric_.ToLogString();
- metric_.Reset();
- }
- return disp;
-}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
index b18a751..78a035a 100644
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
@@ -21,21 +21,23 @@
#include "singa/neuralnet/loss_layer.h"
#include "singa/utils/blob.h"
+#include "singa/utils/math_blob.h"
#include "singa/utils/math_kernel.h"
namespace singa {
void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
- CudnnSoftmaxLayer::Setup(conf, vector<Layer*> {srclayers.at(0)});
- topk_ = conf.softmaxloss_conf().topk();
- loss_ = accuracy_ = 0.0f;
- counter_ = 0;
+ softmax_.Setup(conf, vector<Layer*> {srclayers.at(0)});
+ data_.Reshape(softmax_.data(this).shape());
+ data_.ShareData(*softmax_.mutable_data(this), false);
+ batchsize_ = data_.shape(0);
+ dim_ = data_.count() / batchsize_;
+ LOG(ERROR) << batchsize_ << " " << dim_;
}
void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
- CudnnSoftmaxLayer::ComputeFeature(flag, srclayers);
+ softmax_.ComputeFeature(flag, srclayers);
// compute loss
- float *prob = data_.mutable_gpu_data();
Blob<int> label(batchsize_);
int *labelptr = label.mutable_cpu_data();
@@ -46,9 +48,10 @@ void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
Blob<float> loss(batchsize_);
+ const float *prob = data_.gpu_data();
singa_gpu_softmaxloss_forward(batchsize_, dim_, prob, label.gpu_data(),
loss.mutable_gpu_data());
-
+ loss_ += Asum(loss);
counter_++;
}
@@ -66,15 +69,18 @@ void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
labelptr[i] = srclayers[1]->aux_data(this)[i];
}
- singa_gpu_softmaxloss_backward(batchsize_, dim_, scale_, label.gpu_data(),
+ singa_gpu_softmaxloss_backward(batchsize_, dim_, 1.0f, label.gpu_data(),
gsrcptr);
+ Scale(1.0f / batchsize_, gsrcblob);
}
const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
- string disp = "Loss = " + std::to_string(loss_ / counter_)
- + ", accuracy = " + std::to_string(accuracy_ / counter_);
+ if (debug)
+ return Layer::ToString(debug, flag);
+
+ string disp = "Loss = " + std::to_string(loss_ / counter_);
counter_ = 0;
- loss_ = accuracy_ = 0;
+ loss_ = 0;
return disp;
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/loss_layer/euclidean.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc
index b6aa12a..49179d6 100644
--- a/src/neuralnet/loss_layer/euclidean.cc
+++ b/src/neuralnet/loss_layer/euclidean.cc
@@ -50,7 +50,8 @@ void EuclideanLossLayer::ComputeFeature(int flag,
loss += (input_dptr[i] - reconstruct_dptr[i]) *
(input_dptr[i] - reconstruct_dptr[i]);
}
- metric_.Add("loss", loss / srclayers[0]->data(this).shape()[0]);
+ loss_ += loss / srclayers[0]->data(this).shape()[0];
+ counter_ ++;
}
void EuclideanLossLayer::ComputeGradient(int flag,
@@ -67,5 +68,13 @@ void EuclideanLossLayer::ComputeGradient(int flag,
Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count()));
gsrc /= srclayers[0]->data(this).shape()[0];
}
+const std::string EuclideanLossLayer::ToString(bool debug, int flag) {
+ if (debug)
+ return Layer::ToString(debug, flag);
+ string disp = "Loss = " + std::to_string(loss_ / counter_);
+ counter_ = 0;
+ loss_ = 0;
+ return disp;
+}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index 0f3d5bf..ce858ac 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -79,8 +79,9 @@ void SoftmaxLossLayer::ComputeFeature(int flag,
probptr += dim_;
}
CHECK_EQ(probptr, prob.dptr + prob.shape.Size());
- metric_.Add("loss", loss * scale_ / (1.0f * batchsize_));
- metric_.Add("accuracy", precision * scale_ / (1.0f * batchsize_));
+ loss_ += loss * scale_ / (1.0f * batchsize_);
+ accuracy_ += precision * scale_ / (1.0f * batchsize_);
+ counter_ ++;
}
void SoftmaxLossLayer::ComputeGradient(int flag,
@@ -95,5 +96,14 @@ void SoftmaxLossLayer::ComputeGradient(int flag,
Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count()));
gsrc *= scale_ / (1.0f * batchsize_);
}
+const std::string SoftmaxLossLayer::ToString(bool debug, int flag) {
+ if (debug)
+ return Layer::ToString(debug, flag);
+ string disp = "Loss = " + std::to_string(loss_ / counter_)
+ + ", accuracy = " + std::to_string(accuracy_ / counter_);
+ counter_ = 0;
+ loss_ = accuracy_ = 0;
+ return disp;
+}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index b7944e7..aabc361 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -155,7 +155,7 @@ std::string NeuralNet::ToAdjacency() {
}
*/
-void NeuralNet::ShareParamsFrom(NeuralNet* other) {
+void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
for (auto& layer : layers_) {
auto otherlayer = other->name2layer(layer->name());
if (otherlayer != nullptr) {
@@ -163,7 +163,7 @@ void NeuralNet::ShareParamsFrom(NeuralNet* other) {
const auto& params = layer->GetParams();
CHECK_EQ(params.size(), otherparams.size());
for (size_t i = 0; i < params.size(); i++) {
- params[i]->ShareFrom(*otherparams[i]);
+ params[i]->ShareFrom(*otherparams[i], cpu_only);
}
}
}
@@ -416,7 +416,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
const string share_from = param->share_from();
if (param->share_from() != "") {
if (name2param.find(share_from) != name2param.end()) {
- param->ShareFrom(*name2param.at(param->share_from()));
+ param->ShareFrom(*name2param.at(param->share_from()), false);
} else {
LOG(FATAL) << "No param with the name (share_from) " << share_from;
}
@@ -430,7 +430,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
auto params = (*it)->GetParams();
CHECK_EQ(params.size(), owner_params.size());
for (size_t i = 0; i < params.size(); i++)
- params.at(i)->ShareFrom(*owner_params.at(i));
+ params.at(i)->ShareFrom(*owner_params.at(i), true);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/neuron_layer/cudnn_softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cc b/src/neuralnet/neuron_layer/cudnn_softmax.cc
index 21d17c4..a1a492e 100644
--- a/src/neuralnet/neuron_layer/cudnn_softmax.cc
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cc
@@ -29,15 +29,15 @@ void CudnnSoftmaxLayer::InitCudnn() {
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
- num_softmax_per_instance_,
- count_per_softmax_,
+ dim_,
+ 1,
1));
CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
- num_softmax_per_instance_,
- count_per_softmax_,
+ dim_,
+ 1,
1));
}
@@ -46,12 +46,13 @@ void CudnnSoftmaxLayer::ComputeFeature(int flag,
if (!has_init_cudnn_)
InitCudnn();
const float alpha = 1.0f, beta = 0.0f;
+ CHECK_EQ(srclayers.at(0)->data(this).shape().size(), 2);
CHECK_CUDNN(cudnnSoftmaxForward(handle_,
CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_MODE_CHANNEL,
+ CUDNN_SOFTMAX_MODE_INSTANCE,
&alpha,
src_desc_,
- srclayers[0]->data(this).gpu_data(),
+ srclayers.at(0)->data(this).gpu_data(),
&beta,
my_desc_,
data_.mutable_gpu_data()));
@@ -62,7 +63,7 @@ void CudnnSoftmaxLayer::ComputeGradient(int flag,
const float alpha = 1.f, beta = 0.f;
CHECK_CUDNN(cudnnSoftmaxBackward(handle_,
CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_MODE_CHANNEL,
+ CUDNN_SOFTMAX_MODE_INSTANCE,
&alpha,
my_desc_,
data_.gpu_data(),
@@ -70,6 +71,6 @@ void CudnnSoftmaxLayer::ComputeGradient(int flag,
grad_.gpu_data(),
&beta,
src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
+ srclayers.at(0)->mutable_grad(this)->mutable_gpu_data()));
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/neuron_layer/rbm.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/rbm.cc b/src/neuralnet/neuron_layer/rbm.cc
index fadd1df..69ffa62 100644
--- a/src/neuralnet/neuron_layer/rbm.cc
+++ b/src/neuralnet/neuron_layer/rbm.cc
@@ -111,10 +111,11 @@ void RBMVisLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
for (int i = 0; i < pos_data_.count(); i++) {
err += (dptr[i] - rcns[i]) * (dptr[i] - rcns[i]);
}
- metric_.Add("Squared Error", err / batchsize_);
+ error_ += err / batchsize_;
}
first_gibbs_ = false;
}
+ counter_ += batchsize_;
}
void RBMVisLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
@@ -133,6 +134,15 @@ void RBMVisLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
gweight -= dot(hid_pos.T(), vis_pos);
gweight /= batchsize_;
}
+const std::string RBMVisLayer::ToString(bool debug, int flag) {
+ if (debug)
+ return Layer::ToString(debug, flag);
+
+ string disp = "Squared Error = " + std::to_string(error_ / counter_);
+ counter_ = 0;
+ error_ = 0;
+ return disp;
+}
/**************** Implementation for RBMHidLayer********************/
RBMHidLayer::~RBMHidLayer() {
delete weight_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/neuron_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/softmax.cc b/src/neuralnet/neuron_layer/softmax.cc
index 44c64f8..4a09241 100644
--- a/src/neuralnet/neuron_layer/softmax.cc
+++ b/src/neuralnet/neuron_layer/softmax.cc
@@ -38,11 +38,13 @@ void SoftmaxLayer::Setup(const LayerProto& proto,
CHECK_EQ(srclayers.size(), 1);
NeuronLayer::Setup(proto, srclayers);
const auto& srcdata = srclayers[0]->data(this);
- batchsize_ = data_.shape()[0];
+ batchsize_ = srcdata.shape()[0];
+ dim_ = srcdata.count() / batchsize_;
+ /*
num_softmax_per_instance_ = proto.softmax_conf().num_softmax_per_instance();
- count_per_softmax_ = data_.count() / batchsize_ / num_softmax_per_instance_;
- data_.Reshape(vector<int>{batchsize_, num_softmax_per_instance_,
- count_per_softmax_});
+ count_per_softmax_ = srcdata.count() / batchsize_ / num_softmax_per_instance_;
+ */
+ data_.Reshape(batchsize_, dim_);
grad_.ReshapeLike(data_);
}
@@ -58,6 +60,7 @@ void SoftmaxLayer::ComputeFeature(int flag,
void SoftmaxLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
int batchsize = data_.shape()[0];
+ LOG(FATAL) << "not implemented";
for (int n = 0; n < batchsize; n++) {
// TODO(wangwei) finish the code using new math API
// gxi=[(gyi+gyi*yi)-\sum_k(gyk*yk)]*yi
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/neuralnet/output_layer/accuracy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/accuracy.cc b/src/neuralnet/output_layer/accuracy.cc
new file mode 100644
index 0000000..21107df
--- /dev/null
+++ b/src/neuralnet/output_layer/accuracy.cc
@@ -0,0 +1,61 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include <algorithm>
+#include "singa/neuralnet/output_layer.h"
+
+namespace singa {
+
+void AccuracyLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ CHECK_EQ(srclayers.size(), 2);
+ ArgSortLayer::Setup(proto, vector<Layer*>{srclayers.at(0)});
+}
+
+void AccuracyLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ ArgSortLayer::ComputeFeature(flag, vector<Layer*>{srclayers.at(0)});
+ const auto& label = srclayers[1]->aux_data(this);
+ int ncorrect = 0;
+ for (int n = 0; n < batchsize_; n++) {
+ const float* pos = data_.cpu_data() + topk_ * n;
+ // check if true label is in top k predictions
+ for (int k = 0; k < topk_; k++) {
+ if (pos[k] == label[n]) {
+ ncorrect++;
+ break;
+ }
+ }
+ }
+ accuracy_ += ncorrect * 1.0f / batchsize_;
+ counter_ ++;
+}
+
+const std::string AccuracyLayer::ToString(bool debug, int flag) {
+ if (debug)
+ return Layer::ToString(debug, flag);
+
+ string disp = "accuracy = " + std::to_string(accuracy_ / counter_);
+ counter_ = 0;
+ accuracy_ = 0;
+ return disp;
+}
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 88caa44..ff27c05 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -602,6 +602,7 @@ enum LayerType {
kRGBImage = 10;
// Neuron layers
// - Feature transformation
+ kAccuracy = 36;
kArgSort = 35;
kConvolution = 1;
kCConvolution = 27;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/server.cc
----------------------------------------------------------------------
diff --git a/src/server.cc b/src/server.cc
index 9af5fc4..003d4e3 100644
--- a/src/server.cc
+++ b/src/server.cc
@@ -153,7 +153,7 @@ Msg* Server::HandlePut(Msg **msg) {
shard_[slice_id] = new ParamEntry(num_shares, param);
// must set version after HandlePutMsg which allocates the memory
param->set_version(version);
- param->set_local_version(version);
+ param->set_last_version(version);
param->set_id(slice_id);
// allocate blob for param sync between groups.
if (slice2group_[slice_id] != grp_id_) {
@@ -201,16 +201,18 @@ const vector<Msg*> Server::HandleUpdate(Msg **msg) {
// extract and aggregate gradients
param->ParseUpdateMsgs(request);
updater_->Update(step, param, 1.0f / entry->num_total);
- param->set_local_version(param->local_version() + 1);
+ param->set_version(param->version() + 1);
// response to all shares of this param
for (auto response : param->GenUpdateResponseMsgs(&request, false)) {
- response->set_trgt(trgt_val, param->local_version());
+ response->set_trgt(trgt_val, param->version());
ret.push_back(response);
}
entry->num_update = 0;
n_updates_[sliceid]++;
// sync with master group after at least sync_freq local updates
// the last check is to avoid sending msg to stopped servers
+ // may send the update steps on this server since last sync, i.e.,
+ // version-last_version
if (slice2group_[sliceid] != grp_id_
&& n_updates_[sliceid] >= Cluster::Get()->sync_freq()
&& n_pending_sync_[sliceid] <= Cluster::Get()->sync_freq()) {
@@ -221,7 +223,7 @@ const vector<Msg*> Server::HandleUpdate(Msg **msg) {
int addr = Addr(slice2group_[sliceid], slice2server_[sliceid], kServer);
Msg* sync = new Msg(Addr(grp_id_, id_, kServer), addr);
sync->set_type(kSyncRequest);
- sync->set_trgt(trgt_val, param->local_version());
+ sync->set_trgt(trgt_val, param->version());
sync->AddFrame(tmp.dptr, param->size() * sizeof(float));
Copy(tmp, cur);
ret.push_back(sync);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/stub.cc
----------------------------------------------------------------------
diff --git a/src/stub.cc b/src/stub.cc
index a3605f7..184a1f3 100644
--- a/src/stub.cc
+++ b/src/stub.cc
@@ -270,7 +270,8 @@ void Stub::HandleGetResponse(ParamEntry* entry, Msg** msg) {
int sliceid = SliceID((*msg)->trgt_val());
auto param = entry->shares.at(0);
if (param->ParseGetResponseMsg(*msg, sliceid-param->slice_start()))
- param->set_version(version);
+ for (auto *p : entry->shares)
+ p->set_version(version);
DeleteMsg(msg);
}
@@ -279,7 +280,8 @@ void Stub::HandleUpdateResponse(ParamEntry* entry, Msg** msg) {
int sliceid = SliceID((*msg)->trgt_val());
auto param = entry->shares.at(0);
if (param->ParseUpdateResponseMsg(*msg, sliceid-param->slice_start()))
- param->set_version(version);
+ for (auto *p : entry->shares)
+ p->set_version(version);
DeleteMsg(msg);
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index 0dc797e..9607683 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -187,12 +187,15 @@ void SyncedMemory::to_gpu() {
template <typename Dtype>
void Blob<Dtype>::Reshape(const std::vector<int>& shape) {
+ int count = count_;
count_ = 1;
shape_ = shape;
for (size_t i = 0; i < shape.size(); ++i) {
CHECK(shape[i]);
count_ *= shape[i];
}
+ if (count > 0)
+ CHECK_EQ(count, count_);
if (count_ > capacity_) {
capacity_ = count_;
data_.reset(new SyncedMemory(capacity_ * sizeof(Dtype)));
@@ -262,9 +265,12 @@ void Blob<Dtype>::SetValue(Dtype v) {
ptr[i] = v;
}
template <typename Dtype>
-void Blob<Dtype>::ShareData(const Blob& other) {
+void Blob<Dtype>::ShareData(const Blob& other, bool cpu_only) {
CHECK_EQ(count_, other.count());
- data_ = other.data_;
+ if (cpu_only)
+ data_->set_cpu_data(other.cpu_data());
+ else
+ data_ = other.data_;
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/utils/math_kernel.cu
----------------------------------------------------------------------
diff --git a/src/utils/math_kernel.cu b/src/utils/math_kernel.cu
index 9c54520..3650c09 100644
--- a/src/utils/math_kernel.cu
+++ b/src/utils/math_kernel.cu
@@ -37,8 +37,7 @@ void kernel_softmax_loss(const float *prob, const int *label , float *loss,
int index = blockIdx.x * blockDim.x + threadIdx.x;
int num_threads = blockDim.x * gridDim.x;
for (; index < n; index += num_threads) {
- const int label_value = static_cast<int>(label[index]);
- float prob_of_truth = prob[index * dim + label_value];
+ float prob_of_truth = prob[index * dim + label[index]];
loss[index] -= log(max(prob_of_truth, FLT_MIN));
}
}
@@ -49,8 +48,8 @@ void kernel_softmax_gradient(float *grad, const int *label ,
int index = blockIdx.x * blockDim.x + threadIdx.x;
int num_threads = blockDim.x * gridDim.x;
for (; index < n; index += num_threads) {
- int pos = index * dim + static_cast<int>(label[index]);
- grad[pos] = (grad[pos] - 1.0f) * scale / (1.0 * n);
+ int pos = index * dim + label[index];
+ grad[pos] = (grad[pos] - 1.0f) * scale;
}
}
@@ -87,7 +86,20 @@ void kernel_sum_vec(float *data, float *sum , int n) {
}
__global__
-void kernel_sum_by_col(const float *src_mat_data,
+void kernel_sum_col(const float *src_mat_data,
+ float *dst_vec_data, int rows, int cols, int stride) {
+ int index = blockIdx.x * blockDim.x + threadIdx.x;
+ int num_threads = blockDim.x * gridDim.x;
+ for (; index < rows; index += num_threads) {
+ dst_vec_data[index] = 0.0f;
+ for (int k = 0; k < cols; k++) {
+ dst_vec_data[index] += src_mat_data[index * stride + k];
+ }
+ }
+}
+
+__global__
+void kernel_sum_row(const float *src_mat_data,
float *dst_vec_data, int rows, int cols, int stride) {
int j = blockIdx.x;
int THREADS = blockDim.x;
@@ -119,19 +131,9 @@ void kernel_sum_by_col(const float *src_mat_data,
__syncthreads();
dst_vec_data[j] = aux[0];
-}
-__global__
-void kernel_sum_by_row(const float *src_mat_data,
- float *dst_vec_data, int rows, int cols, int stride) {
- int index = blockIdx.x * blockDim.x + threadIdx.x;
- int num_threads = blockDim.x * gridDim.x;
- for (; index < rows; index += num_threads) {
- dst_vec_data[index] = 0.0f;
- for (int k = 0; k < cols; k++) {
- dst_vec_data[index] += src_mat_data[index * stride + k];
- }
- }
+
+
}
__global__
@@ -337,21 +339,21 @@ void singa_gpu_sum_vec(float *data, float *sum , int n) {
kernel_sum_vec<<<num_blocks, threads_per_block>>>(data, sum, n);
}
-void singa_gpu_sum_by_col(const float *src_mat_data, float *dst_vec_data,
+void singa_gpu_sum_col(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride) {
int threads_per_block = rows > CU1DBLOCK ? CU1DBLOCK : rows;
int num_blocks = cols;
- kernel_sum_by_col<<<num_blocks, threads_per_block>>>(src_mat_data,
+ kernel_sum_col<<<num_blocks, threads_per_block>>>(src_mat_data,
dst_vec_data, rows, cols, stride);
}
-void singa_gpu_sum_by_row(const float *src_mat_data, float *dst_vec_data,
+void singa_gpu_sum_row(const float *src_mat_data, float *dst_vec_data,
int rows, int cols, int stride) {
int threads_per_block = cols > CU1DBLOCK ? CU1DBLOCK : cols;
int num_blocks = rows;
- kernel_sum_by_row<<<num_blocks, threads_per_block>>>(src_mat_data,
+ kernel_sum_row<<<num_blocks, threads_per_block>>>(src_mat_data,
dst_vec_data, rows, cols, stride);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/utils/param.cc
----------------------------------------------------------------------
diff --git a/src/utils/param.cc b/src/utils/param.cc
index 54fe2aa..09f519b 100644
--- a/src/utils/param.cc
+++ b/src/utils/param.cc
@@ -151,7 +151,7 @@ void Param::SliceParams(int num, const vector<Param*>& params) {
}
void Param::Setup(const vector<int>& shape) {
- data_ = std::make_shared<Blob<float>>(shape);
+ data_.Reshape(shape);
grad_.Reshape(shape);
history_.Reshape(shape);
}
@@ -162,17 +162,18 @@ void Param::InitValues() {
void Param::InitValues(int version) {
ParamGenerator* gen = ParamGenerator::Create(proto_.init());
- gen->Fill(data_.get());
+ gen->Fill(&data_);
set_version(version);
}
-void Param::ShareFrom(const Param& other) {
+void Param::ShareFrom(const Param& other, bool cpu_only) {
proto_.set_owner(other.owner());
- if (data_ != nullptr)
- CHECK(data_->shape() == other.data_->shape());
- data_ = other.data_;
+ CHECK(data_.shape() == other.data_.shape());
+ data_.ShareData(other.data_, cpu_only);
if (grad_.count() == 0)
- grad_.Reshape(data_->shape());
+ grad_.Reshape(data_.shape());
+ version_ = other.version_;
+ last_version_ = other.last_version_;
slice_start_ = other.slice_start_;
num_slices_ = other.num_slices_;
slice_offset_ = other.slice_offset_;
@@ -183,11 +184,11 @@ void Param::ShareFrom(const Param& other) {
}
void Param::FromProto(const BlobProto& blob) {
- data_->FromProto(blob);
+ data_.FromProto(blob);
}
void Param::ToProto(BlobProto* blob) {
- data_->ToProto(blob);
+ data_.ToProto(blob);
}
void Param::AddSlice(int slice_id, int size) {
@@ -225,7 +226,7 @@ Msg* Param::GenGetMsg(bool copy, int idx) {
CHECK_LT(idx, num_slices_);
Msg* msg = new Msg();
msg->set_type(kGet);
- msg->AddFormatFrame("ip", copy, data_->cpu_data() + slice_offset_[idx]);
+ msg->AddFormatFrame("ip", copy, data_.cpu_data() + slice_offset_[idx]);
pending_get_[idx] = true;
num_pending_requests_++;
return msg;
@@ -239,7 +240,7 @@ Msg* Param::GenUpdateMsg(bool copy, int idx) {
void* ptr = grad_.mutable_cpu_data() + slice_offset_[idx];
// to change the head of SyncMem to cpu; otherwise, the updated parameter
// // values would not be synced to gpu (since the head is at gpu).
- data_->mutable_cpu_data();
+ data_.mutable_cpu_data();
if (copy) {
msg->AddFrame(ptr, slice_size_[idx]*sizeof(float));
} else {
@@ -254,9 +255,9 @@ Msg* Param::GenUpdateMsg(bool copy, int idx) {
Msg* Param::GenSyncMsg(int offset, int size) {
Msg* msg = new Msg();
msg->set_type(kSyncRequest);
- msg->set_trgt(ParamTrgt(-1, id()), local_version());
+ msg->set_trgt(ParamTrgt(-1, id()), last_version());
// always copy data because syn is between server groups in diff procs
- msg->AddFrame(mutable_cpu_data(), data_->count()*sizeof(float));
+ msg->AddFrame(mutable_cpu_data(), data_.count()*sizeof(float));
return msg;
}
@@ -278,7 +279,7 @@ Msg* Param::HandlePutMsg(Msg** msg, bool reserve) {
CHECK_EQ(size * sizeof(float), (*msg)->FrameSize());
memcpy(mutable_cpu_data(), (*msg)->FrameData(), size * sizeof(float));
} else {
- data_->set_cpu_data(ptr);
+ data_.set_cpu_data(ptr);
}
if (!reserve) DeleteMsg(msg);
return nullptr;
@@ -292,7 +293,7 @@ Msg* Param::HandleGetMsg(Msg** msg, bool reserve) {
(*msg)->ParseFormatFrame("ip", ©, &ptr);
if (copy) {
(*msg)->AddFrame(mutable_cpu_data(), sizeof(float) * size());
- } else if (ptr != data_->cpu_data()) {
+ } else if (ptr != data_.cpu_data()) {
// this case reflects following situation:
// worker 0 and server are in the same process, while worker 1 is not.
// worker 1 "put" data into server, so server need to allocate memory.
@@ -300,8 +301,8 @@ Msg* Param::HandleGetMsg(Msg** msg, bool reserve) {
// 1. copy the data to the worker0 provided space
// 2. change its own pointer to that space in order to share memory
// in this case, the server always points to last worker's space
- memcpy(ptr, data_->cpu_data(), sizeof(float) * size());
- data_->set_cpu_data(ptr);
+ memcpy(ptr, data_.cpu_data(), sizeof(float) * size());
+ data_.set_cpu_data(ptr);
}
// else the mem space is shared among all worker and servers
Msg* ret = nullptr;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/81747603/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index a41e3a8..333408d 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -64,10 +64,11 @@ void Worker::Run() {
// setup gpu device
auto context = Singleton<Context>::Instance();
int device = context->device_id(std::this_thread::get_id());
- LOG(ERROR) << "Worker (group = " << grp_id_ <<", id = " << id_
- << ") start on device " << device;
+ LOG(ERROR) << "Worker (group = " << grp_id_ <<", id = " << id_ << ") "
+ << " start on " << (device >= 0 ? "GPU " + std::to_string(device) : "CPU");
if (device >= 0)
context->ActivateDevice(device);
+
auto cluster = Cluster::Get();
int svr_grp = grp_id_ / cluster->nworker_groups_per_server_group();
CHECK(cluster->runtime()->JoinSGroup(grp_id_, id_, svr_grp));
@@ -76,12 +77,12 @@ void Worker::Run() {
InitNetParams(job_conf_, train_net_);
while (!StopNow(step_)) {
if (ValidateNow(step_) && val_net_ != nullptr) {
- CollectAll(step_, val_net_);
+ CollectAll(step_, train_net_);
LOG(ERROR) << "Validation @ step " + std::to_string(step_);
Test(job_conf_.validate_steps(), kVal, val_net_);
}
if (TestNow(step_) && test_net_ != nullptr) {
- CollectAll(step_, test_net_);
+ CollectAll(step_, train_net_);
LOG(ERROR) << "Test @ step " + std::to_string(step_);
Test(job_conf_.test_steps(), kTest, test_net_);
}
@@ -195,8 +196,8 @@ void Worker::InitNetParams(const JobProto& job_conf, NeuralNet* net) {
}
// warmup training before put params to servers
- for (; step_ < job_conf.warmup_steps(); step_++)
- TrainOneBatch(step_, net);
+ // for (; step_ < job_conf.warmup_steps(); step_++)
+ // TrainOneBatch(step_, net);
for (auto layer : net->layers()) {
if (layer->partition_id() == id_)
for (auto param : layer->GetParams())
@@ -213,6 +214,7 @@ void Worker::InitNetParams(const JobProto& job_conf, NeuralNet* net) {
}
}
+
void Worker::Checkpoint(int step, const std::string& folder, NeuralNet* net) {
BlobProtos bps;
for (auto layer : net->layers()) {
@@ -261,7 +263,7 @@ int Worker::Get(int step, Param* param) {
}
int Worker::Update(int step, Param* param) {
- param->set_local_version(param->version());
+ param->set_last_version(param->version());
if (dealer_ == nullptr) {
LOG(WARNING) << "Null dealer in worker (" << grp_id_ << ", " << id_ << ")";
return 1;
@@ -286,7 +288,7 @@ int Worker::CollectAll(int step, NeuralNet* net) {
}
int Worker::Collect(int step, Param* param) {
- while (param->version() <= param->local_version())
+ while (param->version() <= param->last_version())
std::this_thread::sleep_for(std::chrono::milliseconds(kCollectSleepTime));
return 1;
}
[12/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
1. Fix bugs from rebase onto latest master.
The LOG(FATAL) inside mutable_grad and mutable_data for input/output/loss layers
are commmented out. Because other layers may call check mutable_xxx() == nullptr.
2. Update Makefile.am; run make test;
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/f8be9afa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/f8be9afa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/f8be9afa
Branch: refs/heads/master
Commit: f8be9afac09882ae85e905c9e679b6effd642f5c
Parents: 5728c24
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri Dec 11 10:59:33 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:48:24 2015 +0800
----------------------------------------------------------------------
Makefile.am | 29 ++++---
examples/cifar10/job.conf | 7 +-
include/singa/neuralnet/connection_layer.h | 17 ++--
.../singa/neuralnet/connection_layer/slice.h | 49 -----------
.../singa/neuralnet/connection_layer/split.h | 48 -----------
include/singa/neuralnet/layer.h | 12 +--
include/singa/neuralnet/neuron_layer.h | 34 +++++---
include/singa/neuralnet/neuron_layer/dummy.h | 51 -----------
include/singa/neuralnet/neuron_layer/rbm.h | 89 --------------------
include/singa/neuralnet/neuron_layer/sigmoid.h | 44 ----------
include/singa/utils/blob.h | 2 +-
include/singa/utils/param.h | 2 +-
src/neuralnet/connection_layer/bridge.cc | 7 +-
src/neuralnet/connection_layer/split.cc | 2 +-
src/neuralnet/loss_layer/cudnn_softmaxloss.cc | 2 +-
src/neuralnet/neuralnet.cc | 6 +-
src/neuralnet/neuron_layer/dummy.cc | 2 +-
src/proto/job.proto | 4 +-
src/test/test_connection_layers.cc | 10 +--
src/utils/blob.cc | 8 +-
src/utils/param.cc | 24 +++---
21 files changed, 90 insertions(+), 359 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index bbb3497..470ea8a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@
#* under the License.
#*
#*************************************************************/
-
+
ACLOCAL_AMFLAGS = -I config
AUTOMAKE_OPTIONS = foreign subdir-objects
@@ -51,7 +51,7 @@ PROTO_PYS := tool/python/pb2/singa_pb2.py \
CUDA_SRCS := src/utils/math_kernel.cu
PY_SRCS := tool/python/singa/driver_wrap.cxx
- src/driver.cc
+ src/driver.cc
SINGA_SRCS := src/driver.cc \
src/server.cc \
@@ -68,22 +68,24 @@ SINGA_SRCS := src/driver.cc \
src/neuralnet/input_layer/record.cc \
src/neuralnet/input_layer/deprecated.cc \
src/neuralnet/input_layer/store.cc \
+ src/neuralnet/output_layer/accuracy.cc \
src/neuralnet/output_layer/argsort.cc \
src/neuralnet/output_layer/csv.cc \
src/neuralnet/output_layer/record.cc \
src/neuralnet/loss_layer/euclidean.cc \
src/neuralnet/loss_layer/softmax.cc \
+ src/neuralnet/neuron_layer/activation.cc \
src/neuralnet/neuron_layer/convolution.cc \
- src/neuralnet/neuron_layer/dummy.cc \
src/neuralnet/neuron_layer/dropout.cc \
+ src/neuralnet/neuron_layer/dummy.cc \
src/neuralnet/neuron_layer/inner_product.cc \
src/neuralnet/neuron_layer/lrn.cc \
src/neuralnet/neuron_layer/pooling.cc \
src/neuralnet/neuron_layer/rbm.cc \
src/neuralnet/neuron_layer/relu.cc \
src/neuralnet/neuron_layer/sigmoid.cc \
- src/neuralnet/neuron_layer/stanh.cc \
src/neuralnet/neuron_layer/softmax.cc \
+ src/neuralnet/neuron_layer/stanh.cc \
src/neuralnet/neuralnet.cc \
src/comm/socket.cc \
src/comm/msg.cc \
@@ -155,11 +157,11 @@ TEST_SRCS := include/gtest/gtest_main.cc \
src/test/test_csv_input_layer.cc
#EXTRA_PROGRAMS = $(PROGS)
-EXTRA_PROGRAMS = singatest test
+EXTRA_PROGRAMS = singatest test
#EXTRA_LTLIBRARIES = $(LTLIBS)
EXTRA_LTLIBRARIES = libgtest.la _driver.la
-lib_LTLIBRARIES = libsinga.la $(LTLIBS)
+lib_LTLIBRARIES = libsinga.la $(LTLIBS)
bin_PROGRAMS = singa singatool $(PROGS)
pydir = $(CURDIR)/tool/python/singa/
py_LTLIBRARIES = $(PY_PROGS)
@@ -167,7 +169,7 @@ py_LTLIBRARIES = $(PY_PROGS)
#lib_LTLIBRARIES = libsinga.la
libsinga_la_SOURCES = $(PROTO_SRCS) $(SINGA_SRCS)
-libsinga_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
+libsinga_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
if LMDB
libsinga_la_CXXFLAGS += -DUSE_LMDB
endif
@@ -181,7 +183,7 @@ endif
#bin_PROGRAMS = singa
singa_SOURCES = src/main.cc
-singa_CXXFLAGS = $(DEFAULT_FLAGS) -MMD -I$(top_srcdir)/include
+singa_CXXFLAGS = $(DEFAULT_FLAGS) -MMD -I$(top_srcdir)/include
singa_LDFLAGS = -lsinga \
-lglog \
-lprotobuf \
@@ -203,7 +205,7 @@ endif
#bin_PROGRAMS += singatool
singatool_SOURCES = src/utils/tool.cc
singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \
- -funroll-loops -DTHREADED -I$(top_srcdir)/include
+ -funroll-loops -DTHREADED -I$(top_srcdir)/include
singatool_LDFLAGS = -lsinga \
-lglog \
-lprotobuf \
@@ -220,7 +222,7 @@ endif
#bin_PROGRAMS += test
singatest_SOURCES = $(GTEST_HDRS) $(TEST_SRCS)
-singatest_CXXFLAGS = $(DEFAULT_FLAGS) -I$(top_srcdir)/include
+singatest_CXXFLAGS = $(DEFAULT_FLAGS) -I$(top_srcdir)/include
singatest_LDADD = ./libgtest.la
singatest_LDFLAGS = -lsinga \
-lglog \
@@ -241,8 +243,8 @@ singatest_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
_driver_la_SOURCES = $(PY_SRCS)
-_driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include $(PYFLAGS)
-_driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir)
+_driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include $(PYFLAGS)
+_driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir)
if DCUDA
_driver_la_CXXFLAGS += $(CUDA_CFLAGS)
@@ -281,7 +283,7 @@ install-pyLTLIBRARIES: $(py_LTLIBRARIES)
touch tool/python/singa/__init__.py
cp -f .libs/_driver.so tool/python/singa/
-uninstall-pyLTLIBRARIES:
+uninstall-pyLTLIBRARIES:
rm -f tool/python/singa/__init__.py
rm -f tool/python/singa/_driver.so
@@ -296,3 +298,4 @@ $(PROTO_HDRS) $(PROTO_SRCS): $(PROTOS)
mkdir -p $(top_srcdir)/include/singa/proto/
cp $(top_srcdir)/src/proto/*.pb.h $(top_srcdir)/include/singa/proto/
@echo
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 1dad0f7..d20b452 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -1,11 +1,10 @@
name: "cifar10-convnet"
-train_steps: 5
+train_steps: 1000
test_steps: 100
-test_freq: 0
+test_freq: 200
#validate_steps: 100
#validate_freq: 300
-disp_freq: 1
-debug: true
+disp_freq: 50
#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
train_one_batch {
alg: kBP
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/connection_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h
index 0b14a94..0cbe940 100644
--- a/include/singa/neuralnet/connection_layer.h
+++ b/include/singa/neuralnet/connection_layer.h
@@ -93,15 +93,14 @@ class ConcateLayer : public ConnectionLayer {
*/
class SliceLayer : public ConnectionLayer {
public:
+ ~SliceLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- std::vector<Blob<float>> datavec_;
- std::vector<Blob<float>> gradvec_;
- int slice_dim_;
- int slice_num_;
+ const Blob<float>& data(const Layer* from) const override;
+ const Blob<float>& grad(const Layer* from) const override;
+ Blob<float>* mutable_data(const Layer* from) override;
+ Blob<float>* mutable_grad(const Layer* from) override;
};
/**
@@ -113,12 +112,12 @@ class SliceLayer : public ConnectionLayer {
*/
class SplitLayer : public ConnectionLayer {
public:
+ ~SplitLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- protected:
- Blob<float> grads_;
+ const Blob<float>& grad(const Layer* from) const override;
+ Blob<float>* mutable_grad(const Layer* from) override;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/connection_layer/slice.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/slice.h b/include/singa/neuralnet/connection_layer/slice.h
deleted file mode 100644
index 023ebc1..0000000
--- a/include/singa/neuralnet/connection_layer/slice.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_CONNECTION_LAYER_SLICE_H_
-#define SINGA_NEURALNET_CONNECTION_LAYER_SLICE_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-/**
- * Connect a single (src) layer with multiple (dst) layers.
- *
- * It slices the feature Blob (i.e., matrix) of the src layer on one dimension.
- * The sliced feature Blobs will be fed into dst layers.
- */
-class SliceLayer : public ConnectionLayer {
- public:
- ~SliceLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- const Blob<float>& data(const Layer* from) const override;
- const Blob<float>& grad(const Layer* from) const override;
- Blob<float>* mutable_data(const Layer* from) override;
- Blob<float>* mutable_grad(const Layer* from) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_CONNECTION_LAYER_SLICE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/connection_layer/split.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/split.h b/include/singa/neuralnet/connection_layer/split.h
deleted file mode 100644
index 959d1a3..0000000
--- a/include/singa/neuralnet/connection_layer/split.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_CONNECTION_LAYER_SPLIT_H_
-#define SINGA_NEURALNET_CONNECTION_LAYER_SPLIT_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-
-namespace singa {
-/**
- * Connect a single (src) layer with multiple dst layers.
- *
- * It replicates the feature Blob of the src layer.
- * Each replicated feature Blob will be fed into one dst layer.
- * It aggregates gradients set by all dst layers and set it to the src layer.
- */
-class SplitLayer : public ConnectionLayer {
- public:
- ~SplitLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- const Blob<float>& grad(const Layer* from) const override;
- Blob<float>* mutable_grad(const Layer* from) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_CONNECTION_LAYER_SPLIT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/layer.h b/include/singa/neuralnet/layer.h
index 599203f..3dbb3fc 100644
--- a/include/singa/neuralnet/layer.h
+++ b/include/singa/neuralnet/layer.h
@@ -283,12 +283,12 @@ class InputLayer : virtual public Layer {
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
ConnectionType dst_layer_connection() const override { return kOneToMany; }
Blob<float>* mutable_grad(const Layer* layer) override {
- LOG(FATAL) << "Input layer has no gradient blob";
return nullptr;
+ // LOG(FATAL) << "Input layer has no gradient blob";
}
const Blob<float>& grad(const Layer* from) const override {
- LOG(FATAL) << "Input layer has no gradient blob";
return grad_;
+ // LOG(FATAL) << "Input layer has no gradient blob";
}
};
@@ -309,12 +309,12 @@ class NeuronLayer : virtual public Layer {
class LossLayer : virtual public Layer {
public:
Blob<float>* mutable_grad(const Layer* layer) override {
- LOG(FATAL) << "Loss layer has no gradient blob";
return nullptr;
+ // LOG(FATAL) << "Loss layer has no gradient blob";
}
const Blob<float>& grad(const Layer* from) const override {
- LOG(FATAL) << "Loss layer has no gradient blob";
return grad_;
+ // LOG(FATAL) << "Loss layer has no gradient blob";
}
};
@@ -325,12 +325,12 @@ class OutputLayer : virtual public Layer {
public:
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
Blob<float>* mutable_grad(const Layer* layer) override {
- LOG(FATAL) << "Output layer has no gradient blob";
return nullptr;
+ // LOG(FATAL) << "Output layer has no gradient blob";
}
const Blob<float>& grad(const Layer* from) const override {
- LOG(FATAL) << "Output layer has no gradient blob";
return grad_;
+ // LOG(FATAL) << "Output layer has no gradient blob";
}
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 39f4d69..17891e1 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -109,6 +109,24 @@ class DropoutLayer : public NeuronLayer {
*/
Blob<float> mask_;
};
+/**
+ * This layer is dummy and do no real work.
+ * It is used for testing purpose only.
+ *
+ * Use it as input layer, it will generate random data;
+ * Use it as output layer, it will generate random grad;
+ * Use it as neuron layer, it will replicates data and grad.
+ */
+class DummyLayer: public Layer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ private:
+ bool input_ = false; // use as input layer
+ bool output_ = false; // use as output layer
+};
+
/**
* Layer that applys linear transformations as
@@ -356,16 +374,10 @@ class CudnnSoftmaxLayer : public SoftmaxLayer, public CudnnLayer {
/**
* Base layer for RBM models.
*/
-class RBMLayer: virtual public NeuronLayer {
+class RBMLayer: virtual public Layer {
public:
virtual ~RBMLayer() {}
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- const Blob<float>& neg_data(const Layer* layer) {
- return neg_data_;
- }
- Blob<float>* mutable_neg_data(const Layer* layer) {
- return &neg_data_;
- }
const std::vector<Param*> GetParams() const override {
std::vector<Param*> params{weight_, bias_};
return params;
@@ -382,16 +394,16 @@ class RBMLayer: virtual public NeuronLayer {
int batchsize_;
bool first_gibbs_;
Param* weight_, *bias_;
-
+ Blob<float> pos_data_;
Blob<float> neg_data_;
Blob<float> neg_sample_;
- Blob<float> sample_;
+ Blob<float> pos_sample_;
};
/**
* RBM visible layer
*/
-class RBMVisLayer: public RBMLayer {
+class RBMVisLayer: public RBMLayer, public LossLayer {
public:
~RBMVisLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
@@ -402,9 +414,9 @@ class RBMVisLayer: public RBMLayer {
private:
RBMLayer* hid_layer_;
Layer* input_layer_;
-
float error_ = 0.0f;
int counter_ = 0;
+
};
/**
* RBM hidden layer
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/neuron_layer/dummy.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/dummy.h b/include/singa/neuralnet/neuron_layer/dummy.h
deleted file mode 100644
index 3177b7e..0000000
--- a/include/singa/neuralnet/neuron_layer/dummy.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_DUMMY_H_
-#define SINGA_NEURALNET_NEURON_LAYER_DUMMY_H_
-
-#include <random>
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * This layer is dummy and do no real work.
- * It is used for testing purpose only.
- *
- * Use it as input layer, it will generate random data;
- * Use it as output layer, it will generate random grad;
- * Use it as neuron layer, it will replicates data and grad.
- */
-class DummyLayer: public Layer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
- private:
- bool input_ = false; // use as input layer
- bool output_ = false; // use as output layer
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_DUMMY_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/neuron_layer/rbm.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/rbm.h b/include/singa/neuralnet/neuron_layer/rbm.h
deleted file mode 100644
index 432c499..0000000
--- a/include/singa/neuralnet/neuron_layer/rbm.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_RBM_H_
-#define SINGA_NEURALNET_NEURON_LAYER_RBM_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * Base layer for RBM models.
- */
-class RBMLayer: virtual public Layer {
- public:
- virtual ~RBMLayer() {}
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- const std::vector<Param*> GetParams() const override {
- std::vector<Param*> params{weight_, bias_};
- return params;
- }
- virtual Blob<float>* Sample(int flat);
-
- protected:
- //! if ture, sampling according to guassian distribution
- bool gaussian_;
- //! dimension of the hidden layer
- int hdim_;
- //! dimension of the visible layer
- int vdim_;
- int batchsize_;
- bool first_gibbs_;
- Param* weight_, *bias_;
- Blob<float> pos_data_;
- Blob<float> neg_data_;
- Blob<float> neg_sample_;
- Blob<float> pos_sample_;
-};
-
-/**
- * RBM visible layer
- */
-class RBMVisLayer: public RBMLayer, public LossLayer {
- public:
- ~RBMVisLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- RBMLayer* hid_layer_;
- Layer* input_layer_;
-};
-/**
- * RBM hidden layer
- */
-class RBMHidLayer: public RBMLayer {
- public:
- ~RBMHidLayer();
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- RBMLayer *vis_layer_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_RBM_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/neuralnet/neuron_layer/sigmoid.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/sigmoid.h b/include/singa/neuralnet/neuron_layer/sigmoid.h
deleted file mode 100644
index 3cf80e7..0000000
--- a/include/singa/neuralnet/neuron_layer/sigmoid.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_NEURON_LAYER_SIGMOID_H_
-#define SINGA_NEURALNET_NEURON_LAYER_SIGMOID_H_
-
-#include <vector>
-#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-namespace singa {
-/**
- * This layer apply Sigmoid function to neuron activations.
- * f(x)=1/(1+exp(-x))
- * f'(x)=f(x)*(1-f(x))
- */
-class SigmoidLayer: public Layer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_NEURON_LAYER_SIGMOID_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index b260862..7e1e516 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -247,7 +247,7 @@ class Blob {
* data_ field. For training with multi-gpu cards, cpu_only must be true,
* becuase gpu memory cannot be shared among different devices.
*/
- void ShareData(const Blob& other, bool cpu_only = true);
+ void ShareData(Blob* other, bool cpu_only = true);
/*
void Swap(Blob& other);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/include/singa/utils/param.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/param.h b/include/singa/utils/param.h
index 33b61ff..415490e 100644
--- a/include/singa/utils/param.h
+++ b/include/singa/utils/param.h
@@ -146,7 +146,7 @@ class Param {
* @param cpu_only if true, share only cpu memory (used for training with
* multi-gpu cards); else, share both cpu and gpu memory.
*/
- void ShareFrom(const Param& other, bool cpu_only);
+ void ShareFrom(Param* other, bool cpu_only);
/**
* Init param values from checkpoint blob.
*/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/neuralnet/connection_layer/bridge.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/bridge.cc b/src/neuralnet/connection_layer/bridge.cc
index 200a3f9..a2302ab 100644
--- a/src/neuralnet/connection_layer/bridge.cc
+++ b/src/neuralnet/connection_layer/bridge.cc
@@ -19,8 +19,7 @@
*
*************************************************************/
-<<<<<<< HEAD
-#include "singa/neuralnet/connection_layer/bridge.h"
+#include "singa/neuralnet/connection_layer.h"
#include "singa/comm/msg.h"
namespace singa {
@@ -70,8 +69,8 @@ void BridgeSrcLayer::Setup(const LayerProto& conf,
Layer::Setup(conf, srclayers);
data_.Reshape(srclayers[0]->data(this).shape());
grad_.ReshapeLike(data_);
- data_.ShareData(srclayers[0]->data(this));
- grad_.ShareData(srclayers[0]->grad(this));
+ data_.ShareData(srclayers[0]->mutable_data(this));
+ grad_.ShareData(srclayers[0]->mutable_grad(this));
}
void BridgeSrcLayer::ComputeFeature(int flag, const vector<Layer*>& srcs) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index 7ee24fa..5b9db5b 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -36,7 +36,7 @@ void SplitLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
data_.Reshape(srclayers[0]->data(this).shape());
- data_.ShareData(srclayers[0]->data(this));
+ data_.ShareData(srclayers[0]->mutable_data(this));
CHECK_GT(num_partitions(), 0);
// add num_partitions()-1 more grad blobs
for (int i = 1; i < num_partitions(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
index 78a035a..1fe228b 100644
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
+++ b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
@@ -29,7 +29,7 @@ void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
softmax_.Setup(conf, vector<Layer*> {srclayers.at(0)});
data_.Reshape(softmax_.data(this).shape());
- data_.ShareData(*softmax_.mutable_data(this), false);
+ data_.ShareData(softmax_.mutable_data(this), false);
batchsize_ = data_.shape(0);
dim_ = data_.count() / batchsize_;
LOG(ERROR) << batchsize_ << " " << dim_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index aabc361..eb8c270 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -163,7 +163,7 @@ void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
const auto& params = layer->GetParams();
CHECK_EQ(params.size(), otherparams.size());
for (size_t i = 0; i < params.size(); i++) {
- params[i]->ShareFrom(*otherparams[i], cpu_only);
+ params[i]->ShareFrom(otherparams[i], cpu_only);
}
}
}
@@ -416,7 +416,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
const string share_from = param->share_from();
if (param->share_from() != "") {
if (name2param.find(share_from) != name2param.end()) {
- param->ShareFrom(*name2param.at(param->share_from()), false);
+ param->ShareFrom(name2param.at(param->share_from()), false);
} else {
LOG(FATAL) << "No param with the name (share_from) " << share_from;
}
@@ -430,7 +430,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
auto params = (*it)->GetParams();
CHECK_EQ(params.size(), owner_params.size());
for (size_t i = 0; i < params.size(); i++)
- params.at(i)->ShareFrom(*owner_params.at(i), true);
+ params.at(i)->ShareFrom(owner_params.at(i), true);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/neuralnet/neuron_layer/dummy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc
index 8d165f7..9b352dd 100644
--- a/src/neuralnet/neuron_layer/dummy.cc
+++ b/src/neuralnet/neuron_layer/dummy.cc
@@ -19,7 +19,7 @@
*
*************************************************************/
-#include "singa/neuralnet/neuron_layer/dummy.h"
+#include "singa/neuralnet/neuron_layer.h"
#include <glog/logging.h>
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index ff27c05..80752c3 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -191,14 +191,14 @@ message LayerProto {
optional string user_type =21;
// proto for the specific layer
- // configuration for activation layer
+ // configuration for input layers
optional ActivationProto activation_conf = 54;
// configuration for argsort layer
optional ArgSortProto argsort_conf = 52;
// configuration for convolution layer
optional ConvolutionProto convolution_conf = 30;
// configuration for dummy layer
- optional DummyProto dummy_conf = 53;
+ optional DummyProto dummy_conf = 55;
// configuration for dropout layer
optional DropoutProto dropout_conf = 33;
// configuration for inner product layer
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index 2415fcd..a10d3be 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -25,11 +25,11 @@
#include "gtest/gtest.h"
#include "singa/comm/msg.h"
#include "singa/comm/socket.h"
-#include "singa/neuralnet/connection_layer/bridge.h"
-#include "singa/neuralnet/connection_layer/concate.h"
-#include "singa/neuralnet/connection_layer/slice.h"
-#include "singa/neuralnet/connection_layer/split.h"
-#include "singa/neuralnet/neuron_layer/dummy.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/neuron_layer.h"
#include "singa/proto/job.pb.h"
using namespace singa;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index 9607683..735226d 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -265,12 +265,12 @@ void Blob<Dtype>::SetValue(Dtype v) {
ptr[i] = v;
}
template <typename Dtype>
-void Blob<Dtype>::ShareData(const Blob& other, bool cpu_only) {
- CHECK_EQ(count_, other.count());
+void Blob<Dtype>::ShareData(Blob* other, bool cpu_only) {
+ CHECK_EQ(count_, other->count());
if (cpu_only)
- data_->set_cpu_data(other.cpu_data());
+ data_->set_cpu_data(other->mutable_cpu_data());
else
- data_ = other.data_;
+ data_ = other->data_;
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f8be9afa/src/utils/param.cc
----------------------------------------------------------------------
diff --git a/src/utils/param.cc b/src/utils/param.cc
index 09f519b..097fa61 100644
--- a/src/utils/param.cc
+++ b/src/utils/param.cc
@@ -166,21 +166,21 @@ void Param::InitValues(int version) {
set_version(version);
}
-void Param::ShareFrom(const Param& other, bool cpu_only) {
- proto_.set_owner(other.owner());
- CHECK(data_.shape() == other.data_.shape());
- data_.ShareData(other.data_, cpu_only);
+void Param::ShareFrom(Param* other, bool cpu_only) {
+ proto_.set_owner(other->owner());
+ CHECK(data_.shape() == other->data_.shape());
+ data_.ShareData(&(other->data_), cpu_only);
if (grad_.count() == 0)
grad_.Reshape(data_.shape());
- version_ = other.version_;
- last_version_ = other.last_version_;
- slice_start_ = other.slice_start_;
- num_slices_ = other.num_slices_;
- slice_offset_ = other.slice_offset_;
- slice_size_ = other.slice_size_;
+ version_ = other->version_;
+ last_version_ = other->last_version_;
+ slice_start_ = other->slice_start_;
+ num_slices_ = other->num_slices_;
+ slice_offset_ = other->slice_offset_;
+ slice_size_ = other->slice_size_;
// change pending list size equal to slice size
- pending_get_.resize(other.pending_get_.size());
- pending_update_.resize(other.pending_update_.size());
+ pending_get_.resize(other->pending_get_.size());
+ pending_update_.resize(other->pending_update_.size());
}
void Param::FromProto(const BlobProto& blob) {
[02/19] incubator-singa git commit: SINGA-100 Implement layers using
CUDNN for GPU training
Posted by wa...@apache.org.
SINGA-100 Implement layers using CUDNN for GPU training
Fix errors from compiling cpu code and cudnn code.
tmp commit. Trying to run cudnn for gpu training and Mshadow for cpu training.
1. finish the CudnnSoftmaxLossLayer
2. finish the DropoutLayer
3. add ActivationLayer, which leads to link errors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/49293a68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/49293a68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/49293a68
Branch: refs/heads/master
Commit: 49293a6812cbd2e74c3130af17d619a0e3bf0e89
Parents: af1bf50
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Tue Nov 24 15:20:39 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 11 11:01:29 2015 +0800
----------------------------------------------------------------------
Makefile.am | 11 ++--
Makefile.gpu | 17 ++---
include/mshadow/tensor_random.h | 4 +-
include/singa/neuralnet/input_layer.h | 1 +
include/singa/neuralnet/loss_layer.h | 17 +++++
include/singa/neuralnet/neuron_layer.h | 13 ++--
include/singa/utils/blob.h | 4 +-
include/singa/utils/math_addr.h | 26 ++++++++
include/singa/utils/math_blob.h | 67 ++++++++++++++++++--
include/singa/worker.h | 4 +-
src/driver.cc | 30 ++-------
src/neuralnet/neuron_layer/argsort.cc | 56 ----------------
src/neuralnet/neuron_layer/convolution.cc | 8 +--
src/neuralnet/neuron_layer/cudnn_activation.cu | 18 +++---
src/neuralnet/neuron_layer/cudnn_convolution.cu | 24 +++----
src/neuralnet/neuron_layer/cudnn_lrn.cu | 56 +++++++---------
src/neuralnet/neuron_layer/cudnn_pooling.cu | 15 +++--
src/neuralnet/neuron_layer/cudnn_softmax.cu | 16 ++---
src/neuralnet/neuron_layer/dropout.cc | 19 +++---
src/neuralnet/neuron_layer/pooling.cc | 65 +++++++++++++------
src/neuralnet/neuron_layer/softmax.cc | 8 +--
src/neuralnet/output_layer/argsort.cc | 56 ++++++++++++++++
src/proto/job.proto | 53 ++++++++++++++--
src/utils/blob.cc | 25 +-------
24 files changed, 369 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 835f66d..bbb3497 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -68,11 +68,11 @@ SINGA_SRCS := src/driver.cc \
src/neuralnet/input_layer/record.cc \
src/neuralnet/input_layer/deprecated.cc \
src/neuralnet/input_layer/store.cc \
+ src/neuralnet/output_layer/argsort.cc \
src/neuralnet/output_layer/csv.cc \
src/neuralnet/output_layer/record.cc \
src/neuralnet/loss_layer/euclidean.cc \
src/neuralnet/loss_layer/softmax.cc \
- src/neuralnet/neuron_layer/argsort.cc \
src/neuralnet/neuron_layer/convolution.cc \
src/neuralnet/neuron_layer/dummy.cc \
src/neuralnet/neuron_layer/dropout.cc \
@@ -118,8 +118,11 @@ SINGA_HDRS := include/singa.h \
include/worker.h \
include/stub.h \
include/neuralnet/layer.h \
- include/neuralnet/output_layer/csv.h \
- include/neuralnet/output_layer/record.h \
+ include/neuralnet/output_layer.h \
+ include/neuralnet/input_layer.h \
+ include/neuralnet/loss_layer.h \
+ include/neuralnet/neuron_layer.h \
+ include/neuralnet/connection_layer.h \
include/neuralnet/neuralnet.h \
include/singa/comm/msg.h \
include/singa/comm/socket.h \
@@ -127,12 +130,12 @@ SINGA_HDRS := include/singa.h \
include/singa/io/kvfile.h \
include/singa/io/kvfile_store.h \
include/singa/io/textfile_store.h \
+ include/mshadow/cxxnet_op.h \
include/mshadow/tensor_expr.h \
include/mshadow/tensor_container.h \
include/mshadow/tensor_expr_ext.h \
include/mshadow/tensor.h \
include/mshadow/tensor_io.h \
- include/mshadow/cxxnet_op.h \
include/mshadow/tensor_base.h \
include/mshadow/tensor_random.h
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/Makefile.gpu
----------------------------------------------------------------------
diff --git a/Makefile.gpu b/Makefile.gpu
index 35b81b9..9cbbc42 100644
--- a/Makefile.gpu
+++ b/Makefile.gpu
@@ -24,12 +24,13 @@ HOME_DIR := /home/wangwei/local
CUDA_DIR := /usr/local/cuda
-#CUDA_DIR :=
+# must config the cudnn folder if using cudnn
+CUDNN_DIR := /home/wangwei/local/cuda
# Lib folder for system and external libs. You may need to change it.
-LIBRARY_DIRS := $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib $(HOME_DIR)/lib64 $(HOME_DIR)/lib
+LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(HOME_DIR)/local/lib $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib $(CUDNN_DIR)/lib64
# Header folder for system and external libs. You may need to change it.
-INCLUDE_DIRS := $(CUDA_DIR)/include $(HOME_DIR)/include ./include
+INCLUDE_DIRS := $(HOME_DIR)/include ./include $(HOME_DIR)/local/include/zookeeper $(CUDA_DIR)/include $(CUDNN_DIR)/include
# g++ location, should support c++11, tested with 4.8.1
CXX := g++
CUCXX := nvcc
@@ -38,7 +39,7 @@ CUCXX := nvcc
LIBRARIES := glog protobuf openblas zmq czmq zookeeper_mt
ifneq ($(CUDA_DIR),)
- LIBRARIES := $(LIBRARIES) cublas cudart curand
+ LIBRARIES := $(LIBRARIES) cublas cudart curand cudnn
endif
LDFLAGS := $(foreach librarydir, $(LIBRARY_DIRS), -L$(librarydir))\
@@ -48,9 +49,9 @@ BUILD_DIR := .libs
MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
ZK_FLAGS :=-DTHREADED -fpermissive
CXXFLAGS := -O2 -msse3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
- $(MSHADOW_FLAGS) -DCPU_ONLY=1 $(ZK_FLAGS)\
+ $(MSHADOW_FLAGS) $(ZK_FLAGS)\
-funroll-loops $(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
-CUCXXFLAGS := $(MSHADOW_FLAGS) -DUSE_GPU -std=c++11 -G $(CUDA_ARCH) \
+CUCXXFLAGS := -DUSE_CUDNN $(MSHADOW_FLAGS) -std=c++11 $(CUDA_ARCH) \
$(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
#Add device compile option
@@ -84,7 +85,7 @@ TEST_CUDA_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cu")
TEST_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_CUDA_SRCS:.cu=.o)))
-include $(TEST_CUDA_OBJS:%.o=%.P)
-SINGA_CUDA_SRCS := $(shell find src/ \( -path "src/test" \) -prune -o \( -name "*.cu" -type f \) -print )
+SINGA_CUDA_SRCS :=$(shell find src/ -name "*.cu")
SINGA_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_CUDA_SRCS:.cu=.o)))
-include $(SINGA_CUDA_OBJS:%.o=%.P)
@@ -140,7 +141,7 @@ proto: $(PROTO_OBJS)
$(PROTO_SRCS): $(PROTOS)
protoc --proto_path=src/proto --cpp_out=src/proto $(PROTOS)
mkdir -p include/proto/
- cp src/proto/*.pb.h include/proto/
+ cp src/proto/*.pb.h include/singa/proto/
mkdir -p tool/pb2/
touch tool/pb2/__init__.py
protoc --proto_path=src/proto --python_out=tool/pb2/ $(PROTOS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/mshadow/tensor_random.h
----------------------------------------------------------------------
diff --git a/include/mshadow/tensor_random.h b/include/mshadow/tensor_random.h
index 59ef082..ae2836a 100644
--- a/include/mshadow/tensor_random.h
+++ b/include/mshadow/tensor_random.h
@@ -249,8 +249,8 @@ namespace mshadow {
std::mt19937 gen_;
}; // class Random<cpu>
-#ifdef __CUDACC__
-
+#if MSHADOW_USE_CUDA
+// __CUDACC__
/*! \brief GPU random number generator */
template<>
class Random<gpu> {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer.h b/include/singa/neuralnet/input_layer.h
index 72593d5..2825d65 100644
--- a/include/singa/neuralnet/input_layer.h
+++ b/include/singa/neuralnet/input_layer.h
@@ -26,6 +26,7 @@
#include <vector>
#include <thread>
#include "singa/io/store.h"
+#include "singa/io/kvfile.h"
#include "singa/neuralnet/layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/neuralnet/loss_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer.h b/include/singa/neuralnet/loss_layer.h
index e7fcd74..50e6c24 100644
--- a/include/singa/neuralnet/loss_layer.h
+++ b/include/singa/neuralnet/loss_layer.h
@@ -49,8 +49,25 @@ class SoftmaxLossLayer : public LossLayer, public SoftmaxLayer {
private:
float scale_;
+ int topk_, dim_;
+};
+
+#ifdef USE_CUDNN
+class CudnnSoftmaxLossLayer : public LossLayer, public CudnnSoftmaxLayer {
+ public:
+ void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const std::string ToString(bool debug, int flag) override;
+
+ private:
int topk_;
+ int counter_;
+ float loss_, accuracy_;
+
+ CudnnSoftmaxLayer *softmax_;
};
+#endif
} // namespace singa
#endif // SINGA_NEURALNET_LOSS_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 2bf9682..097b209 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -249,6 +249,7 @@ class STanhLayer : public NeuronLayer {
/*************** Layers implemented using cudnn v3 ***************/
#ifdef USE_CUDNN
+#include <cudnn.h>
#define CHECK_CUDNN(x) CHECK_EQ(x, CUDNN_STATUS_SUCCESS)
class CudnnLayer : virtual public NeuronLayer {
@@ -289,9 +290,9 @@ class CudnnActivationLayer : public ActivationLayer, public CudnnLayer {
/**
* Convolution layer implemeneted using cudnn (v3 version backward functions).
*/
-class CuDNNConvLayer : public ConvolutionLayer, public CudnnLayer {
+class CudnnConvLayer : public ConvolutionLayer, public CudnnLayer {
public:
- ~CuDNNConvLayer();
+ ~CudnnConvLayer();
void InitCudnn() override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
@@ -306,7 +307,7 @@ class CuDNNConvLayer : public ConvolutionLayer, public CudnnLayer {
size_t workspace_byte_limit_, workspace_count_;
};
-class CudnnLRNLayer : public LRNLayer {
+class CudnnLRNLayer : public LRNLayer, public CudnnLayer {
public:
~CudnnLRNLayer();
void InitCudnn() override;
@@ -320,9 +321,9 @@ class CudnnLRNLayer : public LRNLayer {
/**
* Pooling layer implemented using cudnn.
*/
-class CuDNNPoolLayer : public PoolingLayer, public CudnnLayer {
+class CudnnPoolLayer : public PoolingLayer, public CudnnLayer {
public:
- ~CuDNNPoolLayer();
+ ~CudnnPoolLayer();
void InitCudnn() override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
@@ -407,4 +408,4 @@ class RBMHidLayer: public RBMLayer {
};
} // namespace singa
-#define SINGA_NEURALNET_NEURON_LAYER_H_
+#endif // SINGA_NEURALNET_NEURON_LAYER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/utils/blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/blob.h b/include/singa/utils/blob.h
index 87a97d5..e7bf995 100644
--- a/include/singa/utils/blob.h
+++ b/include/singa/utils/blob.h
@@ -255,14 +255,14 @@ class Blob {
/**
* @return the size of the k-th dimension.
*/
- inline const int shape(int k) const {
+ inline int shape(int k) const {
CHECK_LT(k, shape_.size());
return shape_.at(k);
}
inline int count() const {
return count_;
}
- inline const int version() const {
+ inline int version() const {
return version_;
}
inline void set_version(int v) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/utils/math_addr.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_addr.h b/include/singa/utils/math_addr.h
index 9b91e70..b21ea45 100644
--- a/include/singa/utils/math_addr.h
+++ b/include/singa/utils/math_addr.h
@@ -124,6 +124,20 @@ void cpu_expand_f(const Dtype * A, const int m, const int n, Dtype * B) {
}
// expand each element in A into a row of B
+template<typename Dtype>
+void cpu_sample_uniform(int n, Dtype low, Dtype high, Dtype* A);
+
+template<>
+inline void cpu_sample_uniform<float>(int n, float low, float high, float* A) {
+
+}
+template<typename Dtype>
+void cpu_sample_gaussian(int n, Dtype mean, Dtype std, Dtype* A);
+
+template<>
+inline void cpu_sample_gaussian<float>(int n, float mean, float std, float* A) {
+
+}
#ifdef USE_GPU
template<typename Dtype>
void gpu_gemm(const Dtype * A, const Dtype * B, const int m, const int n,
@@ -213,6 +227,18 @@ void gpu_expand_f(const Dtype * A, const int m, const int n, Dtype * B) {
Op::CudaMap(A[i], n, B+i*n);
}
}
+
+
+template<typename Dtype>
+void gpu_sample_uniform(int n, Dtype low, Dtype high, Dtype* A) {
+
+}
+
+template<typename Dtype>
+void gpu_sample_gaussian(int n, Dtype mean, Dtype std, Dtype* A) {
+
+}
+
// expand each element in A into a row of B
#endif // USE_GPU
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/utils/math_blob.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/math_blob.h b/include/singa/utils/math_blob.h
index 6428117..ca75205 100644
--- a/include/singa/utils/math_blob.h
+++ b/include/singa/utils/math_blob.h
@@ -24,9 +24,12 @@
#include <vector>
#include <algorithm>
+#include <thread>
#include "singa/utils/blob.h"
#include "singa/utils/singa_op.h"
#include "singa/utils/math_addr.h"
+#include "singa/utils/singleton.h"
+#include "singa/utils/context.h"
namespace singa {
@@ -355,10 +358,21 @@ void Sub(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
* Map(XPU, const Blob<Dtype>&, const Blob<Dtype>&, Blob<Dtype>*).
*/
template<typename Dtype>
-void Mult(XPU xpu, const Blob<Dtype> & A, const Blob<Dtype> & B,
+void Mult(const Blob<Dtype> & A, const Blob<Dtype> & B,
Blob<Dtype> * C) {
- Map<singa::op::Mult<Dtype>>(xpu, A, B, C);
+ //Map<singa::op::Mult<Dtype>>(xpu, A, B, C);
// TODO(wangwei) use MKL's vector func
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1)
+ cpu_e_f<op::Mult<Dtype>>(C->count(), A.cpu_data(), B.cpu_data(),
+ C->mutable_cpu_data());
+ else {
+#ifdef USE_GPU
+ gpu_e_f<op::Mult<Dtype>>(C->count(), A.gpu_data(), B.gpu_data(),
+ C->mutable_gpu_data());
+#endif
+ }
}
/**
@@ -473,7 +487,7 @@ void RepmatRow(XPU xpu, const Blob<Dtype> & A, Blob<Dtype> * B) {
}
/**
- * Sum all columns of matrix A to a column vector B,
+ * Sum all columns of matrix A to a column vector B,
* i.e., Bi = \sum_j {alpha*Aij}+beta*Bi
* Loose shape checking, A.count() % B.count() == 0.
* # columns of A = A.count() / B.count().
@@ -498,7 +512,7 @@ void MVSumCol(XPU xpu, Dtype alpha, Dtype beta, const Blob<Dtype> & A,
}
/**
- * Sum all rows of matrix A to a row vector B,
+ * Sum all rows of matrix A to a row vector B,
* i.e., Bj = \sum_i {alpha*Aij}+beta*Bj
* Loose shape checking, A.count() % B.count() == 0.
* # rows of A = A.count() / B.count().
@@ -574,6 +588,51 @@ Dtype Asum(XPU xpu, const Blob<Dtype>& A) {
#endif
}
+/*************Random Sample***************/
+template<typename Dtype>
+void SampleUniform(Dtype low, Dtype high, Blob<Dtype>* A);
+
+template<>
+inline void SampleUniform<float>(float low, float high, Blob<float> *A) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1)
+ cpu_sample_uniform(A->count(), low, high, A->mutable_cpu_data());
+ else {
+#ifdef USE_GPU
+ gpu_sample_uniform(A->count(), low, high, A->mutable_gpu_data());
+#endif
+ }
+}
+
+template<typename Dtype>
+void SampleGaussian(Dtype mean, Dtype std, Blob<Dtype>* A);
+
+template<>
+inline void SampleGaussian<float>(float low, float high, Blob<float> *A) {
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
+ if (device == -1)
+ cpu_sample_gaussian(A->count(), low, high, A->mutable_cpu_data());
+ else {
+#ifdef USE_GPU
+ gpu_sample_gaussian(A->count(), low, high, A->mutable_gpu_data());
+#endif
+ }
+}
+
+/************** Other functions ****************/
+template<typename Dtype>
+void Softmax(int nb_rows, const Blob<Dtype>& A, Blob<Dtype>* B) {
+ CHECK_GT(nb_rows, 0);
+ CHECK_EQ(A.count() % nb_rows, 0);
+ CHECK_EQ(A.count(), B->count());
+
+#ifdef USE_GPU
+ cpu_softmax(nb_rows, A.count() / nb_rows, A.cpu_data(),
+ B->mutable_cpu_data());
+#endif // USE_GPU
+}
} // end of namespace singa
#endif // SINGA_UTILS_MATH_BLOB_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/include/singa/worker.h
----------------------------------------------------------------------
diff --git a/include/singa/worker.h b/include/singa/worker.h
index 53f2118..8738c27 100644
--- a/include/singa/worker.h
+++ b/include/singa/worker.h
@@ -28,8 +28,8 @@
#include "singa/comm/socket.h"
#include "singa/neuralnet/neuralnet.h"
#include "singa/proto/job.pb.h"
-#include "singa/neuralnet/connection_layer/bridge.h"
-#include "singa/neuralnet/neuron_layer/rbm.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/neuron_layer.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 4ffd9b1..8a48c30 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -33,31 +33,11 @@
#include "singa/stub.h"
#include "singa/worker.h"
-#include "singa/neuralnet/connection_layer/bridge.h"
-#include "singa/neuralnet/connection_layer/concate.h"
-#include "singa/neuralnet/connection_layer/slice.h"
-#include "singa/neuralnet/connection_layer/split.h"
-#include "singa/neuralnet/input_layer/deprecated.h"
-#include "singa/neuralnet/input_layer/csv.h"
-#include "singa/neuralnet/input_layer/image_preprocess.h"
-#include "singa/neuralnet/input_layer/prefetch.h"
-#include "singa/neuralnet/input_layer/record.h"
-#include "singa/neuralnet/input_layer/store.h"
-#include "singa/neuralnet/loss_layer/euclidean.h"
-#include "singa/neuralnet/loss_layer/softmax.h"
-#include "singa/neuralnet/neuron_layer/argsort.h"
-#include "singa/neuralnet/neuron_layer/convolution.h"
-#include "singa/neuralnet/neuron_layer/dropout.h"
-#include "singa/neuralnet/neuron_layer/inner_product.h"
-#include "singa/neuralnet/neuron_layer/lrn.h"
-#include "singa/neuralnet/neuron_layer/pooling.h"
-#include "singa/neuralnet/neuron_layer/rbm.h"
-#include "singa/neuralnet/neuron_layer/relu.h"
-#include "singa/neuralnet/neuron_layer/sigmoid.h"
-#include "singa/neuralnet/neuron_layer/stanh.h"
-#include "singa/neuralnet/neuron_layer/softmax.h"
-#include "singa/neuralnet/output_layer/record.h"
-#include "singa/neuralnet/output_layer/csv.h"
+#include "singa/neuralnet/connection_layer.h"
+#include "singa/neuralnet/input_layer.h"
+#include "singa/neuralnet/loss_layer.h"
+#include "singa/neuralnet/neuron_layer.h"
+#include "singa/neuralnet/output_layer.h"
extern "C" void openblas_set_num_threads(int num);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/argsort.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/argsort.cc b/src/neuralnet/neuron_layer/argsort.cc
deleted file mode 100644
index d1775c0..0000000
--- a/src/neuralnet/neuron_layer/argsort.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/output_layer.h"
-#include <algorithm>
-
-namespace singa {
-
-void ArgSortLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- NeuronLayer::Setup(proto, srclayers);
- batchsize_ = srclayers[0]->data(this).shape()[0];
- dim_ = srclayers[0]->data(this).count() / batchsize_;
- topk_ = proto.argsort_conf().topk();
- data_.Reshape(vector<int>{batchsize_, topk_});
-}
-
-void ArgSortLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- // TODO(wangwei) check flag to ensure it is not called in training phase
- const float* srcptr = srclayers.at(0)->data(this).cpu_data();
- float* ptr = data_.mutable_cpu_data();
- for (int n = 0; n < batchsize_; n++) {
- vector<std::pair<float, int> > vec;
- for (int j = 0; j < dim_; ++j)
- vec.push_back(std::make_pair(srcptr[j], j));
- std::partial_sort(vec.begin(), vec.begin() + topk_, vec.end(),
- std::greater<std::pair<float, int> >());
-
- for (int j = 0; j < topk_; ++j)
- ptr[j] = static_cast<float> (vec.at(j).second);
- ptr += topk_;
- srcptr += dim_;
- }
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/convolution.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/convolution.cc b/src/neuralnet/neuron_layer/convolution.cc
index 4247bee..2b4c6d9 100644
--- a/src/neuralnet/neuron_layer/convolution.cc
+++ b/src/neuralnet/neuron_layer/convolution.cc
@@ -106,7 +106,7 @@ void ConvolutionLayer::ComputeFeature(int flag,
auto weight = Tensor2(weight_->mutable_data());
auto bias = Tensor1(bias_->mutable_data());
for (int n = 0; n < batchsize_; n++) {
- if (pad_ > 0)
+ if (pad_x_ > 0)
col = expr::unpack_patch2col(pad(src[n], pad_x_), kernel_x_, stride_x_);
else
col = expr::unpack_patch2col(src[n], kernel_x_, stride_x_);
@@ -131,11 +131,11 @@ void ConvolutionLayer::ComputeGradient(int flag,
gbias = expr::sumall_except_dim<1>(grad);
gweight = 0.0f;
Shape<3> padshp(gsrc.shape.SubShape());
- padshp[0] += 2 * pad_;
- padshp[1] += 2 * pad_;
+ padshp[0] += 2 * pad_y_;
+ padshp[1] += 2 * pad_x_;
Shape<2> imgshp = Shape2(height_, width_);
for (int n = 0; n < batchsize_; n++) {
- if (pad_ > 0)
+ if (pad_x_ > 0)
col = expr::unpack_patch2col(pad(src[n], pad_x_), kernel_x_, stride_x_);
else
col = expr::unpack_patch2col(src[n], kernel_x_, stride_x_);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/cudnn_activation.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_activation.cu b/src/neuralnet/neuron_layer/cudnn_activation.cu
index f77a8a8..e8a7b41 100644
--- a/src/neuralnet/neuron_layer/cudnn_activation.cu
+++ b/src/neuralnet/neuron_layer/cudnn_activation.cu
@@ -27,14 +27,14 @@ void CudnnActivationLayer::InitCudnn() {
CudnnLayer::InitCudnn();
// TODO(wangwei) make the mode case insensitive
- if (layer_conf_.activation_conf().mode() == "sigmoid")
+ if (layer_conf_.activation_conf().type() == SIGMOID)
mode_ = CUDNN_ACTIVATION_SIGMOID;
- else if (layer_conf_.activation_conf().mode() == "tanh")
+ else if (layer_conf_.activation_conf().type() == TANH)
mode_ = CUDNN_ACTIVATION_TANH;
- else if (layer_conf_.activation_conf().mode() == "relu")
+ else if (layer_conf_.activation_conf().type() == RELU)
mode_ = CUDNN_ACTIVATION_RELU;
else {
- LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().mode();
+ LOG(FATAL) << "Unkown activation: " << layer_conf_.activation_conf().type();
}
const auto& shape = data_.shape();
@@ -48,7 +48,7 @@ void CudnnActivationLayer::InitCudnn() {
stride[i] = 1;
for (--i; i >= 0; i--) {
sdim[i] = shape[i];
- stride[i] = shape[i + 1] * stride[i + 1]
+ stride[i] = shape[i + 1] * stride[i + 1];
}
CHECK_CUDNN(cudnnSetTensorNdDescriptor(src_desc_,
CUDNN_DATA_FLOAT,
@@ -75,7 +75,7 @@ void CudnnActivationLayer::ComputeFeature(int flag,
mode_,
&alpha,
src_desc_,
- srclayers[0].data(this)->gpu_data(),
+ srclayers[0]->data(this).gpu_data(),
&beta,
my_desc_,
data_.mutable_gpu_data()));
@@ -88,13 +88,13 @@ void CudnnActivationLayer::ComputeGradient(int flag,
mode_,
&alpha,
my_desc_,
- data_.gpu_data()
+ data_.gpu_data(),
my_desc_,
grad_.gpu_data(),
src_desc_,
- srclayers[0].data(this)->gpu_data(),
+ srclayers[0]->data(this).gpu_data(),
&beta,
src_desc_,
- srclayers[0].mutable_grad(this)->mutable_gpu_data()));
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/cudnn_convolution.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_convolution.cu b/src/neuralnet/neuron_layer/cudnn_convolution.cu
index debe4c3..13e9f65 100644
--- a/src/neuralnet/neuron_layer/cudnn_convolution.cu
+++ b/src/neuralnet/neuron_layer/cudnn_convolution.cu
@@ -70,7 +70,7 @@ void CudnnConvLayer::InitCudnn() {
channels_,
height_,
width_));
- CHECK_CUDNN(cudnnSetTensor4dDescriptor(data_desc_,
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
@@ -126,7 +126,7 @@ void CudnnConvLayer::InitCudnn() {
filter_desc_,
bp_filter_alg_,
&bp_filter_byte));
- workspace_count_ = std::max(std::max(fp_bypte, bp_data_byte), bp_filter_byte)
+ workspace_count_ = std::max(std::max(fp_byte, bp_data_byte), bp_filter_byte)
/ sizeof(float) + 1;
}
@@ -134,30 +134,30 @@ void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (!has_init_cudnn_)
InitCudnn();
float alpha = 1.f, beta = 0.f;
- Blob<float> workspace(vector<int>{workspace_count_});
- CUDNN_CHECK(cudnnConvolutionForward(handle_,
+ Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
+ CHECK_CUDNN(cudnnConvolutionForward(handle_,
&alpha,
src_desc_,
srclayers[0]->data(this).gpu_data(),
filter_desc_,
weight_->data().gpu_data(),
- conv_descs_,
+ conv_desc_,
fp_alg_,
workspace.mutable_gpu_data(),
workspace_count_ * sizeof(float),
&beta,
- data_desc_,
+ my_desc_,
data_.mutable_gpu_data()));
if (bias_) {
beta = 1.f;
- CUDNN_CHECK(cudnnAddTensor(handle_,
+ CHECK_CUDNN(cudnnAddTensor(handle_,
CUDNN_ADD_SAME_C,
&alpha,
bias_desc_,
bias_->data().gpu_data(),
&beta,
- data_desc_,
+ my_desc_,
data_.mutable_gpu_data()));
}
}
@@ -165,11 +165,11 @@ void CudnnConvLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
{
float alpha = 1.f, beta = 0.f;
- Blob<float> workspace(vector<int>{workspace_count_});
+ Blob<float> workspace(vector<int>{static_cast<int>(workspace_count_)});
if (bias_) {
CHECK_CUDNN(cudnnConvolutionBackwardBias(handle_,
&alpha,
- data_desc_,
+ my_desc_,
grad_.gpu_data(),
&beta,
bias_desc_,
@@ -178,7 +178,7 @@ void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
CHECK_CUDNN(cudnnConvolutionBackwardFilter_v3(handle_,
&alpha,
src_desc_,
- srclayers[0]->data(this)->gpu_data(),
+ srclayers[0]->data(this).gpu_data(),
my_desc_,
grad_.gpu_data(),
conv_desc_,
@@ -191,7 +191,7 @@ void CudnnConvLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
CHECK_CUDNN(cudnnConvolutionBackwardData_v3(handle_,
&alpha,
filter_desc_,
- weight_->data()->gpu_data(),
+ weight_->data().gpu_data(),
my_desc_,
grad_.gpu_data(),
conv_desc_,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/cudnn_lrn.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_lrn.cu b/src/neuralnet/neuron_layer/cudnn_lrn.cu
index 7cb111a..4a2b695 100644
--- a/src/neuralnet/neuron_layer/cudnn_lrn.cu
+++ b/src/neuralnet/neuron_layer/cudnn_lrn.cu
@@ -23,72 +23,64 @@
namespace singa {
CudnnLRNLayer::~CudnnLRNLayer() {
- if (!init_cudnn_) {
+ if (has_init_cudnn_) {
cudnnDestroyLRNDescriptor(norm_desc_);
}
}
-void CudnnLRNLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- LRNLayer::Setup(proto, srclayers);
- mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
-}
-
void CudnnLRNLayer::InitCudnn() {
- CudnnLayer::InitCudnn(srclayers);
- CHECK_EQ(cudnnCreateLRNDescriptor(&norm_desc_), CUDNN_STATUS_SUCCESS);
- CHECK_EQ(cudnnSetLRNDescriptor(norm_desc_,
+ mode_ = CUDNN_LRN_CROSS_CHANNEL_DIM1;
+ CudnnLayer::InitCudnn();
+ CHECK_CUDNN(cudnnCreateLRNDescriptor(&norm_desc_));
+ CHECK_CUDNN(cudnnSetLRNDescriptor(norm_desc_,
lsize_,
alpha_,
beta_,
- knorm_), CUDNN_STATUS_SUCCESS);
- CHECK_EQ(cudnnCreateTensorDescriptor(&src_desc_), CUDNN_STATUS_SUCCESS);
- CHECK_EQ(cudnnSetTensor4dDescriptor(src_desc_,
+ knorm_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&src_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
channels_,
height_,
- width_), CUDNN_STATUS_SUCCESS);
- CHECK_EQ(cudnnCreateTensorDescriptor(&my_desc_), CUDNN_STATUS_SUCCESS);
- CHECK_EQ(cudnnSetTensor4dDescriptor(my_desc_,
+ width_));
+ CHECK_CUDNN(cudnnCreateTensorDescriptor(&my_desc_));
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
channels_,
height_,
- width_), CUDNN_STATUS_SUCCESS);
+ width_));
}
-void ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (init_cudnn_) {
+void CudnnLRNLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (!has_init_cudnn_)
InitCudnn();
- init_cudnn_ = false;
- }
- CHECK_EQ(cudnnLRNCrossChannelForward(handle_,
+ CHECK_CUDNN(cudnnLRNCrossChannelForward(handle_,
norm_desc_,
mode_,
- &alpha,
+ &alpha_,
src_desc_,
srclayers[0]->data(this).gpu_data(),
- &beta,
+ &beta_,
my_desc_,
- data_.mutable_gpu_data()), CUDNN_STATUS_SUCCESS);
+ data_.mutable_gpu_data()));
}
-void ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(cudnnLRNCrossChannelBackward(handle_,
+void CudnnLRNLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
+ CHECK_CUDNN(cudnnLRNCrossChannelBackward(handle_,
norm_desc_,
mode_,
- &alpha,
+ &alpha_,
my_desc_, // ???
data_.gpu_data(),
my_desc_,
- grad_.gpu_data()
+ grad_.gpu_data(),
src_desc_,
srclayers[0]->data(this).gpu_data(),
- &beta,
+ &beta_,
src_desc_,
- srclayers[0]->mutable_grad(this)->mutable_gpu_data()),
- CUDNN_STATUS_SUCCESS);
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/cudnn_pooling.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_pooling.cu b/src/neuralnet/neuron_layer/cudnn_pooling.cu
index 619998f..ffdfb3b 100644
--- a/src/neuralnet/neuron_layer/cudnn_pooling.cu
+++ b/src/neuralnet/neuron_layer/cudnn_pooling.cu
@@ -23,13 +23,13 @@
namespace singa {
-CuDNNPoolLayer::~CuDNNPoolLayer() {
+CudnnPoolLayer::~CudnnPoolLayer() {
if (has_init_cudnn_) {
- CHECK_EQ(cudnnDestroyPoolingDescriptor(pool_desc_), CUDNN_STATUS_SUCCESS);
+ CHECK_CUDNN(cudnnDestroyPoolingDescriptor(pool_desc_));
}
}
-void CuDNNPoolLayer::InitCudnn() {
+void CudnnPoolLayer::InitCudnn() {
CudnnLayer::InitCudnn();
CHECK_CUDNN(cudnnCreatePoolingDescriptor(&pool_desc_));
CHECK_CUDNN(cudnnSetTensor4dDescriptor(src_desc_,
@@ -57,9 +57,10 @@ void CuDNNPoolLayer::InitCudnn() {
pad_x_,
stride_y_,
stride_x_));
+
}
-void CuDNNPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+void CudnnPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (!has_init_cudnn_)
InitCudnn();
float alpha = 1.0f, beta = 0.0f;
@@ -71,11 +72,11 @@ void CuDNNPoolLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
src_desc_,
srclayers[0]->data(this).gpu_data(),
&beta,
- data_desc_,
+ my_desc_,
data_.mutable_gpu_data()));
}
-void CuDNNPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
+void CudnnPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
{
float alpha = 1.0f, beta = 0.0f;
CHECK_CUDNN(cudnnPoolingBackward(handle_,
@@ -89,7 +90,7 @@ void CuDNNPoolLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers)
srclayers[0]->data(this).gpu_data(),
&beta,
src_desc_,
- srclayers[0]->mutable_grad(this).mutable_gpu_data()));
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/cudnn_softmax.cu
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/cudnn_softmax.cu b/src/neuralnet/neuron_layer/cudnn_softmax.cu
index f7e8abe..7fade3e 100644
--- a/src/neuralnet/neuron_layer/cudnn_softmax.cu
+++ b/src/neuralnet/neuron_layer/cudnn_softmax.cu
@@ -32,7 +32,7 @@ void CudnnSoftmaxLayer::InitCudnn() {
num_softmax_per_instance_,
count_per_softmax_,
1));
- CHECK_EQ(cudnnSetTensor4dDescriptor(my_desc_,
+ CHECK_CUDNN(cudnnSetTensor4dDescriptor(my_desc_,
CUDNN_TENSOR_NCHW,
CUDNN_DATA_FLOAT,
batchsize_,
@@ -45,10 +45,10 @@ void CudnnSoftmaxLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
if (!has_init_cudnn_)
InitCudnn();
- float alpha = 1.0f, beta = 0.0f;
- CHECK_CUDNN(CudnnSoftmaxForward(handle_,
+ const float alpha = 1.0f, beta = 0.0f;
+ CHECK_CUDNN(cudnnSoftmaxForward(handle_,
CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_CHANNEL,
+ CUDNN_SOFTMAX_MODE_CHANNEL,
&alpha,
src_desc_,
srclayers[0]->data(this).gpu_data(),
@@ -59,10 +59,10 @@ void CudnnSoftmaxLayer::ComputeFeature(int flag,
void CudnnSoftmaxLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
- float alpha = 1.f, beta = 0.f;
- CHECK_CUDNN(CudnnSoftmaxForward(handle_,
+ const float alpha = 1.f, beta = 0.f;
+ CHECK_CUDNN(cudnnSoftmaxBackward(handle_,
CUDNN_SOFTMAX_ACCURATE,
- CUDNN_SOFTMAX_CHANNEL,
+ CUDNN_SOFTMAX_MODE_CHANNEL,
&alpha,
my_desc_,
data_.gpu_data(),
@@ -70,6 +70,6 @@ void CudnnSoftmaxLayer::ComputeGradient(int flag,
grad_.gpu_data(),
&beta,
src_desc_,
- srclayers[0]->mutable_grad(this).mutable_gpu_data()));
+ srclayers[0]->mutable_grad(this)->mutable_gpu_data()));
}
} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/dropout.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dropout.cc b/src/neuralnet/neuron_layer/dropout.cc
index 6158a6c..ad8c10f 100644
--- a/src/neuralnet/neuron_layer/dropout.cc
+++ b/src/neuralnet/neuron_layer/dropout.cc
@@ -23,6 +23,8 @@
#include <glog/logging.h>
#include "singa/utils/singleton.h"
+#include "singa/utils/singa_op.h"
+#include "singa/utils/math_blob.h"
namespace singa {
using std::vector;
@@ -42,20 +44,17 @@ void DropoutLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
data_.CopyFrom(srclayers[0]->data(this));
return;
}
+
float pkeep = 1 - pdrop_;
- auto mask = Tensor1(&mask_);
- mask = expr::F<op::threshold>(TSingleton<Random<cpu>>::Instance() \
- ->uniform(mask.shape), pkeep) * (1.0f/pkeep);
- auto data = Tensor1(&data_);
- auto src = Tensor1(srclayers[0]->mutable_data(this));
- data = src * mask;
+ Blob<float> rand(data_.count());
+ SampleUniform(0.0f, 1.0f, &rand);
+ // Threashold(pkeep, rand, &mask_);
+ // Scale(1.0f / pkeep, &mask_);
+ Mult(srclayers[0]->data(this), mask_, &data_);
}
void DropoutLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- auto mask = Tensor1(&mask_);
- auto grad = Tensor1(&grad_);
- auto gsrc = Tensor1(srclayers[0]->mutable_grad(this));
- gsrc = grad * mask;
+ Mult(grad_, mask_, srclayers[0]->mutable_grad(this));
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/pooling.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/pooling.cc b/src/neuralnet/neuron_layer/pooling.cc
index fd475d8..2e246fc 100644
--- a/src/neuralnet/neuron_layer/pooling.cc
+++ b/src/neuralnet/neuron_layer/pooling.cc
@@ -34,9 +34,31 @@ void PoolingLayer::Setup(const LayerProto& conf,
Layer::Setup(conf, srclayers);
CHECK_EQ(srclayers.size(), 1);
PoolingProto pool_conf = conf.pooling_conf();
- kernel_ = pool_conf.kernel();
- stride_ = pool_conf.stride();
- CHECK_LT(pad_, kernel_);
+ if (pool_conf.has_kernel()) {
+ kernel_x_ = kernel_y_ = pool_conf.kernel();
+ } else {
+ CHECK(pool_conf.has_kernel_x());
+ CHECK(pool_conf.has_kernel_y());
+ kernel_x_ = pool_conf.kernel_x();
+ kernel_y_ = pool_conf.kernel_y();
+ }
+ if (pool_conf.has_pad()) {
+ pad_x_ = pad_y_ = pool_conf.pad();
+ } else {
+ CHECK(pool_conf.has_pad_x());
+ CHECK(pool_conf.has_pad_y());
+ pad_x_ = pool_conf.pad_x();
+ pad_y_ = pool_conf.pad_y();
+ }
+ if (pool_conf.has_stride()) {
+ stride_x_ = stride_y_ = pool_conf.stride();
+ } else {
+ CHECK(pool_conf.has_stride_x());
+ CHECK(pool_conf.has_stride_y());
+ stride_x_ = pool_conf.stride_x();
+ stride_y_ = pool_conf.stride_y();
+ }
+
pool_ = conf.pooling_conf().pool();
CHECK(pool_ == PoolingProto_PoolMethod_AVG
|| pool_ == PoolingProto_PoolMethod_MAX)
@@ -51,8 +73,10 @@ void PoolingLayer::Setup(const LayerProto& conf,
else
channels_ = 1;
batchsize_ = srcshape[0];
- pooled_height_ = static_cast<int>((height_ - kernel_) / stride_) + 1;
- pooled_width_ = static_cast<int>((width_ - kernel_) / stride_) + 1;
+ pooled_height_ = static_cast<int>(
+ (height_ + 2 * pad_y_- kernel_y_) / stride_y_) + 1;
+ pooled_width_ = static_cast<int>(
+ (width_ + 2* pad_x_ - kernel_x_) / stride_x_) + 1;
data_.Reshape(vector<int>{batchsize_, channels_, pooled_height_,
pooled_width_});
grad_.ReshapeLike(data_);
@@ -62,10 +86,10 @@ void PoolingLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
auto src = Tensor4(srclayers[0]->mutable_data(this));
auto data = Tensor4(&data_);
if (pool_ == PoolingProto_PoolMethod_MAX)
- data = expr::pool<red::maximum>(src, kernel_, stride_);
+ data = expr::pool<red::maximum>(src, kernel_x_, stride_x_);
else if (pool_ == PoolingProto_PoolMethod_AVG)
- data = expr::pool<red::sum>(src, kernel_, stride_)
- * (1.0f / (kernel_ * kernel_));
+ data = expr::pool<red::sum>(src, kernel_x_, stride_x_)
+ * (1.0f / (kernel_x_ * kernel_x_));
}
/*
@@ -78,10 +102,10 @@ void PoolingLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
auto data = Tensor4(&data_);
auto grad = Tensor4(&grad_);
if (pool_ == PoolingProto_PoolMethod_MAX)
- gsrc = expr::unpool<red::maximum>(src, data, grad, kernel_, stride_);
+ gsrc = expr::unpool<red::maximum>(src, data, grad, kernel_x_, stride_x_);
else if (pool_ == PoolingProto_PoolMethod_AVG)
- gsrc = expr::unpool<red::sum>(src, data, grad, kernel_, stride_)
- * (1.0f / (kernel_ * kernel_));
+ gsrc = expr::unpool<red::sum>(src, data, grad, kernel_x_, stride_x_)
+ * (1.0f / (kernel_x_ * kernel_x_));
}
/***************** Implementation of CPoolingLayer ***************/
@@ -95,12 +119,13 @@ void CPoolingLayer::Setup(const LayerProto& conf,
void CPoolingLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
if (pool_ == PoolingProto_PoolMethod_MAX)
ForwardMaxPooling(srclayers[0]->mutable_data(this)->mutable_cpu_data(),
- batchsize_, channels_, height_, width_, kernel_, kernel_, pad_, pad_,
- stride_, stride_, data_.mutable_cpu_data(), mask_.mutable_cpu_data());
+ batchsize_, channels_, height_, width_, kernel_y_, kernel_x_,
+ pad_y_, pad_y_, stride_y_, stride_x_,
+ data_.mutable_cpu_data(), mask_.mutable_cpu_data());
else if (pool_ == PoolingProto_PoolMethod_AVG)
ForwardAvgPooling(srclayers[0]->mutable_data(this)->mutable_cpu_data(),
- batchsize_, channels_, height_, width_, kernel_, kernel_, pad_, pad_,
- stride_, stride_, data_.mutable_cpu_data());
+ batchsize_, channels_, height_, width_, kernel_y_, kernel_x_,
+ pad_y_, pad_x_, stride_y_, stride_y_, data_.mutable_cpu_data());
else
LOG(FATAL) << "unknow pooling method";
}
@@ -108,12 +133,14 @@ void CPoolingLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void CPoolingLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
if (pool_ == PoolingProto_PoolMethod_MAX)
BackwardMaxPooling(grad_.cpu_data(), mask_.cpu_data(), batchsize_,
- channels_, height_, width_, kernel_, kernel_, pad_, pad_,
- stride_, stride_, srclayers[0]->mutable_grad(this)->mutable_cpu_data());
+ channels_, height_, width_, kernel_y_, kernel_x_, pad_y_, pad_x_,
+ stride_y_, stride_y_,
+ srclayers[0]->mutable_grad(this)->mutable_cpu_data());
else if (pool_ == PoolingProto_PoolMethod_AVG)
BackwardAvgPooling(grad_.cpu_data(), batchsize_,
- channels_, height_, width_, kernel_, kernel_, pad_, pad_,
- stride_, stride_, srclayers[0]->mutable_grad(this)->mutable_cpu_data());
+ channels_, height_, width_, kernel_y_, kernel_x_, pad_y_, pad_x_,
+ stride_y_, stride_x_,
+ srclayers[0]->mutable_grad(this)->mutable_cpu_data());
else
LOG(FATAL) << "unknow pooling method";
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/neuron_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/softmax.cc b/src/neuralnet/neuron_layer/softmax.cc
index 83bbc5a..44c64f8 100644
--- a/src/neuralnet/neuron_layer/softmax.cc
+++ b/src/neuralnet/neuron_layer/softmax.cc
@@ -39,8 +39,8 @@ void SoftmaxLayer::Setup(const LayerProto& proto,
NeuronLayer::Setup(proto, srclayers);
const auto& srcdata = srclayers[0]->data(this);
batchsize_ = data_.shape()[0];
- num_softmax_per_instance_ = proto.softmax_conf().softmax_dim();
- count_per_softmax_ = .count() / batchsize_ / num_softmax_per_instance_;
+ num_softmax_per_instance_ = proto.softmax_conf().num_softmax_per_instance();
+ count_per_softmax_ = data_.count() / batchsize_ / num_softmax_per_instance_;
data_.Reshape(vector<int>{batchsize_, num_softmax_per_instance_,
count_per_softmax_});
grad_.ReshapeLike(data_);
@@ -48,8 +48,8 @@ void SoftmaxLayer::Setup(const LayerProto& proto,
void SoftmaxLayer::ComputeFeature(int flag,
const vector<Layer*>& srclayers) {
- int dim = data_.count() / batchsize;
- Shape<2> s = Shape2(batchsize, dim);
+ int dim = data_.count() / batchsize_;
+ Shape<2> s = Shape2(batchsize_, dim);
Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
Tensor<cpu, 2> src(srclayers[0]->mutable_data(this)->mutable_cpu_data(), s);
Softmax(prob, src);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/neuralnet/output_layer/argsort.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/output_layer/argsort.cc b/src/neuralnet/output_layer/argsort.cc
new file mode 100644
index 0000000..63969b3
--- /dev/null
+++ b/src/neuralnet/output_layer/argsort.cc
@@ -0,0 +1,56 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/output_layer.h"
+#include <algorithm>
+
+namespace singa {
+
+void ArgSortLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ CHECK_EQ(srclayers.size(), 1);
+ OutputLayer::Setup(proto, srclayers);
+ batchsize_ = srclayers[0]->data(this).shape()[0];
+ dim_ = srclayers[0]->data(this).count() / batchsize_;
+ topk_ = proto.argsort_conf().topk();
+ data_.Reshape(vector<int>{batchsize_, topk_});
+}
+
+void ArgSortLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ // TODO(wangwei) check flag to ensure it is not called in training phase
+ const float* srcptr = srclayers.at(0)->data(this).cpu_data();
+ float* ptr = data_.mutable_cpu_data();
+ for (int n = 0; n < batchsize_; n++) {
+ vector<std::pair<float, int> > vec;
+ for (int j = 0; j < dim_; ++j)
+ vec.push_back(std::make_pair(srcptr[j], j));
+ std::partial_sort(vec.begin(), vec.begin() + topk_, vec.end(),
+ std::greater<std::pair<float, int> >());
+
+ for (int j = 0; j < topk_; ++j)
+ ptr[j] = static_cast<float> (vec.at(j).second);
+ ptr += topk_;
+ srcptr += dim_;
+ }
+}
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index ca5e546..9d64c4b 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -188,6 +188,8 @@ message LayerProto {
optional string user_type =21;
// proto for the specific layer
+ // configuration for activation layer
+ optional ActivationProto activation_conf = 54;
// configuration for argsort layer
optional ArgSortProto argsort_conf = 52;
// configuration for convolution layer
@@ -216,11 +218,16 @@ message LayerProto {
optional RGBImageProto rgbimage_conf = 39;
// configuration for data layer
optional DataProto sharddata_conf = 32;
+ // configuration for slice layer
+ optional SliceProto slice_conf = 41;
+ // configuration for softmax layer
+ optional SoftmaxProto softmax_conf = 53;
// configuration for softmax loss layer
optional SoftmaxLossProto softmaxloss_conf = 40;
// configuration for store input layers
optional StoreProto store_conf = 51;
+
// overrides the partition dimension for neural net
optional int32 partition_dim = 60 [default = -1];
// names of parameters shared from other layers
@@ -298,6 +305,17 @@ message ParamGenProto {
extensions 101 to 200;
}
+enum ActivationType {
+ RELU = 1;
+ SIGMOID = 2;
+ TANH = 3;
+ STANH = 4;
+}
+
+message ActivationProto {
+ optional ActivationType type = 1 [default = RELU];
+}
+
message RGBImageProto {
// scale factor for each pixel
optional float scale = 1 [default = 1.0];
@@ -327,6 +345,7 @@ message StoreProto {
optional int32 random_skip = 11 [default = 0];
optional bool has_label = 12 [default = true];
}
+
message SoftmaxLossProto {
// computing accuracy against topk results
optional int32 topk = 1 [default = 1];
@@ -341,17 +360,26 @@ message ArgSortProto {
message ConvolutionProto {
// The number of outputs for the layer
- required int32 num_filters = 1;
+ optional int32 num_filters = 1;
// the kernel height/width
- required int32 kernel = 2;
+ optional int32 kernel = 2;
// The padding height/width
optional int32 pad = 30 [default = 0];
// the stride
optional int32 stride = 31 [default = 1];
+ optional int32 kernel_x = 41;
+ optional int32 kernel_y = 42;
+
+ optional int32 pad_x = 44 [default = 0];
+ optional int32 pad_y = 45 [default = 0];
+
+ optional int32 stride_x = 47 [default = 1];
+ optional int32 stride_y = 48 [default = 1];
+
// cudnn workspace size in MB
- optional int32 workspace_byte_limit = 33 [default = 512];
+ optional int32 workspace_byte_limit = 50 [default = 512];
}
message DataProto {
@@ -384,9 +412,9 @@ message MnistProto {
message DummyProto {
// shape of data and grad blobs
- optional bool input = 1 [default = false];
- optional bool output = 2 [default = false];
- repeated int32 shape = 3;
+ optional bool input = 1 [default = false];
+ optional bool output = 2 [default = false];
+ repeated int32 shape = 3;
}
// Message that stores parameters used by DropoutLayer
@@ -424,7 +452,7 @@ message LRNProto {
message PoolingProto {
// The kernel size (square)
- required int32 kernel= 1;
+ optional int32 kernel= 1;
enum PoolMethod {
MAX = 0;
AVG = 1;
@@ -435,8 +463,19 @@ message PoolingProto {
optional uint32 pad = 31 [default = 0];
// The stride
optional uint32 stride = 32 [default = 1];
+
+
+ optional int32 kernel_x = 41;
+ optional int32 kernel_y = 42;
+
+ optional int32 pad_x = 44 [default = 0];
+ optional int32 pad_y = 45 [default = 0];
+
+ optional int32 stride_x = 47 [default = 1];
+ optional int32 stride_y = 48 [default = 1];
}
+
message ReLUProto {
// Ref. Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013).
// Rectifier nonlinearities improve neural network acoustic models.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49293a68/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index 94b1fcb..0dc797e 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -77,28 +77,7 @@ private:\
classname& operator=(const classname&)
#ifndef CPU_ONLY
-// CUDA: various checks for different function calls.
-#define CUDA_CHECK(condition) \
- /* Code block avoids redefinition of cudaError_t error */ \
- do { \
- cudaError_t error = condition; \
- CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \
- } while (0)
-
-#define CUBLAS_CHECK(condition) \
- do { \
- cublasStatus_t status = condition; \
- CHECK_EQ(status, CUBLAS_STATUS_SUCCESS) << " " \
- << caffe::cublasGetErrorString(status); \
- } while (0)
-
-#define CURAND_CHECK(condition) \
- do { \
- curandStatus_t status = condition; \
- CHECK_EQ(status, CURAND_STATUS_SUCCESS) << " " \
- << caffe::curandGetErrorString(status); \
- } while (0)
-
+#include "singa/utils/cuda_utils.h"
#endif // CPU_ONLY
namespace singa {
@@ -187,7 +166,7 @@ void SyncedMemory::to_gpu() {
switch (head_) {
case UNINITIALIZED:
CUDA_CHECK(cudaMalloc(&gpu_ptr_, size_));
- CUDA_CHECK(cudaMemset(gpu_ptr_, 0, N));
+ CUDA_CHECK(cudaMemset(gpu_ptr_, 0, size_));
head_ = HEAD_AT_GPU;
break;
case HEAD_AT_CPU: