You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/10/09 11:38:25 UTC
[09/11] incubator-singa git commit: SINGA-84 Header Files Rearrange
SINGA-84 Header Files Rearrange
Move deprecated input layers into input/deprecated.h and input/deprecated.cc
Add "deprecated" tag in the comments.
Remove the DataShard class in utils/ folder, use it as an alias of io::KVFile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/e469503d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/e469503d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/e469503d
Branch: refs/heads/master
Commit: e469503de96ef3184c983f568b25a2d9710edd96
Parents: 239ed21
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Thu Oct 8 17:53:08 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Thu Oct 8 17:53:08 2015 +0800
----------------------------------------------------------------------
Makefile.am | 9 +-
include/singa/io/kvfile.h | 4 +-
.../singa/neuralnet/connection_layer/bridge.h | 4 -
.../singa/neuralnet/connection_layer/concate.h | 4 -
.../singa/neuralnet/connection_layer/slice.h | 4 -
.../singa/neuralnet/connection_layer/split.h | 4 -
.../singa/neuralnet/input_layer/csv_record.h | 17 +-
include/singa/neuralnet/input_layer/data.h | 76 ----
.../singa/neuralnet/input_layer/deprecated.h | 172 +++++++++
.../neuralnet/input_layer/image_preprocess.h | 17 -
include/singa/neuralnet/input_layer/label.h | 59 ---
include/singa/neuralnet/input_layer/lmdb_data.h | 75 ----
include/singa/neuralnet/input_layer/mnist.h | 62 ----
include/singa/neuralnet/input_layer/parser.h | 65 ----
include/singa/neuralnet/input_layer/prefetch.h | 22 +-
.../singa/neuralnet/input_layer/proto_record.h | 15 -
include/singa/neuralnet/input_layer/rgb_image.h | 66 ----
.../singa/neuralnet/input_layer/shard_data.h | 65 ----
.../singa/neuralnet/input_layer/store_input.h | 14 -
include/singa/neuralnet/loss_layer/euclidean.h | 8 +-
include/singa/neuralnet/loss_layer/softmax.h | 6 +-
.../singa/neuralnet/neuron_layer/convolution.h | 4 -
include/singa/neuralnet/neuron_layer/dropout.h | 5 -
.../neuralnet/neuron_layer/inner_product.h | 6 -
include/singa/neuralnet/neuron_layer/lrn.h | 4 -
include/singa/neuralnet/neuron_layer/pooling.h | 4 -
include/singa/neuralnet/neuron_layer/rbm.h | 4 -
include/singa/neuralnet/neuron_layer/relu.h | 5 -
include/singa/neuralnet/neuron_layer/sigmoid.h | 4 -
include/singa/neuralnet/neuron_layer/stanh.h | 4 -
include/singa/utils/data_shard.h | 171 ---------
src/driver.cc | 9 +-
src/neuralnet/input_layer/deprecated.cc | 360 +++++++++++++++++++
src/neuralnet/input_layer/label.cc | 56 ---
src/neuralnet/input_layer/lmdb_data.cc | 143 --------
src/neuralnet/input_layer/mnist.cc | 86 -----
src/neuralnet/input_layer/parser.cc | 35 --
src/neuralnet/input_layer/prefetch.cc | 2 -
src/neuralnet/input_layer/rgb_image.cc | 131 -------
src/neuralnet/input_layer/shard_data.cc | 80 -----
src/neuralnet/neuron_layer/convolution.cc | 1 -
src/neuralnet/neuron_layer/dropout.cc | 1 -
src/utils/data_shard.cc | 214 -----------
43 files changed, 544 insertions(+), 1553 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 5c341a3..6419980 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,7 +33,6 @@ SINGA_SRCS := src/driver.cc \
src/utils/common.cc \
src/utils/param.cc \
src/utils/updater.cc \
- src/utils/data_shard.cc \
src/utils/blob.cc \
src/utils/image_transform.cc \
src/server.cc \
@@ -44,16 +43,11 @@ SINGA_SRCS := src/driver.cc \
src/neuralnet/connection_layer/concate.cc \
src/neuralnet/connection_layer/slice.cc \
src/neuralnet/connection_layer/split.cc \
- src/neuralnet/input_layer/parser.cc \
src/neuralnet/input_layer/csv_record.cc \
src/neuralnet/input_layer/image_preprocess.cc \
- src/neuralnet/input_layer/label.cc \
- src/neuralnet/input_layer/lmdb_data.cc \
- src/neuralnet/input_layer/mnist.cc \
src/neuralnet/input_layer/prefetch.cc \
src/neuralnet/input_layer/proto_record.cc \
- src/neuralnet/input_layer/rgb_image.cc \
- src/neuralnet/input_layer/shard_data.cc \
+ src/neuralnet/input_layer/deprecated.cc \
src/neuralnet/input_layer/store_input.cc \
src/neuralnet/loss_layer/euclidean.cc \
src/neuralnet/loss_layer/softmax.cc \
@@ -80,7 +74,6 @@ SINGA_HDRS := include/singa/singa.h \
include/singa/utils/param.h \
include/singa/utils/common.h \
include/singa/utils/factory.h \
- include/singa/utils/data_shard.h \
include/singa/utils/singleton.h \
include/singa/utils/graph.h \
include/singa/utils/blob.h \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/io/kvfile.h
----------------------------------------------------------------------
diff --git a/include/singa/io/kvfile.h b/include/singa/io/kvfile.h
index 27dd35e..6d9a709 100644
--- a/include/singa/io/kvfile.h
+++ b/include/singa/io/kvfile.h
@@ -175,8 +175,8 @@ class KVFile {
/**
* @deprecated {ShardData is deprecated! Use KVFile}.
-using ShardData = KVFile;
-*/
+ */
+using DataShard = io::KVFile;
} // namespace singa
#endif // SINGA_IO_KVFILE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/bridge.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/bridge.h b/include/singa/neuralnet/connection_layer/bridge.h
index 333d6c9..5f224b3 100644
--- a/include/singa/neuralnet/connection_layer/bridge.h
+++ b/include/singa/neuralnet/connection_layer/bridge.h
@@ -25,10 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * \file this file includes the declarations of layers that inherit the
- * base ConnectionLayer.
- */
namespace singa {
class BridgeLayer : virtual public ConnectionLayer {
public:
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/concate.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h
index 9c5180b..5875835 100644
--- a/include/singa/neuralnet/connection_layer/concate.h
+++ b/include/singa/neuralnet/connection_layer/concate.h
@@ -25,10 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * \file this file includes the declarations of layers that inherit the
- * base ConnectionLayer.
- */
namespace singa {
/**
* Connect multiple (src) layers with a single (dst) layer.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/slice.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/slice.h b/include/singa/neuralnet/connection_layer/slice.h
index 87b1754..615685b 100644
--- a/include/singa/neuralnet/connection_layer/slice.h
+++ b/include/singa/neuralnet/connection_layer/slice.h
@@ -25,10 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * \file this file includes the declarations of layers that inherit the
- * base ConnectionLayer.
- */
namespace singa {
/**
* Connect a single (src) layer with multiple (dst) layers.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/connection_layer/split.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/split.h b/include/singa/neuralnet/connection_layer/split.h
index 391473a..7557766 100644
--- a/include/singa/neuralnet/connection_layer/split.h
+++ b/include/singa/neuralnet/connection_layer/split.h
@@ -25,10 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * \file this file includes the declarations of layers that inherit the
- * base ConnectionLayer.
- */
namespace singa {
/**
* Connect a single (src) layer with multiple dst layers.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/csv_record.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/csv_record.h b/include/singa/neuralnet/input_layer/csv_record.h
index 1c50ceb..2653b63 100644
--- a/include/singa/neuralnet/input_layer/csv_record.h
+++ b/include/singa/neuralnet/input_layer/csv_record.h
@@ -24,23 +24,8 @@
#include <string>
#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
#include "singa/neuralnet/input_layer/store_input.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
+
namespace singa {
using std::string;
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/data.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/data.h b/include/singa/neuralnet/input_layer/data.h
deleted file mode 100644
index 46ffbf7..0000000
--- a/include/singa/neuralnet/input_layer/data.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_DATA_H_
-#define SINGA_NEURALNET_INPUT_LAYER_DATA_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc.
- */
-class DataLayer: virtual public InputLayer {
- public:
- Blob<float>* mutable_data(const Layer* layer) override { return nullptr; }
- ConnectionType dst_layer_connection() const override {
- return kOneToMany;
- }
-
- inline int batchsize() const { return batchsize_; }
- virtual const Record& sample() const {
- return sample_;
- }
- /**
- * @return the loaded records
- */
- virtual const std::vector<Record>& records() const {
- return records_;
- }
-
- protected:
- int random_skip_;
- int batchsize_;
- Record sample_;
- std::vector<Record> records_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_DATA_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/deprecated.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/deprecated.h b/include/singa/neuralnet/input_layer/deprecated.h
new file mode 100644
index 0000000..bd2f60c
--- /dev/null
+++ b/include/singa/neuralnet/input_layer/deprecated.h
@@ -0,0 +1,172 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
+#define SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
+
+#include <string>
+#include <vector>
+#include "singa/neuralnet/layer.h"
+#include "singa/io/kvfile.h"
+namespace singa {
+using std::string;
+using std::vector;
+/**
+ * @deprecated please use the StoreInputLayer.
+ *
+ * Base layer for reading ::Record from local Shard, HDFS, lmdb, etc.
+ */
+class DataLayer: virtual public InputLayer {
+ public:
+ Blob<float>* mutable_data(const Layer* layer) override { return nullptr; }
+ ConnectionType dst_layer_connection() const override {
+ return kOneToMany;
+ }
+
+ inline int batchsize() const { return batchsize_; }
+ virtual const Record& sample() const {
+ return sample_;
+ }
+ /**
+ * @return the loaded records
+ */
+ virtual const std::vector<Record>& records() const {
+ return records_;
+ }
+
+ protected:
+ int random_skip_;
+ int batchsize_;
+ Record sample_;
+ std::vector<Record> records_;
+};
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer.
+ *
+ * Layer for loading Record from DataShard.
+ */
+class ShardDataLayer : public DataLayer {
+ public:
+ ~ShardDataLayer();
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ DataShard* shard_;
+};
+/**
+ * @deprecated please use the subclasses of StoreInputLayer.
+ *
+ * Layer for loading Record from LMDB.
+ */
+#ifdef USE_LMDB
+#include <lmdb.h>
+class LMDBDataLayer : public DataLayer {
+ public:
+ ~LMDBDataLayer();
+
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void OpenLMDB(const std::string& path);
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
+ SingleLabelImageRecord* record);
+
+ private:
+ MDB_env* mdb_env_;
+ MDB_dbi mdb_dbi_;
+ MDB_txn* mdb_txn_;
+ MDB_cursor* mdb_cursor_;
+ MDB_val mdb_key_, mdb_value_;
+};
+#endif
+
+/******************Parser layers***************/
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer which load and parse
+ * data in a single layer.
+ *
+ * Base layer for parsing the input records into Blobs.
+ */
+class ParserLayer : public InputLayer {
+ public:
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+ void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
+ ConnectionType dst_layer_connection() const override {
+ return kOneToMany;
+ }
+ /**
+ * Parse records from DataLayer into blob.
+ */
+ virtual void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) = 0;
+};
+/**
+ *
+ * @deprecated Please use the SingleLabelRecordLayer which parses both feature
+ * and label for each record. Its aux_data() function returns the parsed labels.
+ *
+ * Derived from ParserLayer to parse label in SingaleLabelImageRecord loaded by
+ * ShardDataLayer.
+ */
+class LabelLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+};
+
+/**
+ * @deprecated Please use the subclasses of StoreInputLayer.
+ *
+ * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord.
+ */
+class MnistLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+
+ protected:
+ float norm_a_, norm_b_;
+};
+/**
+ * @deprecated please use the ImagePreprocessLayer which preprocess image
+ * feature from data Blob of source layers.
+ *
+ * Derived from ParserLayer to parse RGB image feature from
+ * SingaleLabelImageRecord.
+ */
+class RGBImageLayer : public ParserLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ParseRecords(int flag, const std::vector<Record>& records,
+ Blob<float>* blob) override;
+
+ private:
+ float scale_;
+ int cropsize_;
+ bool mirror_;
+ Blob<float> mean_;
+};
+} // namespace singa
+
+#endif // SINGA_NEURALNET_INPUT_LAYER_DEPRECATED_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/image_preprocess.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/image_preprocess.h b/include/singa/neuralnet/input_layer/image_preprocess.h
index 2407990..f3bf90a 100644
--- a/include/singa/neuralnet/input_layer/image_preprocess.h
+++ b/include/singa/neuralnet/input_layer/image_preprocess.h
@@ -22,26 +22,9 @@
#ifndef SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_
#define SINGA_NEURALNET_INPUT_LAYER_IMAGE_PREPROCESS_H_
-#include <string>
#include <vector>
-#include "singa/io/store.h"
#include "singa/neuralnet/layer.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
namespace singa {
-using std::string;
using std::vector;
/**
* Do preprocessing for images, including cropping, mirroring, resizing.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/label.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/label.h b/include/singa/neuralnet/input_layer/label.h
deleted file mode 100644
index 1c40275..0000000
--- a/include/singa/neuralnet/input_layer/label.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_LABEL_H_
-#define SINGA_NEURALNET_INPUT_LAYER_LABEL_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer/parser.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Derived from ParserLayer to parse label from SingaleLabelImageRecord.
- */
-class LabelLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_LABEL_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/lmdb_data.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/lmdb_data.h b/include/singa/neuralnet/input_layer/lmdb_data.h
deleted file mode 100644
index bc26ab0..0000000
--- a/include/singa/neuralnet/input_layer/lmdb_data.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_
-#define SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer/data.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Layer for loading Record from LMDB.
- *
- * It is derived from DataLayer.
- */
-#ifdef USE_LMDB
-#include <lmdb.h>
-class LMDBDataLayer : public DataLayer {
- public:
- ~LMDBDataLayer();
-
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void OpenLMDB(const std::string& path);
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record);
-
- private:
- MDB_env* mdb_env_;
- MDB_dbi mdb_dbi_;
- MDB_txn* mdb_txn_;
- MDB_cursor* mdb_cursor_;
- MDB_val mdb_key_, mdb_value_;
-};
-#endif
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_LMDB_DATA_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/mnist.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/mnist.h b/include/singa/neuralnet/input_layer/mnist.h
deleted file mode 100644
index 7c91173..0000000
--- a/include/singa/neuralnet/input_layer/mnist.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_MNIST_H_
-#define SINGA_NEURALNET_INPUT_LAYER_MNIST_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer/parser.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Derived from ParserLayer to parse MNIST feature from SingaleLabelImageRecord.
- */
-class MnistLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-
- protected:
- float norm_a_, norm_b_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_MNIST_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/parser.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/parser.h b/include/singa/neuralnet/input_layer/parser.h
deleted file mode 100644
index de52b0a..0000000
--- a/include/singa/neuralnet/input_layer/parser.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_PARSER_H_
-#define SINGA_NEURALNET_INPUT_LAYER_PARSER_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Base layer for parsing the input records into Blobs.
- */
-class ParserLayer : public InputLayer {
- public:
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
- void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
- ConnectionType dst_layer_connection() const override {
- return kOneToMany;
- }
- /**
- * Parse records from DataLayer into blob.
- */
- virtual void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) = 0;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_PARSER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/prefetch.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/prefetch.h b/include/singa/neuralnet/input_layer/prefetch.h
index 038f7e2..d30521d 100644
--- a/include/singa/neuralnet/input_layer/prefetch.h
+++ b/include/singa/neuralnet/input_layer/prefetch.h
@@ -24,30 +24,14 @@
#include <string>
#include <vector>
-#include "singa/io/store.h"
+#include <thread>
#include "singa/neuralnet/layer.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
namespace singa {
-using std::string;
using std::vector;
/**
- * Layer for prefetching data records and parsing them.
+ * TODO(wangwei) Layer for prefetching data records and parsing them.
*
- * The data loading and parsing work is done by internal DataLayer and
- * ParserLayer respectively. This layer controls the prefetching thread, i.e.,
+ * This layer controls the prefetching thread, i.e.,
* creating and joining the prefetching thread.
*/
class PrefetchLayer : public Layer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/proto_record.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/proto_record.h b/include/singa/neuralnet/input_layer/proto_record.h
index bfc9934..6a4ad0c 100644
--- a/include/singa/neuralnet/input_layer/proto_record.h
+++ b/include/singa/neuralnet/input_layer/proto_record.h
@@ -25,22 +25,7 @@
#include <string>
#include <vector>
#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
#include "singa/neuralnet/input_layer/store_input.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
namespace singa {
using std::string;
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/rgb_image.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/rgb_image.h b/include/singa/neuralnet/input_layer/rgb_image.h
deleted file mode 100644
index c4d6f18..0000000
--- a/include/singa/neuralnet/input_layer/rgb_image.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_
-#define SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer/parser.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Derived from ParserLayer to parse RGB image feature from
- * SingaleLabelImageRecord.
- */
-class RGBImageLayer : public ParserLayer {
- public:
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ParseRecords(int flag, const std::vector<Record>& records,
- Blob<float>* blob) override;
-
- private:
- float scale_;
- int cropsize_;
- bool mirror_;
- Blob<float> mean_;
-};
-
-} // namespace singa
-
-#endif // SINGA_NEURALNET_INPUT_LAYER_RGB_IMAGE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/shard_data.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/shard_data.h b/include/singa/neuralnet/input_layer/shard_data.h
deleted file mode 100644
index 13ea9d8..0000000
--- a/include/singa/neuralnet/input_layer/shard_data.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_NEURALNET_INPUT_LAYER_SHARD_DATA_H_
-#define SINGA_NEURALNET_INPUT_LAYER_SHARD_DATA_H_
-
-#include <string>
-#include <vector>
-#include "singa/io/store.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer/data.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
-namespace singa {
-using std::string;
-using std::vector;
-/**
- * Layer for loading Record from DataShard.
- *
- * It is derived from DataLayer.
- */
-class ShardDataLayer : public DataLayer {
- public:
- ~ShardDataLayer();
-
- void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
- void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- DataShard* shard_;
-};
-
-} // namespace singa
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/input_layer/store_input.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/input_layer/store_input.h b/include/singa/neuralnet/input_layer/store_input.h
index dd63be6..86fbaf2 100644
--- a/include/singa/neuralnet/input_layer/store_input.h
+++ b/include/singa/neuralnet/input_layer/store_input.h
@@ -26,20 +26,6 @@
#include <vector>
#include "singa/io/store.h"
#include "singa/neuralnet/layer.h"
-#include "singa/utils/data_shard.h"
-/**
- * \file this file includes the declarations of input layers that inherit the
- * base InputLayer to load input features.
- *
- * The feature loading phase can be implemented using a single layer or
- * separated into DataLayer (for loading features as records) and ParserLayer
- * (for parsing features from records). SINGA has provided some subclasses of
- * DataLayer and ParserLayer.
- *
- * Data prefetching can be implemented as a sub-class of InputLayer.
- * SINGA provides a built-in PrefetchLayer which embeds DataLayer and
- * ParserLayer.
- */
namespace singa {
using std::string;
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/loss_layer/euclidean.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer/euclidean.h b/include/singa/neuralnet/loss_layer/euclidean.h
index 57703e0..4d50dc5 100644
--- a/include/singa/neuralnet/loss_layer/euclidean.h
+++ b/include/singa/neuralnet/loss_layer/euclidean.h
@@ -25,15 +25,11 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * @file this file includes the declarations of layers that inherit the base
- * LossLayer for measuring the objective training loss.
- */
namespace singa {
using std::vector;
/**
- * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is for prediction
- * t is for ground truth.
+ * Squared Euclidean loss as @f$0.5 ||p - t||^2@f$, where p is prediction
+ * result, t is the ground truth.
*/
class EuclideanLossLayer : public LossLayer {
public:
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/loss_layer/softmax.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/loss_layer/softmax.h b/include/singa/neuralnet/loss_layer/softmax.h
index 7a57517..c497377 100644
--- a/include/singa/neuralnet/loss_layer/softmax.h
+++ b/include/singa/neuralnet/loss_layer/softmax.h
@@ -25,10 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-/**
- * @file this file includes the declarations of layers that inherit the base
- * LossLayer for measuring the objective training loss.
- */
namespace singa {
using std::vector;
/**
@@ -43,7 +39,7 @@ class SoftmaxLossLayer : public LossLayer {
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
/**
- * softmax is not recommendeded for partition because it requires the whole
+ * This layer is not recommendeded for partition because it requires the whole
* src layer for normalization.
*/
ConnectionType src_neuron_connection(int k) const override {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/convolution.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/convolution.h b/include/singa/neuralnet/neuron_layer/convolution.h
index a73131d..2ffbf46 100644
--- a/include/singa/neuralnet/neuron_layer/convolution.h
+++ b/include/singa/neuralnet/neuron_layer/convolution.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
/**
* Convolution layer.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/dropout.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/dropout.h b/include/singa/neuralnet/neuron_layer/dropout.h
index 1fe189e..8c1883c 100644
--- a/include/singa/neuralnet/neuron_layer/dropout.h
+++ b/include/singa/neuralnet/neuron_layer/dropout.h
@@ -24,12 +24,7 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
class DropoutLayer : public NeuronLayer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/inner_product.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/inner_product.h b/include/singa/neuralnet/neuron_layer/inner_product.h
index e109bb3..bf3fdee 100644
--- a/include/singa/neuralnet/neuron_layer/inner_product.h
+++ b/include/singa/neuralnet/neuron_layer/inner_product.h
@@ -24,12 +24,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
-#include "singa/proto/job.pb.h"
-
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
class InnerProductLayer : public NeuronLayer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/lrn.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/lrn.h b/include/singa/neuralnet/neuron_layer/lrn.h
index ba983a4..42b10ed 100644
--- a/include/singa/neuralnet/neuron_layer/lrn.h
+++ b/include/singa/neuralnet/neuron_layer/lrn.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
/**
* Local Response Normalization edge
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/pooling.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/pooling.h b/include/singa/neuralnet/neuron_layer/pooling.h
index d8f0f68..a66a4ef 100644
--- a/include/singa/neuralnet/neuron_layer/pooling.h
+++ b/include/singa/neuralnet/neuron_layer/pooling.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
class PoolingLayer : public NeuronLayer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/rbm.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/rbm.h b/include/singa/neuralnet/neuron_layer/rbm.h
index c2e1140..7c6f81a 100644
--- a/include/singa/neuralnet/neuron_layer/rbm.h
+++ b/include/singa/neuralnet/neuron_layer/rbm.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
/**
* Base layer for RBM models.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/relu.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/relu.h b/include/singa/neuralnet/neuron_layer/relu.h
index 735be7c..48a40a8 100644
--- a/include/singa/neuralnet/neuron_layer/relu.h
+++ b/include/singa/neuralnet/neuron_layer/relu.h
@@ -25,11 +25,6 @@
#include <vector>
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
class ReLULayer : public NeuronLayer {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/sigmoid.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/sigmoid.h b/include/singa/neuralnet/neuron_layer/sigmoid.h
index 2d32206..88f15e9 100644
--- a/include/singa/neuralnet/neuron_layer/sigmoid.h
+++ b/include/singa/neuralnet/neuron_layer/sigmoid.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
/**
* This layer apply Sigmoid function to neuron activations.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/neuralnet/neuron_layer/stanh.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/stanh.h b/include/singa/neuralnet/neuron_layer/stanh.h
index e25f44c..08edb7d 100644
--- a/include/singa/neuralnet/neuron_layer/stanh.h
+++ b/include/singa/neuralnet/neuron_layer/stanh.h
@@ -26,10 +26,6 @@
#include "singa/neuralnet/layer.h"
#include "singa/proto/job.pb.h"
-/**
- * \file this file includes the declarations neuron layer classes that conduct
- * the transformation of features.
- */
namespace singa {
/**
* This layer apply scaled Tanh function to neuron activations.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/include/singa/utils/data_shard.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/data_shard.h b/include/singa/utils/data_shard.h
deleted file mode 100644
index 7d69ae5..0000000
--- a/include/singa/utils/data_shard.h
+++ /dev/null
@@ -1,171 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#ifndef SINGA_UTILS_DATA_SHARD_H_
-#define SINGA_UTILS_DATA_SHARD_H_
-
-#include <google/protobuf/message.h>
-#include <fstream>
-#include <string>
-#include <unordered_set>
-
-namespace singa {
-
-/**
- * Data shard stores training/validation/test tuples.
- * Every worker node should have a training shard (validation/test shard
- * is optional). The shard file for training is
- * singa::Cluster::workspace()/train/shard.dat; The shard file for validation
- * is singa::Cluster::workspace()/train/shard.dat; Similar path for test.
- *
- * shard.dat consists of a set of unordered tuples. Each tuple is
- * encoded as [key_len key record_len val] (key_len and record_len are of type
- * uint32, which indicate the bytes of key and record respectively.
- *
- * When Shard obj is created, it will remove the last key if the record size
- * and key size do not match because the last write of tuple crashed.
- *
- * TODO
- * 1. split one shard into multiple shards.
- * 2. add threading to prefetch and parse records
- *
- */
-class DataShard {
- public:
- enum {
- // read only mode used in training
- kRead = 0,
- // write mode used in creating shard (will overwrite previous one)
- kCreate = 1,
- // append mode, e.g. used when previous creating crashes
- kAppend = 2
- };
-
- /**
- * Init the shard obj.
- *
- * @param folder Shard folder (path excluding shard.dat) on worker node
- * @param mode Shard open mode, Shard::kRead, Shard::kWrite or Shard::kAppend
- * @param bufsize Batch bufsize bytes data for every disk op (read or write),
- * default is 100MB
- */
- DataShard(const std::string& folder, int mode);
- DataShard(const std::string& folder, int mode, int capacity);
- ~DataShard();
-
- /**
- * read next tuple from the shard.
- *
- * @param key Tuple key
- * @param val Record of type Message
- * @return false if read unsuccess, e.g., the tuple was not inserted
- * completely.
- */
- bool Next(std::string* key, google::protobuf::Message* val);
- /**
- * read next tuple from the shard.
- *
- * @param key Tuple key
- * @param val Record of type string
- * @return false if read unsuccess, e.g., the tuple was not inserted
- * completely.
- */
- bool Next(std::string* key, std::string* val);
- /**
- * Append one tuple to the shard.
- *
- * @param key e.g., image path
- * @param val
- * @return false if unsucess, e.g., inserted before
- */
- bool Insert(const std::string& key, const google::protobuf::Message& tuple);
- /**
- * Append one tuple to the shard.
- *
- * @param key e.g., image path
- * @param val
- * @return false if unsucess, e.g., inserted before
- */
- bool Insert(const std::string& key, const std::string& tuple);
- /**
- * Move the read pointer to the head of the shard file.
- * Used for repeated reading.
- */
- void SeekToFirst();
- /**
- * Flush buffered data to disk.
- * Used only for kCreate or kAppend.
- */
- void Flush();
- /**
- * Iterate through all tuples to get the num of all tuples.
- *
- * @return num of tuples
- */
- int Count();
- /**
- * @return path to shard file
- */
- inline std::string path() { return path_; }
-
- protected:
- /**
- * Read the next key and prepare buffer for reading value.
- *
- * @param key
- * @return length (i.e., bytes) of value field.
- */
- int Next(std::string* key);
- /**
- * Setup the disk pointer to the right position for append in case that
- * the pervious write crashes.
- *
- * @param path shard path.
- * @return offset (end pos) of the last success written record.
- */
- int PrepareForAppend(const std::string& path);
- /**
- * Read data from disk if the current data in the buffer is not a full field.
- *
- * @param size size of the next field.
- */
- bool PrepareNextField(int size);
-
- private:
- char mode_ = 0;
- std::string path_ = "";
- // either ifstream or ofstream
- std::fstream fdat_;
- // to avoid replicated record
- std::unordered_set<std::string> keys_;
- // internal buffer
- char* buf_ = nullptr;
- // offset inside the buf_
- int offset_ = 0;
- // allocated bytes for the buf_
- int capacity_ = 0;
- // bytes in buf_, used in reading
- int bufsize_ = 0;
-};
-
-} // namespace singa
-
-#endif // SINGA_UTILS_DATA_SHARD_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 5f81325..90d7bde 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -37,17 +37,11 @@
#include "singa/neuralnet/connection_layer/concate.h"
#include "singa/neuralnet/connection_layer/slice.h"
#include "singa/neuralnet/connection_layer/split.h"
-#include "singa/neuralnet/input_layer/parser.h"
+#include "singa/neuralnet/input_layer/deprecated.h"
#include "singa/neuralnet/input_layer/csv_record.h"
-#include "singa/neuralnet/input_layer/data.h"
#include "singa/neuralnet/input_layer/image_preprocess.h"
-#include "singa/neuralnet/input_layer/label.h"
-#include "singa/neuralnet/input_layer/lmdb_data.h"
-#include "singa/neuralnet/input_layer/mnist.h"
#include "singa/neuralnet/input_layer/prefetch.h"
#include "singa/neuralnet/input_layer/proto_record.h"
-#include "singa/neuralnet/input_layer/rgb_image.h"
-#include "singa/neuralnet/input_layer/shard_data.h"
#include "singa/neuralnet/input_layer/store_input.h"
#include "singa/neuralnet/loss_layer/euclidean.h"
#include "singa/neuralnet/loss_layer/softmax.h"
@@ -62,6 +56,7 @@
#include "singa/neuralnet/neuron_layer/stanh.h"
#include "singa/neuralnet/output_layer/output_layer.h"
+
extern "C" void openblas_set_num_threads(int num);
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
new file mode 100644
index 0000000..b0baf1d
--- /dev/null
+++ b/src/neuralnet/input_layer/deprecated.cc
@@ -0,0 +1,360 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/input_layer/deprecated.h"
+#include "mshadow/tensor.h"
+namespace singa {
+
+using namespace mshadow;
+using mshadow::cpu;
+using mshadow::Shape4;
+using mshadow::Tensor;
+
+using std::string;
+using std::vector;
+
+ShardDataLayer::~ShardDataLayer() {
+ if (shard_ != nullptr)
+ delete shard_;
+ shard_ = nullptr;
+}
+
+void ShardDataLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ Layer::Setup(proto, srclayers);
+ shard_ = new DataShard(proto.sharddata_conf().path(), DataShard::kRead);
+ string key;
+ shard_->Next(&key, &sample_);
+ delete shard_;
+ shard_ = nullptr;
+ batchsize_ = proto.sharddata_conf().batchsize();
+ if (partition_dim() == 0)
+ batchsize_ /= proto.num_partitions();
+ records_.resize(batchsize_);
+ random_skip_ = proto.sharddata_conf().random_skip();
+}
+
+void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (shard_ == nullptr)
+ shard_ = new DataShard(layer_conf_.sharddata_conf().path(),
+ DataShard::kRead);
+ if (random_skip_) {
+ int nskip = rand() % random_skip_;
+ LOG(INFO) << "Random Skip " << nskip << " records, there are "
+ << shard_->Count() << " records in total";
+ string key;
+ for (int i = 0; i < nskip; i++) {
+ shard_->Next(&key, &sample_);
+ }
+ random_skip_ = 0;
+ }
+ for (auto& record : records_) {
+ string key;
+ if (!shard_->Next(&key, &record)) {
+ shard_->SeekToFirst();
+ CHECK(shard_->Next(&key, &record));
+ }
+ }
+}
+
+/*****************LMDB data layer*******************/
+#ifdef USE_LMDB
+LMDBDataLayer::~LMDBDataLayer() {
+ mdb_cursor_close(mdb_cursor_);
+ mdb_txn_abort(mdb_txn_);
+ mdb_cursor_ = nullptr;
+}
+
+void LMDBDataLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ Layer::Setup(proto, srclayers);
+ OpenLMDB(proto.lmdbdata_conf().path());
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT),
+ MDB_SUCCESS);
+ mdb_cursor_close(mdb_cursor_);
+ mdb_txn_abort(mdb_txn_);
+ mdb_cursor_ = nullptr;
+ CaffeDatum datum;
+ datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
+ SingleLabelImageRecord* record = sample_.mutable_image();
+ ConvertCaffeDatumToRecord(datum, record);
+ batchsize_ = proto.lmdbdata_conf().batchsize();
+ if (partition_dim() == 0)
+ batchsize_ /= proto.num_partitions();
+ records_.resize(batchsize_);
+ random_skip_ = proto.lmdbdata_conf().random_skip();
+}
+
+void LMDBDataLayer::OpenLMDB(const std::string& path) {
+ CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
+ CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
+ CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(),
+ MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path;
+ CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
+ << "mdb_txn_begin failed";
+ CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
+ << "mdb_open failed";
+ CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
+ << "mdb_cursor_open failed";
+ LOG(INFO) << "Opening lmdb " << path;
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
+ MDB_SUCCESS) << "mdb_cursor_get failed";
+}
+
+void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ if (mdb_cursor_ == nullptr)
+ OpenLMDB(layer_conf_.lmdbdata_conf().path());
+ if (random_skip_) {
+ int nskip = rand() % random_skip_;
+ int n = 0;
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
+ n++;
+ LOG(INFO) << "Random Skip " << nskip << " records of total "
+ << n << "records";
+ // We have reached the end. Restart from the first.
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ for (int i = 0; i < nskip; i++) {
+ if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
+ // We have reached the end. Restart from the first.
+ DLOG(INFO) << "Restarting data prefetching from start.";
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ }
+ }
+ random_skip_ = 0;
+ }
+ CaffeDatum datum;
+ for (auto& record : records_) {
+ SingleLabelImageRecord* image = record.mutable_image();
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
+ datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
+ ConvertCaffeDatumToRecord(datum, image);
+ if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
+ // We have reached the end. Restart from the first.
+ DLOG(INFO) << "Restarting data prefetching from start.";
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ }
+ }
+}
+
+void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum,
+ SingleLabelImageRecord* record) {
+ record->set_label(datum.label());
+ record->clear_shape();
+ if (datum.has_channels())
+ record->add_shape(datum.channels());
+ if (datum.has_height())
+ record->add_shape(datum.height());
+ if (datum.has_width())
+ record->add_shape(datum.width());
+ if (datum.has_data())
+ record->set_pixel(datum.data());
+ if (datum.float_data_size()) {
+ record->clear_data();
+ for (float x : datum.float_data())
+ record->add_data(x);
+ }
+}
+#endif
+
+/***************Parser layer*******************/
+void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
+ CHECK_EQ(srclayers.size(), 1);
+ auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin());
+ ParseRecords(flag, datalayer->records(), &data_);
+}
+
+/**********Mnist Layer************/
+void MnistLayer::ParseRecords(int flag, const vector<Record>& records,
+ Blob<float>* blob) {
+ LOG_IF(ERROR, records.size() == 0) << "Empty records to parse";
+ int ndim = records.at(0).image().shape_size();
+ int inputsize = records.at(0).image().shape(ndim-1);
+ CHECK_EQ(inputsize, blob->shape()[2]);
+
+ float* dptr = blob->mutable_cpu_data();
+ for (const Record& record : records) {
+ const SingleLabelImageRecord& imagerecord = record.image();
+ if (imagerecord.pixel().size()) {
+ string pixel = imagerecord.pixel();
+ for (int i = 0, k = 0; i < inputsize; i++) {
+ for (int j = 0; j < inputsize; j++) {
+ // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of
+ // time to debug this
+ float x = static_cast<float>(static_cast<uint8_t>(pixel[k++]));
+ x = x / norm_a_-norm_b_;
+ *dptr = x;
+ dptr++;
+ }
+ }
+ } else {
+ for (int i = 0, k = 0; i < inputsize; i++) {
+ for (int j = 0; j < inputsize; j++) {
+ *dptr = imagerecord.data(k++) / norm_a_ - norm_b_;
+ dptr++;
+ }
+ }
+ }
+ }
+ CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count());
+}
+
+void MnistLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ Layer::Setup(proto, srclayers);
+ CHECK_EQ(srclayers.size(), 1);
+ int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
+ Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
+ norm_a_ = proto.mnist_conf().norm_a();
+ norm_b_ = proto.mnist_conf().norm_b();
+ int ndim = sample.image().shape_size();
+ CHECK_GE(ndim, 2);
+ int s = sample.image().shape(ndim - 1);
+ CHECK_EQ(s, sample.image().shape(ndim - 2));
+ data_.Reshape(vector<int>{batchsize, 1, s, s});
+}
+
+/**********RGB image layer****************/
+void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
+ Blob<float>* blob) {
+ const vector<int>& s = blob->shape();
+ Tensor<cpu, 4> images(data_.mutable_cpu_data(),
+ Shape4(s[0], s[1], s[2], s[3]));
+ const SingleLabelImageRecord& r = records.at(0).image();
+ Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2)));
+ AllocSpace(raw_image);
+ Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3]));
+ if (cropsize_)
+ AllocSpace(croped_image);
+ int rid = 0;
+ const float* meandptr = mean_.cpu_data();
+ for (const Record& record : records) {
+ auto image = images[rid];
+ bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain);
+ bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
+ float* dptr = nullptr;
+ if (do_crop || do_mirror)
+ dptr = raw_image.dptr;
+ else
+ dptr = image.dptr;
+ if (record.image().pixel().size()) {
+ string pixel = record.image().pixel();
+ for (size_t i = 0; i < pixel.size(); i++)
+ dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i]));
+ } else {
+ memcpy(dptr, record.image().data().data(),
+ sizeof(float) * record.image().data_size());
+ }
+ for (int i = 0; i < mean_.count(); i++)
+ dptr[i] -= meandptr[i];
+ if (do_crop) {
+ int hoff = rand() % (r.shape(1) - cropsize_);
+ int woff = rand() % (r.shape(2) - cropsize_);
+ Shape<2> cropshape = Shape2(cropsize_, cropsize_);
+ if (do_mirror) {
+ croped_image = expr::crop(raw_image, cropshape, hoff, woff);
+ image = expr::mirror(croped_image);
+ } else {
+ image = expr::crop(raw_image, cropshape, hoff, woff);
+ }
+ } else if (do_mirror) {
+ image = expr::mirror(raw_image);
+ }
+ rid++;
+ }
+ if (scale_)
+ images = images * scale_;
+ FreeSpace(raw_image);
+ if (cropsize_)
+ FreeSpace(croped_image);
+}
+
+void RGBImageLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ ParserLayer::Setup(proto, srclayers);
+ CHECK_EQ(srclayers.size(), 1);
+ scale_ = proto.rgbimage_conf().scale();
+ cropsize_ = proto.rgbimage_conf().cropsize();
+ mirror_ = proto.rgbimage_conf().mirror();
+ int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
+ Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
+ vector<int> shape;
+ shape.push_back(batchsize);
+ for (int x : sample.image().shape()) {
+ shape.push_back(x);
+ }
+ CHECK_EQ(shape.size(), 4);
+ if (cropsize_) {
+ shape[2] = cropsize_;
+ shape[3] = cropsize_;
+ }
+ data_.Reshape(shape);
+ mean_.Reshape({shape[1], shape[2], shape[3]});
+ if (proto.rgbimage_conf().has_meanfile()) {
+ if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) {
+ CaffeBlob mean;
+ ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
+ CHECK_EQ(mean_.count(), mean.data_size());
+ memcpy(mean_.mutable_cpu_data(), mean.data().data(),
+ sizeof(float)*mean.data_size());
+ } else {
+ SingleLabelImageRecord mean;
+ ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
+ CHECK_EQ(mean_.count(), mean.data_size());
+ memcpy(mean_.mutable_cpu_data(), mean.data().data(),
+ sizeof(float)*mean.data_size());
+ }
+ } else {
+ memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count());
+ }
+}
+
+/*************Label layer *************/
+
+void LabelLayer::Setup(const LayerProto& proto,
+ const vector<Layer*>& srclayers) {
+ Layer::Setup(proto, srclayers);
+ CHECK_EQ(srclayers.size(), 1);
+ int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
+ data_.Reshape(vector<int>{batchsize});
+}
+
+void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
+ Blob<float>* blob) {
+ int rid = 0;
+ float *label = blob->mutable_cpu_data();
+ for (const Record& record : records) {
+ label[rid++] = record.image().label();
+ // CHECK_LT(record.image().label(),10);
+ }
+ CHECK_EQ(rid, blob->shape()[0]);
+}
+
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/label.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/label.cc b/src/neuralnet/input_layer/label.cc
deleted file mode 100644
index 6de9324..0000000
--- a/src/neuralnet/input_layer/label.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer/label.h"
-#include "singa/neuralnet/input_layer/data.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-using mshadow::Shape4;
-using mshadow::Tensor;
-
-using std::string;
-using std::vector;
-
-void LabelLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- data_.Reshape(vector<int>{batchsize});
-}
-
-void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- int rid = 0;
- float *label = blob->mutable_cpu_data();
- for (const Record& record : records) {
- label[rid++] = record.image().label();
- // CHECK_LT(record.image().label(),10);
- }
- CHECK_EQ(rid, blob->shape()[0]);
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/lmdb_data.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/lmdb_data.cc b/src/neuralnet/input_layer/lmdb_data.cc
deleted file mode 100644
index 37da538..0000000
--- a/src/neuralnet/input_layer/lmdb_data.cc
+++ /dev/null
@@ -1,143 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer/lmdb_data.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-using mshadow::Shape4;
-using mshadow::Tensor;
-
-using std::string;
-using std::vector;
-
-#ifdef USE_LMDB
-LMDBDataLayer::~LMDBDataLayer() {
- mdb_cursor_close(mdb_cursor_);
- mdb_txn_abort(mdb_txn_);
- mdb_cursor_ = nullptr;
-}
-
-void LMDBDataLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- OpenLMDB(proto.lmdbdata_conf().path());
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT),
- MDB_SUCCESS);
- mdb_cursor_close(mdb_cursor_);
- mdb_txn_abort(mdb_txn_);
- mdb_cursor_ = nullptr;
- CaffeDatum datum;
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- SingleLabelImageRecord* record = sample_.mutable_image();
- ConvertCaffeDatumToRecord(datum, record);
- batchsize_ = proto.lmdbdata_conf().batchsize();
- if (partition_dim() == 0)
- batchsize_ /= proto.num_partitions();
- records_.resize(batchsize_);
- random_skip_ = proto.lmdbdata_conf().random_skip();
-}
-
-void LMDBDataLayer::OpenLMDB(const std::string& path) {
- CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
- CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
- CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(),
- MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path;
- CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
- << "mdb_txn_begin failed";
- CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
- << "mdb_open failed";
- CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
- << "mdb_cursor_open failed";
- LOG(INFO) << "Opening lmdb " << path;
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
- MDB_SUCCESS) << "mdb_cursor_get failed";
-}
-
-void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (mdb_cursor_ == nullptr)
- OpenLMDB(layer_conf_.lmdbdata_conf().path());
- if (random_skip_) {
- int nskip = rand() % random_skip_;
- int n = 0;
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
- n++;
- LOG(INFO) << "Random Skip " << nskip << " records of total "
- << n << "records";
- // We have reached the end. Restart from the first.
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- for (int i = 0; i < nskip; i++) {
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
- random_skip_ = 0;
- }
- CaffeDatum datum;
- for (auto& record : records_) {
- SingleLabelImageRecord* image = record.mutable_image();
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- ConvertCaffeDatumToRecord(datum, image);
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
-}
-
-void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record) {
- record->set_label(datum.label());
- record->clear_shape();
- if (datum.has_channels())
- record->add_shape(datum.channels());
- if (datum.has_height())
- record->add_shape(datum.height());
- if (datum.has_width())
- record->add_shape(datum.width());
- if (datum.has_data())
- record->set_pixel(datum.data());
- if (datum.float_data_size()) {
- record->clear_data();
- for (float x : datum.float_data())
- record->add_data(x);
- }
-}
-#endif
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/mnist.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/mnist.cc b/src/neuralnet/input_layer/mnist.cc
deleted file mode 100644
index 41e4390..0000000
--- a/src/neuralnet/input_layer/mnist.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer/mnist.h"
-#include "singa/neuralnet/input_layer/data.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-using mshadow::Shape4;
-using mshadow::Tensor;
-
-using std::string;
-using std::vector;
-
-void MnistLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- LOG_IF(ERROR, records.size() == 0) << "Empty records to parse";
- int ndim = records.at(0).image().shape_size();
- int inputsize = records.at(0).image().shape(ndim-1);
- CHECK_EQ(inputsize, blob->shape()[2]);
-
- float* dptr = blob->mutable_cpu_data();
- for (const Record& record : records) {
- const SingleLabelImageRecord& imagerecord = record.image();
- if (imagerecord.pixel().size()) {
- string pixel = imagerecord.pixel();
- for (int i = 0, k = 0; i < inputsize; i++) {
- for (int j = 0; j < inputsize; j++) {
- // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of
- // time to debug this
- float x = static_cast<float>(static_cast<uint8_t>(pixel[k++]));
- x = x / norm_a_-norm_b_;
- *dptr = x;
- dptr++;
- }
- }
- } else {
- for (int i = 0, k = 0; i < inputsize; i++) {
- for (int j = 0; j < inputsize; j++) {
- *dptr = imagerecord.data(k++) / norm_a_ - norm_b_;
- dptr++;
- }
- }
- }
- }
- CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count());
-}
-
-void MnistLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
- norm_a_ = proto.mnist_conf().norm_a();
- norm_b_ = proto.mnist_conf().norm_b();
- int ndim = sample.image().shape_size();
- CHECK_GE(ndim, 2);
- int s = sample.image().shape(ndim - 1);
- CHECK_EQ(s, sample.image().shape(ndim - 2));
- data_.Reshape(vector<int>{batchsize, 1, s, s});
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/parser.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/parser.cc b/src/neuralnet/input_layer/parser.cc
deleted file mode 100644
index dfd9d8e..0000000
--- a/src/neuralnet/input_layer/parser.cc
+++ /dev/null
@@ -1,35 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer/parser.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
-#include "singa/neuralnet/input_layer/data.h"
-namespace singa {
-
-void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin());
- ParseRecords(flag, datalayer->records(), &data_);
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/prefetch.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/prefetch.cc b/src/neuralnet/input_layer/prefetch.cc
index e726838..9c1d7ee 100644
--- a/src/neuralnet/input_layer/prefetch.cc
+++ b/src/neuralnet/input_layer/prefetch.cc
@@ -21,8 +21,6 @@
#include "singa/neuralnet/input_layer/prefetch.h"
#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
namespace singa {
using namespace mshadow;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e469503d/src/neuralnet/input_layer/rgb_image.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/rgb_image.cc b/src/neuralnet/input_layer/rgb_image.cc
deleted file mode 100644
index 3763932..0000000
--- a/src/neuralnet/input_layer/rgb_image.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer/rgb_image.h"
-#include "singa/neuralnet/input_layer/data.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/tokenizer.h"
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-using mshadow::Shape4;
-using mshadow::Tensor;
-
-using std::string;
-using std::vector;
-
-void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- const vector<int>& s = blob->shape();
- Tensor<cpu, 4> images(data_.mutable_cpu_data(),
- Shape4(s[0], s[1], s[2], s[3]));
- const SingleLabelImageRecord& r = records.at(0).image();
- Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2)));
- AllocSpace(raw_image);
- Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3]));
- if (cropsize_)
- AllocSpace(croped_image);
- int rid = 0;
- const float* meandptr = mean_.cpu_data();
- for (const Record& record : records) {
- auto image = images[rid];
- bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain);
- bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
- float* dptr = nullptr;
- if (do_crop || do_mirror)
- dptr = raw_image.dptr;
- else
- dptr = image.dptr;
- if (record.image().pixel().size()) {
- string pixel = record.image().pixel();
- for (size_t i = 0; i < pixel.size(); i++)
- dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i]));
- } else {
- memcpy(dptr, record.image().data().data(),
- sizeof(float) * record.image().data_size());
- }
- for (int i = 0; i < mean_.count(); i++)
- dptr[i] -= meandptr[i];
- if (do_crop) {
- int hoff = rand() % (r.shape(1) - cropsize_);
- int woff = rand() % (r.shape(2) - cropsize_);
- Shape<2> cropshape = Shape2(cropsize_, cropsize_);
- if (do_mirror) {
- croped_image = expr::crop(raw_image, cropshape, hoff, woff);
- image = expr::mirror(croped_image);
- } else {
- image = expr::crop(raw_image, cropshape, hoff, woff);
- }
- } else if (do_mirror) {
- image = expr::mirror(raw_image);
- }
- rid++;
- }
- if (scale_)
- images = images * scale_;
- FreeSpace(raw_image);
- if (cropsize_)
- FreeSpace(croped_image);
-}
-
-void RGBImageLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- ParserLayer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- scale_ = proto.rgbimage_conf().scale();
- cropsize_ = proto.rgbimage_conf().cropsize();
- mirror_ = proto.rgbimage_conf().mirror();
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
- vector<int> shape;
- shape.push_back(batchsize);
- for (int x : sample.image().shape()) {
- shape.push_back(x);
- }
- CHECK_EQ(shape.size(), 4);
- if (cropsize_) {
- shape[2] = cropsize_;
- shape[3] = cropsize_;
- }
- data_.Reshape(shape);
- mean_.Reshape({shape[1], shape[2], shape[3]});
- if (proto.rgbimage_conf().has_meanfile()) {
- if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) {
- CaffeBlob mean;
- ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
- CHECK_EQ(mean_.count(), mean.data_size());
- memcpy(mean_.mutable_cpu_data(), mean.data().data(),
- sizeof(float)*mean.data_size());
- } else {
- SingleLabelImageRecord mean;
- ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
- CHECK_EQ(mean_.count(), mean.data_size());
- memcpy(mean_.mutable_cpu_data(), mean.data().data(),
- sizeof(float)*mean.data_size());
- }
- } else {
- memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count());
- }
-}
-
-} // namespace singa