You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/08/11 17:13:55 UTC
svn commit: r1695323 - in /incubator/singa/site/trunk/content/markdown/docs:
communication.md data.md neuralnet.md rnn.md
Author: wangwei
Date: Tue Aug 11 15:13:55 2015
New Revision: 1695323
URL: http://svn.apache.org/r1695323
Log:
merge conflict files; newly added docs including communication, rnn, neuralnet; data.md is updated.
Added:
incubator/singa/site/trunk/content/markdown/docs/data.md
Modified:
incubator/singa/site/trunk/content/markdown/docs/communication.md
incubator/singa/site/trunk/content/markdown/docs/neuralnet.md
incubator/singa/site/trunk/content/markdown/docs/rnn.md
Modified: incubator/singa/site/trunk/content/markdown/docs/communication.md
URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/content/markdown/docs/communication.md?rev=1695323&r1=1695322&r2=1695323&view=diff
==============================================================================
--- incubator/singa/site/trunk/content/markdown/docs/communication.md (original)
+++ incubator/singa/site/trunk/content/markdown/docs/communication.md Tue Aug 11 15:13:55 2015
@@ -5,7 +5,7 @@ ___
Different messaging libraries has different benefits and drawbacks. For instance,
MPI provides fast message passing between GPUs (using GPUDirect), but does not
support fault-tolerance well. On the contrary, systems using ZeroMQ can be
-fault-tolerant. But ZeroMQ does not support GPUDirect. The AllReduce function
+fault-tolerant, but does not support GPUDirect. The AllReduce function
of MPI is also missing in ZeroMQ which is efficient for data aggregation for
distributed training. In Singa, we provide general messaging APIs for
communication between threads within a process and across processes, and let
@@ -35,17 +35,17 @@ supported </object>
<p><strong> Fig.2 - Logical message format</strong></p>
Fig.2 shows the logical message format which has two parts, the header and the
-content. The message header includes the sender and receiver's ID consisting of
+content. The message header includes the sender's and receiver's IDs, each consisting of
the group ID and the worker/server ID within the group. The stub forwards
messages by looking up an address table based on the receiver's ID.
-There are two sets of messages, their message types are:
+There are two sets of messages according to the message type defined below.
* kGet/kPut/kRequest/kSync for messages about parameters
* kFeaBlob/kGradBlob for messages about transferring feature and gradient
blobs of one layer to its neighboring layer
-There is a target ID in the header. If the message is related to parameters,
+There is a target ID in the header. If the message body is parameters,
the target ID is then the parameter ID. Otherwise the message is related to
layer feature or gradient, and the target ID consists of the layer ID and the
blob ID of that layer. The message content has multiple frames to store the
@@ -53,53 +53,205 @@ parameter or feature data.
The API for the base Msg is:
- class Msg{
+ /**
+ * Msg used to transfer Param info (gradient or value), feature blob, etc
+ * between workers, stubs and servers.
+ *
+ * Each msg has a source addr and dest addr identified by a unique integer.
+ * It is also associated with a target field (value and version) for ease of
+ * getting some meta info (e.g., parameter id) from the msg.
+ *
+ * Other data is added into the message as frames.
+ */
+ class Msg {
public:
+ ~Msg();
+ Msg();
/**
- * Destructor to free memory
+ * Construct the msg providing source and destination addr.
*/
- virtual ~Msg()=0;
+ Msg(int src, int dst);
/**
- * @param group_id worker/server group id
- * @param id worker/server id within the group
- * @param flag 0 for server, 1 for worker, 2 for stub
- */
- virtual void set_src(int group_id, int id, int flag)=0;
- virtual void set_dst(int group_id, int id, int flag)=0;
- virtual void src_group_id()=0;
- virtual int src_id() const=0;
- virtual int dst_group_id() const=0;
- virtual int dst_id() const=0;
- virtual int src_flag() const=0;
- virtual int dst_flag() const=0;
- virtual void set_type(int type)=0;
- virtual void set_target(int param_id)=0;
- virtual void set_target(int layer_id, blob_id)=0;
- virtual int type() const=0;
- /**
- * @return true if the msg is about parameter, otherwise false
- */
- virtual bool is_param_msg() const=0;
- virtual int param_id() const=0;
- virtual int layer_id() const=0;
- virtual int blob_id() const=0;
-
- virtual void add_frame(void*, int nBytes)=0;
- virtual int frame_size()=0;
- virtual void* frame_data()=0;
+ * Copy constructor.
+ */
+ Msg(const Msg& msg);
+ /**
+ * Swap the src/dst addr
+ */
+ void SwapAddr();
+ /**
+ * Add a frame (a chunk of bytes) into the message
+ */
+ void AddFrame(const void* addr, int nBytes);
+ /**
+ * @return num of bytes of the current frame.
+ */
+ int FrameSize();
+ /**
+ * @return the pointer to the current frame data.
+ */
+ void* FrameData();
+ /**
+ * @return the data of the current frame as c string
+ */
+ char* FrameStr();
+ /**
+ * Move the cursor to the first frame.
+ */
+ void FirstFrame();
+ /**
+ * Move the cursor to the last frame.
+ */
+ void LastFrame();
/**
* Move the cursor to the next frame
* @return true if the next frame is not NULL; otherwise false
*/
- virtual bool next_frame()=0;
- virtual int SerializeTo(string* buf);
- virtual int ParseFrom(const string& buf);
+ bool NextFrame();
+ /**
+ * Add a 'format' frame to the msg (like CZMQ's zsock_send).
+ *
+ * The format is a string that defines the type of each field.
+ * The format can contain any of these characters, each corresponding to
+ * one or two arguments:
+ * i = int (signed)
+ * 1 = uint8_t
+ * 2 = uint16_t
+ * 4 = uint32_t
+ * 8 = uint64_t
+ * p = void * (sends the pointer value, only meaningful over inproc)
+ * s = char**
+ *
+ * Returns size of the added content.
+ */
+ int AddFormatFrame(const char *format, ...);
+ /**
+ * Parse the current frame added using AddFormatFrame(const char*, ...).
+ *
+ * The format is a string that defines the type of each field.
+ * The format can contain any of these characters, each corresponding to
+ * one or two arguments:
+ * i = int (signed)
+ * 1 = uint8_t
+ * 2 = uint16_t
+ * 4 = uint32_t
+ * 8 = uint64_t
+ * p = void * (sends the pointer value, only meaningful over inproc)
+ * s = char**
+ *
+ * Returns size of the parsed content.
+ */
+ int ParseFormatFrame(const char* format, ...);
+
+ #ifdef USE_ZMQ
+ void ParseFromZmsg(zmsg_t* msg);
+ zmsg_t* DumpToZmsg();
+ #endif
+
+ /**
+ * @return msg size in terms of bytes, ignore meta info.
+ */
+ int size() const;
+ /**
+ * Set source addr.
+ * @param addr unique identify one worker/server/stub in the current job
+ */
+ void set_src(int addr) { src_ = addr; }
+ /**
+ * @return source addr.
+ */
+ int src() const { return src_; }
+ /**
+ * Set destination addr.
+ * @param addr unique identify one worker/server/stub in the current job
+ */
+ void set_dst(int addr) { dst_ = addr; }
+ /**
+ * @return dst addr.
+ */
+ int dst() const { return dst_; }
+ /**
+ * Set msg type, e.g., kPut, kGet, kUpdate, kRequest
+ */
+ void set_type(int type) { type_ = type; }
+ /**
+ * @return msg type.
+ */
+ int type() const { return type_; }
+ /**
+ * Set msg target.
+ *
+ * One msg has a target to identify some entity in worker/server/stub.
+ * The target is associated with a version, e.g., Param version.
+ */
+ void set_trgt(int val, int version) {
+ trgt_val_ = val;
+ trgt_version_ = version;
+ }
+ int trgt_val() const {
+ return trgt_val_;
+ }
+ int trgt_version() const {
+ return trgt_version_;
+ }
+
};
+In order for a Msg object to be routed, the source and dest address should be attached.
+This is achieved by calling the set_src and set_dst methods of the Msg object.
+The address parameter passed to these two methods can be manipulated via a set of
+helper functions, shown as below.
+
+ /**
+ * Wrapper to generate message address
+ * @param grp worker/server group id
+ * @param id_or_proc worker/server id or procs id
+ * @param type msg type
+ */
+ inline int Addr(int grp, int id_or_proc, int type) {
+ return (grp << 16) | (id_or_proc << 8) | type;
+ }
+
+ /**
+ * Parse group id from addr.
+ *
+ * @return group id
+ */
+ inline int AddrGrp(int addr) {
+ return addr >> 16;
+ }
+ /**
+ * Parse worker/server id from addr.
+ *
+ * @return id
+ */
+ inline int AddrID(int addr) {
+ static const int mask = (1 << 8) - 1;
+ return (addr >> 8) & mask;
+ }
+
+ /**
+ * Parse worker/server procs from addr.
+ *
+ * @return procs id
+ */
+ inline int AddrProc(int addr) {
+ return AddrID(addr);
+ }
+ /**
+ * Parse msg type from addr
+ * @return msg type
+ */
+ inline int AddrType(int addr) {
+ static const int mask = (1 << 8) -1;
+ return addr & mask;
+ }
+
+
### Socket
In Singa, there are two types of sockets, the Dealer Socket and the Router
-Socket. The names are from ZeroMQ. All connections are of the same type, i.e.,
+Socket, whose names are adapted from ZeroMQ. All connections are of the same type, i.e.,
Dealer<-->Router. The communication between dealers and routers are
asynchronous. In other words, one Dealer
socket can talk with multiple Router sockets, and one Router socket can talk
@@ -110,95 +262,130 @@ with multiple Dealer sockets.
The basic functions of a Singa Socket is to send and receive messages. The APIs
are:
- class Socket{
+ class SocketInterface {
public:
+ virtual ~SocketInterface() {}
/**
- * @param args depending on the underlying implementation.
- */
- Socket(void* args);
+ * Send a message to connected socket(s), non-blocking. The message
+ * will be deallocated after sending, thus should not be used after
+ * calling Send();
+ *
+ * @param msg The message to be sent
+ * @return 1 for success queuing the message for sending, 0 for failure
+ */
+ virtual int Send(Msg** msg) = 0;
/**
- * Send a message to connected socket(s), non-blocking. The message will
- * be deallocated after sending, thus should not be used after calling Send();
- * @param the message to be sent
- * @param dst the identifier of the connected socket. By default, it is
- * -1, which means sending this message to all connected sockets.
- * @return 1 for success queuing the message for sending, 0 for failure
- */
- virtual int Send(Msg** msg, int dst=-1)=0;
+ * Receive a message from any connected socket.
+ *
+ * @return a message pointer if success; nullptr if failure
+ */
+ virtual Msg* Receive() = 0;
/**
- * Receive a message
- * @return a message pointer if success; nullptr if failure
+ * @return Identifier of the implementation dependent socket. E.g., zsock_t*
+ * for ZeroMQ implementation and rank for MPI implementation.
*/
- virtual Message* Receive()=0;
+ virtual void* InternalID() const = 0;
};
- class Poller{
+A poller class is provided to enable asynchronous communication between routers and dealers.
+One can register a set of SocketInterface objects with a poller instance via calling its Add method, and
+then call the Wait method of this poll object to wait for the registered SocketInterface objects to be ready
+for sending and receiving messages. The APIs of the poller class is shown below.
+
+ class Poller {
public:
+ Poller();
+ Poller(SocketInterface* socket);
/**
- * Add a socket for polling; Multiple sockets can be polled together by
- * adding them into the same poller.
- */
- void Add(Socket* socket);
+ * Add a socket for polling; Multiple sockets can be polled together by
+ * adding them into the same poller.
+ */
+ void Add(SocketInterface* socket);
+ /**
+ * Poll for all sockets added into this poller.
+ * @param timeout Stop after this number of mseconds
+ * @return pointer To the socket if it has one message in the receiving
+ * queue; nullptr if no message in any sockets,
+ */
+ SocketInterface* Wait(int duration);
+
/**
- * Poll for all sockets added into this poller.
- * @param duration stop after this number of milliseconds
- * @return pointer to the socket if it has one message in the receiving
- * queue; nullptr if no message in any sockets,
+ * @return true if the poller is terminated due to process interupt
*/
- Socket* Poll(int duation);
+ virtual bool Terminated();
};
+
#### Dealer Socket
The Dealer socket inherits from the base Socket. In Singa, every Dealer socket
only connects to one Router socket as shown in Fig.1. The connection is set up
by connecting the Dealer socket to the endpoint of a Router socket.
- class Dealer : public Socket{
+ class Dealer : public SocketInterface {
public:
- /**
- * Blocking operation to setup the connection with the router, called
- * only once.
- * @param endpoint identifier of the router. For intra-process
- * connection, the endpoint follows the format of ZeroMQ, i.e.,
- * starting with "inproc://"; in Singa, since each process has one
- * router, hence we can fix the endpoint to be "inproc://router" for
- * intra-process. For inter-process, the endpoint follows ZeroMQ's
- * format, i.e., IP:port, where IP is the connected process.
- * @return 1 connection sets up successfully; 0 otherwise
- /
- int Connect(string endpoint);
/*
- * Since the Dealer socket connects to only one router, it must send to
- * the connected router, thus the dst argument is useless.
+ * @param id Local dealer ID within a procs if the dealer is from worker or
+ * server thread, starts from 1 (0 is used by the router); or the connected
+ * remote procs ID for inter-process dealers from the stub thread.
*/
- virtual int Send(Msg** msg, int dst=-1);
- virtual Message* Receive();
+ Dealer();
+ explicit Dealer(int id);
+ ~Dealer() override;
+ /**
+ * Setup the connection with the router.
+ *
+ * @param endpoint Identifier of the router. For intra-process
+ * connection, the endpoint follows the format of ZeroMQ, i.e.,
+ * starting with "inproc://"; in Singa, since each process has one
+ * router, hence we can fix the endpoint to be "inproc://router" for
+ * intra-process. For inter-process, the endpoint follows ZeroMQ's
+ * format, i.e., IP:port, where IP is the connected process.
+ * @return 1 connection sets up successfully; 0 otherwise
+ */
+ int Connect(const std::string& endpoint);
+ int Send(Msg** msg) override;
+ Msg* Receive() override;
+ void* InternalID() const override;
};
#### Router Socket
The Router socket inherits from the base Socket. One Router socket connects to
-at least one Dealer socket.
+at least one Dealer socket. Upon receiving a message, the router forwards it to
+the appropriate dealer according to the receiver's ID of this message.
- class Router : public Socket{
- /**
- * Blocking operation to setup the connection with dealers.
- * It automatically binds to the endpoint for intra-process communication,
- * i.e., "inproc://router".
- *
- * @param endpoint the identifier for the Dealer socket in other process
- * to connect. It has the format IP:Port, where IP is the host machine.
- * If endpoint is empty, it means that all connections are
- * intra-process connection.
- * @param expected_connections total number of connections. This function
- * exits after receiving this number of connections from dealers or after
- * a timeout (1 minutes).
- * @return number of connected dealers.
- */
- int Bind(string endpoint, int expected_connections);
- virtual int Send(Msg** msg, int dst=-1);
- virtual Message* Receive();
+ class Router : public SocketInterface {
+ public:
+ Router();
+ /**
+ * There is only one router per procs, hence its local id is 0 and is not set
+ * explicitly.
+ *
+ * @param bufsize Buffer at most this number of messages
+ */
+ explicit Router(int bufsize);
+ ~Router() override;
+ /**
+ * Setup the connection with dealers.
+ *
+ * It automatically binds to the endpoint for intra-process communication,
+ * i.e., "inproc://router".
+ *
+ * @param endpoint The identifier for the Dealer socket in other process
+ * to connect. It has the format IP:Port, where IP is the host machine.
+ * If endpoint is empty, it means that all connections are
+ * intra-process connection.
+ * @return number of connected dealers.
+ */
+ int Bind(const std::string& endpoint);
+ /**
+ * If the destination socket has not connected yet, buffer this the message.
+ */
+ int Send(Msg** msg) override;
+ Msg* Receive() override;
+ void* InternalID() const override;
+
};
### Implementation
@@ -253,7 +440,7 @@ passed to all sockets' constructor via *
};
For inter-process communication, we serialize the message and call MPI's
-send/receive functions to transferring them. All inter-process connections are
+send/receive functions to transfer them. All inter-process connections are
setup by MPI at the beginning. Consequently, the Connect and Bind functions do
nothing for both inter-process and intra-process communication.
@@ -340,4 +527,4 @@ All messages in SINGA are of multi-frame
5. After a PMServer processes a request, it generates a message with the format similar to (3) but with extra frame indicating if the message is to be routed back to a worker (a response message) or to route to another server (a SYNC request).
6. When a request is re-queued, the PMServer generates a message and sends it directly to the server's front-end socket. The re-queued request seen by the server's main thread consists of all the frames in (3), followed by a REQUEUED frame, and finally by another frame generated by the ROUTER socket identifying connection from the PMServer instance. The main thread then strips off these additional two frames before forwarding it to another PMServer instance like another ordinary request.
--->
+-->
\ No newline at end of file
Added: incubator/singa/site/trunk/content/markdown/docs/data.md
URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/content/markdown/docs/data.md?rev=1695323&view=auto
==============================================================================
--- incubator/singa/site/trunk/content/markdown/docs/data.md (added)
+++ incubator/singa/site/trunk/content/markdown/docs/data.md Tue Aug 11 15:13:55 2015
@@ -0,0 +1,134 @@
+--- trunk/content/markdown/docs/data.md (revision 1693059)
++++ trunk/content/markdown/docs/data.md (working copy)
+@@ -1,18 +1,125 @@
+ ## Data Preparation
+
+-To submit a training job, users need to convert raw data (e.g., images, text
+-documents) into records that can be recognized by SINGA. SINGA uses a DataLayer
+-to load these records into memory and uses ParserLayer to parse features (e.g.,
+-image pixels and labels) from these records. The records could be organized and
+-stored using many different ways, e.g., using a light database, or a file or
+-HDFS, as long as there is a corresponding DataLayer that can load the records.
++To submit a training job, users need to convert raw data (e.g., images, text documents) into records that can be recognized by SINGA. SINGA uses a DataLayer
++to load these records into memory and uses ParserLayer to parse features (e.g., image pixels and labels) from these records. The records could be organized and
++stored using many different ways, e.g., a file, a light database, or HDFS, as long as there is a corresponding DataLayer that can load the records.
+
+ ### DataShard
+
++To create shard for your own data, users may need to implement or modify the following files
+
++- user.proto
++- create_shard.cc
++- Makefile
+
++**1. Define record**
++
++Record class is inherited from Message class whose format follows Google protocol buffers. Please refer to the [Tutorial][1].
++
++Your record will be defined in a file, "user.proto", as an extension of Record written in "SINGA_ROOT/src/proto/common.proto".
++
++(a) Declare the user-defined record as an extension of Record. Please refer to the [Tutorial][2] for Extension.
++
++ package singa;
++
++ import "common.proto"; // required to import common.proto
++
++ extend Record {
++ optional UserRecord1 user_record1 = 100; // unique extension field id, reserved for users (e.g., 100-199)
++ optional UserRecord2 user_record2 = 101; // unique extension field id, reserved for users (e.g., 100-199)
++ }
++
++(b) Define the user own record
++
++ message UserRecord1 {
++ repeated int userVAR1 = 1; // unique field id
++ optional string userVAR2 = 2; // unique field id
++ ...
++ }
++
++ message UserRecord2 {
++ ...
++ }
++
++*Note*
++
++There is an alternative way to define the proto extension.
++
++ message UserRecord {
++ extend Record {
++ optional UserRecord user_record = 100; // unique extension field id, reserved for users (e.g., 100-199)
++ }
++ repeated int userVAR1 = 1; // unique field id
++ optional string userVAR2 = 2; // unique field id
++ ...
++ }
++
++In this way, you should be careful of the scope of fields and how to access the fields, which are different from the above.
++
++**2. Compile proto/Singa**
++
++Once you defined protos, you need to compile them as follow.
++
++ protoc -I=IMPORT_DIR --cpp_out=DST_DIR [proto1] [proto2] ...
++
++ - IMPORT_DIR: path to contain proto files
++ - DST_DIR: path to generate C++ codes (.pb.h and .pb.cc)
++
++Assume that users will implement their own [Layers][3] as well. The following commands help you to compile Singa sources as well as proto.
++
++ cd SINGA_ROOT
++ ./configure
++ make
++
++**3. Create shard**
++
++(a) Create a folder for dataset and shard, e.g., we call it "USER_DATA". (This folder would be the same as WORKSPACE.)
++
++(b) Source files, e.g., "create_shard.cc", will be in "SINGA_ROOT/USER_DATA/"
++
++DataShard object
++
++ singa::DataShard myShard( outputpath, mode );
++
++ - `string outputpath`, where user wants to create shard.
++ - `int mode := kRead | kCreate | kAppend`, is defined in "SINGA_ROOT/include/utils/data_shard.h"
++
++Record object
++
++ singa::Record record;
++
++ singa::UserRecord *myRecord = record.MutableExtension(singa::user_record);
++
++ - `MutableExtension()` is templated method, generated after compile at Step 2 and processed at runtime.
++
++(c) Set/Add values into your record
++
++ myRecord->add_userVAR1( int_val ); // for repeated field
++ myRecord->set_userVAR2( string_val );
++
++(d) Write the record to shard
++
++ myShard.Insert( key, myRecord );
++- `String key`, will be a unique id for a message
++
++**Note: Preparing Makefile**
++
++There are some notes when you write your own Makefile
++
++- Specify `outputpath` of output directory for shard.
++
++- Make sure you copy "user.proto" into "SINGA_ROOT/src/proto".
++
++Examples of creating shard can be found at [RNNLM][4].
++
+ ### LMDB
+
+
+
+ ### HDFS
++
++
++ [1]: https://developers.google.com/protocol-buffers/docs/cpptutorial
++ [2]: https://developers.google.com/protocol-buffers/docs/reference/cpp-generated?hl=en#extension
++ [3]: http://singa.incubator.apache.org/docs/layer.html
++ [4]: http://singa.incubator.apache.org/docs/examples.html
\ No newline at end of file
Modified: incubator/singa/site/trunk/content/markdown/docs/neuralnet.md
URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/content/markdown/docs/neuralnet.md?rev=1695323&r1=1695322&r2=1695323&view=diff
==============================================================================
--- incubator/singa/site/trunk/content/markdown/docs/neuralnet.md (original)
+++ incubator/singa/site/trunk/content/markdown/docs/neuralnet.md Tue Aug 11 15:13:55 2015
@@ -0,0 +1,81 @@
+NeuralNet
+===
+
+This page will introduce the implement of neuralnet in Singa.
+It contains:
+
+[Introduction]()
+
+[NeuralNet class in detail]()
+
+[How it works]()
+
+###Introduction
+**NeuralNet** represents a neural net consisting of a set of unidirectionally connected layers. Properties and connections of layers are specified by users. The NeuralNet object is passed as an argument to the **TrainOneBatch function**.
+
+###NeuralNet class in detail
+**Main member functions list**:
+
+- *public*:
+
+ static void RegisterLayers()
+ //Register Layers, i.e., map layer type to layer class
+
+ static shared_ptr<NeuralNet> Create(const NetProto& np, Phase phase, int num)
+ //Create the neural network for training, test or validation.
+ //Parameters for test/validation net can share those from training after setup (done outside of this funcion).
+
+ explicit NeuralNet(NetProto netproto, int npartitions = 1);
+ //construct the net structure from protocol buffer.
+
+ std::string ToAdjacency();
+ //To display the adjacency layers
+
+ void ShareParamsFrom(shared_ptr<NeuralNet> other)
+ //Share memory of parameter values from other neuralnet
+
+- *protected*:
+
+ Graph* CreateGraph(const NetProto& netproto, int npartitions);
+ //Create a neural net graph, one node for each layer.
+ //Partition the graph if npartitions > 1, each layer is sliced according to its own partition setting.
+
+ void CreateNetFromGraph(Graph* graph, int npartitions)
+ //Create neural net from graph, one layer per node.
+
+ void PrepareDataStructures()
+ //prepare data structures, e.g., params_, layers_, etc.
+
+###How it works
+**Layer connections** in NeuralNet are not represented explicitly, instead each layer records its own source layers as specified by users.
+
+**An example of NeuralNet configuration**:
+
+ layer: name: "softmax loss"
+ type: SoftmaxLossLayer
+ srclayer: "hidden"
+ "label parser"
+
+ layer: name: "hidden"
+ type: HiddenLayer
+ srclayer: "data"
+ shape: 3
+
+ layer: name: "image parser"
+ type: ImageParserLayer
+
+ layer: name: "label parser"
+ type: LabelParserLayer
+
+ layer: name: "data"
+ type: kData
+ path: "./train.shard"
+
+<img src = "../images/model_configure.png" style = "width: 400px"> Fig. 1:
+Model Configure</img>
+
+**Different model** categories have different types of **layer connections**. However, they can be **unified** using directed edges as follows.
+
+- For **feed-forward models**, nothing needs to be done as their connections are already directed.
+- For **undirected models**, users need to replace each edge with two directed edges.
+- For **recurrent models**, users can unroll a recurrent layer into directed-connecting sub-layers.
\ No newline at end of file
Modified: incubator/singa/site/trunk/content/markdown/docs/rnn.md
URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/content/markdown/docs/rnn.md?rev=1695323&r1=1695322&r2=1695323&view=diff
==============================================================================
--- incubator/singa/site/trunk/content/markdown/docs/rnn.md (original)
+++ incubator/singa/site/trunk/content/markdown/docs/rnn.md Tue Aug 11 15:13:55 2015
@@ -0,0 +1,123 @@
+## Recurrent neural networks (RNN)
+
+Example files for RNN can be found in "SINGA_ROOT/examples/rnnlm", which we assume to be WORKSPACE.
+
+### Create DataShard
+
+(a) Define your own record. Please refer to [Data Preparation][1] for details.
+
+Records for RNN example are defined in "user.proto" as an extension.
+
+ package singa;
+
+ import "common.proto"; // Record message for SINGA is defined
+ import "job.proto"; // Layer message for SINGA is defined
+
+ extend Record {
+ optional WordClassRecord wordclass = 101;
+ optional SingleWordRecord singleword = 102;
+ }
+
+ message WordClassRecord {
+ optional int32 class_index = 1; // the index of this class
+ optional int32 start = 2; // the index of the start word in this class;
+ optional int32 end = 3; // the index of the end word in this class
+ }
+
+ message SingleWordRecord {
+ optional string word = 1;
+ optional int32 word_index = 2; // the index of this word in the vocabulary
+ optional int32 class_index = 3; // the index of the class corresponding to this word
+ }
+
+(b) Download raw data
+
+This example downloads rnnlm-0.4b from [www.rnnlm.org][2] by a command
+
+ make download
+
+The raw data is stored in a folder "rnnlm-0.4b/train" and "rnnlm-0.4b/test".
+
+(c) Create data shard for training and testing
+
+Data shards (e.g., "shard.dat") will be created in "rnnlm_class_shard", "rnnlm_vocab_shard", "rnnlm_word_shard_train" and "rnnlm_word_shard_test" by a command
+
+ make create
+
+
+### Define Layers
+
+Similar to records, layers are also defined in "user.proto" as an extension.
+
+ package singa;
+
+ import "common.proto"; // Record message for SINGA is defined
+ import "job.proto"; // Layer message for SINGA is defined
+
+ //For implementation of RNNLM application
+ extend LayerProto {
+ optional RnnlmComputationProto rnnlmcomputation_conf = 201;
+ optional RnnlmSigmoidProto rnnlmsigmoid_conf = 202;
+ optional RnnlmInnerproductProto rnnlminnerproduct_conf = 203;
+ optional RnnlmWordinputProto rnnlmwordinput_conf = 204;
+ optional RnnlmDataProto rnnlmdata_conf = 207;
+ }
+
+ // 1-Message that stores parameters used by RnnlmComputationLayer
+ message RnnlmComputationProto {
+ optional bool bias_term = 1 [default = true]; // use bias vector or not
+ }
+
+ // 2-Message that stores parameters used by RnnlmSigmoidLayer
+ message RnnlmSigmoidProto {
+ optional bool bias_term = 1 [default = true]; // use bias vector or not
+ }
+
+ // 3-Message that stores parameters used by RnnlmInnerproductLayer
+ message RnnlmInnerproductProto {
+ required int32 num_output = 1; // number of outputs for the layer
+ optional bool bias_term = 30 [default = true]; // use bias vector or not
+ }
+
+ // 4-Message that stores parameters used by RnnlmWordinputLayer
+ message RnnlmWordinputProto {
+ required int32 word_length = 1; // vector length for each input word
+ optional bool bias_term = 30 [default = true]; // use bias vector or not
+ }
+
+ // 5-Message that stores parameters used by RnnlmWordparserLayer - nothing needs to be configured
+ //message RnnlmWordparserProto {
+ //}
+
+ // 6-Message that stores parameters used by RnnlmClassparserLayer - nothing needs to be configured
+ //message RnnlmClassparserProto {
+ //}
+
+ // 7-Message that stores parameters used by RnnlmDataLayer
+ message RnnlmDataProto {
+ required string class_path = 1; // path to the data file/folder, absolute or relative to the workspace
+ required string word_path = 2;
+ required int32 window_size = 3; // window size.
+ }
+
+
+### Configure Job
+
+Job configuration is written in "job.conf".
+
+Note: Extended field names should be embraced with square-parenthesis [], e.g., [singa.rnnlmdata_conf].
+
+
+### Run Training
+
+Start training by the following commands
+
+ cd SINGA_ROOT
+ ./bin/singa-run.sh -workspace=examples/rnnlm
+
+
+
+
+
+ [1]: http://singa.incubator.apache.org/docs/data.html
+ [2]: www.rnnlm.org
\ No newline at end of file