You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by jx...@apache.org on 2017/08/26 18:37:13 UTC
[incubator-mxnet] branch master updated: unit test for csv iter,
doc update for libsvmiter (#7623)
This is an automated email from the ASF dual-hosted git repository.
jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 2e6ef8c unit test for csv iter, doc update for libsvmiter (#7623)
2e6ef8c is described below
commit 2e6ef8c88006a8c80dd5f7be3cc4c6ed56cbcaae
Author: Haibin Lin <li...@gmail.com>
AuthorDate: Sat Aug 26 11:37:10 2017 -0700
unit test for csv iter, doc update for libsvmiter (#7623)
* add unit test for csv iter
* fix lint
* add libsvm to mxnet.io doc
* update libsvm doc
---
docs/api/python/io.md | 1 +
python/mxnet/io.py | 3 +-
src/io/inst_vector.h | 2 +-
src/io/iter_libsvm.cc | 90 +++++++++++++++++++++-------------------
tests/python/unittest/test_io.py | 21 ++++++++++
5 files changed, 73 insertions(+), 44 deletions(-)
diff --git a/docs/api/python/io.md b/docs/api/python/io.md
index 15f8aa3..ce8245b 100644
--- a/docs/api/python/io.md
+++ b/docs/api/python/io.md
@@ -56,6 +56,7 @@ A detailed tutorial is available at
io.NDArrayIter
io.CSVIter
+ io.LibSVMIter
io.ImageRecordIter
io.ImageRecordUInt8Iter
io.MNISTIter
diff --git a/python/mxnet/io.py b/python/mxnet/io.py
index 314a2b2..b169681 100644
--- a/python/mxnet/io.py
+++ b/python/mxnet/io.py
@@ -189,6 +189,7 @@ class DataIter(object):
--------
NDArrayIter : Data-iterator for MXNet NDArray or numpy-ndarray objects.
CSVIter : Data-iterator for csv data.
+ LibSVMIter : Data-iterator for libsvm data.
ImageIter : Data-iterator for images.
"""
def __init__(self, batch_size=0):
@@ -721,7 +722,7 @@ class MXDataIter(DataIter):
"""A python wrapper a C++ data iterator.
This iterator is the Python wrapper to all native C++ data iterators, such
- as `CSVIter, `ImageRecordIter`, `MNISTIter`, etc. When initializing
+ as `CSVIter`, `ImageRecordIter`, `MNISTIter`, etc. When initializing
`CSVIter` for example, you will get an `MXDataIter` instance to use in your
Python code. Calls to `next`, `reset`, etc will be delegated to the
underlying C++ data iterators.
diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h
index 6dc7bdf..afa19e2 100644
--- a/src/io/inst_vector.h
+++ b/src/io/inst_vector.h
@@ -169,7 +169,7 @@ struct TBlobBatch {
}
/*! \brief destructor */
~TBlobBatch() {
- delete inst_index;
+ delete[] inst_index;
}
}; // struct TBlobBatch
diff --git a/src/io/iter_libsvm.cc b/src/io/iter_libsvm.cc
index 803d19e..8e53e6f 100644
--- a/src/io/iter_libsvm.cc
+++ b/src/io/iter_libsvm.cc
@@ -198,19 +198,21 @@ class LibSVMIter: public SparseIIterator<DataInst> {
DMLC_REGISTER_PARAMETER(LibSVMIterParam);
MXNET_REGISTER_IO_ITER(LibSVMIter)
-.describe(R"code(Returns the LibSVM file iterator. This iterator is experimental and
-should be used with care.
+.describe(R"code(Returns the libsvm file iterator which returns sparse data with `csr`
+storage type. This iterator is experimental and should be used with care.
-The input data is similar to libsvm file format, except that the indices are expected to be
-zero-based instead of one-based. Details of the libsvm format are available at
-`https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/`
+The input data is stored in a format similar to libsvm file format, except that the indices
+are expected to be zero-based instead of one-based. Details of the libsvm format are available
+at `https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/`
In this function, the `data_shape` parameter is used to set the shape of each line of the data.
The dimension of both `data_shape` and `label_shape` are expected to be 1.
When `label_libsvm` is set to ``NULL``, both data and label are read from the same file specified
-by `data_libsvm`. Otherwise, data is read from `data_libsvm` and label from `label_libsvm`,
-in this case, if `data_libsvm` contains label, it will ignored.
+by `data_libsvm`. In this case, the data is stored in `csr` storage type, while the label is a 1D
+dense array. Otherwise, data is read from `data_libsvm` and label from `label_libsvm`,
+in this case, both data and label are stored in csr storage type. If `data_libsvm` contains label,
+it will ignored.
The `LibSVMIter` only support `round_batch` parameter set to ``True`` for now. So, if `batch_size`
is 3 and there are 4 total rows in libsvm file, 2 more examples
@@ -221,58 +223,62 @@ If ``data_libsvm = 'data/'`` is set, then all the files in this directory will b
Examples::
- // Contents of libsvm file ``data.t``.
+ # Contents of libsvm file ``data.t``.
1.0 0:0.5 2:1.2
-2.0
-3.0 0:0.6 1:2.4 2:1.2
4 2:-1.2
- // Creates a `LibSVMIter` with `batch_size`=3.
- LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,),
- batch_size = 3)
-
- // The first batch (data and label)
- [[ 0.5 0. 1.2 ]
- [ 0. 0. 0. ]
- [ 0.6 2.4 1.2 ]]
-
+ # Creates a `LibSVMIter` with `batch_size`=3.
+ >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), batch_size = 3)
+ # The data of the first batch is stored in csr storage type
+ >>> batch = data_iter.next()
+ >>> csr = batch.data[0]
+ <CSRNDArray 3x3 @cpu(0)>
+ >>> csr.asnumpy()
+ [[ 0.5 0. 1.2 ]
+ [ 0. 0. 0. ]
+ [ 0.6 2.4 1.2]]
+ # The label of first batch
+ >>> label = batch.label[0]
+ >>> label
[ 1. -2. -3.]
+ <NDArray 3 @cpu(0)>
- // The second batch (data and label)
+ >>> second_batch = data_iter.next()
+ # The data of the second batch
+ >>> second_batch.data[0].asnumpy()
[[ 0. 0. -1.2 ]
[ 0.5 0. 1.2 ]
[ 0. 0. 0. ]]
-
+ # The label of the second batch
+ >>> second_batch.label[0].asnumpy()
[ 4. 1. -2.]
- // Contents of libsvm file ``label.t``
+ # Contents of libsvm file ``label.t``
1.0
-2.0 0:0.125
-3.0 2:1.2
4 1:1.0 2:-1.2
- // Creates a `LibSVMIter` with specified label file
- LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,),
- label_libsvm = 'label.t', label_shape = (3,), batch_size = 3)
-
- // Two batches of data read from the above iterator are as follows(data and label):
- // The first batch
- [[ 0.5 0. 1.2 ]
- [ 0. 0. 0. ]
- [ 0.6 2.4 1.2 ]]
-
- [[ 0. 0. 0. ]
- [ 0.125 0. 0. ]
- [ 0. 0. 1.2 ]]
-
- // The second batch
- [[ 0. 0. -1.2 ]
- [ 0.5 0. 1.2 ]
- [ 0. 0. 0. ]]
-
- [[ 0. 1. -1.2 ]
- [ 0. 0. 0. ]
- [ 0.125 0. 0. ]]
+ # Creates a `LibSVMIter` with specified label file
+ >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,),
+ label_libsvm = 'label.t', label_shape = (3,), batch_size = 3)
+
+ # Both data and label are in csr storage type
+ >>> batch = data_iter.next()
+ >>> csr_data = batch.data[0]
+ <CSRNDArray 3x3 @cpu(0)>
+ >>> csr_data.asnumpy()
+ [[ 0.5 0. 1.2 ]
+ [ 0. 0. 0. ]
+ [ 0.6 2.4 1.2 ]]
+ >>> csr_label = batch.label[0]
+ <CSRNDArray 3x3 @cpu(0)>
+ >>> csr_label.asnumpy()
+ [[ 0. 0. 0. ]
+ [ 0.125 0. 0. ]
+ [ 0. 0. 1.2 ]]
)code" ADD_FILELINE)
.add_arguments(LibSVMIterParam::__FIELDS__())
diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py
index a543463..fb8aa2a 100644
--- a/tests/python/unittest/test_io.py
+++ b/tests/python/unittest/test_io.py
@@ -257,6 +257,26 @@ def test_LibSVMIter():
check_libSVMIter_synthetic()
check_libSVMIter_news_data()
+def test_CSVIter():
+ def check_CSVIter_synthetic():
+ cwd = os.getcwd()
+ data_path = os.path.join(cwd, 'data.t')
+ label_path = os.path.join(cwd, 'label.t')
+ with open(data_path, 'w') as fout:
+ for i in range(1000):
+ fout.write(','.join(['1' for _ in range(8*8)]) + '\n')
+ with open(label_path, 'w') as fout:
+ for i in range(1000):
+ fout.write('0\n')
+
+ data_train = mx.io.CSVIter(data_csv=data_path, data_shape=(8,8),
+ label_csv=label_path, batch_size=100)
+ expected = mx.nd.ones((100, 8, 8))
+ for batch in iter(data_train):
+ assert_almost_equal(data_train.getdata().asnumpy(), expected.asnumpy())
+
+ check_CSVIter_synthetic()
+
if __name__ == "__main__":
test_NDArrayIter()
if h5py:
@@ -265,3 +285,4 @@ if __name__ == "__main__":
test_Cifar10Rec()
test_LibSVMIter()
test_NDArrayIter_csr()
+ test_CSVIter()
--
To stop receiving notification emails like this one, please contact
['"commits@mxnet.apache.org" <co...@mxnet.apache.org>'].