You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2017/05/26 05:26:50 UTC
incubator-singa git commit: SINGA-318 - Extend ImageBatchIter to read
image list file without meta info
Repository: incubator-singa
Updated Branches:
refs/heads/master 3415099a9 -> 5aecb47ed
SINGA-318 - Extend ImageBatchIter to read image list file without meta info
To read the image list file which contains only the image path per line.
The returned value of function next() consists of x (ndarray of images)
and y([None] * batchsize).
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5aecb47e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5aecb47e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5aecb47e
Branch: refs/heads/master
Commit: 5aecb47ed25c0d4e30dd7278d1a68c1db393b31d
Parents: 3415099
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri May 26 12:11:51 2017 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri May 26 12:11:51 2017 +0800
----------------------------------------------------------------------
examples/imagenet/resnet/README.md | 2 +-
python/singa/data.py | 23 ++++++++++++++++-------
2 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5aecb47e/examples/imagenet/resnet/README.md
----------------------------------------------------------------------
diff --git a/examples/imagenet/resnet/README.md b/examples/imagenet/resnet/README.md
index 4a0f4da..9ea12cd 100644
--- a/examples/imagenet/resnet/README.md
+++ b/examples/imagenet/resnet/README.md
@@ -31,7 +31,7 @@ In this example, we convert Residual Networks trained on [Torch](https://github.
$ python serve.py --parameter_file resnet-18.pickle --model resnet --depth 18 &
The parameter files for the following model and depth configuration pairs are provided:
- * resnet (original resnet), [18](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-101.tar.gz)|[34](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-34.tar.gz)|[101](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-101.tar.gz)|[152](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-152.tar.gz)
+ * resnet (original resnet), [18](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-18.tar.gz)|[34](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-34.tar.gz)|[101](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-101.tar.gz)|[152](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-152.tar.gz)
* addbn (resnet with a batch normalization layer after the addition), [50](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-50.tar.gz)
* wrn (wide resnet), [50](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/wrn-50-2.tar.gz)
* preact (resnet with pre-activation) [200](https://s3-ap-southeast-1.amazonaws.com/dlfile/resnet/resnet-200.tar.gz)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5aecb47e/python/singa/data.py
----------------------------------------------------------------------
diff --git a/python/singa/data.py b/python/singa/data.py
index ec4aa97..492d218 100644
--- a/python/singa/data.py
+++ b/python/singa/data.py
@@ -66,7 +66,8 @@ class ImageBatchIter:
meta_info should not contain the delimiter. If the meta_info
of each image is just the label index, then we will parse the
label index into a numpy array with length=batchsize
- (for compatibility); otherwise, we return a list of meta_info
+ (for compatibility); otherwise, we return a list of meta_info;
+ if meta info is available, we return a list of None.
batch_size(int): num of samples in one mini-batch
image_transform: a function for image augmentation; it accepts the full
image path and outputs a list of augmented images.
@@ -103,6 +104,9 @@ class ImageBatchIter:
x, y = self.queue.get() # dequeue one mini-batch
return x, y
+ def stop(self):
+ self.end();
+
def end(self):
if self.p is not None:
self.stop = True
@@ -111,12 +115,17 @@ class ImageBatchIter:
def run(self):
img_list = []
- is_labelindex = True
+ is_label_index = True
for line in open(self.img_list_file, 'r'):
item = line.strip('\n').split(self.delimiter)
- if not item[1].strip().isdigit(): # the meta info is not label index
- is_labelindex = False
- img_list.append((item[0].strip(), item[1].strip()))
+ if len(item) < 2:
+ is_label_index = False
+ img_list.append((item[0].strip(), None))
+ else:
+ if not item[1].strip().isdigit():
+ # the meta info is not label index
+ is_label_index = False
+ img_list.append((item[0].strip(), item[1].strip()))
index = 0 # index for the image
if self.shuffle:
random.shuffle(img_list)
@@ -134,7 +143,7 @@ class ImageBatchIter:
for img in aug_images:
ary = np.asarray(img.convert('RGB'), dtype=np.float32)
x.append(ary.transpose(2, 0, 1))
- if is_labelindex:
+ if is_label_index:
y.append(int(img_meta))
else:
y.append(img_meta)
@@ -145,7 +154,7 @@ class ImageBatchIter:
if self.shuffle:
random.shuffle(img_list)
# enqueue one mini-batch
- if is_labelindex:
+ if is_label_index:
self.queue.put((np.asarray(x), np.asarray(y, dtype=np.int32)))
else:
self.queue.put((np.asarray(x), y))