You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by wk...@apache.org on 2019/03/07 03:13:21 UTC
[incubator-mxnet] branch master updated: [MXNET-1291] solve pylint
errors in examples with issue no.12205 (#13848)
This is an automated email from the ASF dual-hosted git repository.
wkcn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new f4ab2d7 [MXNET-1291] solve pylint errors in examples with issue no.12205 (#13848)
f4ab2d7 is described below
commit f4ab2d759c8d13cd69faada2d0bc93a53dbe77b5
Author: Neil Chien <cc...@cs.ccu.edu.tw>
AuthorDate: Thu Mar 7 11:13:02 2019 +0800
[MXNET-1291] solve pylint errors in examples with issue no.12205 (#13848)
* [issue_12205 - PART1]
* [issue_12205 - PART2]
* Update example/cnn_chinese_text_classification/data_helpers.py
Co-Authored-By: cchung100m <cc...@cs.ccu.edu.tw>
* Update example/distributed_training/cifar10_dist.py
Co-Authored-By: cchung100m <cc...@cs.ccu.edu.tw>
* unify the style
unify the style
* unify the style
unify the style
* Update model.py
* unify the style here
unify the style here
* Remove the preposition 'with' in comment
Remove the preposition 'with' in comment 'Train module using Caffe operator in MXNet'
* Unify the style of comments
Unify the style of comments suggested by @sandeep-krishnamurthy
* add CONTRIBUTORS.md
* retrigger the CI
---
CONTRIBUTORS.md | 2 +-
example/bayesian-methods/bdk_demo.py | 1 -
example/caffe/caffe_net.py | 4 +-
example/caffe/train_model.py | 4 +-
.../data_helpers.py | 3 +-
.../cnn_chinese_text_classification/text_cnn.py | 9 +-
example/cnn_text_classification/data_helpers.py | 8 +-
example/cnn_text_classification/text_cnn.py | 162 +++++++++++++++------
example/ctc/captcha_generator.py | 45 ++----
example/ctc/ctc_metrics.py | 20 ++-
example/ctc/hyperparams.py | 6 +-
example/ctc/lstm_ocr_infer.py | 7 +-
example/ctc/lstm_ocr_train.py | 4 +-
example/ctc/multiproc_data.py | 36 ++---
example/ctc/ocr_iter.py | 17 ++-
example/ctc/ocr_predict.py | 22 +--
example/deep-embedded-clustering/autoencoder.py | 29 ++--
example/deep-embedded-clustering/data.py | 8 +-
example/deep-embedded-clustering/dec.py | 31 ++--
example/deep-embedded-clustering/model.py | 3 +-
example/distributed_training/cifar10_dist.py | 78 +++++++---
21 files changed, 288 insertions(+), 211 deletions(-)
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 0a7eb42..584940b 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -214,8 +214,8 @@ List of Contributors
* [Dang Trung Kien](https://github.com/kiendang)
* [Zach Boldyga](https://github.com/zboldyga)
* [Gordon Reid](https://github.com/gordon1992)
-
* [Ming Yang](http://ufoym.com)
+* [Neo Chien](https://github.com/cchung100m)
Label Bot
---------
diff --git a/example/bayesian-methods/bdk_demo.py b/example/bayesian-methods/bdk_demo.py
index 83a4319..bd883d2 100644
--- a/example/bayesian-methods/bdk_demo.py
+++ b/example/bayesian-methods/bdk_demo.py
@@ -162,7 +162,6 @@ def dev(gpu_id=None):
return mx.gpu(gpu_id) if gpu_id else mx.cpu()
-
def run_mnist_SGD(num_training=50000, gpu_id=None):
X, Y, X_test, Y_test = load_mnist(num_training)
minibatch_size = 100
diff --git a/example/caffe/caffe_net.py b/example/caffe/caffe_net.py
index 6796fca..803efda 100644
--- a/example/caffe/caffe_net.py
+++ b/example/caffe/caffe_net.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
"""Generate helper functions to load Caffe into MXNet"""
import argparse
import mxnet as mx
@@ -86,8 +87,7 @@ def get_network_from_json_file(file_name):
def parse_args():
- """Parse the arguments
- """
+ """Parse the arguments"""
parser = argparse.ArgumentParser(description='train an image classifier on mnist')
parser.add_argument('--network', type=str, default='lenet',
help='the cnn to use (mlp | lenet | <path to network json file>')
diff --git a/example/caffe/train_model.py b/example/caffe/train_model.py
index 16b1867..d7dfd5d 100644
--- a/example/caffe/train_model.py
+++ b/example/caffe/train_model.py
@@ -14,14 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Train module with using Caffe operator in MXNet"""
+"""Train module using Caffe operator in MXNet"""
import os
import logging
import mxnet as mx
def fit(args, network, data_loader, eval_metrics=None, batch_end_callback=None):
- """Train the model with using Caffe operator in MXNet"""
+ """Train the model using Caffe operator in MXNet"""
# kvstore
kv = mx.kvstore.create(args.kv_store)
diff --git a/example/cnn_chinese_text_classification/data_helpers.py b/example/cnn_chinese_text_classification/data_helpers.py
index 49bb3d5..af3c62f 100644
--- a/example/cnn_chinese_text_classification/data_helpers.py
+++ b/example/cnn_chinese_text_classification/data_helpers.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-"""Help functions to support for implementing CNN + Highway Network for Chinese Text Classification in MXNet"""
+"""Helper functions to support for implementing CNN + Highway Network for Chinese Text Classification in MXNet"""
import codecs
import itertools
diff --git a/example/cnn_chinese_text_classification/text_cnn.py b/example/cnn_chinese_text_classification/text_cnn.py
index ce70681..f17ccb2 100644
--- a/example/cnn_chinese_text_classification/text_cnn.py
+++ b/example/cnn_chinese_text_classification/text_cnn.py
@@ -66,7 +66,6 @@ parser.add_argument('--save-period', type=int, default=10,
def save_model():
"""Save cnn model
-
Returns
----------
callback: A callback function that can be passed as epoch_end_callback to fit
@@ -78,11 +77,9 @@ def save_model():
def highway(data):
"""Construct highway net
-
Parameters
----------
data:
-
Returns
----------
Highway Networks
@@ -104,7 +101,6 @@ def highway(data):
def data_iter(batch_size, num_embed, pre_trained_word2vec=False):
"""Construct data iter
-
Parameters
----------
batch_size: int
@@ -166,7 +162,6 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size,
num_label=2, filter_list=None, num_filter=100,
dropout=0.0, pre_trained_word2vec=False):
"""Generate network symbol
-
Parameters
----------
batch_size: int
@@ -178,7 +173,7 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size,
num_filter: int
dropout: int
pre_trained_word2vec: boolean
- identify the pre-trained layers or not
+ identify the pre-trained layers or not
Returns
----------
sm: symbol
@@ -236,7 +231,6 @@ def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size,
def train(symbol_data, train_iterator, valid_iterator, data_column_names, target_names):
"""Train cnn model
-
Parameters
----------
symbol_data: symbol
@@ -291,7 +285,6 @@ class CustomInit(Initializer):
"""https://mxnet.incubator.apache.org/api/python/optimization.html#mxnet.initializer.register
Create and register a custom initializer that
Initialize the weight and bias with custom requirements
-
"""
weightMethods = ["normal", "uniform", "orthogonal", "xavier"]
biasMethods = ["costant"]
diff --git a/example/cnn_text_classification/data_helpers.py b/example/cnn_text_classification/data_helpers.py
index 093da7b..78dacb0 100644
--- a/example/cnn_text_classification/data_helpers.py
+++ b/example/cnn_text_classification/data_helpers.py
@@ -15,7 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-"""Help functions to support for implementing CNN + Highway Network for Text Classification in MXNet"""
+"""
+Help functions to support for implementing CNN + Highway Network for Text Classification in MXNet
+"""
import itertools
import os
@@ -108,7 +110,9 @@ def build_input_data(sentences, labels, vocabulary):
def build_input_data_with_word2vec(sentences, labels, word2vec_list):
- """Map sentences and labels to vectors based on a pretrained word2vec"""
+ """
+ Map sentences and labels to vectors based on a pretrained word2vec
+ """
x_vec = []
for sent in sentences:
vec = []
diff --git a/example/cnn_text_classification/text_cnn.py b/example/cnn_text_classification/text_cnn.py
index 9ad9443..79c3945 100644
--- a/example/cnn_text_classification/text_cnn.py
+++ b/example/cnn_text_classification/text_cnn.py
@@ -19,12 +19,13 @@
# -*- coding: utf-8 -*-
-import sys
+"""Implementing CNN + Highway Network for Text Classification in MXNet"""
+
import os
-import mxnet as mx
-import numpy as np
-import argparse
import logging
+import argparse
+import numpy as np
+import mxnet as mx
import data_helpers
logging.basicConfig(level=logging.DEBUG)
@@ -54,26 +55,55 @@ parser.add_argument('--disp-batches', type=int, default=50,
parser.add_argument('--save-period', type=int, default=10,
help='save checkpoint for every n epochs')
+
def save_model():
+ """Save cnn model
+
+ Returns
+ ----------
+ callback: A callback function that can be passed as epoch_end_callback to fit
+ """
if not os.path.exists("checkpoint"):
os.mkdir("checkpoint")
return mx.callback.do_checkpoint("checkpoint/checkpoint", args.save_period)
+
def data_iter(batch_size, num_embed, pre_trained_word2vec=False):
+ """Construct data iter
+
+ Parameters
+ ----------
+ batch_size: int
+ num_embed: int
+ pre_trained_word2vec: boolean
+ identify the pre-trained layers or not
+ Returns
+ ----------
+ train_set: DataIter
+ Train DataIter
+ valid: DataIter
+ Valid DataIter
+ sentences_size: int
+ array dimensions
+ embedded_size: int
+ array dimensions
+ vocab_size: int
+ array dimensions
+ """
print('Loading data...')
if pre_trained_word2vec:
word2vec = data_helpers.load_pretrained_word2vec('data/rt.vec')
x, y = data_helpers.load_data_with_word2vec(word2vec)
- # reshpae for convolution input
+ # reshape for convolution input
x = np.reshape(x, (x.shape[0], 1, x.shape[1], x.shape[2]))
- embed_size = x.shape[-1]
- sentence_size = x.shape[2]
- vocab_size = -1
+ embedded_size = x.shape[-1]
+ sentences_size = x.shape[2]
+ vocabulary_size = -1
else:
x, y, vocab, vocab_inv = data_helpers.load_data()
- embed_size = num_embed
- sentence_size = x.shape[1]
- vocab_size = len(vocab)
+ embedded_size = num_embed
+ sentences_size = x.shape[1]
+ vocabulary_size = len(vocab)
# randomly shuffle data
np.random.seed(10)
@@ -87,27 +117,53 @@ def data_iter(batch_size, num_embed, pre_trained_word2vec=False):
print('Train/Valid split: %d/%d' % (len(y_train), len(y_dev)))
print('train shape:', x_train.shape)
print('valid shape:', x_dev.shape)
- print('sentence max words', sentence_size)
- print('embedding size', embed_size)
- print('vocab size', vocab_size)
+ print('sentence max words', sentences_size)
+ print('embedding size', embedded_size)
+ print('vocab size', vocabulary_size)
- train = mx.io.NDArrayIter(
+ train_set = mx.io.NDArrayIter(
x_train, y_train, batch_size, shuffle=True)
valid = mx.io.NDArrayIter(
x_dev, y_dev, batch_size)
- return (train, valid, sentence_size, embed_size, vocab_size)
+ return train_set, valid, sentences_size, embedded_size, vocabulary_size
-def sym_gen(batch_size, sentence_size, num_embed, vocab_size,
- num_label=2, filter_list=[3, 4, 5], num_filter=100,
+
+def sym_gen(batch_size, sentences_size, num_embed, vocabulary_size,
+ num_label=2, filter_list=None, num_filter=100,
dropout=0.0, pre_trained_word2vec=False):
+ """Generate network symbol
+
+ Parameters
+ ----------
+ batch_size: int
+ sentences_size: int
+ num_embed: int
+ vocabulary_size: int
+ num_label: int
+ filter_list: list
+ num_filter: int
+ dropout: int
+ pre_trained_word2vec: boolean
+ identify the pre-trained layers or not
+ Returns
+ ----------
+ sm: symbol
+ data: list of str
+ data names
+ softmax_label: list of str
+ label names
+ """
input_x = mx.sym.Variable('data')
input_y = mx.sym.Variable('softmax_label')
# embedding layer
if not pre_trained_word2vec:
- embed_layer = mx.sym.Embedding(data=input_x, input_dim=vocab_size, output_dim=num_embed, name='vocab_embed')
- conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentence_size, num_embed))
+ embed_layer = mx.sym.Embedding(data=input_x,
+ input_dim=vocabulary_size,
+ output_dim=num_embed,
+ name='vocab_embed')
+ conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentences_size, num_embed))
else:
conv_input = input_x
@@ -116,7 +172,7 @@ def sym_gen(batch_size, sentence_size, num_embed, vocab_size,
for i, filter_size in enumerate(filter_list):
convi = mx.sym.Convolution(data=conv_input, kernel=(filter_size, num_embed), num_filter=num_filter)
relui = mx.sym.Activation(data=convi, act_type='relu')
- pooli = mx.sym.Pooling(data=relui, pool_type='max', kernel=(sentence_size - filter_size + 1, 1), stride=(1,1))
+ pooli = mx.sym.Pooling(data=relui, pool_type='max', kernel=(sentences_size - filter_size + 1, 1), stride=(1, 1))
pooled_outputs.append(pooli)
# combine all pooled outputs
@@ -141,20 +197,39 @@ def sym_gen(batch_size, sentence_size, num_embed, vocab_size,
return sm, ('data',), ('softmax_label',)
-def train(symbol, train_iter, valid_iter, data_names, label_names):
- devs = mx.cpu() if args.gpus is None or args.gpus is '' else [
- mx.gpu(int(i)) for i in args.gpus.split(',')]
- module = mx.mod.Module(symbol, data_names=data_names, label_names=label_names, context=devs)
- module.fit(train_data = train_iter,
- eval_data = valid_iter,
- eval_metric = 'acc',
- kvstore = args.kv_store,
- optimizer = args.optimizer,
- optimizer_params = { 'learning_rate': args.lr },
- initializer = mx.initializer.Uniform(0.1),
- num_epoch = args.num_epochs,
- batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches),
- epoch_end_callback = save_model())
+
+def train(symbol_data, train_iterator, valid_iterator, data_column_names, target_names):
+ """Train cnn model
+
+ Parameters
+ ----------
+ symbol_data: symbol
+ train_iterator: DataIter
+ Train DataIter
+ valid_iterator: DataIter
+ Valid DataIter
+ data_column_names: list of str
+ Defaults to ('data') for a typical model used in image classification
+ target_names: list of str
+ Defaults to ('softmax_label') for a typical model used in image classification
+ """
+ devs = mx.cpu() # default setting
+ if args.gpus is not None:
+ for i in args.gpus.split(','):
+ mx.gpu(int(i))
+ devs = mx.gpu()
+ module = mx.mod.Module(symbol_data, data_names=data_column_names, label_names=target_names, context=devs)
+ module.fit(train_data=train_iterator,
+ eval_data=valid_iterator,
+ eval_metric='acc',
+ kvstore=args.kv_store,
+ optimizer=args.optimizer,
+ optimizer_params={'learning_rate': args.lr},
+ initializer=mx.initializer.Uniform(0.1),
+ num_epoch=args.num_epochs,
+ batch_end_callback=mx.callback.Speedometer(args.batch_size, args.disp_batches),
+ epoch_end_callback=save_model())
+
if __name__ == '__main__':
# parse args
@@ -162,14 +237,17 @@ if __name__ == '__main__':
# data iter
train_iter, valid_iter, sentence_size, embed_size, vocab_size = data_iter(args.batch_size,
- args.num_embed,
- args.pretrained_embedding)
+ args.num_embed,
+ args.pretrained_embedding)
# network symbol
symbol, data_names, label_names = sym_gen(args.batch_size,
- sentence_size,
- embed_size,
- vocab_size,
- num_label=2, filter_list=[3, 4, 5], num_filter=100,
- dropout=args.dropout, pre_trained_word2vec=args.pretrained_embedding)
+ sentence_size,
+ embed_size,
+ vocab_size,
+ num_label=2,
+ filter_list=[3, 4, 5],
+ num_filter=100,
+ dropout=args.dropout,
+ pre_trained_word2vec=args.pretrained_embedding)
# train cnn model
train(symbol, train_iter, valid_iter, data_names, label_names)
diff --git a/example/ctc/captcha_generator.py b/example/ctc/captcha_generator.py
index 97fab40..8a3d726 100644
--- a/example/ctc/captcha_generator.py
+++ b/example/ctc/captcha_generator.py
@@ -15,22 +15,20 @@
# specific language governing permissions and limitations
# under the License.
""" Helper classes for multiprocess captcha image generation
-
This module also provides script for saving captcha images to file using CLI.
"""
from __future__ import print_function
import random
+import numpy as np
from captcha.image import ImageCaptcha
import cv2
from multiproc_data import MPData
-import numpy as np
class CaptchaGen(object):
- """
- Generates a captcha image
+ """Generates a captcha image
"""
def __init__(self, h, w, font_paths):
"""
@@ -48,8 +46,7 @@ class CaptchaGen(object):
self.w = w
def image(self, captcha_str):
- """
- Generate a greyscale captcha image representing number string
+ """Generate a greyscale captcha image representing number string
Parameters
----------
@@ -71,8 +68,7 @@ class CaptchaGen(object):
class DigitCaptcha(object):
- """
- Provides shape() and get() interface for digit-captcha image generation
+ """Provides shape() and get() interface for digit-captcha image generation
"""
def __init__(self, font_paths, h, w, num_digit_min, num_digit_max):
"""
@@ -95,8 +91,7 @@ class DigitCaptcha(object):
@property
def shape(self):
- """
- Returns shape of the image data generated
+ """Returns shape of the image data generated
Returns
-------
@@ -105,8 +100,7 @@ class DigitCaptcha(object):
return self.captcha.h, self.captcha.w
def get(self):
- """
- Get an image from the queue
+ """Get an image from the queue
Returns
-------
@@ -117,9 +111,8 @@ class DigitCaptcha(object):
@staticmethod
def get_rand(num_digit_min, num_digit_max):
- """
- Generates a character string of digits. Number of digits are
- between self.num_digit_min and self.num_digit_max
+ """Generates a character string of digits. Number of digits are
+ between self.num_digit_min and self.num_digit_max
Returns
-------
str
@@ -131,8 +124,7 @@ class DigitCaptcha(object):
return buf
def _gen_sample(self):
- """
- Generate a random captcha image sample
+ """Generate a random captcha image sample
Returns
-------
(numpy.ndarray, str)
@@ -143,13 +135,10 @@ class DigitCaptcha(object):
class MPDigitCaptcha(DigitCaptcha):
- """
- Handles multi-process captcha image generation
+ """Handles multi-process captcha image generation
"""
def __init__(self, font_paths, h, w, num_digit_min, num_digit_max, num_processes, max_queue_size):
- """
-
- Parameters
+ """Parameters
----------
font_paths: list of str
List of path to ttf font files
@@ -170,14 +159,11 @@ class MPDigitCaptcha(DigitCaptcha):
self.mp_data = MPData(num_processes, max_queue_size, self._gen_sample)
def start(self):
- """
- Starts the processes
- """
+ """Starts the processes"""
self.mp_data.start()
def get(self):
- """
- Get an image from the queue
+ """Get an image from the queue
Returns
-------
@@ -187,9 +173,7 @@ class MPDigitCaptcha(DigitCaptcha):
return self.mp_data.get()
def reset(self):
- """
- Resets the generator by stopping all processes
- """
+ """Resets the generator by stopping all processes"""
self.mp_data.reset()
@@ -197,6 +181,7 @@ if __name__ == '__main__':
import argparse
def main():
+ """Program entry point"""
parser = argparse.ArgumentParser()
parser.add_argument("font_path", help="Path to ttf font file")
parser.add_argument("output", help="Output filename including extension (e.g. 'sample.jpg')")
diff --git a/example/ctc/ctc_metrics.py b/example/ctc/ctc_metrics.py
index 0db680a..5f51025 100644
--- a/example/ctc/ctc_metrics.py
+++ b/example/ctc/ctc_metrics.py
@@ -22,14 +22,17 @@ import numpy as np
class CtcMetrics(object):
+ """Module for calculating the prediction accuracy during training. Two accuracy measures are implemented:
+ A simple accuracy measure that calculates number of correct predictions divided by total number of predictions
+ and a second accuracy measure based on sum of Longest Common Sequence(LCS) ratio of all predictions divided by total
+ number of predictions
+ """
def __init__(self, seq_len):
self.seq_len = seq_len
@staticmethod
def ctc_label(p):
- """
- Iterates through p, identifying non-zero and non-repeating values, and returns them in a list
- Parameters
+ """Iterates through p, identifying non-zero and non-repeating values, and returns them in a list Parameters
----------
p: list of int
@@ -42,7 +45,7 @@ class CtcMetrics(object):
for i, _ in enumerate(p):
c1 = p1[i]
c2 = p1[i+1]
- if c2 == 0 or c2 == c1:
+ if c2 in (0, c1):
continue
ret.append(c2)
return ret
@@ -65,12 +68,16 @@ class CtcMetrics(object):
return 0
P = np.array(list(p)).reshape((1, len(p)))
L = np.array(list(l)).reshape((len(l), 1))
- M = np.int32(P == L)
+ M = np.ndarray(shape=(len(P), len(L)), dtype=np.int32)
for i in range(M.shape[0]):
for j in range(M.shape[1]):
up = 0 if i == 0 else M[i-1, j]
left = 0 if j == 0 else M[i, j-1]
- M[i, j] = max(up, left, M[i, j] if (i == 0 or j == 0) else M[i, j] + M[i-1, j-1])
+
+ if i == 0 or j == 0:
+ M[i, j] = max(up, left, M[i, j])
+ else:
+ M[i, j] = M[i, j] + M[i - 1, j - 1]
return M.max()
def accuracy(self, label, pred):
@@ -111,4 +118,3 @@ class CtcMetrics(object):
total += 1.0
assert total == batch_size
return hit / total
-
diff --git a/example/ctc/hyperparams.py b/example/ctc/hyperparams.py
index cdcb874..ef24401 100644
--- a/example/ctc/hyperparams.py
+++ b/example/ctc/hyperparams.py
@@ -20,9 +20,7 @@ from __future__ import print_function
class Hyperparams(object):
- """
- Hyperparameters for LSTM network
- """
+ """Hyperparameters for LSTM network"""
def __init__(self):
# Training hyper parameters
self._train_epoch_size = 30000
@@ -48,7 +46,7 @@ class Hyperparams(object):
@property
def batch_size(self):
return self._batch_size
-
+
@property
def num_epoch(self):
return self._num_epoch
diff --git a/example/ctc/lstm_ocr_infer.py b/example/ctc/lstm_ocr_infer.py
index 80de2c7..eec48ea 100644
--- a/example/ctc/lstm_ocr_infer.py
+++ b/example/ctc/lstm_ocr_infer.py
@@ -20,12 +20,12 @@ from __future__ import print_function
import argparse
+import numpy as np
from ctc_metrics import CtcMetrics
import cv2
from hyperparams import Hyperparams
import lstm
import mxnet as mx
-import numpy as np
from ocr_iter import SimpleBatch
@@ -46,8 +46,7 @@ def lstm_init_states(batch_size):
def load_module(prefix, epoch, data_names, data_shapes):
- """
- Loads the model from checkpoint specified by prefix and epoch, binds it
+ """Loads the model from checkpoint specified by prefix and epoch, binds it
to an executor, and sets its parameters and returns a mx.mod.Module
"""
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
@@ -64,6 +63,7 @@ def load_module(prefix, epoch, data_names, data_shapes):
def main():
+ """Program entry point"""
parser = argparse.ArgumentParser()
parser.add_argument("path", help="Path to the CAPTCHA image file")
parser.add_argument("--prefix", help="Checkpoint prefix [Default 'ocr']", default='ocr')
@@ -86,7 +86,6 @@ def main():
# Predictions are 1 to 10 for digits 0 to 9 respectively (prediction 0 means no-digit)
prediction = [p - 1 for p in prediction]
print("Digits:", prediction)
- return
if __name__ == '__main__':
diff --git a/example/ctc/lstm_ocr_train.py b/example/ctc/lstm_ocr_train.py
index 2c25f7e..49d9531 100644
--- a/example/ctc/lstm_ocr_train.py
+++ b/example/ctc/lstm_ocr_train.py
@@ -42,7 +42,7 @@ def get_fonts(path):
def parse_args():
- # Parse command line arguments
+ """Parse command line arguments"""
parser = argparse.ArgumentParser()
parser.add_argument("font_path", help="Path to ttf font file or directory containing ttf files")
parser.add_argument("--loss", help="'ctc' or 'warpctc' loss [Default 'ctc']", default='ctc')
@@ -56,6 +56,7 @@ def parse_args():
def main():
+ """Program entry point"""
args = parse_args()
if not any(args.loss == s for s in ['ctc', 'warpctc']):
raise ValueError("Invalid loss '{}' (must be 'ctc' or 'warpctc')".format(args.loss))
@@ -122,4 +123,3 @@ def main():
if __name__ == '__main__':
main()
-
diff --git a/example/ctc/multiproc_data.py b/example/ctc/multiproc_data.py
index c5f8da5..313ab4e 100644
--- a/example/ctc/multiproc_data.py
+++ b/example/ctc/multiproc_data.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+"""Contains a class for handling multi-process data generation"""
+
from __future__ import print_function
from ctypes import c_bool
import multiprocessing as mp
@@ -25,12 +27,9 @@ except ImportError:
from Queue import Full as QFullExcept
from Queue import Empty as QEmptyExcept
-import numpy as np
-
class MPData(object):
- """
- Handles multi-process data generation.
+ """Handles multi-process data generation.
Operation:
- call start() to start the data generation
@@ -38,9 +37,7 @@ class MPData(object):
- call reset() to stop data generation
"""
def __init__(self, num_processes, max_queue_size, fn):
- """
-
- Parameters
+ """Parameters
----------
num_processes: int
Number of processes to spawn
@@ -56,22 +53,12 @@ class MPData(object):
self.fn = fn
def start(self):
- """
- Starts the processes
- Parameters
- ----------
- fn: function
-
- """
- """
- Starts the processes
- """
+ """Starts the processes"""
self._init_proc()
@staticmethod
def _proc_loop(proc_id, alive, queue, fn):
- """
- Thread loop for generating data
+ """Thread loop for generating data
Parameters
----------
@@ -102,9 +89,7 @@ class MPData(object):
queue.close()
def _init_proc(self):
- """
- Start processes if not already started
- """
+ """Start processes if not already started"""
if not self.proc:
self.proc = [
mp.Process(target=self._proc_loop, args=(i, self.alive, self.queue, self.fn))
@@ -115,8 +100,7 @@ class MPData(object):
p.start()
def get(self):
- """
- Get a datum from the queue
+ """Get a datum from the queue
Returns
-------
@@ -127,9 +111,7 @@ class MPData(object):
return self.queue.get()
def reset(self):
- """
- Resets the generator by stopping all processes
- """
+ """Resets the generator by stopping all processes"""
self.alive.value = False
qsize = 0
try:
diff --git a/example/ctc/ocr_iter.py b/example/ctc/ocr_iter.py
index 1432e92..5a02f38 100644
--- a/example/ctc/ocr_iter.py
+++ b/example/ctc/ocr_iter.py
@@ -23,7 +23,11 @@ import mxnet as mx
class SimpleBatch(object):
- def __init__(self, data_names, data, label_names=list(), label=list()):
+ """Batch class for getting label data
+ Operation:
+ - call get_label() to start label data generation
+ """
+ def __init__(self, data_names, data, label_names=None, label=None):
self._data = data
self._label = label
self._data_names = data_names
@@ -59,20 +63,17 @@ class SimpleBatch(object):
def get_label(buf):
ret = np.zeros(4)
- for i in range(len(buf)):
- ret[i] = 1 + int(buf[i])
+ for i, element in enumerate(buf):
+ ret[i] = 1 + int(element)
if len(buf) == 3:
ret[3] = 0
return ret
class OCRIter(mx.io.DataIter):
- """
- Iterator class for generating captcha image data
- """
+ """Iterator class for generating captcha image data"""
def __init__(self, count, batch_size, lstm_init_states, captcha, name):
- """
- Parameters
+ """Parameters
----------
count: int
Number of batches to produce for one epoch
diff --git a/example/ctc/ocr_predict.py b/example/ctc/ocr_predict.py
index 2cf1967..c5c691b 100644
--- a/example/ctc/ocr_predict.py
+++ b/example/ctc/ocr_predict.py
@@ -21,18 +21,12 @@ from __future__ import print_function
import argparse
import sys
-import cv2
import numpy as np
-import mxnet as mx
-from collections import namedtuple
-from ocr_iter import SimpleBatch
-from captcha_generator import DigitCaptcha
-from ctc_metrics import CtcMetrics
-import lstm
-from hyperparams import Hyperparams
+import cv2
class lstm_ocr_model(object):
+ """LSTM network for predicting the Optical Character Recognition"""
# Keep Zero index for blank. (CTC request it)
CONST_CHAR = '0123456789'
@@ -67,6 +61,16 @@ class lstm_ocr_model(object):
all_shapes_dict)
def forward_ocr(self, img_):
+ """Forward the image through the LSTM network model
+
+ Parameters
+ ----------
+ img_: int of array
+
+ Returns
+ ----------
+ label_list: string of list
+ """
img_ = cv2.resize(img_, (80, 30))
img_ = img_.transpose(1, 0)
print(img_.shape)
@@ -92,7 +96,7 @@ class lstm_ocr_model(object):
for i in range(len(label_list)):
c1 = label_list2[i]
c2 = label_list2[i+1]
- if c2 == 0 or c2 == c1:
+ if c2 in (0, c1):
continue
ret.append(c2)
# change to ascii
diff --git a/example/deep-embedded-clustering/autoencoder.py b/example/deep-embedded-clustering/autoencoder.py
index 096f045..c756344 100644
--- a/example/deep-embedded-clustering/autoencoder.py
+++ b/example/deep-embedded-clustering/autoencoder.py
@@ -19,9 +19,8 @@
from __future__ import print_function
import logging
-
-import mxnet as mx
import numpy as np
+import mxnet as mx
import model
from solver import Solver, Monitor
@@ -95,18 +94,18 @@ class AutoEncoderModel(model.MXModel):
else:
x = mx.symbol.LinearRegressionOutput(data=x, label=data)
- args = {'encoder_%d_weight'%istack: mx.nd.empty((num_hidden, num_input), self.xpu),
- 'encoder_%d_bias'%istack: mx.nd.empty((num_hidden,), self.xpu),
- 'decoder_%d_weight'%istack: mx.nd.empty((num_input, num_hidden), self.xpu),
- 'decoder_%d_bias'%istack: mx.nd.empty((num_input,), self.xpu),}
- args_grad = {'encoder_%d_weight'%istack: mx.nd.empty((num_hidden, num_input), self.xpu),
- 'encoder_%d_bias'%istack: mx.nd.empty((num_hidden,), self.xpu),
- 'decoder_%d_weight'%istack: mx.nd.empty((num_input, num_hidden), self.xpu),
- 'decoder_%d_bias'%istack: mx.nd.empty((num_input,), self.xpu),}
- args_mult = {'encoder_%d_weight'%istack: 1.0,
- 'encoder_%d_bias'%istack: 2.0,
- 'decoder_%d_weight'%istack: 1.0,
- 'decoder_%d_bias'%istack: 2.0,}
+ args = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu),
+ 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu),
+ 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu),
+ 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), }
+ args_grad = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu),
+ 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu),
+ 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu),
+ 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), }
+ args_mult = {'encoder_%d_weight' % istack: 1.0,
+ 'encoder_%d_bias' % istack: 2.0,
+ 'decoder_%d_weight' % istack: 1.0,
+ 'decoder_%d_bias' % istack: 2.0, }
auxs = {}
if encoder_act == 'sigmoid' and sparseness_penalty:
auxs['sparse_encoder_%d_moving_avg' % istack] = mx.nd.ones(num_hidden, self.xpu) * 0.5
@@ -203,4 +202,4 @@ class AutoEncoderModel(model.MXModel):
last_batch_handle='pad')
Y = list(model.extract_feature(
self.loss, self.args, self.auxs, data_iter, X.shape[0], self.xpu).values())[0]
- return np.mean(np.square(Y-X))/2.0
\ No newline at end of file
+ return np.mean(np.square(Y-X))/2.0
diff --git a/example/deep-embedded-clustering/data.py b/example/deep-embedded-clustering/data.py
index 3649990..4ae3271 100644
--- a/example/deep-embedded-clustering/data.py
+++ b/example/deep-embedded-clustering/data.py
@@ -18,10 +18,8 @@
# pylint: disable=missing-docstring
from __future__ import print_function
-import os
-
-import mxnet as mx
import numpy as np
+import mxnet as mx
def get_mnist():
@@ -35,7 +33,3 @@ def get_mnist():
X = X[p].reshape((X.shape[0], -1)).astype(np.float32)*5
Y = Y[p]
return X, Y
-
-
-
-
diff --git a/example/deep-embedded-clustering/dec.py b/example/deep-embedded-clustering/dec.py
index d759470..8fb3891 100644
--- a/example/deep-embedded-clustering/dec.py
+++ b/example/deep-embedded-clustering/dec.py
@@ -17,27 +17,29 @@
# pylint: skip-file
from __future__ import print_function
-import sys
+
import os
-import mxnet as mx
+import logging
import numpy as np
-import data
-from scipy.spatial.distance import cdist
from sklearn.cluster import KMeans
+from scipy.spatial.distance import cdist
+import mxnet as mx
+import data
import model
from autoencoder import AutoEncoderModel
from solver import Solver, Monitor
-import logging
+
def cluster_acc(Y_pred, Y):
from sklearn.utils.linear_assignment_ import linear_assignment
assert Y_pred.size == Y.size
D = max(Y_pred.max(), Y.max())+1
- w = np.zeros((D,D), dtype=np.int64)
+ w = np.zeros((D, D), dtype=np.int64)
for i in range(Y_pred.size):
w[Y_pred[i], int(Y[i])] += 1
ind = linear_assignment(w.max() - w)
- return sum([w[i,j] for i,j in ind])*1.0/Y_pred.size, w
+ return sum([w[i, j] for i, j in ind])*1.0/Y_pred.size, w
+
class DECModel(model.MXModel):
class DECLoss(mx.operator.NumpyOp):
@@ -81,12 +83,12 @@ class DECModel(model.MXModel):
sep = X.shape[0]*9//10
X_train = X[:sep]
X_val = X[sep:]
- ae_model = AutoEncoderModel(self.xpu, [X.shape[1],500,500,2000,10], pt_dropout=0.2)
+ ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10], pt_dropout=0.2)
if not os.path.exists(save_to+'_pt.arg'):
ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0,
- lr_scheduler=mx.lr_scheduler.FactorScheduler(20000,0.1))
+ lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1))
ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0,
- lr_scheduler=mx.lr_scheduler.FactorScheduler(20000,0.1))
+ lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1))
ae_model.save(save_to+'_pt.arg')
logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train))
logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val))
@@ -98,9 +100,9 @@ class DECModel(model.MXModel):
label = mx.sym.Variable('label')
self.feature = self.ae_model.encoder
self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec')
- self.args.update({k:v for k,v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()})
+ self.args.update({k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()})
self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu)
- self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k,v in self.args.items()})
+ self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items()})
self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args})
self.num_centers = num_centers
@@ -117,6 +119,7 @@ class DECModel(model.MXModel):
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01)
+
def ce(label, pred):
return np.sum(label*np.log(label/(pred+0.000001)))/label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
@@ -125,6 +128,7 @@ class DECModel(model.MXModel):
train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff}, batch_size=batch_size,
shuffle=False, last_batch_handle='roll_over')
self.y_pred = np.zeros((X.shape[0]))
+
def refresh(i):
if i%update_interval == 0:
z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0]
@@ -155,6 +159,7 @@ class DECModel(model.MXModel):
else:
return -1
+
def mnist_exp(xpu):
X, Y = data.get_mnist()
if not os.path.isdir('data'):
@@ -167,7 +172,7 @@ def mnist_exp(xpu):
logging.info(str(acc))
logging.info('Best Clustering ACC: %f at update_interval: %d'%(np.max(acc), 10*(2**np.argmax(acc))))
+
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
mnist_exp(mx.gpu(0))
-
diff --git a/example/deep-embedded-clustering/model.py b/example/deep-embedded-clustering/model.py
index b388c55..ac029c1 100644
--- a/example/deep-embedded-clustering/model.py
+++ b/example/deep-embedded-clustering/model.py
@@ -20,7 +20,6 @@ from __future__ import print_function
import numpy as np
import mxnet as mx
-
try:
import cPickle as pickle
except ImportError:
@@ -54,7 +53,7 @@ def extract_feature(sym, args, auxs, data_iter, N, xpu=mx.cpu()):
class MXModel(object):
- def __init__(self, *args, xpu=mx.cpu(), **kwargs):
+ def __init__(self, xpu=mx.cpu(), *args, **kwargs):
self.xpu = xpu
self.loss = None
self.args = {}
diff --git a/example/distributed_training/cifar10_dist.py b/example/distributed_training/cifar10_dist.py
index 506afbb..d3ba515 100644
--- a/example/distributed_training/cifar10_dist.py
+++ b/example/distributed_training/cifar10_dist.py
@@ -17,15 +17,17 @@
# specific language governing permissions and limitations
# under the License.
+"""cifar10_dist.py contains code that trains a ResNet18 network using distributed training"""
+
from __future__ import print_function
-import random, sys
+import sys
+import random
+import numpy as np
import mxnet as mx
from mxnet import autograd, gluon, kv, nd
from mxnet.gluon.model_zoo import vision
-import numpy as np
-
# Create a distributed key-value store
store = kv.create('dist')
@@ -45,11 +47,13 @@ batch_size = batch_size_per_gpu * gpus_per_machine
# Create the context (a list of all GPUs to be used for training)
ctx = [mx.gpu(i) for i in range(gpus_per_machine)]
+
# Convert to float 32
# Having channel as the first dimension makes computation more efficient. Hence the (2,0,1) transpose.
# Dividing by 255 normalizes the input between 0 and 1
def transform(data, label):
- return nd.transpose(data.astype(np.float32), (2,0,1))/255, label.astype(np.float32)
+ return nd.transpose(data.astype(np.float32), (2, 0, 1))/255, label.astype(np.float32)
+
class SplitSampler(gluon.data.sampler.Sampler):
""" Split the dataset into `num_parts` parts and sample from the part with index `part_index`
@@ -80,14 +84,14 @@ class SplitSampler(gluon.data.sampler.Sampler):
def __len__(self):
return self.part_len
+
# Load the training data
-train_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True, transform=transform),
- batch_size,
- sampler=SplitSampler(50000, store.num_workers, store.rank))
+train_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=True, transform=transform), batch_size,
+ sampler=SplitSampler(50000, store.num_workers, store.rank))
-# Load the test data
+# Load the test data
test_data = gluon.data.DataLoader(gluon.data.vision.CIFAR10(train=False, transform=transform),
- batch_size, shuffle=False)
+ batch_size, shuffle=False)
# Use ResNet from model zoo
net = vision.resnet18_v1()
@@ -98,12 +102,25 @@ net.collect_params().initialize(mx.init.Xavier(), ctx=ctx)
# SoftmaxCrossEntropy is the most common choice of loss function for multiclass classification
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
-# Use Adam optimizer. Ask trainer to use the distributer kv store.
+# Use Adam optimizer. Ask trainer to use the distributor kv store.
trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': .001}, kvstore=store)
+
# Evaluate accuracy of the given network using the given data
-def evaluate_accuracy(data_iterator, net):
+def evaluate_accuracy(data_iterator, network):
+ """ Measure the accuracy of ResNet
+
+ Parameters
+ ----------
+ data_iterator: Iter
+ examples of dataset
+ network:
+ ResNet
+ Returns
+ ----------
+ tuple of array element
+ """
acc = mx.metric.Accuracy()
# Iterate through data and label
@@ -115,7 +132,7 @@ def evaluate_accuracy(data_iterator, net):
# Get network's output which is a probability distribution
# Apply argmax on the probability distribution to get network's classification.
- output = net(data)
+ output = network(data)
predictions = nd.argmax(output, axis=1)
# Give network's prediction and the correct label to update the metric
@@ -124,38 +141,54 @@ def evaluate_accuracy(data_iterator, net):
# Return the accuracy
return acc.get()[1]
+
# We'll use cross entropy loss since we are doing multiclass classification
loss = gluon.loss.SoftmaxCrossEntropyLoss()
+
# Run one forward and backward pass on multiple GPUs
-def forward_backward(net, data, label):
+def forward_backward(network, data, label):
# Ask autograd to remember the forward pass
with autograd.record():
# Compute the loss on all GPUs
- losses = [loss(net(X), Y) for X, Y in zip(data, label)]
+ losses = [loss(network(X), Y) for X, Y in zip(data, label)]
# Run the backward pass (calculate gradients) on all GPUs
for l in losses:
l.backward()
+
# Train a batch using multiple GPUs
-def train_batch(batch, ctx, net, trainer):
+def train_batch(batch_list, context, network, gluon_trainer):
+ """ Training with multiple GPUs
+ Parameters
+ ----------
+ batch_list: List
+ list of dataset
+ context: List
+ a list of all GPUs to be used for training
+ network:
+ ResNet
+ gluon_trainer:
+ rain module of gluon
+ """
# Split and load data into multiple GPUs
- data = batch[0]
- data = gluon.utils.split_and_load(data, ctx)
+ data = batch_list[0]
+ data = gluon.utils.split_and_load(data, context)
# Split and load label into multiple GPUs
- label = batch[1]
- label = gluon.utils.split_and_load(label, ctx)
+ label = batch_list[1]
+ label = gluon.utils.split_and_load(label, context)
# Run the forward and backward pass
- forward_backward(net, data, label)
+ forward_backward(network, data, label)
# Update the parameters
- this_batch_size = batch[0].shape[0]
- trainer.step(this_batch_size)
+ this_batch_size = batch_list[0].shape[0]
+ gluon_trainer.step(this_batch_size)
+
# Run as many epochs as required
for epoch in range(epochs):
@@ -173,4 +206,3 @@ for epoch in range(epochs):
test_accuracy = evaluate_accuracy(test_data, net)
print("Epoch %d: Test_acc %f" % (epoch, test_accuracy))
sys.stdout.flush()
-