You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by in...@apache.org on 2018/04/23 18:20:19 UTC
[incubator-mxnet] branch master updated: [MXNET-307] Add tutorial
tests to the CI (#10608)
This is an automated email from the ASF dual-hosted git repository.
indhub pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 6c3e1a4 [MXNET-307] Add tutorial tests to the CI (#10608)
6c3e1a4 is described below
commit 6c3e1a4ba44ccdceabfd28b933303f4f1a1a0337
Author: ThomasDelteil <th...@gmail.com>
AuthorDate: Mon Apr 23 11:20:05 2018 -0700
[MXNET-307] Add tutorial tests to the CI (#10608)
* Initial attempt at linking tutorials in CI
* Update CI to link tutorials
* update
* update typos
* Update ubuntu_scala.sh
trying to find out why sometimes it crashes
* trigger build
* trigger build
* link_tutorial_ci
* fix tutorials
* Update test_tutorials.py
* trigger build
* set seed for test
---
Jenkinsfile | 22 +++
ci/build.py | 27 ++-
ci/docker/Dockerfile.build.ubuntu_gpu | 6 +
ci/docker/install/ubuntu_docs.sh | 2 +-
ci/docker/install/ubuntu_python.sh | 4 +-
ci/docker/install/ubuntu_scala.sh | 8 +-
.../{ubuntu_docs.sh => ubuntu_tutorials.sh} | 8 +-
ci/docker/runtime_functions.sh | 23 +++
docs/build_version_doc/README.md | 2 +
docs/mxdoc.py | 6 +-
docs/tutorials/basic/module.md | 2 -
docs/tutorials/gluon/datasets.md | 23 ++-
docs/tutorials/onnx/fine_tuning_gluon.md | 2 +-
docs/tutorials/onnx/inference_on_onnx_model.md | 2 +-
docs/tutorials/sparse/csr.md | 3 -
docs/tutorials/sparse/train.md | 5 +-
docs/tutorials/unsupervised_learning/gan.md | 36 ++--
.../tutorials/vision/large_scale_classification.md | 12 +-
tests/nightly/Jenkinsfile | 28 ---
tests/nightly/test_tutorial.py | 138 --------------
tests/nightly/test_tutorial_config.txt | 31 ----
tests/tutorials/test_sanity_tutorials.py | 83 +++++++++
tests/tutorials/test_tutorials.py | 200 +++++++++++++++++++++
23 files changed, 418 insertions(+), 255 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6c39736..a4bf2c4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -722,6 +722,28 @@ try {
}
}
}
+ },
+ 'tutorial tests Python 2 GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/it-tutorials-py2') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ unpack_lib('gpu')
+ sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python2_gpu"
+ }
+ }
+ }
+ },
+ 'tutorial tests Python 3 GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/it-tutorials-py3') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ unpack_lib('gpu')
+ sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python3_gpu"
+ }
+ }
+ }
}
}
diff --git a/ci/build.py b/ci/build.py
index ee36876..6d8d014 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -89,13 +89,19 @@ def buildir() -> str:
return os.path.join(get_mxnet_root(), "build")
-def container_run(platform: str, docker_binary: str, command: List[str], dry_run: bool = False, into_container: bool = False) -> str:
+def container_run(platform: str,
+ docker_binary: str,
+ shared_memory_size: str,
+ command: List[str],
+ dry_run: bool = False,
+ into_container: bool = False) -> str:
tag = get_docker_tag(platform)
mx_root = get_mxnet_root()
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
- runlist = [docker_binary, 'run', '--rm',
+ runlist = [docker_binary, 'run', '--rm', '-t',
+ '--shm-size={}'.format(shared_memory_size),
'-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
'-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
@@ -157,6 +163,11 @@ def main() -> int:
help="Use nvidia docker",
action='store_true')
+ parser.add_argument("--shm-size",
+ help="Size of the shared memory /dev/shm allocated in the container (e.g '1g')",
+ default='500m',
+ dest="shared_memory_size")
+
parser.add_argument("-l", "--list",
help="List platforms",
action='store_true')
@@ -176,11 +187,11 @@ def main() -> int:
args = parser.parse_args()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
+ shared_memory_size = args.shared_memory_size
print("into container: {}".format(args.into_container))
if args.list:
list_platforms()
-
elif args.platform:
platform = args.platform
build_docker(platform, docker_binary)
@@ -190,15 +201,15 @@ def main() -> int:
tag = get_docker_tag(platform)
if command:
- container_run(platform, docker_binary, command)
+ container_run(platform, docker_binary, shared_memory_size, command)
elif args.print_docker_run:
- print(container_run(platform, docker_binary, [], True))
+ print(container_run(platform, docker_binary, shared_memory_size, [], True))
elif args.into_container:
- container_run(platform, docker_binary, [], False, True)
+ container_run(platform, docker_binary, shared_memory_size, [], False, True)
else:
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
logging.info("No command specified, trying default build: %s", ' '.join(cmd))
- container_run(platform, docker_binary, cmd)
+ container_run(platform, docker_binary, shared_memory_size, cmd)
elif args.all:
platforms = get_platforms()
@@ -211,7 +222,7 @@ def main() -> int:
build_platform = "build_{}".format(platform)
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
shutil.rmtree(buildir(), ignore_errors=True)
- container_run(platform, docker_binary, cmd)
+ container_run(platform, docker_binary, shared_memory_size, cmd)
plat_buildir = os.path.join(get_mxnet_root(), build_platform)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 625d570..547f984 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -44,6 +44,12 @@ COPY install/ubuntu_llvm.sh /work/
RUN /work/ubuntu_llvm.sh
COPY install/ubuntu_caffe.sh /work/
RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_onnx.sh /work/
+RUN /work/ubuntu_onnx.sh
+COPY install/ubuntu_docs.sh /work/
+RUN /work/ubuntu_docs.sh
+COPY install/ubuntu_tutorials.sh /work/
+RUN /work/ubuntu_tutorials.sh
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh
index bb8026f..3729f99 100755
--- a/ci/docker/install/ubuntu_docs.sh
+++ b/ci/docker/install/ubuntu_docs.sh
@@ -25,4 +25,4 @@ wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
apt-get install -y doxygen libatlas-base-dev graphviz pandoc
-pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
+pip install sphinx==1.5.6 CommonMark==0.5.4 breathe mock recommonmark==0.4.0 pypandoc beautifulsoup4
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh
index 948d8a2..554000d 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -29,5 +29,5 @@ wget -nv https://bootstrap.pypa.io/get-pip.py
python3 get-pip.py
python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py scipy
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py scipy
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy
diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh
index ee5554d..ce17a78 100755
--- a/ci/docker/install/ubuntu_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -23,9 +23,7 @@
set -ex
# install libraries for mxnet's scala package on ubuntu
apt-get install -y software-properties-common
-add-apt-repository -y ppa:webupd8team/java
apt-get update
-echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
-apt-get install -y oracle-java8-installer
-apt-get install -y oracle-java8-set-default
-apt-get update && apt-get install -y maven
\ No newline at end of file
+apt-get install -y openjdk-8-jdk
+apt-get install -y openjdk-8-jre
+apt-get update && apt-get install -y maven
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_tutorials.sh
similarity index 76%
copy from ci/docker/install/ubuntu_docs.sh
copy to ci/docker/install/ubuntu_tutorials.sh
index bb8026f..886ce93 100755
--- a/ci/docker/install/ubuntu_docs.sh
+++ b/ci/docker/install/ubuntu_tutorials.sh
@@ -21,8 +21,6 @@
# the whole docker cache for the image
set -ex
-wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
- dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-apt-get install -y doxygen libatlas-base-dev graphviz pandoc
-pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
+apt-get install graphviz python-opencv
+pip2 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz
+pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz
\ No newline at end of file
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 44de137..d6a1d6c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -349,6 +349,7 @@ sanity_check() {
tools/license_header.py check
make cpplint rcpplint jnilint
make pylint
+ nosetests-3.4 tests/tutorials/test_sanity_tutorials.py
}
@@ -385,6 +386,28 @@ unittest_ubuntu_python2_gpu() {
nosetests-2.7 --verbose tests/python/gpu
}
+tutorialtest_ubuntu_python3_gpu() {
+ set -ex
+ cd /work/mxnet/docs
+ export MXNET_DOCS_BUILD_MXNET=0
+ make html
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ export PYTHONPATH=/work/mxnet/python/
+ export MXNET_TUTORIAL_TEST_KERNEL=python3
+ cd /work/mxnet/tests/tutorials && nosetests-3.4 test_tutorials.py --nologcapture
+}
+
+tutorialtest_ubuntu_python2_gpu() {
+ set -ex
+ cd /work/mxnet/docs
+ export MXNET_DOCS_BUILD_MXNET=0
+ make html
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ export PYTHONPATH=/work/mxnet/python/
+ export MXNET_TUTORIAL_TEST_KERNEL=python2
+ cd /work/mxnet/tests/tutorials && nosetests-3.4 test_tutorials.py --nologcapture
+}
+
unittest_ubuntu_python3_gpu() {
set -ex
export PYTHONPATH=./python/
diff --git a/docs/build_version_doc/README.md b/docs/build_version_doc/README.md
index 221618b..6e768a3 100644
--- a/docs/build_version_doc/README.md
+++ b/docs/build_version_doc/README.md
@@ -133,6 +133,8 @@ From the MXNet source root run:
make docs USE_OPENMP=1
```
+If you set `MXNET_DOCS_BUILD_MXNET=0`, the doc generation will skip the build step. This is useful when you have already built MXNet locally.
+
The files from `make docs` are viewable in `docs/_build/html/`.
**NOTE:** `make docs` doesn't add any version information, and the versions dropdown in the top level navigation is not present. UI bugs can be introduced when the versions dropdown is included, so just testing with `make docs` may be insufficient.
diff --git a/docs/mxdoc.py b/docs/mxdoc.py
index 7bec720..9be7160 100644
--- a/docs/mxdoc.py
+++ b/docs/mxdoc.py
@@ -367,7 +367,11 @@ def add_buttons(app, docname, source):
# source[i] = '\n'.join(lines)
def setup(app):
- app.connect("builder-inited", build_mxnet)
+
+ # If MXNET_DOCS_BUILD_MXNET is set something different than 1
+ # Skip the build step
+ if os.getenv('MXNET_DOCS_BUILD_MXNET', '1') == '1':
+ app.connect("builder-inited", build_mxnet)
app.connect("builder-inited", generate_doxygen)
app.connect("builder-inited", build_scala_docs)
# skipped to build r, it requires to install latex, which is kinds of too heavy
diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md
index 11a3ffb..dbc7b7a 100644
--- a/docs/tutorials/basic/module.md
+++ b/docs/tutorials/basic/module.md
@@ -313,5 +313,3 @@ assert score[0][1] > 0.77, "Achieved accuracy (%f) is less than expected (0.77)"
<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
-
-
diff --git a/docs/tutorials/gluon/datasets.md b/docs/tutorials/gluon/datasets.md
index 21eac78..0b0038d 100644
--- a/docs/tutorials/gluon/datasets.md
+++ b/docs/tutorials/gluon/datasets.md
@@ -47,13 +47,14 @@ We get a tuple of a data sample and its corresponding label, which makes sense b
A [`DataLoader`](https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataloader#mxnet.gluon.data.DataLoader) is used to create mini-batches of samples from a [`Dataset`](https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataset#mxnet.gluon.data.Dataset), and provides a convenient iterator interface for looping these batches. It's typically much more efficient to pass a mini-batch of data through a neural network than a single sample at a time, be [...]
-Another benefit of using [`DataLoader`](https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataloader#mxnet.gluon.data.DataLoader) is the ability to easily load data in parallel using [`multiprocessing`](https://docs.python.org/3.6/library/multiprocessing.html). Just set the `num_workers` parameter to the number of CPUs avaliable on your machine for maximum performance.
+Another benefit of using [`DataLoader`](https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataloader#mxnet.gluon.data.DataLoader) is the ability to easily load data in parallel using [`multiprocessing`](https://docs.python.org/3.6/library/multiprocessing.html). You can set the `num_workers` parameter to the number of CPUs avalaible on your machine for maximum performance, or limit it to a lower number to spare resources.
```python
from multiprocessing import cpu_count
+CPU_COUNT = cpu_count()
-data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, num_workers=cpu_count())
+data_loader = mx.gluon.data.DataLoader(dataset, batch_size=5, num_workers=CPU_COUNT)
for X_batch, y_batch in data_loader:
print("X_batch has shape {}, and y_batch has shape {}".format(X_batch.shape, y_batch.shape))
@@ -120,8 +121,8 @@ If you have more complex shuffling requirements (e.g. when handling sequential d
```python
batch_size = 32
-train_data_loader = mx.gluon.data.DataLoader(train_dataset, batch_size, shuffle=True, num_workers=cpu_count())
-valid_data_loader = mx.gluon.data.DataLoader(valid_dataset, batch_size, num_workers=cpu_count())
+train_data_loader = mx.gluon.data.DataLoader(train_dataset, batch_size, shuffle=True, num_workers=CPU_COUNT)
+valid_data_loader = mx.gluon.data.DataLoader(valid_dataset, batch_size, num_workers=CPU_COUNT)
```
With both `DataLoader`s defined, we can now train a model to classify each image and evaluate the validation loss at each epoch. Our Fashion MNIST dataset has 10 classes including shirt, dress, sneakers, etc. We define a simple fully connected network with a softmax output and use cross entropy as our loss.
@@ -139,19 +140,23 @@ def construct_net():
return net
# construct and initialize network.
-ctx = mx.cpu()
+ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu()
+
net = construct_net()
net.hybridize()
-net.initialize(mx.init.Xavier())
+net.initialize(mx.init.Xavier(), ctx=ctx)
# define loss and trainer.
criterion = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
+```
+
+```python
+
epochs = 5
for epoch in range(epochs):
-
# training loop (with autograd and trainer steps, etc.)
- cumulative_train_loss = mx.nd.array([0])
+ cumulative_train_loss = mx.nd.zeros(1, ctx=ctx)
training_samples = 0
for batch_idx, (data, label) in enumerate(train_data_loader):
data = data.as_in_context(ctx).reshape((-1, 784)) # 28*28=784
@@ -166,7 +171,7 @@ for epoch in range(epochs):
train_loss = cumulative_train_loss.asscalar()/training_samples
# validation loop
- cumulative_valid_loss = mx.nd.array([0])
+ cumulative_valid_loss = mx.nd.zeros(1, ctx)
valid_samples = 0
for batch_idx, (data, label) in enumerate(valid_data_loader):
data = data.as_in_context(ctx).reshape((-1, 784)) # 28*28=784
diff --git a/docs/tutorials/onnx/fine_tuning_gluon.md b/docs/tutorials/onnx/fine_tuning_gluon.md
index c301542..fc940fc 100644
--- a/docs/tutorials/onnx/fine_tuning_gluon.md
+++ b/docs/tutorials/onnx/fine_tuning_gluon.md
@@ -430,4 +430,4 @@ plot_predictions(caltech101_images_test, result, categories, TOP_P)
**Great!** The network classified these images correctly after being fine-tuned on a dataset that contains images of `wrench`, `dolphin` and `lotus`
-<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
+<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
\ No newline at end of file
diff --git a/docs/tutorials/onnx/inference_on_onnx_model.md b/docs/tutorials/onnx/inference_on_onnx_model.md
index bdda820..f342dad 100644
--- a/docs/tutorials/onnx/inference_on_onnx_model.md
+++ b/docs/tutorials/onnx/inference_on_onnx_model.md
@@ -265,4 +265,4 @@ We show that in our next tutorial:
- [Fine-tuning an ONNX Model using the modern imperative MXNet/Gluon](http://mxnet.incubator.apache.org/tutorials/onnx/fine_tuning_gluon.html)
-<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
+<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
\ No newline at end of file
diff --git a/docs/tutorials/sparse/csr.md b/docs/tutorials/sparse/csr.md
index 84ff41e..c2842ac 100644
--- a/docs/tutorials/sparse/csr.md
+++ b/docs/tutorials/sparse/csr.md
@@ -535,6 +535,3 @@ except mx.MXNetError as err:
<!-- INSERT SOURCE DOWNLOAD BUTTONS -->
-
-
-
diff --git a/docs/tutorials/sparse/train.md b/docs/tutorials/sparse/train.md
index 6475f2e..0232281 100644
--- a/docs/tutorials/sparse/train.md
+++ b/docs/tutorials/sparse/train.md
@@ -36,6 +36,9 @@ We can specify the `stype` of a variable as "csr" or "row_sparse" to hold sparse
```python
import mxnet as mx
+
+mx.random.seed(42) # set the seed for repeatability
+
# Create a variable to hold an NDArray
a = mx.sym.Variable('a')
# Create a variable to hold a CSRNDArray
@@ -48,7 +51,7 @@ c = mx.sym.Variable('c', stype='row_sparse')
- (<Symbol a>, <Symbol b>, <Symbol c>)
+`(<Symbol a>, <Symbol b>, <Symbol c>)` <!--notebook-skip-line-->
diff --git a/docs/tutorials/unsupervised_learning/gan.md b/docs/tutorials/unsupervised_learning/gan.md
index 1c99a02..1556bf6 100644
--- a/docs/tutorials/unsupervised_learning/gan.md
+++ b/docs/tutorials/unsupervised_learning/gan.md
@@ -1,3 +1,4 @@
+
# Generative Adversarial Network (GAN)
Generative Adversarial Networks (GANs) are a class of algorithms used in unsupervised learning - you don't need labels for your dataset in order to train a GAN.
@@ -44,10 +45,9 @@ This example is designed to be trained on a single GPU. Training this network on
To complete this tutorial, you need:
- MXNet
-- Python 2.7, and the following libraries for Python:
+- Python, and the following libraries for Python:
- Numpy - for matrix math
- OpenCV - for image manipulation
- - Scikit-learn - to easily get the MNIST dataset
- Matplotlib - to visualize the output
## The Data
@@ -57,30 +57,40 @@ We need two pieces of data to train the DCGAN:
The Generator network will use the random numbers as the input to produce the images of handwritten digits, and the Discriminator network will use images of handwritten digits from the MNIST dataset to determine if images produced by the Generator are realistic.
-We are going to use the python library, scikit-learn, to get the MNIST dataset. Scikit-learn comes with a function that gets the dataset for us, which we will then manipulate to create the training and testing inputs.
-
The MNIST dataset contains 70,000 images of handwritten digits. Each image is 28x28 pixels in size. To create random numbers, we're going to create a custom MXNet data iterator, which will returns random numbers from a normal distribution as we need then.
## Prepare the Data
### 1. Preparing the MNSIT dataset
-Let us start by preparing the handwritten digits from the MNIST dataset. We import the fetch_mldata function from scikit-learn, and use it to get the MNSIT dataset. Notice that it's shape is 70000x784. This contains 70000 images, one per row and 784 pixels of each image in the columns of each row. Each image is 28x28 pixels, but has been flattened so that all 784 pixels are represented in a single list.
+Let us start by preparing the handwritten digits from the MNIST dataset.
+```python
+import mxnet as mx
+import numpy as np
+
+mnist_train = mx.gluon.data.vision.datasets.MNIST(train=True)
+mnist_test = mx.gluon.data.vision.datasets.MNIST(train=False)
+```
```python
-from sklearn.datasets import fetch_mldata
-mnist = fetch_mldata('MNIST original')
+# The downloaded data is of type `Dataset` which are
+# Well suited to work with the new Gluon interface but less
+# With the older symbol API, used in this tutorial.
+# Therefore we convert them to numpy array first
+X = np.zeros((70000, 28, 28))
+for i, (data, label) in enumerate(mnist_train):
+ X[i] = data.asnumpy()[:,:,0]
+for i, (data, label) in enumerate(mnist_test):
+ X[len(mnist_train)+i] = data.asnumpy()[:,:,0]
```
-Next, we will randomize the handwritten digits by using numpy to create random permutations on the dataset on the rows (images). We will then reshape the dataset from 70000x786 to 70000x28x28, so that every image in the dataset is arranged into a 28x28 grid, where each cell in the grid represents 1 pixel of the image.
+Next, we will randomize the handwritten digits by using numpy to create random permutations on the dataset on the rows (images). Every image in the dataset is arranged into a 28x28 grid, where each cell in the grid represents 1 pixel of the image.
```python
-import numpy as np
#Use a seed so that we get the same random permutation each time
np.random.seed(1)
-p = np.random.permutation(mnist.data.shape[0])
-X = mnist.data[p]
-X = X.reshape((70000, 28, 28))
+p = np.random.permutation(X.shape[0])
+X = X[p]
```
Since the DCGAN that we're creating takes in a 64x64 image as the input, we will use OpenCV to resize the each 28x28 image to 64x64 images:
```python
@@ -90,7 +100,7 @@ X = np.asarray([cv2.resize(x, (64,64)) for x in X])
Each pixel in the 64x64 image is represented by a number between 0-255, that represents the intensity of the pixel. However, we want to input numbers between -1 and 1 into the DCGAN, as suggested by the [research paper](https://arxiv.org/abs/1511.06434). To rescale the pixel values, we will divide it by (255/2). This changes the scale to 0-2. We then subtract by 1 to get them in the range of -1 to 1.
```python
-X = X.astype(np.float32)/(255.0/2) - 1.0
+X = X.astype(np.float32, copy=False)/(255.0/2) - 1.0
```
Ultimately, images are fed into the neural net through a 70000x3x64x64 array but they are currently in a 70000x64x64 array. We need to add 3 channels to the images. Typically, when we are working with the images, the 3 channels represent the red, green, and blue (RGB) components of each image. Since the MNIST dataset is grayscale, we only need 1 channel to represent the dataset. We will pad the other channels with 0's:
diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md
index 4f40598..aac03e4 100644
--- a/docs/tutorials/vision/large_scale_classification.md
+++ b/docs/tutorials/vision/large_scale_classification.md
@@ -68,7 +68,7 @@ n00120010
```
### Remove uncommon classes for transfer learning (optional)
-A common reason to train a network on ImageNet data is to use it for transfer learning (including feature extraction or fine-tuning other models). According to [this](https://arxiv.org/pdf/1608.08614v1.pdf) study, classes with too few images don’t help in transfer learning. So, we could remove classes with fewer than a certain number of images. The following code will remove classes with less than 500 images.
+A common reason to train a network on ImageNet data is to use it for transfer learning (including feature extraction or fine-tuning other models). According to [this](https://arxiv.org/pdf/1608.08614v1.pdf) study, classes with too few images don't help in transfer learning. So, we could remove classes with fewer than a certain number of images. The following code will remove classes with less than 500 images.
```
BAK=${ROOT}_filtered
@@ -85,7 +85,7 @@ done
```
### Generate a validation set
-To ensure we don’t overfit the data, we will create a validation set separate from the training set. During training, we will monitor loss on the validation set frequently. We create the validation set by picking fifty random images from each class and moving them to the validation set.
+To ensure we don't overfit the data, we will create a validation set separate from the training set. During training, we will monitor loss on the validation set frequently. We create the validation set by picking fifty random images from each class and moving them to the validation set.
```
VAL_ROOT=${ROOT}_val
mkdir -p ${VAL_ROOT}
@@ -100,7 +100,7 @@ done
```
### Pack images into record files
-While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV’s python module needs to be installed in the system.
+While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV's python module needs to be installed in the system.
Set the environment variable `MXNET` to point to the MXNet installation directory and `NAME` to the name of the dataset. Here, we assume MXNet is installed at `~/mxnet`
@@ -157,7 +157,7 @@ We will use 16 machines (P2.16x instances), each containing 16 GPUs (Tesla K80).
AWS CloudFormation makes it very easy to create deep learning clusters. We follow instructions from [this](https://aws.amazon.com/blogs/compute/distributed-deep-learning-made-easy/) page and create a deep learning cluster with 16 P2.16x instances.
-We load the data and code in the first machine (we’ll refer to this machine as master). We share both the data and code to other machines using EFS.
+We load the data and code in the first machine (we'll refer to this machine as master). We share both the data and code to other machines using EFS.
If you are setting up your cluster manually, without using AWS CloudFormation, remember to do the following:
1. Compile MXNet using `USE_DIST_KVSTORE=1` to enable distributed training.
@@ -177,7 +177,7 @@ If you are setting up your cluster manually, without using AWS CloudFormation, r
...
ubuntu@ip-10-0-1-199:~$
```
- One way to do this is to use ssh agent forwarding. Please check [this](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) page to learn how to set this up. In short, you’ll configure all machines to login using a particular certificate (mycert.pem) which is present on your local machine. You then login to the master using the certificate and the `-A` switch to enable agent forwarding. Now, from the master, you should be able to [...]
+ One way to do this is to use ssh agent forwarding. Please check [this](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) page to learn how to set this up. In short, you'll configure all machines to login using a particular certificate (mycert.pem) which is present on your local machine. You then login to the master using the certificate and the `-A` switch to enable agent forwarding. Now, from the master, you should be able to [...]
### Run Training
After the cluster is setup, login to master and run the following command from ${MXNET}/example/image-classification
@@ -258,7 +258,7 @@ It is often straightforward to achieve a reasonable validation accuracy, but ach
- Check [this](http://mxnet.io/faq/perf.html) page for more details.
### Memory
-If the batch size is too big, it can exhaust GPU memory. If this happens, you’ll see the error message “cudaMalloc failed: out of memory” or something similar. There are a couple of ways to fix this:
+If the batch size is too big, it can exhaust GPU memory. If this happens, you'll see the error message "cudaMalloc failed: out of memory" or something similar. There are a couple of ways to fix this:
- Reduce the batch size.
- Set the environment variable `MXNET_BACKWARD_DO_MIRROR` to 1. It reduces the memory consumption by trading off speed. For example, with batch size 64, inception-v3 uses 10G memory and trains 30 image/sec on a single K80 GPU. When mirroring is enabled, with 10G GPU memory consumption, we can run inception-v3 using batch size of 128. The cost is that, the speed reduces to 27 images/sec.
diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile
deleted file mode 100644
index 361e6d7..0000000
--- a/tests/nightly/Jenkinsfile
+++ /dev/null
@@ -1,28 +0,0 @@
-// -*- mode: groovy -*-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-// Runs nightly builds
-
-stage("Hello World") {
- node('mxnetlinux') {
- sh "echo 'Hello World'"
- }
-}
\ No newline at end of file
diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py
deleted file mode 100644
index 3c9470c..0000000
--- a/tests/nightly/test_tutorial.py
+++ /dev/null
@@ -1,138 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name
-"""
- This script converts all python tutorials into python script
- and tests whether there is any warning or error.
- After running python script, it will also convert markdown files
- to notebooks to make sure notebook execution has no error.
-"""
-import os
-import warnings
-import imp
-import shutil
-import time
-import argparse
-import traceback
-import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor
-import sys
-
-fail_dict = {}
-TIME_OUT = 1800
-
-def test_tutorial_nb(file_path, workingdir, kernel=None):
- """Run tutorial jupyter notebook to catch any execution error.
-
- Parameters
- ----------
- file_path : str
- path of tutorial .ipynb file
- workingdir: str
- path of the directory to run the tutorial in
- kernel: str
- Default None
- name of the kernel to use, if none, will use first kernel
- in the list
- """
- tutorial_name = os.path.basename(file_path)
- sys.stdout.write('Testing {}...'.format(file_path))
- sys.stdout.flush()
- tick = time.time()
- notebook = nbformat.read(file_path + '.ipynb', as_version=4)
- if kernel:
- eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
- else:
- eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
- success = True
- try:
- eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}})
- except Exception as err:
- err_msg = str(err)
- fail_dict[tutorial_name] = err_msg
- success = False
- finally:
- output_file = os.path.join(workingdir, "output.txt")
- output_nb = open(output_file, mode='w')
- nbformat.write(notebook, output_nb)
- output_nb.close()
- output_nb = open(output_file, mode='r')
- for line in output_nb:
- if "Warning:" in line:
- success = False
- if tutorial_name in fail_dict:
- fail_dict[tutorial_name] += "\n"+line
- else:
- fail_dict[tutorial_name] = "Warning:\n"+line
- sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick ))
- sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed'))
- sys.stdout.flush()
-
-
-if __name__ == "__main__":
- tutorial_dir = os.path.join('..','..','docs', '_build', 'html', 'tutorials')
- tick = time.time()
-
- parser = argparse.ArgumentParser()
- parser.add_argument("--tutorial", help="tutorial to test, if not set, read from test_tutorial_config.txt")
- parser.add_argument("--kernel", help="name of the jupyter kernel to use for the test")
- parser.add_argument("--no-cache", help="clean the temp directory", action="store_true", dest="no_cache")
- args = parser.parse_args()
-
-
- tutorial_list = []
- if args.tutorial:
- tutorial_list.append(args.tutorial)
- else:
- with open('test_tutorial_config.txt') as config_file:
- for line in config_file:
- tutorial_list.append(line.lstrip().rstrip())
-
- temp_dir = 'tmp_notebook'
- if args.no_cache:
- print("Cleaning and setting up temp directory '{}'".format(temp_dir))
- shutil.rmtree(temp_dir, ignore_errors=True)
-
- kernel = args.kernel if args.kernel else None
-
- for tutorial in tutorial_list:
- file_dir = os.path.join(*([tutorial_dir]+tutorial.split('/')))
- working_dir = os.path.join(*([temp_dir]+tutorial.split('/')))
- if not os.path.isdir(working_dir):
- os.makedirs(working_dir)
- test_tutorial_nb(file_dir, working_dir, kernel)
-
- fail_num = len(fail_dict)
- success_num = len(tutorial_list) - fail_num
- print("Test Summary Start")
- print("%d tutorials tested:" % (len(tutorial_list)))
- for tutorial in tutorial_list:
- print(tutorial)
- print("\n%d tests failed:" % (fail_num))
- for tutorial, msg in fail_dict.items():
- print(tutorial + ":")
- print(msg)
- print("Test Summary End")
- print("Stats start")
- print("[Passed: %d of %d]" % (success_num, len(tutorial_list)))
- print("Total time: {:.2f}s".format(time.time()-tick))
- print("Stats end")
-
- if fail_num > 0:
- exit(1)
-
diff --git a/tests/nightly/test_tutorial_config.txt b/tests/nightly/test_tutorial_config.txt
deleted file mode 100644
index be249b0..0000000
--- a/tests/nightly/test_tutorial_config.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-basic/ndarray
-basic/ndarray_indexing
-basic/symbol
-basic/module
-basic/data
-gluon/customop
-gluon/data_augmentation
-gluon/datasets
-gluon/ndarray
-gluon/mnist
-gluon/autograd
-gluon/gluon
-gluon/hybrid
-nlp/cnn
-onnx/super_resolution
-onnx/fine_tuning_gluon
-onnx/inference_on_onnx_model
-python/matrix_factorization
-python/linear-regression
-python/mnist
-python/predict_image
-python/data_augmentation
-python/data_augmentation_with_masks
-python/kvstore
-python/types_of_data_augmentation
-sparse/row_sparse
-sparse/csr
-sparse/train
-speech_recognition/ctc
-unsupervised_learning/gan
-vision/large_scale_classification
diff --git a/tests/tutorials/test_sanity_tutorials.py b/tests/tutorials/test_sanity_tutorials.py
new file mode 100644
index 0000000..eec6f4e
--- /dev/null
+++ b/tests/tutorials/test_sanity_tutorials.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import glob
+import os
+import re
+
+# White list of tutorials that should not have a
+# 'Download jupyter notebook' button or be added to the
+# automated test suite.
+# Rules to be in the whitelist:
+# - not a python tutorial
+whitelist = ['c++/basics.md',
+ 'embedded/wine_detector.md',
+ 'r/CallbackFunction.md',
+ 'r/charRnnModel.md',
+ 'r/classifyRealImageWithPretrainedModel.md',
+ 'r/CustomIterator.md',
+ 'r/CustomLossFunction.md',
+ 'r/fiveMinutesNeuralNetwork.md',
+ 'r/index.md',
+ 'r/mnistCompetition.md',
+ 'r/ndarray.md',
+ 'r/symbol.md',
+ 'scala/char_lstm.md',
+ 'scala/mnist.md',
+ 'scala/README.md',
+ 'scala/mxnet_scala_on_intellij.md']
+whitelist_set = set(whitelist)
+
+def test_tutorial_downloadable():
+ """
+ Make sure that every tutorial that isn't in the whitelist has the placeholder
+ that enables notebook download
+ """
+ download_button_string = '<!-- INSERT SOURCE DOWNLOAD BUTTONS -->'
+
+ tutorial_path = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', 'tutorials')
+ tutorials = glob.glob(os.path.join(tutorial_path, '**', '*.md'))
+
+ for tutorial in tutorials:
+ with open(tutorial, 'r') as file:
+ lines= file.readlines()
+ last = lines[-1]
+ second_last = lines[-2]
+ downloadable = download_button_string in last or download_button_string in second_last
+ friendly_name = '/'.join(tutorial.split('/')[-2:])
+ if not downloadable and friendly_name not in whitelist_set:
+ print(last, second_last)
+ assert False, "{} is missing <!-- INSERT SOURCE DOWNLOAD BUTTONS --> as its last line".format(friendly_name)
+
+def test_tutorial_tested():
+ """
+ Make sure that every tutorial that isn't in the whitelist
+ has been added to the tutorial test file
+ """
+ tutorial_test_file = os.path.join(os.path.dirname(__file__), 'test_tutorials.py')
+ f = open(tutorial_test_file, 'r')
+ tutorial_test_text = '\n'.join(f.readlines())
+ tutorial_path = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', 'tutorials')
+ tutorials = glob.glob(os.path.join(tutorial_path, '**', '*.md'))
+
+ tested_tutorials = set(re.findall(r"assert _test_tutorial_nb\('(.*)'\)", tutorial_test_text))
+ for tutorial in tutorials:
+ friendly_name = '/'.join(tutorial.split('/')[-2:]).split('.')[0]
+ if friendly_name not in tested_tutorials and friendly_name+".md" not in whitelist_set:
+ assert False, "{} has not been added to the tests/tutorials/test_tutorials.py test_suite".format(friendly_name)
+
+
diff --git a/tests/tutorials/test_tutorials.py b/tests/tutorials/test_tutorials.py
new file mode 100644
index 0000000..0949da7
--- /dev/null
+++ b/tests/tutorials/test_tutorials.py
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name
+"""
+ This file tests and ensures that all tutorials notebooks run
+ without warning or exception.
+
+ env variable MXNET_TUTORIAL_TEST_KERNEL controls which kernel to use
+ when running the notebook. e.g:
+ `export MXNET_TUTORIAL_TEST_KERNEL=python2`
+
+ env variable MXNET_TUTORIAL_TEST_NO_CACHE controls whether to clean the
+ temporary directory in which the notebook was run and re-download any
+ resource file. The default behaviour is to not clean the directory. Set to '1'
+ to force clean the directory. e.g:
+ `export MXNET_TUTORIAL_TEST_NO_CACHE=1`
+ NB: in the real CI, the tests will re-download everything since they start from
+ a clean workspace.
+"""
+import os
+import warnings
+import imp
+import shutil
+import time
+import argparse
+import traceback
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor
+import sys
+
+
+# Maximum 7 minutes per test
+# Reaching timeout causes a test failure
+TIME_OUT = 7*60
+# Pin to ipython version 4
+IPYTHON_VERSION = 4
+temp_dir = 'tmp_notebook'
+
+def _test_tutorial_nb(tutorial):
+ """Run tutorial jupyter notebook to catch any execution error.
+
+ Parameters
+ ----------
+ tutorial : str
+ tutorial name in folder/tutorial format
+ """
+
+ tutorial_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', '_build', 'html', 'tutorials')
+ tutorial_path = os.path.join(*([tutorial_dir] + tutorial.split('/')))
+
+ # see env variable docs in the doc string of the file
+ kernel = os.getenv('MXNET_TUTORIAL_TEST_KERNEL', None)
+ no_cache = os.getenv('MXNET_TUTORIAL_TEST_NO_CACHE', False)
+
+ working_dir = os.path.join(*([temp_dir] + tutorial.split('/')))
+
+ if no_cache == '1':
+ print("Cleaning and setting up temp directory '{}'".format(working_dir))
+ shutil.rmtree(temp_dir, ignore_errors=True)
+
+ errors = []
+ notebook = None
+ if not os.path.isdir(working_dir):
+ os.makedirs(working_dir)
+ try:
+ notebook = nbformat.read(tutorial_path + '.ipynb', as_version=IPYTHON_VERSION)
+ if kernel is not None:
+ eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
+ else:
+ eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
+ nb, stuff = eprocessor.preprocess(notebook, {'metadata': {'path': working_dir}})
+ print(stuff)
+ except Exception as err:
+ err_msg = str(err)
+ errors.append(err_msg)
+ finally:
+ if notebook is not None:
+ output_file = os.path.join(working_dir, "output.txt")
+ nbformat.write(notebook, output_file)
+ output_nb = open(output_file, mode='r')
+ for line in output_nb:
+ if "Warning:" in line:
+ errors.append("Warning:\n"+line)
+ if len(errors) > 0:
+ print('\n'.join(errors))
+ return False
+ return True
+
+
+
+def test_basic_ndarray():
+ assert _test_tutorial_nb('basic/ndarray')
+
+def test_basic_ndarray_indexing():
+ assert _test_tutorial_nb('basic/ndarray_indexing')
+
+def test_basic_symbol():
+ assert _test_tutorial_nb('basic/symbol')
+
+def test_basic_module():
+ assert _test_tutorial_nb('basic/module')
+
+def test_basic_data():
+ assert _test_tutorial_nb('basic/data')
+
+def test_gluon_customop():
+ assert _test_tutorial_nb('gluon/customop')
+
+def test_gluon_data_augmentation():
+ assert _test_tutorial_nb('gluon/data_augmentation')
+
+def test_gluon_datasets():
+ assert _test_tutorial_nb('gluon/datasets')
+
+def test_gluon_naming():
+ assert _test_tutorial_nb('gluon/naming')
+
+def test_gluon_ndarray():
+ assert _test_tutorial_nb('gluon/ndarray')
+
+def test_gluon_mnist():
+ assert _test_tutorial_nb('gluon/mnist')
+
+def test_gluon_autograd():
+ assert _test_tutorial_nb('gluon/autograd')
+
+def test_gluon_gluon():
+ assert _test_tutorial_nb('gluon/gluon')
+
+def test_gluon_hybrid():
+ assert _test_tutorial_nb('gluon/hybrid')
+
+def test_nlp_cnn():
+ assert _test_tutorial_nb('nlp/cnn')
+
+def test_onnx_super_resolution():
+ assert _test_tutorial_nb('onnx/super_resolution')
+
+def test_onnx_fine_tuning_gluon():
+ assert _test_tutorial_nb('onnx/fine_tuning_gluon')
+
+def test_onnx_inference_on_onnx_model():
+ assert _test_tutorial_nb('onnx/inference_on_onnx_model')
+
+def test_python_matrix_factorization():
+ assert _test_tutorial_nb('python/matrix_factorization')
+
+def test_python_linear_regression() :
+ assert _test_tutorial_nb('python/linear-regression')
+
+def test_python_mnist():
+ assert _test_tutorial_nb('python/mnist')
+
+def test_python_predict_image():
+ assert _test_tutorial_nb('python/predict_image')
+
+def test_python_data_augmentation():
+ assert _test_tutorial_nb('python/data_augmentation')
+
+def test_python_data_augmentation_with_masks():
+ assert _test_tutorial_nb('python/data_augmentation_with_masks')
+
+def test_python_kvstore():
+ assert _test_tutorial_nb('python/kvstore')
+
+def test_python_types_of_data_augmentation():
+ assert _test_tutorial_nb('python/types_of_data_augmentation')
+
+def test_sparse_row_sparse():
+ assert _test_tutorial_nb('sparse/row_sparse')
+
+def test_sparse_csr():
+ assert _test_tutorial_nb('sparse/csr')
+
+def test_sparse_train():
+ assert _test_tutorial_nb('sparse/train')
+
+def test_speech_recognition_ctc():
+ assert _test_tutorial_nb('speech_recognition/ctc')
+
+def test_unsupervised_learning_gan():
+ assert _test_tutorial_nb('unsupervised_learning/gan')
+
+def test_vision_large_scale_classification():
+ assert _test_tutorial_nb('vision/large_scale_classification')
--
To stop receiving notification emails like this one, please contact
indhub@apache.org.