You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ns...@apache.org on 2018/03/23 20:22:48 UTC
[incubator-mxnet] branch master updated: [MXNET-53]Image classifier
for scala-infer package (#10054)
This is an automated email from the ASF dual-hosted git repository.
nswamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new f386d91 [MXNET-53]Image classifier for scala-infer package (#10054)
f386d91 is described below
commit f386d91b59a0d792cdfdc708ed93a29b063da33c
Author: Roshani Nagmote <ro...@gmail.com>
AuthorDate: Fri Mar 23 13:22:43 2018 -0700
[MXNET-53]Image classifier for scala-infer package (#10054)
* Image classifier for infer package
---
scala-package/examples/pom.xml | 36 +++-
.../imageclassifier/get_resnet_data.sh | 41 ++++
.../imageclassifier/run_classifier_example.sh | 36 ++++
.../imageclassifier/ImageClassifierExample.scala | 126 +++++++++++++
.../inferexample/imageclassifier/README.md | 88 +++++++++
.../examples/src/test/resources/log4j.properties | 24 +++
.../ImageClassifierExampleSuite.scala | 70 +++++++
scala-package/infer/pom.xml | 7 +-
.../ml/dmlc/mxnet/infer/ImageClassifier.scala | 208 +++++++++++++++++++++
.../ml/dmlc/mxnet/infer/ImageClassifierSuite.scala | 152 +++++++++++++++
10 files changed, 783 insertions(+), 5 deletions(-)
diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml
index 0a9c0b0..5919b3e 100644
--- a/scala-package/examples/pom.xml
+++ b/scala-package/examples/pom.xml
@@ -15,6 +15,24 @@
<profiles>
<profile>
+ <id>osx-x86_64-cpu</id>
+ <properties>
+ <platform>osx-x86_64-cpu</platform>
+ </properties>
+ </profile>
+ <profile>
+ <id>linux-x86_64-cpu</id>
+ <properties>
+ <platform>linux-x86_64-cpu</platform>
+ </properties>
+ </profile>
+ <profile>
+ <id>linux-x86_64-gpu</id>
+ <properties>
+ <platform>linux-x86_64-gpu</platform>
+ </properties>
+ </profile>
+ <profile>
<id>release</id>
<build>
<plugins>
@@ -108,12 +126,21 @@
<artifactId>scala-maven-plugin</artifactId>
</plugin>
<plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <configuration>
+ <argLine>
+ -Djava.library.path=${project.parent.basedir}/native/${platform}/target \
+ -Dlog4j.configuration=file://${project.basedir}/src/test/resources/log4j.properties
+ </argLine>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
-
<dependencies>
<dependency>
<groupId>ml.dmlc.mxnet</groupId>
@@ -123,7 +150,7 @@
</dependency>
<dependency>
<groupId>ml.dmlc.mxnet</groupId>
- <artifactId>mxnet-infer</artifactId>
+ <artifactId>mxnet-infer_${scala.binary.version}</artifactId>
<version>1.2.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
@@ -147,5 +174,10 @@
<artifactId>opencv</artifactId>
<version>2.4.9-7</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.7.5</version>
+ </dependency>
</dependencies>
</project>
diff --git a/scala-package/examples/scripts/inferexample/imageclassifier/get_resnet_data.sh b/scala-package/examples/scripts/inferexample/imageclassifier/get_resnet_data.sh
new file mode 100755
index 0000000..0fbd323
--- /dev/null
+++ b/scala-package/examples/scripts/inferexample/imageclassifier/get_resnet_data.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd)
+
+data_path=$MXNET_ROOT/scripts/inferexample/models/resnet-152/
+
+image_path=$MXNET_ROOT/scripts/inferexample/images/
+
+if [ ! -d "$data_path" ]; then
+ mkdir -p "$data_path"
+fi
+
+if [ ! -d "$image_path" ]; then
+ mkdir -p "$image_path"
+fi
+
+if [ ! -f "$data_path" ]; then
+ wget http://data.mxnet.io/models/imagenet-11k/resnet-152/resnet-152-0000.params -P $data_path
+ wget http://data.mxnet.io/models/imagenet-11k/resnet-152/resnet-152-symbol.json -P $data_path
+ wget http://data.mxnet.io/models/imagenet-11k/synset.txt -P $data_path
+ wget https://s3.amazonaws.com/model-server/inputs/kitten.jpg -P $image_path
+fi
diff --git a/scala-package/examples/scripts/inferexample/imageclassifier/run_classifier_example.sh b/scala-package/examples/scripts/inferexample/imageclassifier/run_classifier_example.sh
new file mode 100755
index 0000000..d8c4c3e
--- /dev/null
+++ b/scala-package/examples/scripts/inferexample/imageclassifier/run_classifier_example.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+MXNET_ROOT=$(cd "$(dirname $0)/../../../../.."; pwd)
+CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*
+
+# model dir
+MODEL_PATH_PREFIX=$1
+# input image
+INPUT_IMG=$2
+# which input image dir
+INPUT_DIR=$3
+
+java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASS_PATH \
+ ml.dmlc.mxnetexamples.inferexample.imageclassifier.ImageClassifierExample \
+ --model-path-prefix $MODEL_PATH_PREFIX \
+ --input-image $INPUT_IMG \
+ --input-dir $INPUT_DIR
diff --git a/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExample.scala b/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExample.scala
new file mode 100644
index 0000000..22c49e9
--- /dev/null
+++ b/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExample.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ml.dmlc.mxnetexamples.inferexample.imageclassifier
+
+import ml.dmlc.mxnet.Shape
+import org.kohsuke.args4j.{CmdLineParser, Option}
+import org.slf4j.LoggerFactory
+
+import ml.dmlc.mxnet.{DType, DataDesc}
+import ml.dmlc.mxnet.infer.ImageClassifier
+
+import scala.collection.JavaConverters._
+import java.io.File
+
+/**
+ * Example showing usage of Infer package to do inference on resnet-152 model
+ * Follow instructions in README.md to run this example.
+ */
+object ImageClassifierExample {
+ private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExample])
+
+ def runInferenceOnSingleImage(modelPathPrefix: String, inputImagePath: String):
+ IndexedSeq[IndexedSeq[(String, Float)]] = {
+ val dType = DType.Float32
+ val inputShape = Shape(1, 3, 224, 224)
+
+ val inputDescriptor = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW"))
+
+ // Create object of ImageClassifier class
+ val imgClassifier: ImageClassifier = new
+ ImageClassifier(modelPathPrefix, inputDescriptor)
+
+ // Loading single image from file and getting BufferedImage
+ val img = ImageClassifier.loadImageFromFile(inputImagePath)
+
+ // Running inference on single image
+ val output = imgClassifier.classifyImage(img, Some(5))
+
+ output
+ }
+
+ def runInferenceOnBatchOfImage(modelPathPrefix: String, inputImageDir: String):
+ IndexedSeq[IndexedSeq[(String, Float)]] = {
+ val dType = DType.Float32
+ val inputShape = Shape(1, 3, 224, 224)
+
+ val inputDescriptor = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW"))
+
+ // Create object of ImageClassifier class
+ val imgClassifier: ImageClassifier = new
+ ImageClassifier(modelPathPrefix, inputDescriptor)
+
+ // Loading batch of images from the directory path
+ val imgList = ImageClassifier.loadInputBatch(inputImageDir)
+
+ // Running inference on batch of images loaded in previous step
+ val outputList = imgClassifier.classifyImageBatch(imgList, Some(5))
+
+ outputList
+ }
+
+ def main(args: Array[String]): Unit = {
+ val inst = new ImageClassifierExample
+ val parser: CmdLineParser = new CmdLineParser(inst)
+ try {
+ parser.parseArgument(args.toList.asJava)
+
+ val modelPathPrefix = if (inst.modelPathPrefix == null) System.getenv("MXNET_DATA_DIR")
+ else inst.modelPathPrefix
+
+ val inputImagePath = if (inst.inputImagePath == null) System.getenv("MXNET_DATA_DIR")
+ else inst.inputImagePath
+
+ val inputImageDir = if (inst.inputImageDir == null) System.getenv("MXNET_DATA_DIR")
+ else inst.inputImageDir
+
+ val singleOutput = runInferenceOnSingleImage(modelPathPrefix, inputImagePath)
+
+ // Printing top 5 class probabilities
+ for (i <- singleOutput) {
+ printf("Classes with top 5 probability = %s \n", i)
+ }
+
+ val batchOutput = runInferenceOnBatchOfImage(modelPathPrefix, inputImageDir)
+
+ val d = new File(inputImageDir)
+ val filenames = d.listFiles.filter(_.isFile).toList
+
+ // Printing filename and inference class with top 5 probabilities
+ for ((f, inferOp) <- (filenames zip batchOutput)) {
+ printf("Input image %s ", f)
+ printf("Class with probability =%s \n", inferOp)
+ }
+ } catch {
+ case ex: Exception => {
+ logger.error(ex.getMessage, ex)
+ parser.printUsage(System.err)
+ sys.exit(1)
+ }
+ }
+ }
+}
+
+class ImageClassifierExample {
+ @Option(name = "--model-path-prefix", usage = "the input model directory")
+ private val modelPathPrefix: String = "/resnet-152/resnet-152"
+ @Option(name = "--input-image", usage = "the input image")
+ private val inputImagePath: String = "/images/kitten.jpg"
+ @Option(name = "--input-dir", usage = "the input batch of images directory")
+ private val inputImageDir: String = "/images/"
+}
diff --git a/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/README.md b/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/README.md
new file mode 100644
index 0000000..99e8edb
--- /dev/null
+++ b/scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/README.md
@@ -0,0 +1,88 @@
+# Image Classification
+
+This folder contains an example for image classification with the [MXNet Scala Infer API](https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer).
+The goal of image classification is to identify the objects contained in images.
+The following example shows recognized object classes with corresponding probabilities using a pre-trained model.
+
+
+## Contents
+
+1. [Prerequisites](#prerequisites)
+2. [Download artifacts](#download-artifacts)
+3. [Run the image inference example](#run-the-image-inference-example)
+4. [Pretrained models](#pretrained-models)
+5. [Infer APIs](#infer-api-details)
+6. [Next steps](#next-steps)
+
+
+## Prerequisites
+
+1. MXNet
+2. MXNet Scala Package
+3. [IntelliJ IDE (or alternative IDE) project setup](http://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html) with the MXNet Scala Package
+4. wget
+
+
+## Download Artifacts
+
+For this tutorial, you can get the model and sample input image by running following bash file. This script will use `wget` to download these artifacts from AWS S3.
+
+From the `scala-package/examples/scripts/inferexample/imageclassifier/` folder run:
+
+```bash
+./get_resnet_data.sh
+```
+
+**Note**: You may need to run `chmod +x get_resnet_data.sh` before running this script.
+
+
+## Run the Image Inference Example
+
+Now that you have the model files and the test kitten image, you can run the following script to pass the necessary parameters to the JDK to run this inference example.
+
+```bash
+./run_classifier_example.sh \
+../resnet/resnet-152 ../images/kitten.jpg ../images/
+```
+
+**Notes**:
+* These are relative paths to this script.
+* You may need to run `chmod +x run_predictor_example.sh` before running this script.
+
+There are few options which you can provide to run the example. Use the `--help` argument to list them.
+
+```bash
+./run_predictor_example.sh --help
+```
+
+The available arguments are as follows:
+
+| Argument | Comments |
+| ----------------------------- | ---------------------------------------- |
+| `model-dir` | Folder path with prefix to the model (including json, params, and any synset file). |
+| `input-image` | The image to run inference on. |
+| `input-dir` | The directory of images to run inference on. |
+
+* You must use `model-dir`.
+* You must use `input-image` and `input-dir` as this example shows single image inference as well as batch inference together.
+
+
+## Pretrained Models
+
+The MXNet project repository provides several [pre-trained models on various datasets](https://github.com/apache/incubator-mxnet/tree/master/example/image-classification#pre-trained-models) and examples on how to train them. You may use the [modelzoo.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/common/modelzoo.py) helper script to download these models. Many ImageNet models may be also be downloaded directly from [http://data.mxnet.io/models/imag [...]
+
+
+## Infer API Details
+
+This example uses the [ImageClassifier](https://github.com/apache/incubator-mxnet/blob/master/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/ImageClassifier.scala)
+class provided by the [MXNet Scala Infer API](https://github.com/apache/incubator-mxnet/tree/master/scala-package/infer).
+It provides methods to load the images, create a NDArray out of a `BufferedImage`, and run prediction using the following Infer APIs:
+* [Classifier](https://github.com/apache/incubator-mxnet/blob/master/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/Classifier.scala)
+* [Predictor](https://github.com/apache/incubator-mxnet/blob/master/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/Predictor.scala)
+
+
+## Next Steps
+
+Check out the following related tutorials and examples for the Infer API:
+
+* [Single Shot Detector with the MXNet Scala Infer API](../objectdetector/README.md)
diff --git a/scala-package/examples/src/test/resources/log4j.properties b/scala-package/examples/src/test/resources/log4j.properties
new file mode 100644
index 0000000..ef523cb
--- /dev/null
+++ b/scala-package/examples/src/test/resources/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# for development debugging
+log4j.rootLogger = info, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target = System.out
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c] [%p] - %m%n
diff --git a/scala-package/examples/src/test/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExampleSuite.scala
new file mode 100644
index 0000000..18608b0
--- /dev/null
+++ b/scala-package/examples/src/test/scala/ml/dmlc/mxnetexamples/inferexample/imageclassifier/ImageClassifierExampleSuite.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ml.dmlc.mxnetexamples.inferexample.imageclassifier
+
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
+import org.slf4j.LoggerFactory
+
+import java.io.File
+import sys.process.Process
+
+/**
+ * Integration test for imageClassifier example.
+ * This will run as a part of "make scalatest"
+ */
+class ImageClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
+ private val logger = LoggerFactory.getLogger(classOf[ImageClassifierExampleSuite])
+
+ test("testImageClassifierExample") {
+ logger.info("Downloading resnet-18 model")
+
+ val tempDirPath = System.getProperty("java.io.tmpdir")
+ logger.info("tempDirPath: %s".format(tempDirPath))
+
+ Process("wget http://data.mxnet.io/models/imagenet/resnet/18-layers/resnet-18-symbol.json " +
+ "-P " + tempDirPath + "/resnet18/ -q") !
+
+ Process("wget http://data.mxnet.io/models/imagenet/resnet/18-layers/resnet-18-0000.params " +
+ "-P " + tempDirPath + "/resnet18/ -q") !
+
+ Process("wget http://data.mxnet.io/models/imagenet/resnet/synset.txt -P " + tempDirPath +
+ "/resnet18/ -q") !
+
+ Process("wget " +
+ "https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg " +
+ "-P " + tempDirPath + "/inputImages/") !
+
+ val modelDirPath = tempDirPath + File.separator + "resnet18/"
+ val inputImagePath = tempDirPath + File.separator +
+ "inputImages/Pug-Cookie.jpg"
+ val inputImageDir = tempDirPath + File.separator + "inputImages/"
+
+ val output = ImageClassifierExample.runInferenceOnSingleImage(modelDirPath + "resnet-18",
+ inputImagePath)
+
+ assert(output(0).toList.head._1 === "n02110958 pug, pug-dog")
+
+ val outputList = ImageClassifierExample.runInferenceOnBatchOfImage(modelDirPath + "resnet-18",
+ inputImageDir)
+
+ assert(outputList(0).toList.head._1 === "n02110958 pug, pug-dog")
+
+ Process("rm -rf " + modelDirPath + " " + inputImageDir) !
+
+ }
+}
diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml
index 3ae8f6c..d8c1097 100644
--- a/scala-package/infer/pom.xml
+++ b/scala-package/infer/pom.xml
@@ -2,14 +2,15 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mxnet-parent_2.11</artifactId>
<groupId>ml.dmlc.mxnet</groupId>
<version>1.2.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
</parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>mxnet-infer</artifactId>
+ <artifactId>mxnet-infer_2.11</artifactId>
<name>MXNet Scala Package - Inference</name>
<profiles>
@@ -81,4 +82,4 @@
<scope>test</scope>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/ImageClassifier.scala b/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/ImageClassifier.scala
new file mode 100644
index 0000000..45c4e76
--- /dev/null
+++ b/scala-package/infer/src/main/scala/ml/dmlc/mxnet/infer/ImageClassifier.scala
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ml.dmlc.mxnet.infer
+
+import ml.dmlc.mxnet.{DataDesc, NDArray, Shape}
+
+import scala.collection.mutable.ListBuffer
+
+// scalastyle:off
+import java.awt.image.BufferedImage
+// scalastyle:on
+import java.io.File
+import javax.imageio.ImageIO
+
+
+/**
+ * A class for Image classification tasks.
+ * Contains helper methods.
+ *
+ * @param modelPathPrefix PathPrefix from where to load the symbol, parameters and synset.txt
+ * Example: file://model-dir/resnet-152(containing resnet-152-symbol.json
+ * file://model-dir/synset.txt
+ * @param inputDescriptors Descriptors defining the input node names, shape,
+ * layout and Type parameters
+ */
+class ImageClassifier(modelPathPrefix: String,
+ inputDescriptors: IndexedSeq[DataDesc])
+ extends Classifier(modelPathPrefix,
+ inputDescriptors) {
+
+ val classifier: Classifier = getClassifier(modelPathPrefix, inputDescriptors)
+
+ protected[infer] val inputLayout = inputDescriptors.head.layout
+
+ require(inputDescriptors.nonEmpty, "Please provide input descriptor")
+ require(inputDescriptors.head.layout == "NCHW", "Provided layout doesn't match NCHW format")
+
+ protected[infer] val inputShape = inputDescriptors.head.shape
+
+ // Considering 'NCHW' as default layout when not provided
+ // Else get axis according to the layout
+ // [TODO] if layout is different than the bufferedImage layout,
+ // transpose to match the inputdescriptor shape
+ protected[infer] val batch = inputShape(inputLayout.indexOf('N'))
+ protected[infer] val channel = inputShape(inputLayout.indexOf('C'))
+ protected[infer] val height = inputShape(inputLayout.indexOf('H'))
+ protected[infer] val width = inputShape(inputLayout.indexOf('W'))
+
+ /**
+ * To classify the image according to the provided model
+ *
+ * @param inputImage PathPrefix of the input image
+ * @param topK Get top k elements with maximum probability
+ * @return List of list of tuples of (class, probability)
+ */
+ def classifyImage(inputImage: BufferedImage,
+ topK: Option[Int] = None): IndexedSeq[IndexedSeq[(String, Float)]] = {
+
+ val scaledImage = ImageClassifier.reshapeImage(inputImage, width, height)
+ val pixelsNDArray = ImageClassifier.bufferedImageToPixels(scaledImage, inputShape)
+ inputImage.flush()
+ scaledImage.flush()
+
+ val output = super.classifyWithNDArray(IndexedSeq(pixelsNDArray), topK)
+
+ handler.execute(pixelsNDArray.dispose())
+
+ IndexedSeq(output(0))
+ }
+
+ /**
+ * To classify batch of input images according to the provided model
+ *
+ * @param inputBatch Input batch of Buffered images
+ * @param topK Get top k elements with maximum probability
+ * @return List of list of tuples of (class, probability)
+ */
+ def classifyImageBatch(inputBatch: Traversable[BufferedImage], topK: Option[Int] = None):
+ IndexedSeq[IndexedSeq[(String, Float)]] = {
+
+ val imageBatch = ListBuffer[NDArray]()
+ for (image <- inputBatch) {
+ val scaledImage = ImageClassifier.reshapeImage(image, width, height)
+ val pixelsNDArray = ImageClassifier.bufferedImageToPixels(scaledImage, inputShape)
+ imageBatch += pixelsNDArray
+ }
+ val op = NDArray.concatenate(imageBatch)
+
+ val result = super.classifyWithNDArray(IndexedSeq(op), topK)
+ handler.execute(op.dispose())
+ handler.execute(imageBatch.foreach(_.dispose()))
+
+ result
+ }
+
+ def getClassifier(modelPathPrefix: String, inputDescriptors: IndexedSeq[DataDesc]): Classifier = {
+ new Classifier(modelPathPrefix, inputDescriptors)
+ }
+}
+
+object ImageClassifier {
+
+ /**
+ * Reshape the input image to a new shape
+ *
+ * @param img input image
+ * @param newWidth rescale to new width
+ * @param newHeight rescale to new height
+ * @return Rescaled BufferedImage
+ */
+ def reshapeImage(img: BufferedImage, newWidth: Int, newHeight: Int): BufferedImage = {
+ val resizedImage = new BufferedImage(newWidth, newHeight, BufferedImage.TYPE_INT_RGB)
+ val g = resizedImage.createGraphics()
+ g.drawImage(img, 0, 0, newWidth, newHeight, null)
+ g.dispose()
+
+ resizedImage
+ }
+
+ /**
+ * Convert input BufferedImage to NDArray of input shape
+ *
+ * <p>
+ * Note: Caller is responsible to dispose the NDArray
+ * returned by this method after the use.
+ *
+ * @param resizedImage BufferedImage to get pixels from
+ * @param inputImageShape Should be same as inputDescriptor shape
+ * @return NDArray pixels array
+ */
+ def bufferedImageToPixels(resizedImage: BufferedImage, inputImageShape: Shape): NDArray = {
+ // Get height and width of the image
+ val w = resizedImage.getWidth()
+ val h = resizedImage.getHeight()
+
+ // get an array of integer pixels in the default RGB color mode
+ val pixels = resizedImage.getRGB(0, 0, w, h, null, 0, w)
+
+ // 3 times height and width for R,G,B channels
+ val result = new Array[Float](3 * h * w)
+
+ var row = 0
+ // copy pixels to array vertically
+ while (row < h) {
+ var col = 0
+ // copy pixels to array horizontally
+ while (col < w) {
+ val rgb = pixels(row * w + col)
+ // getting red color
+ result(0 * h * w + row * w + col) = (rgb >> 16) & 0xFF
+ // getting green color
+ result(1 * h * w + row * w + col) = (rgb >> 8) & 0xFF
+ // getting blue color
+ result(2 * h * w + row * w + col) = rgb & 0xFF
+ col += 1
+ }
+ row += 1
+ }
+ resizedImage.flush()
+
+ // creating NDArray according to the input shape
+ val pixelsArray = NDArray.array(result, shape = inputImageShape)
+ pixelsArray
+ }
+
+ /**
+ * Loading input batch of images
+ * @param inputImagePath Path of single input image
+ * @return BufferedImage Buffered image
+ */
+ def loadImageFromFile(inputImagePath: String): BufferedImage = {
+ val img = ImageIO.read(new File(inputImagePath))
+ img
+ }
+
+ /**
+ * Loading input batch of images
+ * @param inputImageDirPath
+ * @return List of buffered images
+ */
+ def loadInputBatch(inputImageDirPath: String): List[BufferedImage] = {
+ val dir = new File(inputImageDirPath)
+ require(dir.exists && dir.isDirectory,
+ "input image directory: %s not found".format(inputImageDirPath))
+
+ val inputBatch = ListBuffer[BufferedImage]()
+ for (imgFile: File <- dir.listFiles()){
+ val img = ImageIO.read(imgFile)
+ inputBatch += img
+ }
+ inputBatch.toList
+ }
+}
diff --git a/scala-package/infer/src/test/scala/ml/dmlc/mxnet/infer/ImageClassifierSuite.scala b/scala-package/infer/src/test/scala/ml/dmlc/mxnet/infer/ImageClassifierSuite.scala
new file mode 100644
index 0000000..96fc800
--- /dev/null
+++ b/scala-package/infer/src/test/scala/ml/dmlc/mxnet/infer/ImageClassifierSuite.scala
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ml.dmlc.mxnet.infer
+
+import ml.dmlc.mxnet.{DType, DataDesc, Shape, NDArray}
+
+import org.mockito.Matchers._
+import org.mockito.Mockito
+import org.scalatest.{BeforeAndAfterAll}
+
+// scalastyle:off
+import java.awt.image.BufferedImage
+// scalastyle:on
+
+/**
+ * Unit tests for ImageClassifier
+ */
+class ImageClassifierSuite extends ClassifierSuite with BeforeAndAfterAll {
+
+ class MyImageClassifier(modelPathPrefix: String,
+ inputDescriptors: IndexedSeq[DataDesc])
+ extends ImageClassifier(modelPathPrefix, inputDescriptors) {
+
+ override def getPredictor(): MyClassyPredictor = {
+ Mockito.mock(classOf[MyClassyPredictor])
+ }
+
+ override def getClassifier(modelPathPrefix: String, inputDescriptors:
+ IndexedSeq[DataDesc]): Classifier = {
+ Mockito.mock(classOf[Classifier])
+ }
+
+ def getSynset(): IndexedSeq[String] = synset
+ }
+
+ test("ImageClassifierSuite-testRescaleImage") {
+ val image1 = new BufferedImage(100, 200, BufferedImage.TYPE_BYTE_GRAY)
+ val image2 = ImageClassifier.reshapeImage(image1, 1000, 2000)
+
+ assert(image2.getWidth === 1000)
+ assert(image2.getHeight === 2000)
+ }
+
+ test("ImageClassifierSuite-testConvertBufferedImageToNDArray") {
+ val dType = DType.Float32
+ val inputDescriptor = IndexedSeq[DataDesc](new DataDesc(modelPath, Shape(1, 3, 2, 2),
+ dType, "NCHW"))
+
+ val image1 = new BufferedImage(100, 200, BufferedImage.TYPE_BYTE_GRAY)
+ val image2 = ImageClassifier.reshapeImage(image1, 2, 2)
+
+ val result = ImageClassifier.bufferedImageToPixels(image2, Shape(1, 3, 2, 2))
+
+ assert(result.shape == inputDescriptor(0).shape)
+ }
+
+ test("ImageClassifierSuite-testWithInputImage") {
+ val dType = DType.Float32
+ val inputDescriptor = IndexedSeq[DataDesc](new DataDesc(modelPath, Shape(1, 3, 512, 512),
+ dType, "NCHW"))
+
+ val inputImage = new BufferedImage(224, 224, BufferedImage.TYPE_INT_RGB)
+
+ val testImageClassifier: ImageClassifier =
+ new MyImageClassifier(modelPath, inputDescriptor)
+
+ val predictExpected: IndexedSeq[Array[Float]] =
+ IndexedSeq[Array[Float]](Array(.98f, 0.97f, 0.96f, 0.99f))
+
+ val synset = testImageClassifier.synset
+
+ val predictExpectedOp : List[(String, Float)] =
+ List[(String, Float)]((synset(1), .98f), (synset(2), .97f),
+ (synset(3), .96f), (synset(0), .99f))
+
+ val predictExpectedND: NDArray = NDArray.array(predictExpected.flatten.toArray, Shape(1, 4))
+
+ Mockito.doReturn(IndexedSeq(predictExpectedND)).when(testImageClassifier.predictor)
+ .predictWithNDArray(any(classOf[IndexedSeq[NDArray]]))
+
+ Mockito.doReturn(IndexedSeq(predictExpectedOp)).when(testImageClassifier.classifier)
+ .classifyWithNDArray(any(classOf[IndexedSeq[NDArray]]), Some(anyInt()))
+
+ val predictResult: IndexedSeq[IndexedSeq[(String, Float)]] =
+ testImageClassifier.classifyImage(inputImage, Some(4))
+
+ for(i <- predictExpected.indices) {
+ assertResult(predictExpected(i).sortBy(-_)) {
+ predictResult(i).map(_._2).toArray
+ }
+ }
+ }
+
+ test("ImageClassifierSuite-testWithInputBatchImage") {
+ val dType = DType.Float32
+ val inputDescriptor = IndexedSeq[DataDesc](new DataDesc(modelPath, Shape(1, 3, 512, 512),
+ dType, "NCHW"))
+
+ val inputImage = new BufferedImage(224, 224, BufferedImage.TYPE_INT_RGB)
+ val imageBatch = IndexedSeq[BufferedImage](inputImage, inputImage)
+
+ val testImageClassifier: ImageClassifier =
+ new MyImageClassifier(modelPath, inputDescriptor)
+
+ val predictExpected: IndexedSeq[Array[Array[Float]]] =
+ IndexedSeq[Array[Array[Float]]](Array(Array(.98f, 0.97f, 0.96f, 0.99f),
+ Array(.98f, 0.97f, 0.96f, 0.99f)))
+
+ val synset = testImageClassifier.synset
+
+ val predictExpectedOp : List[List[(String, Float)]] =
+ List[List[(String, Float)]](List((synset(1), .98f), (synset(2), .97f),
+ (synset(3), .96f), (synset(0), .99f)),
+ List((synset(1), .98f), (synset(2), .97f),
+ (synset(3), .96f), (synset(0), .99f)))
+
+ val predictExpectedND: NDArray = NDArray.array(predictExpected.flatten.flatten.toArray,
+ Shape(2, 4))
+
+ Mockito.doReturn(IndexedSeq(predictExpectedND)).when(testImageClassifier.predictor)
+ .predictWithNDArray(any(classOf[IndexedSeq[NDArray]]))
+
+ Mockito.doReturn(IndexedSeq(predictExpectedOp)).when(testImageClassifier.classifier)
+ .classifyWithNDArray(any(classOf[IndexedSeq[NDArray]]), Some(anyInt()))
+
+ val result: IndexedSeq[IndexedSeq[(String, Float)]] =
+ testImageClassifier.classifyImageBatch(imageBatch, Some(4))
+
+ for (i <- predictExpected.indices) {
+ for (idx <- predictExpected(i).indices) {
+ assertResult(predictExpected(i)(idx).sortBy(-_)) {
+ result(i).map(_._2).toArray
+ }
+ }
+ }
+ }
+}
--
To stop receiving notification emails like this one, please contact
nswamy@apache.org.