You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/07/16 05:57:04 UTC
[GitHub] nswamy closed pull request #11292: [MXNET-531] CNN Examples for
Scala new API
nswamy closed pull request #11292: [MXNET-531] CNN Examples for Scala new API
URL: https://github.com/apache/incubator-mxnet/pull/11292
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/CNNTextClassification.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/CNNTextClassification.scala
index 796a92f7029..674c81459f0 100644
--- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/CNNTextClassification.scala
+++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/CNNTextClassification.scala
@@ -17,66 +17,69 @@
package org.apache.mxnetexamples.cnntextclassification
+import org.apache.mxnet.optimizer.RMSProp
+import org.apache.mxnet.{Context, Executor, Model, NDArray, Optimizer, Shape, Symbol, Uniform}
import org.kohsuke.args4j.{CmdLineParser, Option}
import org.slf4j.LoggerFactory
+
import scala.collection.JavaConverters._
-import org.apache.mxnet.Uniform
-import org.apache.mxnet.Context
-import org.apache.mxnet.Symbol
-import org.apache.mxnet.Shape
-import org.apache.mxnet.NDArray
-import org.apache.mxnet.Executor
-import org.apache.mxnet.optimizer.RMSProp
-import org.apache.mxnet.Optimizer
-import org.apache.mxnet.Model
import scala.util.Random
/**
- * An Implementation of the paper
- * Convolutional Neural Networks for Sentence Classification
- * by Yoon Kim
- * @author Depeng Liang
- */
+ * An Implementation of the paper
+ * Convolutional Neural Networks for Sentence Classification
+ */
object CNNTextClassification {
private val logger = LoggerFactory.getLogger(classOf[CNNTextClassification])
case class CNNModel(cnnExec: Executor, symbol: Symbol, data: NDArray, label: NDArray,
- argsDict: Map[String, NDArray], gradDict: Map[String, NDArray])
+ argsDict: Map[String, NDArray], gradDict: Map[String, NDArray])
def makeTextCNN(sentenceSize: Int, numEmbed: Int, batchSize: Int,
- numLabel: Int = 2, filterList: Array[Int] = Array(3, 4, 5), numFilter: Int = 100,
- dropout: Float = 0.5f): Symbol = {
+ numLabel: Int = 2, filterList: Array[Int] = Array(3, 4, 5), numFilter: Int = 100,
+ dropout: Float = 0.5f): Symbol = {
val inputX = Symbol.Variable("data")
val inputY = Symbol.Variable("softmax_label")
val polledOutputs = filterList.map { filterSize =>
- val conv = Symbol.Convolution()()(
- Map("data" -> inputX, "kernel" -> s"($filterSize, $numEmbed)", "num_filter" -> numFilter))
- val relu = Symbol.Activation()()(Map("data" -> conv, "act_type" -> "relu"))
- val pool = Symbol.Pooling()()(Map("data" -> relu, "pool_type" -> "max",
- "kernel" -> s"(${sentenceSize - filterSize + 1}, 1)", "stride" -> "(1,1)"))
+ val conv = Symbol.api.Convolution(data = Some(inputX),
+ kernel = new Shape(filterSize, numEmbed), num_filter = numFilter)
+ val relu = Symbol.api.Activation(data = Some(conv), act_type = "relu")
+ val pool = Symbol.api.Pooling(data = Some(relu), pool_type = Some("max"),
+ kernel = Some(new Shape(sentenceSize - filterSize + 1, 1)), stride = Some(new Shape(1, 1)))
+ relu.dispose()
+ conv.dispose()
pool
}
val totalFilters = numFilter * filterList.length
- val concat = Symbol.Concat()(polledOutputs: _*)(Map("dim" -> 1))
- val hPool = Symbol.Reshape()()(Map("data" -> concat,
- "target_shape" -> s"($batchSize, $totalFilters)"))
+ // val concat = Symbol.Concat()(polledOutputs: _*)(Map("dim" -> 1))
+ val concat = Symbol.api.concat(data = polledOutputs,
+ num_args = polledOutputs.length, dim = Some(1))
+ val hPool = Symbol.api.reshape(data = Some(concat),
+ target_shape = Some(new Shape(batchSize, totalFilters)))
val hDrop = {
- if (dropout > 0f) Symbol.Dropout()()(Map("data" -> hPool, "p" -> dropout))
+ if (dropout > 0f) Symbol.api.Dropout(data = Some(hPool), p = Some(dropout))
else hPool
}
- val fc = Symbol.FullyConnected()()(Map("data" -> hDrop, "num_hidden" -> numLabel))
- val sm = Symbol.SoftmaxOutput()()(Map("data" -> fc, "label" -> inputY))
+ val fc = Symbol.api.FullyConnected(data = Some(hDrop), num_hidden = numLabel)
+ val sm = Symbol.api.SoftmaxOutput(data = Some(fc), label = Some(inputY))
+ fc.dispose()
+ hDrop.dispose()
+ hPool.dispose()
+ concat.dispose()
+ polledOutputs.foreach(_.dispose())
+ inputX.dispose()
+ inputY.dispose()
sm
}
def setupCnnModel(ctx: Context, batchSize: Int, sentenceSize: Int, numEmbed: Int,
- numLabel: Int = 2, numFilter: Int = 100, filterList: Array[Int ] = Array(3, 4, 5),
- dropout: Float = 0.5f): CNNModel = {
+ numLabel: Int = 2, numFilter: Int = 100, filterList: Array[Int ] = Array(3, 4, 5),
+ dropout: Float = 0.0f): CNNModel = {
val cnn = makeTextCNN(sentenceSize, numEmbed, batchSize,
numLabel, filterList, numFilter, dropout)
@@ -84,11 +87,11 @@ object CNNTextClassification {
val auxNames = cnn.listAuxiliaryStates()
val (argShapes, outShapes, auxShapes) = cnn.inferShape(
- Map("data" -> Shape(batchSize, 1, sentenceSize, numEmbed)))
+ Map("data" -> Shape(batchSize, 1, sentenceSize, numEmbed)))
val argsDict = argNames.zip(argShapes.map(NDArray.zeros(_, ctx))).toMap
val argsGradDict = argNames.zip(argShapes)
- .filter(x => x._1 != "softmax_label" && x._1 != "data")
- .map(x => x._1 -> NDArray.zeros(x._2, ctx)).toMap
+ .filter(x => x._1 != "softmax_label" && x._1 != "data")
+ .map(x => x._1 -> NDArray.zeros(x._2, ctx)).toMap
val auxDict = auxNames.zip(auxShapes.map(NDArray.zeros(_, ctx))).toMap
val cnnExec = cnn.bind(ctx, argsDict, argsGradDict, "add", auxDict, null, null)
@@ -98,131 +101,166 @@ object CNNTextClassification {
}
def trainCNN(model: CNNModel, trainBatches: Array[Array[Array[Float]]],
- trainLabels: Array[Float], devBatches: Array[Array[Array[Float]]],
- devLabels: Array[Float], batchSize: Int, saveModelPath: String,
- learningRate: Float = 0.001f): Unit = {
- val maxGradNorm = 0.5f
- val epoch = 200
- val initializer = new Uniform(0.1f)
- val opt = new RMSProp(learningRate)
- val updater = Optimizer.getUpdater(opt)
- var start = 0L
- var end = 0L
- var numCorrect = 0f
- var numTotal = 0f
- var factor = 0.5f
- var maxAccuracy = -1f
- var updateRate = 0
-
- val paramBlocks = model.symbol.listArguments()
- .filter(x => x != "data" && x != "softmax_label")
- .zipWithIndex.map { x =>
- initializer(x._1, model.gradDict(x._1))
- val state = opt.createState(x._2, model.argsDict(x._1))
- (x._2, model.argsDict(x._1), model.gradDict(x._1), state, x._1)
- }.toArray
-
- for (iter <- 0 until epoch) {
- start = System.currentTimeMillis()
- numCorrect = 0f
- numTotal = 0f
- updateRate = 0
-
- for (begin <- 0 until trainBatches.length by batchSize) {
- val (batchD, batchL) = {
- if (begin + batchSize <= trainBatches.length) {
- val datas = trainBatches.drop(begin).take(batchSize)
- val labels = trainLabels.drop(begin).take(batchSize)
- (datas, labels)
- } else {
- val right = (begin + batchSize) - trainBatches.length
- val left = trainBatches.length - begin
- val datas = trainBatches.drop(begin).take(left) ++ trainBatches.take(right)
- val labels = trainLabels.drop(begin).take(left) ++ trainLabels.take(right)
- (datas, labels)
+ trainLabels: Array[Float], devBatches: Array[Array[Array[Float]]],
+ devLabels: Array[Float], batchSize: Int, saveModelPath: String,
+ learningRate: Float = 0.001f): Float = {
+ val maxGradNorm = 0.5f
+ val epoch = 10
+ val initializer = new Uniform(0.1f)
+ val opt = new RMSProp(learningRate)
+ val updater = Optimizer.getUpdater(opt)
+ var start = 0L
+ var end = 0L
+ var numCorrect = 0f
+ var numTotal = 0f
+ var factor = 0.5f
+ var maxAccuracy = -1f
+ var updateRate = 0
+
+ val paramBlocks = model.symbol.listArguments()
+ .filter(x => x != "data" && x != "softmax_label")
+ .zipWithIndex.map { x =>
+ initializer(x._1, model.gradDict(x._1))
+ val state = opt.createState(x._2, model.argsDict(x._1))
+ (x._2, model.argsDict(x._1), model.gradDict(x._1), state, x._1)
+ }.toArray
+ var devAcc = 0.0f
+ for (iter <- 0 until epoch) {
+ start = System.currentTimeMillis()
+ numCorrect = 0f
+ numTotal = 0f
+ updateRate = 0
+
+ for (begin <- 0 until trainBatches.length by batchSize) {
+ val (batchD, batchL) = {
+ if (begin + batchSize <= trainBatches.length) {
+ val datas = trainBatches.drop(begin).take(batchSize)
+ val labels = trainLabels.drop(begin).take(batchSize)
+ (datas, labels)
+ } else {
+ val right = (begin + batchSize) - trainBatches.length
+ val left = trainBatches.length - begin
+ val datas = trainBatches.drop(begin).take(left) ++ trainBatches.take(right)
+ val labels = trainLabels.drop(begin).take(left) ++ trainLabels.take(right)
+ (datas, labels)
+ }
+ }
+ numTotal += batchSize
+ model.data.set(batchD.flatten.flatten)
+ model.label.set(batchL)
+
+ model.cnnExec.forward(isTrain = true)
+ model.cnnExec.backward()
+
+ val tmpCorrect = {
+ val predLabel = NDArray.api.argmax_channel(model.cnnExec.outputs(0))
+ val result = predLabel.toArray.zip(batchL).map { predLabel =>
+ if (predLabel._1 == predLabel._2) 1
+ else 0
+ }.sum.toFloat
+ predLabel.dispose()
+ result
+ }
+
+ numCorrect = numCorrect + tmpCorrect
+ val norm = Math.sqrt(paramBlocks.map { case (idx, weight, grad, state, name) =>
+ val temp = NDArray.api.norm(grad / batchSize).disposeDepsExcept(grad)
+ val l2Norm = temp.toScalar
+ temp.dispose()
+ l2Norm * l2Norm
+ }.sum).toFloat
+
+ if (updateRate % 2 == 0) {
+ paramBlocks.foreach { case (idx, weight, grad, state, name) =>
+ if (norm > maxGradNorm) {
+ grad.set(grad.toArray.map(_ * (maxGradNorm / norm)))
+ opt.update(idx, weight, grad, state)
}
+ else opt.update(idx, weight, grad, state)
+ grad.set(0f)
}
+ }
+ updateRate = updateRate + 1
+ }
+
+ // decay learning rate
+ if (iter % 50 == 0 && iter > 0) {
+ factor *= 0.5f
+ opt.setLrScale(paramBlocks.map(_._1 -> factor).toMap)
+ logger.info(s"reset learning to ${opt.learningRate * factor}")
+ }
+ // end of training loop
+ end = System.currentTimeMillis()
+ logger.info(s"Iter $iter Train: Time: ${(end - start) / 1000}," +
+ s"Training Accuracy: ${numCorrect / numTotal * 100}%")
+
+ // eval on dev set
+ numCorrect = 0f
+ numTotal = 0f
+ for (begin <- 0 until devBatches.length by batchSize) {
+ if (begin + batchSize <= devBatches.length) {
numTotal += batchSize
+ val (batchD, batchL) = {
+ val datas = devBatches.drop(begin).take(batchSize)
+ val labels = devLabels.drop(begin).take(batchSize)
+ (datas, labels)
+ }
+
model.data.set(batchD.flatten.flatten)
model.label.set(batchL)
- model.cnnExec.forward(isTrain = true)
- model.cnnExec.backward()
+ model.cnnExec.forward(isTrain = false)
val tmpCorrect = {
- val predLabel = NDArray.argmax_channel(model.cnnExec.outputs(0))
- predLabel.toArray.zip(batchL).map { predLabel =>
+ val predLabel = NDArray.api.argmax_channel(model.cnnExec.outputs(0))
+ val result = predLabel.toArray.zip(batchL).map { predLabel =>
if (predLabel._1 == predLabel._2) 1
else 0
}.sum.toFloat
+ predLabel.dispose()
+ result
}
-
numCorrect = numCorrect + tmpCorrect
- val norm = Math.sqrt(paramBlocks.map { case (idx, weight, grad, state, name) =>
- val l2Norm = NDArray.norm(grad / batchSize).toScalar
- l2Norm * l2Norm
- }.sum).toFloat
-
- if (updateRate % 2 == 0) {
- paramBlocks.foreach { case (idx, weight, grad, state, name) =>
- if (norm > maxGradNorm) {
- grad.set(grad.toArray.map(_ * (maxGradNorm / norm)))
- opt.update(idx, weight, grad, state)
- }
- else opt.update(idx, weight, grad, state)
- grad.set(0f)
- }
- }
- updateRate = updateRate + 1
- }
-
- // decay learning rate
- if (iter % 50 == 0 && iter > 0) {
- factor *= 0.5f
- opt.setLrScale(paramBlocks.map(_._1 -> factor).toMap)
- logger.info(s"reset learning to ${opt.learningRate * factor}")
- }
- // end of training loop
- end = System.currentTimeMillis()
- logger.info(s"Iter $iter Train: Time: ${(end - start) / 1000}," +
- s"Training Accuracy: ${numCorrect / numTotal * 100}%")
-
- // eval on dev set
- numCorrect = 0f
- numTotal = 0f
- for (begin <- 0 until devBatches.length by batchSize) {
- if (begin + batchSize <= devBatches.length) {
- numTotal += batchSize
- val (batchD, batchL) = {
- val datas = devBatches.drop(begin).take(batchSize)
- val labels = devLabels.drop(begin).take(batchSize)
- (datas, labels)
- }
-
- model.data.set(batchD.flatten.flatten)
- model.label.set(batchL)
-
- model.cnnExec.forward(isTrain = false)
-
- val tmpCorrect = {
- val predLabel = NDArray.argmax_channel(model.cnnExec.outputs(0))
- predLabel.toArray.zip(batchL).map { predLabel =>
- if (predLabel._1 == predLabel._2) 1
- else 0
- }.sum.toFloat
- }
- numCorrect = numCorrect + tmpCorrect
- }
- }
- val tmpAcc = numCorrect / numTotal
- logger.info(s"Dev Accuracy so far: ${tmpAcc * 100}%")
- if (tmpAcc > maxAccuracy) {
- maxAccuracy = tmpAcc
- Model.saveCheckpoint(s"$saveModelPath/cnn-text-dev-acc-$maxAccuracy",
- iter, model.symbol, model.cnnExec.argDict, model.cnnExec.auxDict)
- logger.info(s"max accuracy on dev so far: ${maxAccuracy * 100}%")
}
}
+ devAcc = numCorrect / numTotal
+ logger.info(s"Dev Accuracy so far: ${devAcc * 100}%")
+ if (devAcc > maxAccuracy) {
+ maxAccuracy = devAcc
+ Model.saveCheckpoint(s"$saveModelPath/cnn-text-dev-acc-$maxAccuracy",
+ iter, model.symbol, model.cnnExec.argDict, model.cnnExec.auxDict)
+ logger.info(s"max accuracy on dev so far: ${maxAccuracy * 100}%")
+ }
+ }
+ devAcc
+ }
+
+ def test(w2vFilePath : String, mrDatasetPath: String,
+ ctx : Context, saveModelPath: String) : Float = {
+ val (numEmbed, word2vec) = DataHelper.loadGoogleModel(w2vFilePath)
+ val (datas, labels) = DataHelper.loadMSDataWithWord2vec(
+ mrDatasetPath, numEmbed, word2vec)
+ // randomly shuffle data
+ val randIdx = Random.shuffle((0 until datas.length).toList)
+ // split train/dev set
+ val (trainDats, devDatas) = {
+ val train = randIdx.dropRight(1000).map(datas(_)).toArray
+ val dev = randIdx.takeRight(1000).map(datas(_)).toArray
+ (train, dev)
+ }
+ val (trainLabels, devLabels) = {
+ val train = randIdx.dropRight(1000).map(labels(_)).toArray
+ val dev = randIdx.takeRight(1000).map(labels(_)).toArray
+ (train, dev)
+ }
+ // reshpae for convolution input
+ val sentenceSize = datas(0).length
+ val batchSize = 100
+ val lr = 0.001f
+ val cnnModel = setupCnnModel(ctx, batchSize, sentenceSize, numEmbed)
+ val result = trainCNN(cnnModel, trainDats, trainLabels, devDatas, devLabels, batchSize,
+ saveModelPath, learningRate = lr)
+ result
}
def main(args: Array[String]): Unit = {
@@ -238,7 +276,7 @@ object CNNTextClassification {
val (datas, labels) = DataHelper.loadMSDataWithWord2vec(
exon.mrDatasetPath, numEmbed, word2vec)
- // randomly shuffle data
+ // randomly shuffle data
val randIdx = Random.shuffle((0 until datas.length).toList)
// split train/dev set
val (trainDats, devDatas) = {
@@ -258,7 +296,7 @@ object CNNTextClassification {
val cnnModel = setupCnnModel(ctx, exon.batchSize, sentenceSize, numEmbed)
trainCNN(cnnModel, trainDats, trainLabels, devDatas, devLabels, exon.batchSize,
- exon.saveModelPath, learningRate = exon.lr)
+ exon.saveModelPath, learningRate = exon.lr)
} catch {
case ex: Exception => {
diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/DataHelper.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/DataHelper.scala
index 0d22fe138d3..6a128b0b5eb 100644
--- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/DataHelper.scala
+++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/DataHelper.scala
@@ -17,18 +17,13 @@
package org.apache.mxnetexamples.cnntextclassification
-import scala.io.Source
-import java.io.BufferedInputStream
-import java.io.FileInputStream
-import java.io.DataInputStream
-import java.io.InputStream
-import org.apache.mxnet.Random
-import org.apache.mxnet.Context
-import org.apache.mxnet.Shape
+import java.io.{BufferedInputStream, DataInputStream, FileInputStream, InputStream}
+import java.nio.charset.CodingErrorAction
+
+import org.apache.mxnet.{Context, Random, Shape}
+
+import scala.io.{Codec, Source}
-/**
- * @author Depeng Liang
- */
object DataHelper {
def cleanStr(str: String): String = {
@@ -52,6 +47,9 @@ object DataHelper {
// Returns split sentences and labels.
def loadMRDataAndLabels(dataPath: String): (Array[Array[String]], Array[Float]) = {
// load data from file
+ implicit val codec = Codec("UTF-8")
+ codec.onMalformedInput(CodingErrorAction.REPLACE)
+ codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
val positiveExamples = {
val lines = Source.fromFile(s"$dataPath/rt-polarity.pos").mkString.split("\n")
lines.map(_.trim())
@@ -157,7 +155,12 @@ object DataHelper {
val xVec = sentences.map { sentence =>
sentence.map { word =>
if (word2vec.contains(word)) word2vec(word)
- else Random.uniform(-0.25f, 0.25f, Shape(embeddingSize), Context.cpu()).toArray
+ else {
+ val temp = Random.uniform(-0.25f, 0.25f, Shape(embeddingSize), Context.cpu())
+ val result = temp.toArray
+ temp.dispose()
+ result
+ }
}
}
xVec
diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md
new file mode 100644
index 00000000000..5e3602e8ab1
--- /dev/null
+++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/cnntextclassification/README.md
@@ -0,0 +1,23 @@
+# CNN Text Classification Example for Scala
+This is the example using Scala type-safe api doing CNN text classification.
+This example is only for Illustration and not modeled to achieve the best accuracy.
+
+Please contribute to improve the dev accuracy of the model.
+
+## Setup
+
+Please configure your maven project using our latest release. An tutorial to do that can be found here:
+[IntelliJ IDE (or alternative IDE) project setup](http://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html)
+
+### Download the training files
+```$xslt
+https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/CNN/rt-polarity.pos
+https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/CNN/rt-polarity.neg
+```
+### Download pretrained Word2Vec Model
+I used the SLIM version, you can try with the full version to see if the accuracy can improve
+```$xslt
+https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/CNN/GoogleNews-vectors-negative300-SLIM.bin
+```
+### Train the model
+Please configure the [args](https://github.com/apache/incubator-mxnet/blob/scala-package/examples/src/main/scala/org/apache/mxnet/examples/cnntextclassification/CNNTextClassification.scala#L299-L312) required for the model here and then run it.
\ No newline at end of file
diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
new file mode 100644
index 00000000000..f7d1332792f
--- /dev/null
+++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/cnntextclassification/CNNClassifierExampleSuite.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mxnetexamples.cnntextclassification
+
+import java.io.File
+import java.net.URL
+
+import org.apache.commons.io.FileUtils
+import org.apache.mxnet.Context
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
+import org.slf4j.LoggerFactory
+
+import scala.sys.process.Process
+
+/**
+ * Integration test for imageClassifier example.
+ * This will run as a part of "make scalatest"
+ */
+class CNNClassifierExampleSuite extends FunSuite with BeforeAndAfterAll {
+ private val logger = LoggerFactory.getLogger(classOf[CNNClassifierExampleSuite])
+
+ test("Example CI - CNN Example") {
+
+ if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
+ System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
+ val context = Context.gpu()
+ val tempDirPath = System.getProperty("java.io.tmpdir")
+ val w2vModelName = "GoogleNews-vectors-negative300-SLIM.bin"
+
+ logger.info("tempDirPath: %s".format(tempDirPath))
+
+ logger.info("Downloading CNN text...")
+ val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala"
+ var tmpFile = new File(tempDirPath + "/CNN/rt-polarity.pos")
+ if (!tmpFile.exists()) {
+ FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/rt-polarity.pos"),
+ tmpFile)
+ }
+ tmpFile = new File(tempDirPath + "/CNN/rt-polarity.neg")
+ if (!tmpFile.exists()) {
+ FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/rt-polarity.neg"),
+ tmpFile)
+ }
+ logger.info("Downloading pretrianed Word2Vec Model, may take a while")
+ tmpFile = new File(tempDirPath + "/CNN/" + w2vModelName)
+ if (!tmpFile.exists()) {
+ FileUtils.copyURLToFile(new URL(baseUrl + "/scala-example-ci/CNN/" + w2vModelName),
+ tmpFile)
+ }
+
+ val modelDirPath = tempDirPath + File.separator + "CNN"
+
+ val output = CNNTextClassification.test(modelDirPath + File.separator + w2vModelName,
+ modelDirPath, context, modelDirPath)
+
+ Process("rm -rf " + modelDirPath) !
+
+ assert(output >= 0.4f)
+ } else {
+ logger.info("Skip this test as it intended for GPU only")
+ }
+ }
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services