You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/10/09 18:42:02 UTC
[GitHub] lanking520 closed pull request #12608: [MXNET-913] Java API ---
Scala NDArray Macros
lanking520 closed pull request #12608: [MXNET-913] Java API --- Scala NDArray Macros
URL: https://github.com/apache/incubator-mxnet/pull/12608
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/NDArray.scala b/scala-package/core/src/main/scala/org/apache/mxnet/NDArray.scala
index 9b6a7dc6654..a0150dc95dc 100644
--- a/scala-package/core/src/main/scala/org/apache/mxnet/NDArray.scala
+++ b/scala-package/core/src/main/scala/org/apache/mxnet/NDArray.scala
@@ -21,6 +21,7 @@ import java.nio.{ByteBuffer, ByteOrder}
import org.apache.mxnet.Base._
import org.apache.mxnet.DType.DType
+import org.apache.mxnet.api.java.JNDArrayAPI
import org.slf4j.LoggerFactory
import scala.collection.mutable
@@ -40,6 +41,8 @@ object NDArray extends NDArrayBase {
val api = NDArrayAPI
+ val japi = JNDArrayAPI
+
private def addDependency(froms: Array[NDArray], tos: Array[NDArray]): Unit = {
froms.foreach { from =>
val weakRef = new WeakReference(from)
diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/api/java/JNDArrayAPI.scala b/scala-package/core/src/main/scala/org/apache/mxnet/api/java/JNDArrayAPI.scala
new file mode 100644
index 00000000000..5d7b7612ce8
--- /dev/null
+++ b/scala-package/core/src/main/scala/org/apache/mxnet/api/java/JNDArrayAPI.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mxnet.api.java
+@AddJNDArrayAPIs(false)
+/**
+ * typesafe Java NDArray API: NDArray.japi._
+ * Main code will be generated during compile time through Macros
+ */
+object JNDArrayAPI extends JNDArrayAPIBase{
+ // TODO: Implement CustomOp for NDArray
+}
diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala
index b4efa659443..21863282a03 100644
--- a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala
+++ b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala
@@ -38,8 +38,9 @@ private[mxnet] object APIDocGenerator{
def main(args: Array[String]) : Unit = {
val FILE_PATH = args(0)
val hashCollector = ListBuffer[String]()
- hashCollector += absClassGen(FILE_PATH, true)
- hashCollector += absClassGen(FILE_PATH, false)
+ hashCollector += absClassGen(FILE_PATH, true, false)
+ hashCollector += absClassGen(FILE_PATH, false, false)
+ hashCollector += absClassGen(FILE_PATH + "/api/java/", false, true)
hashCollector += nonTypeSafeClassGen(FILE_PATH, true)
hashCollector += nonTypeSafeClassGen(FILE_PATH, false)
val finalHash = hashCollector.mkString("\n")
@@ -52,7 +53,7 @@ private[mxnet] object APIDocGenerator{
org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString(digest)
}
- def absClassGen(FILE_PATH : String, isSymbol : Boolean) : String = {
+ def absClassGen(FILE_PATH : String, isSymbol : Boolean, isJava : Boolean) : String = {
// scalastyle:off
val absClassFunctions = getSymbolNDArrayMethods(isSymbol)
// Defines Operators that should not generated
@@ -61,14 +62,14 @@ private[mxnet] object APIDocGenerator{
val absFuncs = absClassFunctions.filterNot(_.name.startsWith("_"))
.filterNot(ele => notGenerated.contains(ele.name))
.map(absClassFunction => {
- val scalaDoc = generateAPIDocFromBackend(absClassFunction)
- val defBody = generateAPISignature(absClassFunction, isSymbol)
+ val scalaDoc = generateAPIDocFromBackend(absClassFunction, true, isJava)
+ val defBody = generateAPISignature(absClassFunction, isSymbol, isJava)
s"$scalaDoc\n$defBody"
})
- val packageName = if (isSymbol) "SymbolAPIBase" else "NDArrayAPIBase"
+ val packageName = (if (isJava) "J" else "") + (if (isSymbol) "SymbolAPIBase" else "NDArrayAPIBase")
val apacheLicence = "/*\n* Licensed to the Apache Software Foundation (ASF) under one or more\n* contributor license agreements. See the NOTICE file distributed with\n* this work for additional information regarding copyright ownership.\n* The ASF licenses this file to You under the Apache License, Version 2.0\n* (the \"License\"); you may not use this file except in compliance with\n* the License. You may obtain a copy of the License at\n*\n* http://www.apache.org/licenses/LICENSE-2.0\n*\n* Unless required by applicable law or agreed to in writing, software\n* distributed under the License is distributed on an \"AS IS\" BASIS,\n* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n* See the License for the specific language governing permissions and\n* limitations under the License.\n*/\n"
val scalaStyle = "// scalastyle:off"
- val packageDef = "package org.apache.mxnet"
+ val packageDef = if (isJava) "package org.apache.mxnet.api.java" else "package org.apache.mxnet"
val imports = "import org.apache.mxnet.annotation.Experimental"
val absClassDef = s"abstract class $packageName"
val finalStr = s"$apacheLicence\n$scalaStyle\n$packageDef\n$imports\n$absClassDef {\n${absFuncs.mkString("\n")}\n}"
@@ -107,7 +108,7 @@ private[mxnet] object APIDocGenerator{
}
// Generate ScalaDoc type
- def generateAPIDocFromBackend(func : absClassFunction, withParam : Boolean = true) : String = {
+ def generateAPIDocFromBackend(func : absClassFunction, withParam : Boolean = true, isJava : Boolean = false) : String = {
val desc = ArrayBuffer[String]()
desc += " * <pre>"
func.desc.split("\n").foreach({ currStr =>
@@ -115,11 +116,14 @@ private[mxnet] object APIDocGenerator{
})
desc += " * </pre>"
val params = func.listOfArgs.map({ absClassArg =>
- val currArgName = absClassArg.argName match {
+ var currArgName = absClassArg.argName match {
case "var" => "vari"
case "type" => "typeOf"
case _ => absClassArg.argName
}
+ if (isJava && absClassArg.isOptional) {
+ currArgName = s"optional_$currArgName"
+ }
s" * @param $currArgName\t\t${absClassArg.argDesc}"
})
val returnType = s" * @return ${func.returnType}"
@@ -130,16 +134,20 @@ private[mxnet] object APIDocGenerator{
}
}
- def generateAPISignature(func : absClassFunction, isSymbol : Boolean) : String = {
+ def generateAPISignature(func : absClassFunction, isSymbol : Boolean, isJava : Boolean = false) : String = {
var argDef = ListBuffer[String]()
func.listOfArgs.foreach(absClassArg => {
- val currArgName = absClassArg.argName match {
+ var currArgName = absClassArg.argName match {
case "var" => "vari"
case "type" => "typeOf"
case _ => absClassArg.argName
}
if (absClassArg.isOptional) {
- argDef += s"$currArgName : Option[${absClassArg.argType}] = None"
+ if (isJava) {
+ argDef += s"optional_$currArgName : ${absClassArg.argType} = null"
+ } else {
+ argDef += s"$currArgName : Option[${absClassArg.argType}] = None"
+ }
}
else {
argDef += s"$currArgName : ${absClassArg.argType}"
@@ -150,7 +158,7 @@ private[mxnet] object APIDocGenerator{
argDef += "name : String = null"
argDef += "attr : Map[String, String] = null"
} else {
- argDef += "out : Option[NDArray] = None"
+ argDef += (if (isJava) "out : org.apache.mxnet.NDArray = null" else "out : Option[NDArray] = None")
returnType = "org.apache.mxnet.NDArrayFuncReturn"
}
val experimentalTag = "@Experimental"
diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala
index 2d3a1c7ec5a..0c96c598abb 100644
--- a/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala
+++ b/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala
@@ -108,7 +108,7 @@ private[mxnet] object NDArrayMacro {
val currArgName = ndarrayarg.argName match {
case "var" => "vari"
case "type" => "typeOf"
- case default => ndarrayarg.argName
+ case _ => ndarrayarg.argName
}
if (ndarrayarg.isOptional) {
argDef += s"${currArgName} : Option[${ndarrayarg.argType}] = None"
diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/api/java/JNDArrayMacro.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/api/java/JNDArrayMacro.scala
new file mode 100644
index 00000000000..20878751bb0
--- /dev/null
+++ b/scala-package/macros/src/main/scala/org/apache/mxnet/api/java/JNDArrayMacro.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mxnet.api.java
+
+import org.apache.mxnet.init.Base._
+import org.apache.mxnet.utils.{CToScalaUtils, OperatorBuildUtils}
+
+import scala.annotation.StaticAnnotation
+import scala.collection.mutable.ListBuffer
+import scala.language.experimental.macros
+import scala.reflect.macros.blackbox
+
+private[mxnet] class AddJNDArrayAPIs(isContrib: Boolean) extends StaticAnnotation {
+ private[mxnet] def macroTransform(annottees: Any*) = macro JNDArrayMacro.typeSafeAPIDefs
+}
+
+private[mxnet] object JNDArrayMacro {
+ case class NDArrayArg(argName: String, argType: String, isOptional : Boolean)
+ case class NDArrayFunction(name: String, listOfArgs: List[NDArrayArg])
+
+ // scalastyle:off havetype
+ def typeSafeAPIDefs(c: blackbox.Context)(annottees: c.Expr[Any]*) = {
+ typeSafeAPIImpl(c)(annottees: _*)
+ }
+ // scalastyle:off havetype
+
+ private val ndarrayFunctions: List[NDArrayFunction] = initNDArrayModule()
+
+ private def typeSafeAPIImpl(c: blackbox.Context)(annottees: c.Expr[Any]*) : c.Expr[Any] = {
+ import c.universe._
+
+ val isContrib: Boolean = c.prefix.tree match {
+ case q"new AddJNDArrayAPIs($b)" => c.eval[Boolean](c.Expr(b))
+ }
+ // Defines Operators that should not generated
+ val notGenerated = Set("Custom")
+
+ val newNDArrayFunctions = {
+ if (isContrib) ndarrayFunctions.filter(
+ func => func.name.startsWith("_contrib_") || !func.name.startsWith("_"))
+ else ndarrayFunctions.filterNot(_.name.startsWith("_"))
+ }.filterNot(ele => notGenerated.contains(ele.name))
+
+ val functionDefs = newNDArrayFunctions.map { ndarrayfunction =>
+
+ // Construct argument field
+ var argDef = ListBuffer[String]()
+ // Construct Implementation field
+ var impl = ListBuffer[String]()
+ impl += "val map = scala.collection.mutable.Map[String, Any]()"
+ impl += "val args = scala.collection.mutable.ArrayBuffer.empty[org.apache.mxnet.NDArray]"
+ ndarrayfunction.listOfArgs.foreach({ ndarrayarg =>
+ // var is a special word used to define variable in Scala,
+ // need to changed to something else in order to make it work
+ var currArgName = ndarrayarg.argName match {
+ case "var" => "vari"
+ case "type" => "typeOf"
+ case default => ndarrayarg.argName
+ }
+ if (ndarrayarg.isOptional) {
+ currArgName = s"optional_$currArgName"
+ argDef += s"$currArgName : ${ndarrayarg.argType} = null"
+ }
+ else {
+ argDef += s"$currArgName : ${ndarrayarg.argType}"
+ }
+ // NDArray arg implementation
+ val returnType = "org.apache.mxnet.NDArray"
+
+ // TODO: Currently we do not add place holder for NDArray
+ // Example: an NDArray operator like the following format
+ // nd.foo(arg1: NDArray(required), arg2: NDArray(Optional), arg3: NDArray(Optional)
+ // If we place nd.foo(arg1, arg3 = arg3), do we need to add place holder for arg2?
+ // What it should be?
+ val base =
+ if (ndarrayarg.argType.equals(returnType)) {
+ s"args += $currArgName"
+ } else if (ndarrayarg.argType.equals(s"Array[$returnType]")){
+ s"args ++= $currArgName"
+ } else {
+ "map(\"" + ndarrayarg.argName + "\") = " + currArgName
+ }
+ impl.append(
+ if (ndarrayarg.isOptional) s"if ($currArgName != null) $base"
+ else base
+ )
+ })
+ // add default out parameter
+ argDef += "out : org.apache.mxnet.NDArray = null"
+ impl += "if (out != null) map(\"out\") = out"
+ // scalastyle:off
+ impl += "org.apache.mxnet.NDArray.genericNDArrayFunctionInvoke(\"" + ndarrayfunction.name + "\", args.toSeq, map.toMap)"
+ // scalastyle:on
+ // Combine and build the function string
+ val returnType = "org.apache.mxnet.NDArrayFuncReturn"
+ var finalStr = s"def ${ndarrayfunction.name}"
+ finalStr += s" (${argDef.mkString(",")}) : $returnType"
+ finalStr += s" = {${impl.mkString("\n")}}"
+ c.parse(finalStr).asInstanceOf[DefDef]
+ }
+
+ structGeneration(c)(functionDefs, annottees : _*)
+ }
+
+ private def structGeneration(c: blackbox.Context)
+ (funcDef : List[c.universe.DefDef], annottees: c.Expr[Any]*)
+ : c.Expr[Any] = {
+ import c.universe._
+ val inputs = annottees.map(_.tree).toList
+ // pattern match on the inputs
+ val modDefs = inputs map {
+ case ClassDef(mods, name, something, template) =>
+ val q = template match {
+ case Template(superMaybe, emptyValDef, defs) =>
+ Template(superMaybe, emptyValDef, defs ++ funcDef)
+ case ex =>
+ throw new IllegalArgumentException(s"Invalid template: $ex")
+ }
+ ClassDef(mods, name, something, q)
+ case ModuleDef(mods, name, template) =>
+ val q = template match {
+ case Template(superMaybe, emptyValDef, defs) =>
+ Template(superMaybe, emptyValDef, defs ++ funcDef)
+ case ex =>
+ throw new IllegalArgumentException(s"Invalid template: $ex")
+ }
+ ModuleDef(mods, name, q)
+ case ex =>
+ throw new IllegalArgumentException(s"Invalid macro input: $ex")
+ }
+ // wrap the result up in an Expr, and return it
+ val result = c.Expr(Block(modDefs, Literal(Constant())))
+ result
+ }
+
+
+
+
+ // List and add all the atomic symbol functions to current module.
+ private def initNDArrayModule(): List[NDArrayFunction] = {
+ val opNames = ListBuffer.empty[String]
+ _LIB.mxListAllOpNames(opNames)
+ opNames.map(opName => {
+ val opHandle = new RefLong
+ _LIB.nnGetOpHandle(opName, opHandle)
+ makeNDArrayFunction(opHandle.value, opName)
+ }).toList
+ }
+
+ // Create an atomic symbol function by handle and function name.
+ private def makeNDArrayFunction(handle: NDArrayHandle, aliasName: String)
+ : NDArrayFunction = {
+ val name = new RefString
+ val desc = new RefString
+ val keyVarNumArgs = new RefString
+ val numArgs = new RefInt
+ val argNames = ListBuffer.empty[String]
+ val argTypes = ListBuffer.empty[String]
+ val argDescs = ListBuffer.empty[String]
+
+ _LIB.mxSymbolGetAtomicSymbolInfo(
+ handle, name, desc, numArgs, argNames, argTypes, argDescs, keyVarNumArgs)
+ val paramStr = OperatorBuildUtils.ctypes2docstring(argNames, argTypes, argDescs)
+ val extraDoc: String = if (keyVarNumArgs.value != null && keyVarNumArgs.value.length > 0) {
+ s"This function support variable length of positional input (${keyVarNumArgs.value})."
+ } else {
+ ""
+ }
+ val realName = if (aliasName == name.value) "" else s"(a.k.a., ${name.value})"
+ val docStr = s"$aliasName $realName\n${desc.value}\n\n$paramStr\n$extraDoc\n"
+ // scalastyle:off println
+ if (System.getenv("MXNET4J_PRINT_OP_DEF") != null
+ && System.getenv("MXNET4J_PRINT_OP_DEF").toLowerCase == "true") {
+ println("NDArray function definition:\n" + docStr)
+ }
+ // scalastyle:on println
+ val argList = argNames zip argTypes map { case (argName, argType) =>
+ val typeAndOption =
+ CToScalaUtils.argumentCleaner(argName, argType, "org.apache.mxnet.NDArray")
+ new NDArrayArg(argName, typeAndOption._1, typeAndOption._2)
+ }
+ new NDArrayFunction(aliasName, argList.toList)
+ }
+}
diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/utils/CToScalaUtils.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/utils/CToScalaUtils.scala
index d0ebe5b1d2c..19a54193cec 100644
--- a/scala-package/macros/src/main/scala/org/apache/mxnet/utils/CToScalaUtils.scala
+++ b/scala-package/macros/src/main/scala/org/apache/mxnet/utils/CToScalaUtils.scala
@@ -28,12 +28,12 @@ private[mxnet] object CToScalaUtils {
case "Symbol" | "NDArray" | "NDArray-or-Symbol" => returnType
case "Symbol[]" | "NDArray[]" | "NDArray-or-Symbol[]" | "SymbolorSymbol[]"
=> s"Array[$returnType]"
- case "float" | "real_t" | "floatorNone" => "org.apache.mxnet.Base.MXFloat"
- case "int" | "intorNone" | "int(non-negative)" => "Int"
- case "long" | "long(non-negative)" => "Long"
- case "double" | "doubleorNone" => "Double"
+ case "float" | "real_t" | "floatorNone" => "java.lang.Float"
+ case "int" | "intorNone" | "int(non-negative)" => "java.lang.Integer"
+ case "long" | "long(non-negative)" => "java.lang.Long"
+ case "double" | "doubleorNone" => "java.lang.Double"
case "string" => "String"
- case "boolean" | "booleanorNone" => "Boolean"
+ case "boolean" | "booleanorNone" => "java.lang.Boolean"
case "tupleof<float>" | "tupleof<double>" | "tupleof<>" | "ptr" | "" => "Any"
case default => throw new IllegalArgumentException(
s"Invalid type for args: $default\nString argType: $argType\nargName: $argName")
diff --git a/scala-package/macros/src/test/scala/org/apache/mxnet/MacrosSuite.scala b/scala-package/macros/src/test/scala/org/apache/mxnet/MacrosSuite.scala
index c3a7c58c1af..4404b0885d5 100644
--- a/scala-package/macros/src/test/scala/org/apache/mxnet/MacrosSuite.scala
+++ b/scala-package/macros/src/test/scala/org/apache/mxnet/MacrosSuite.scala
@@ -36,7 +36,7 @@ class MacrosSuite extends FunSuite with BeforeAndAfterAll {
)
val output = List(
("org.apache.mxnet.Symbol", true),
- ("Int", false),
+ ("java.lang.Integer", false),
("org.apache.mxnet.Shape", true),
("String", true),
("Any", false)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services