From 072dd5a51c813afd789f3b45ed6b515de6fb756d Mon Sep 17 00:00:00 2001 From: Lanking Date: Wed, 18 Jul 2018 09:20:37 -0700 Subject: [PATCH] [MXNET-531] Custom Operator Example for Scala (#11401) Update Custom Operator Example to use new Symbol.api --- .../scala/org/apache/mxnet/NDArrayAPI.scala | 1 + .../scala/org/apache/mxnet/SymbolAPI.scala | 8 + .../apache/mxnetexamples/customop/Data.scala | 7 +- .../customop/ExampleCustomOp.scala | 230 +++++++++--------- .../customop/ExampleCustomOpWithRtc.scala | 222 +++++++++-------- .../apache/mxnetexamples/customop/README.md | 22 ++ .../customop/CustomOpExampleSuite.scala | 86 +++++++ .../org/apache/mxnet/APIDocGenerator.scala | 10 +- .../scala/org/apache/mxnet/NDArrayMacro.scala | 6 +- .../scala/org/apache/mxnet/SymbolMacro.scala | 5 +- 10 files changed, 354 insertions(+), 243 deletions(-) create mode 100644 scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md create mode 100644 scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/NDArrayAPI.scala b/scala-package/core/src/main/scala/org/apache/mxnet/NDArrayAPI.scala index 6136db29d1eb..1d8551c1b1e5 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/NDArrayAPI.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/NDArrayAPI.scala @@ -21,4 +21,5 @@ package org.apache.mxnet * Main code will be generated during compile time through Macros */ object NDArrayAPI extends NDArrayAPIBase { + // TODO: Implement CustomOp for NDArray } diff --git a/scala-package/core/src/main/scala/org/apache/mxnet/SymbolAPI.scala b/scala-package/core/src/main/scala/org/apache/mxnet/SymbolAPI.scala index 56da4fa64cf4..1bfb0559cf96 100644 --- a/scala-package/core/src/main/scala/org/apache/mxnet/SymbolAPI.scala +++ b/scala-package/core/src/main/scala/org/apache/mxnet/SymbolAPI.scala @@ -16,6 +16,8 @@ */ package org.apache.mxnet +import scala.collection.mutable + @AddSymbolAPIs(false) /** @@ -23,4 +25,10 @@ package org.apache.mxnet * Main code will be generated during compile time through Macros */ object SymbolAPI extends SymbolAPIBase { + def Custom (op_type : String, kwargs : mutable.Map[String, Any], + name : String = null, attr : Map[String, String] = null) : Symbol = { + val map = kwargs + map.put("op_type", op_type) + Symbol.createSymbolGeneral("Custom", name, attr, Seq(), map.toMap) + } } diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/Data.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/Data.scala index 7cab3c0bb460..d61269c131ff 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/Data.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/Data.scala @@ -17,13 +17,8 @@ package org.apache.mxnetexamples.customop -import org.apache.mxnet.Shape -import org.apache.mxnet.IO -import org.apache.mxnet.DataIter +import org.apache.mxnet.{DataIter, IO, Shape} -/** - * @author Depeng Liang - */ object Data { // return train and val iterators for mnist def mnistIterator(dataPath: String, batchSize: Int, inputShape: Shape): (DataIter, DataIter) = { diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOp.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOp.scala index d033d6525e50..a4b347959bfe 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOp.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOp.scala @@ -17,37 +17,26 @@ package org.apache.mxnetexamples.customop +import org.apache.mxnet.Callback.Speedometer +import org.apache.mxnet.DType.DType +import org.apache.mxnet.{Accuracy, Context, CustomOp, CustomOpProp, NDArray, Operator, Shape, Symbol, Xavier} +import org.apache.mxnet.optimizer.RMSProp import org.kohsuke.args4j.{CmdLineParser, Option} import org.slf4j.LoggerFactory + import scala.collection.JavaConverters._ -import org.apache.mxnet.Symbol -import org.apache.mxnet.DType.DType -import org.apache.mxnet.DataIter -import org.apache.mxnet.DataBatch -import org.apache.mxnet.NDArray -import org.apache.mxnet.Shape -import org.apache.mxnet.EvalMetric -import org.apache.mxnet.Context -import org.apache.mxnet.Xavier -import org.apache.mxnet.optimizer.RMSProp -import org.apache.mxnet.CustomOp -import org.apache.mxnet.CustomOpProp -import org.apache.mxnet.Operator -import org.apache.mxnet.optimizer.SGD -import org.apache.mxnet.Accuracy -import org.apache.mxnet.Callback.Speedometer +import scala.collection.mutable /** - * Example of CustomOp - * @author Depeng Liang - */ + * Example of CustomOp + */ object ExampleCustomOp { private val logger = LoggerFactory.getLogger(classOf[ExampleCustomOp]) class Softmax(_param: Map[String, String]) extends CustomOp { - override def forward(sTrain: Boolean, req: Array[String], - inData: Array[NDArray], outData: Array[NDArray], aux: Array[NDArray]): Unit = { + override def forward(sTrain: Boolean, req: Array[String], inData: Array[NDArray], + outData: Array[NDArray], aux: Array[NDArray]): Unit = { val xShape = inData(0).shape val x = inData(0).toArray.grouped(xShape(1)).toArray val yArr = x.map { it => @@ -63,8 +52,8 @@ object ExampleCustomOp { } override def backward(req: Array[String], outGrad: Array[NDArray], - inData: Array[NDArray], outData: Array[NDArray], - inGrad: Array[NDArray], aux: Array[NDArray]): Unit = { + inData: Array[NDArray], outData: Array[NDArray], + inGrad: Array[NDArray], aux: Array[NDArray]): Unit = { val l = inData(1).toArray.map(_.toInt) val oShape = outData(0).shape val yArr = outData(0).toArray.grouped(oShape(1)).toArray @@ -86,7 +75,7 @@ object ExampleCustomOp { override def listOutputs(): Array[String] = Array("output") override def inferShape(inShape: Array[Shape]): - (Array[Shape], Array[Shape], Array[Shape]) = { + (Array[Shape], Array[Shape], Array[Shape]) = { val dataShape = inShape(0) val labelShape = Shape(dataShape(0)) val outputShape = dataShape @@ -94,16 +83,113 @@ object ExampleCustomOp { } override def inferType(inType: Array[DType]): - (Array[DType], Array[DType], Array[DType]) = { + (Array[DType], Array[DType], Array[DType]) = { (inType, inType.take(1), null) } override def createOperator(ctx: String, inShapes: Array[Array[Int]], - inDtypes: Array[Int]): CustomOp = new Softmax(this.kwargs) + inDtypes: Array[Int]): CustomOp = new Softmax(this.kwargs) } Operator.register("softmax", new SoftmaxProp) + def test(dataPath : String, ctx : Context) : Float = { + val data = Symbol.Variable("data") + val label = Symbol.Variable("label") + val fc1 = Symbol.api.FullyConnected(data = Some(data), num_hidden = 128, name = "fc1") + val act1 = Symbol.api.Activation (data = Some(fc1), "relu", name = "relu") + val fc2 = Symbol.api.FullyConnected(Some(act1), None, None, 64, name = "fc2") + val act2 = Symbol.api.Activation(data = Some(fc2), "relu", name = "relu2") + val fc3 = Symbol.api.FullyConnected(Some(act2), None, None, 10, name = "fc3") + val kwargs = mutable.Map[String, Any]("label" -> label, "data" -> fc3) + val mlp = Symbol.api.Custom(op_type = "softmax", name = "softmax", kwargs = kwargs) + + val (trainIter, testIter) = + Data.mnistIterator(dataPath, batchSize = 100, inputShape = Shape(784)) + + val datasAndLabels = trainIter.provideData ++ trainIter.provideLabel + val (argShapes, outputShapes, auxShapes) = mlp.inferShape(datasAndLabels) + + val initializer = new Xavier(factorType = "in", magnitude = 2.34f) + val argNames = mlp.listArguments() + val argDict = argNames.zip(argShapes.map(s => NDArray.empty(s, ctx))).toMap + + val gradDict = argNames.zip(argShapes).filter { case (name, shape) => + !datasAndLabels.contains(name) + }.map(x => x._1 -> NDArray.empty(x._2, ctx) ).toMap + + argDict.foreach { case (name, ndArray) => + if (!datasAndLabels.contains(name)) { + initializer.initWeight(name, ndArray) + } + } + + val executor = mlp.bind(ctx, argDict, gradDict) + val lr = 0.001f + val opt = new RMSProp(learningRate = lr, wd = 0.00001f) + val paramsGrads = gradDict.toList.zipWithIndex.map { case ((name, grad), idx) => + (idx, name, grad, opt.createState(idx, argDict(name))) + } + + val evalMetric = new Accuracy + val batchEndCallback = new Speedometer(100, 100) + val numEpoch = 10 + var validationAcc = 0.0f + + for (epoch <- 0 until numEpoch) { + val tic = System.currentTimeMillis + evalMetric.reset() + var nBatch = 0 + var epochDone = false + + trainIter.reset() + while (!epochDone) { + var doReset = true + while (doReset && trainIter.hasNext) { + val dataBatch = trainIter.next() + argDict("data").set(dataBatch.data(0)) + argDict("label").set(dataBatch.label(0)) + executor.forward(isTrain = true) + executor.backward() + paramsGrads.foreach { case (idx, name, grad, optimState) => + opt.update(idx, argDict(name), grad, optimState) + } + evalMetric.update(dataBatch.label, executor.outputs) + nBatch += 1 + batchEndCallback.invoke(epoch, nBatch, evalMetric) + } + if (doReset) { + trainIter.reset() + } + epochDone = true + } + val (name, value) = evalMetric.get + name.zip(value).foreach { case (n, v) => + logger.info(s"Epoch[$epoch] Train-accuracy=$v") + } + val toc = System.currentTimeMillis + logger.info(s"Epoch[$epoch] Time cost=${toc - tic}") + + evalMetric.reset() + testIter.reset() + while (testIter.hasNext) { + val evalBatch = testIter.next() + argDict("data").set(evalBatch.data(0)) + argDict("label").set(evalBatch.label(0)) + executor.forward(isTrain = true) + evalMetric.update(evalBatch.label, executor.outputs) + evalBatch.dispose() + } + val (names, values) = evalMetric.get + names.zip(values).foreach { case (n, v) => + logger.info(s"Epoch[$epoch] Validation-accuracy=$v") + validationAcc = Math.max(validationAcc, v) + } + } + executor.dispose() + validationAcc + } + def main(args: Array[String]): Unit = { val leop = new ExampleCustomOp val parser: CmdLineParser = new CmdLineParser(leop) @@ -115,98 +201,8 @@ object ExampleCustomOp { val dataName = Array("data") val labelName = Array("softmax_label") + test(leop.dataPath, ctx) - val data = Symbol.Variable("data") - val label = Symbol.Variable("label") - val fc1 = Symbol.FullyConnected("fc1")()(Map("data" -> data, "num_hidden" -> 128)) - val act1 = Symbol.Activation("relu1")()(Map("data" -> fc1, "act_type" -> "relu")) - val fc2 = Symbol.FullyConnected("fc2")()(Map("data" -> act1, "num_hidden" -> 64)) - val act2 = Symbol.Activation("relu2")()(Map("data" -> fc2, "act_type" -> "relu")) - val fc3 = Symbol.FullyConnected("fc3")()(Map("data" -> act2, "num_hidden" -> 10)) - val mlp = Symbol.Custom("softmax")()(Map("data" -> fc3, - "label" -> label, "op_type" -> "softmax")) - - val (trainIter, testIter) = - Data.mnistIterator(leop.dataPath, batchSize = 100, inputShape = Shape(784)) - - val datasAndLabels = trainIter.provideData ++ trainIter.provideLabel - val (argShapes, outputShapes, auxShapes) = mlp.inferShape(datasAndLabels) - - val initializer = new Xavier(factorType = "in", magnitude = 2.34f) - val argNames = mlp.listArguments() - val argDict = argNames.zip(argShapes.map(s => NDArray.empty(s, ctx))).toMap - - val gradDict = argNames.zip(argShapes).filter { case (name, shape) => - !datasAndLabels.contains(name) - }.map(x => x._1 -> NDArray.empty(x._2, ctx) ).toMap - - argDict.foreach { case (name, ndArray) => - if (!datasAndLabels.contains(name)) { - initializer.initWeight(name, ndArray) - } - } - - val executor = mlp.bind(ctx, argDict, gradDict) - val lr = 0.001f - val opt = new RMSProp(learningRate = lr, wd = 0.00001f) - val paramsGrads = gradDict.toList.zipWithIndex.map { case ((name, grad), idx) => - (idx, name, grad, opt.createState(idx, argDict(name))) - } - - val evalMetric = new Accuracy - val batchEndCallback = new Speedometer(100, 100) - val numEpoch = 20 - - for (epoch <- 0 until numEpoch) { - val tic = System.currentTimeMillis - evalMetric.reset() - var nBatch = 0 - var epochDone = false - - trainIter.reset() - while (!epochDone) { - var doReset = true - while (doReset && trainIter.hasNext) { - val dataBatch = trainIter.next() - argDict("data").set(dataBatch.data(0)) - argDict("label").set(dataBatch.label(0)) - executor.forward(isTrain = true) - executor.backward() - paramsGrads.foreach { case (idx, name, grad, optimState) => - opt.update(idx, argDict(name), grad, optimState) - } - evalMetric.update(dataBatch.label, executor.outputs) - nBatch += 1 - batchEndCallback.invoke(epoch, nBatch, evalMetric) - } - if (doReset) { - trainIter.reset() - } - epochDone = true - } - val (name, value) = evalMetric.get - name.zip(value).foreach { case (n, v) => - logger.info(s"Epoch[$epoch] Train-accuracy=$v") - } - val toc = System.currentTimeMillis - logger.info(s"Epoch[$epoch] Time cost=${toc - tic}") - - evalMetric.reset() - testIter.reset() - while (testIter.hasNext) { - val evalBatch = testIter.next() - argDict("data").set(evalBatch.data(0)) - argDict("label").set(evalBatch.label(0)) - executor.forward(isTrain = true) - evalMetric.update(evalBatch.label, executor.outputs) - evalBatch.dispose() - } - val (names, values) = evalMetric.get - names.zip(values).foreach { case (n, v) => - logger.info(s"Epoch[$epoch] Validation-accuracy=$v") - } - } - executor.dispose() } catch { case ex: Exception => { logger.error(ex.getMessage, ex) diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOpWithRtc.scala b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOpWithRtc.scala index 030670caca19..c3ac347353df 100644 --- a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOpWithRtc.scala +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/ExampleCustomOpWithRtc.scala @@ -17,31 +17,19 @@ package org.apache.mxnetexamples.customop +import org.apache.mxnet.Callback.Speedometer +import org.apache.mxnet.DType.DType +import org.apache.mxnet.{Accuracy, Context, CustomOp, CustomOpProp, NDArray, Operator, Rtc, Shape, Symbol, Xavier} +import org.apache.mxnet.optimizer.RMSProp import org.kohsuke.args4j.{CmdLineParser, Option} import org.slf4j.LoggerFactory + import scala.collection.JavaConverters._ -import org.apache.mxnet.Symbol -import org.apache.mxnet.DType.DType -import org.apache.mxnet.DataIter -import org.apache.mxnet.DataBatch -import org.apache.mxnet.NDArray -import org.apache.mxnet.Shape -import org.apache.mxnet.EvalMetric -import org.apache.mxnet.Context -import org.apache.mxnet.Xavier -import org.apache.mxnet.optimizer.RMSProp -import org.apache.mxnet.CustomOp -import org.apache.mxnet.CustomOpProp -import org.apache.mxnet.Operator -import org.apache.mxnet.optimizer.SGD -import org.apache.mxnet.Accuracy -import org.apache.mxnet.Callback.Speedometer -import org.apache.mxnet.Rtc +import scala.collection.mutable /** - * Example of CustomOp with Rtc - * @author Depeng Liang - */ + * Example of CustomOp with Rtc + */ object ExampleCustomOpWithRtc { private val logger = LoggerFactory.getLogger(classOf[ExampleCustomOpWithRtc]) @@ -79,8 +67,8 @@ object ExampleCustomOpWithRtc { } override def backward(req: Array[String], outGrad: Array[NDArray], - inData: Array[NDArray], outData: Array[NDArray], - inGrad: Array[NDArray], aux: Array[NDArray]): Unit = { + inData: Array[NDArray], outData: Array[NDArray], + inGrad: Array[NDArray], aux: Array[NDArray]): Unit = { val l = inData(1) val y = outData(0) val dx = inGrad(0) @@ -108,7 +96,7 @@ object ExampleCustomOpWithRtc { override def listOutputs(): Array[String] = Array("output") override def inferShape(inShape: Array[Shape]): - (Array[Shape], Array[Shape], Array[Shape]) = { + (Array[Shape], Array[Shape], Array[Shape]) = { val dataShape = inShape(0) val labelShape = Shape(dataShape(0)) val outputShape = dataShape @@ -116,16 +104,111 @@ object ExampleCustomOpWithRtc { } override def inferType(inType: Array[DType]): - (Array[DType], Array[DType], Array[DType]) = { + (Array[DType], Array[DType], Array[DType]) = { (inType, inType.take(1), null) } override def createOperator(ctx: String, inShapes: Array[Array[Int]], - inDtypes: Array[Int]): CustomOp = new Softmax(this.kwargs) + inDtypes: Array[Int]): CustomOp = new Softmax(this.kwargs) } Operator.register("softmax", new SoftmaxProp) + def test(dataPath : String, ctx : Context) : Float = { + val data = Symbol.Variable("data") + val label = Symbol.Variable("label") + val fc1 = Symbol.api.FullyConnected(data = Some(data), num_hidden = 128, name = "fc1") + val act1 = Symbol.api.Activation (data = Some(fc1), "relu", name = "relu") + val fc2 = Symbol.api.FullyConnected(Some(act1), None, None, 64, name = "fc2") + val act2 = Symbol.api.Activation(data = Some(fc2), "relu", name = "relu2") + val fc3 = Symbol.api.FullyConnected(Some(act2), None, None, 10, name = "fc3") + val kwargs = mutable.Map[String, Any]("label" -> label, "data" -> fc3, + "forwardBlockDim" -> new Shape(1, 1, 1)) + val mlp = Symbol.api.Custom(op_type = "softmax", name = "softmax", kwargs = kwargs) + + val (trainIter, testIter) = + Data.mnistIterator(dataPath, batchSize = 100, inputShape = Shape(784)) + val datasAndLabels = trainIter.provideData ++ trainIter.provideLabel + val (argShapes, outputShapes, auxShapes) = mlp.inferShape(datasAndLabels) + + val initializer = new Xavier(factorType = "in", magnitude = 2.34f) + + val argNames = mlp.listArguments() + val argDict = argNames.zip(argShapes.map(s => NDArray.empty(s, ctx))).toMap + val gradDict = argNames.zip(argShapes).filter { case (name, shape) => + !datasAndLabels.contains(name) + }.map(x => x._1 -> NDArray.empty(x._2, ctx) ).toMap + argDict.foreach { case (name, ndArray) => + if (!datasAndLabels.contains(name)) { + initializer.initWeight(name, ndArray) + } + } + + val executor = mlp.bind(ctx, argDict, gradDict) + val lr = 0.001f + val opt = new RMSProp(learningRate = lr, wd = 0.00001f) + val paramsGrads = gradDict.toList.zipWithIndex.map { case ((name, grad), idx) => + (idx, name, grad, opt.createState(idx, argDict(name))) + } + val evalMetric = new Accuracy + val batchEndCallback = new Speedometer(100, 100) + val numEpoch = 10 + var validationAcc = 0.0f + + for (epoch <- 0 until numEpoch) { + val tic = System.currentTimeMillis + evalMetric.reset() + var nBatch = 0 + var epochDone = false + + trainIter.reset() + while (!epochDone) { + var doReset = true + while (doReset && trainIter.hasNext) { + val dataBatch = trainIter.next() + argDict("data").set(dataBatch.data(0)) + argDict("label").set(dataBatch.label(0)) + executor.forward(isTrain = true) + executor.backward() + paramsGrads.foreach { case (idx, name, grad, optimState) => + opt.update(idx, argDict(name), grad, optimState) + } + evalMetric.update(dataBatch.label, executor.outputs) + nBatch += 1 + batchEndCallback.invoke(epoch, nBatch, evalMetric) + } + if (doReset) { + trainIter.reset() + } + epochDone = true + } + val (name, value) = evalMetric.get + name.zip(value).foreach { case (n, v) => + logger.info(s"Epoch[$epoch] Train-accuracy=$v") + } + val toc = System.currentTimeMillis + logger.info(s"Epoch[$epoch] Time cost=${toc - tic}") + + evalMetric.reset() + testIter.reset() + while (testIter.hasNext) { + val evalBatch = testIter.next() + argDict("data").set(evalBatch.data(0)) + argDict("label").set(evalBatch.label(0)) + executor.forward(isTrain = true) + evalMetric.update(evalBatch.label, executor.outputs) + evalBatch.dispose() + } + val (names, values) = evalMetric.get + names.zip(values).foreach { case (n, v) => + logger.info(s"Epoch[$epoch] Validation-accuracy=$v") + validationAcc = Math.max(validationAcc, v) + } + } + executor.dispose() + validationAcc + } + def main(args: Array[String]): Unit = { val leop = new ExampleCustomOpWithRtc val parser: CmdLineParser = new CmdLineParser(leop) @@ -137,95 +220,8 @@ object ExampleCustomOpWithRtc { val dataName = Array("data") val labelName = Array("softmax_label") + test(leop.dataPath, ctx) - val data = Symbol.Variable("data") - val label = Symbol.Variable("label") - val fc1 = Symbol.FullyConnected("fc1")()(Map("data" -> data, "num_hidden" -> 128)) - val act1 = Symbol.Activation("relu1")()(Map("data" -> fc1, "act_type" -> "relu")) - val fc2 = Symbol.FullyConnected("fc2")()(Map("data" -> act1, "num_hidden" -> 64)) - val act2 = Symbol.Activation("relu2")()(Map("data" -> fc2, "act_type" -> "relu")) - val fc3 = Symbol.FullyConnected("fc3")()(Map("data" -> act2, "num_hidden" -> 10)) - val mlp = Symbol.Custom("softmax")()(Map("data" -> fc3, - "label" -> label, "op_type" -> "softmax", "forwardBlockDim" -> "(1,1,1)")) - - val (trainIter, testIter) = - Data.mnistIterator(leop.dataPath, batchSize = 100, inputShape = Shape(784)) - val datasAndLabels = trainIter.provideData ++ trainIter.provideLabel - val (argShapes, outputShapes, auxShapes) = mlp.inferShape(datasAndLabels) - - val initializer = new Xavier(factorType = "in", magnitude = 2.34f) - - val argNames = mlp.listArguments() - val argDict = argNames.zip(argShapes.map(s => NDArray.empty(s, ctx))).toMap - val gradDict = argNames.zip(argShapes).filter { case (name, shape) => - !datasAndLabels.contains(name) - }.map(x => x._1 -> NDArray.empty(x._2, ctx) ).toMap - argDict.foreach { case (name, ndArray) => - if (!datasAndLabels.contains(name)) { - initializer.initWeight(name, ndArray) - } - } - - val executor = mlp.bind(ctx, argDict, gradDict) - val lr = 0.001f - val opt = new RMSProp(learningRate = lr, wd = 0.00001f) - val paramsGrads = gradDict.toList.zipWithIndex.map { case ((name, grad), idx) => - (idx, name, grad, opt.createState(idx, argDict(name))) - } - val evalMetric = new Accuracy - val batchEndCallback = new Speedometer(100, 100) - val numEpoch = 20 - - for (epoch <- 0 until numEpoch) { - val tic = System.currentTimeMillis - evalMetric.reset() - var nBatch = 0 - var epochDone = false - - trainIter.reset() - while (!epochDone) { - var doReset = true - while (doReset && trainIter.hasNext) { - val dataBatch = trainIter.next() - argDict("data").set(dataBatch.data(0)) - argDict("label").set(dataBatch.label(0)) - executor.forward(isTrain = true) - executor.backward() - paramsGrads.foreach { case (idx, name, grad, optimState) => - opt.update(idx, argDict(name), grad, optimState) - } - evalMetric.update(dataBatch.label, executor.outputs) - nBatch += 1 - batchEndCallback.invoke(epoch, nBatch, evalMetric) - } - if (doReset) { - trainIter.reset() - } - epochDone = true - } - val (name, value) = evalMetric.get - name.zip(value).foreach { case (n, v) => - logger.info(s"Epoch[$epoch] Train-accuracy=$v") - } - val toc = System.currentTimeMillis - logger.info(s"Epoch[$epoch] Time cost=${toc - tic}") - - evalMetric.reset() - testIter.reset() - while (testIter.hasNext) { - val evalBatch = testIter.next() - argDict("data").set(evalBatch.data(0)) - argDict("label").set(evalBatch.label(0)) - executor.forward(isTrain = true) - evalMetric.update(evalBatch.label, executor.outputs) - evalBatch.dispose() - } - val (names, values) = evalMetric.get - names.zip(values).foreach { case (n, v) => - logger.info(s"Epoch[$epoch] Validation-accuracy=$v") - } - } - executor.dispose() } catch { case ex: Exception => { logger.error(ex.getMessage, ex) diff --git a/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md new file mode 100644 index 000000000000..886fa2cc9d46 --- /dev/null +++ b/scala-package/examples/src/main/scala/org/apache/mxnetexamples/customop/README.md @@ -0,0 +1,22 @@ +# Custom Operator Example for Scala +This is the example using Custom Operator for type-safe api of Scala. +In the example, a `Softmax` operator is implemented to run the MNIST example. + +There is also an example using RTC. However, the rtc module is depreciated and no longer can be used. Please contribute to use CudaModule operator to replace the rtc. + +## Setup +### Download the source File +```$xslt +https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/mnist/mnist.zip +``` +### Unzip the file +```$xslt +unzip mnist.zip +``` +### Arguement Configuration +Then you need to define the arguments that you would like to pass in the model: +```$xslt +--data-path +``` + +you can find more in [here](https://github.com/apache/incubator-mxnet/blob/scala-package/examples/src/main/scala/org/apache/mxnet/examples/customop/ExampleCustomOp.scala#L218-L221) \ No newline at end of file diff --git a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala new file mode 100644 index 000000000000..4ba0e1bb87cb --- /dev/null +++ b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/customop/CustomOpExampleSuite.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.mxnetexamples.customop + +import java.io.File +import java.net.URL + +import org.apache.commons.io.FileUtils +import org.apache.mxnet.Context +import org.scalatest.{BeforeAndAfterAll, FunSuite} +import org.slf4j.LoggerFactory + +import scala.sys.process.Process + +class CustomOpExampleSuite extends FunSuite with BeforeAndAfterAll { + private val logger = LoggerFactory.getLogger(classOf[CustomOpExampleSuite]) + + test("Example CI: Test Customop MNIST") { + // This test is CPU only + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + logger.info("CPU test only, skipped...") + } else { + logger.info("Downloading mnist model") + val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" + val tempDirPath = System.getProperty("java.io.tmpdir") + val modelDirPath = tempDirPath + File.separator + "mnist/" + val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") + if (!tmpFile.exists()) { + FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), + tmpFile) + } + // TODO: Need to confirm with Windows + Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + + tempDirPath + "/mnist/") ! + val context = Context.cpu() + val output = ExampleCustomOp.test(modelDirPath, context) + assert(output >= 0.95f) + } + } + + test("Example CI: Test CustomopRtc MNIST") { + // This test is GPU only + // TODO: RTC is depreciated, need to change to CUDA Module + val RTC_fixed = false + if (RTC_fixed) { + if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && + System.getenv("SCALA_TEST_ON_GPU").toInt == 1) { + logger.info("Downloading mnist model") + val baseUrl = "https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci" + val tempDirPath = System.getProperty("java.io.tmpdir") + val modelDirPath = tempDirPath + File.separator + "mnist/" + val tmpFile = new File(tempDirPath + "/mnist/mnist.zip") + if (!tmpFile.exists()) { + FileUtils.copyURLToFile(new URL(baseUrl + "/mnist/mnist.zip"), + tmpFile) + } + // TODO: Need to confirm with Windows + Process("unzip " + tempDirPath + "/mnist/mnist.zip -d " + + tempDirPath + "/mnist/") ! + val context = Context.gpu() + val output = ExampleCustomOpWithRtc.test(modelDirPath, context) + assert(output >= 0.95f) + } else { + logger.info("GPU test only, skipped...") + } + } else { + logger.warn("RTC module is not up to date, please don't use this" + + "\nCreate CudaModule for this") + } + } +} diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala index 9a8ec645f272..b446fb3d5b42 100644 --- a/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala +++ b/scala-package/macros/src/main/scala/org/apache/mxnet/APIDocGenerator.scala @@ -56,7 +56,12 @@ private[mxnet] object APIDocGenerator{ def absClassGen(FILE_PATH : String, isSymbol : Boolean) : String = { // scalastyle:off val absClassFunctions = getSymbolNDArrayMethods(isSymbol) - val absFuncs = absClassFunctions.map(absClassFunction => { + // Defines Operators that should not generated + val notGenerated = Set("Custom") + // TODO: Add Filter to the same location in case of refactor + val absFuncs = absClassFunctions.filterNot(_.name.startsWith("_")) + .filterNot(ele => notGenerated.contains(ele.name)) + .map(absClassFunction => { val scalaDoc = generateAPIDocFromBackend(absClassFunction) val defBody = generateAPISignature(absClassFunction, isSymbol) s"$scalaDoc\n$defBody" @@ -178,9 +183,6 @@ private[mxnet] object APIDocGenerator{ _LIB.mxSymbolGetAtomicSymbolInfo( handle, name, desc, numArgs, argNames, argTypes, argDescs, keyVarNumArgs) - - val realName = if (aliasName == name.value) "" else s"(a.k.a., ${name.value})" - val argList = argNames zip argTypes zip argDescs map { case ((argName, argType), argDesc) => val typeAndOption = CToScalaUtils.argumentCleaner(argName, argType, returnType) new absClassArg(argName, typeAndOption._1, argDesc, typeAndOption._2) diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala index 644bc5c4489d..2d3a1c7ec5af 100644 --- a/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala +++ b/scala-package/macros/src/main/scala/org/apache/mxnet/NDArrayMacro.scala @@ -85,14 +85,16 @@ private[mxnet] object NDArrayMacro { val isContrib: Boolean = c.prefix.tree match { case q"new AddNDArrayAPIs($b)" => c.eval[Boolean](c.Expr(b)) } + // Defines Operators that should not generated + val notGenerated = Set("Custom") val newNDArrayFunctions = { if (isContrib) ndarrayFunctions.filter( func => func.name.startsWith("_contrib_") || !func.name.startsWith("_")) else ndarrayFunctions.filterNot(_.name.startsWith("_")) - } + }.filterNot(ele => notGenerated.contains(ele.name)) - val functionDefs = newNDArrayFunctions map { ndarrayfunction => + val functionDefs = newNDArrayFunctions.map { ndarrayfunction => // Construct argument field var argDef = ListBuffer[String]() diff --git a/scala-package/macros/src/main/scala/org/apache/mxnet/SymbolMacro.scala b/scala-package/macros/src/main/scala/org/apache/mxnet/SymbolMacro.scala index 3e790ef4126b..42aa11781d8f 100644 --- a/scala-package/macros/src/main/scala/org/apache/mxnet/SymbolMacro.scala +++ b/scala-package/macros/src/main/scala/org/apache/mxnet/SymbolMacro.scala @@ -89,13 +89,16 @@ private[mxnet] object SymbolImplMacros { case q"new AddSymbolAPIs($b)" => c.eval[Boolean](c.Expr(b)) } + // Defines Operators that should not generated + val notGenerated = Set("Custom") + // TODO: Put Symbol.api.foo --> Stable APIs // Symbol.contrib.bar--> Contrib APIs val newSymbolFunctions = { if (isContrib) symbolFunctions.filter( func => func.name.startsWith("_contrib_") || !func.name.startsWith("_")) else symbolFunctions.filter(!_.name.startsWith("_")) - } + }.filterNot(ele => notGenerated.contains(ele.name)) val functionDefs = newSymbolFunctions map { symbolfunction =>