You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2018/11/14 14:33:36 UTC
incubator-griffin git commit: [GRIFFIN-209] update param util,
add UT for param util
Repository: incubator-griffin
Updated Branches:
refs/heads/master f1781ab8b -> e16ce3c0e
[GRIFFIN-209] update param util, add UT for param util
Author: Lionel Liu <bh...@163.com>
Closes #452 from bhlx3lyx7/param-util.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/e16ce3c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/e16ce3c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/e16ce3c0
Branch: refs/heads/master
Commit: e16ce3c0e2b345966406d983d86f46c17f86c295
Parents: f1781ab
Author: Lionel Liu <bh...@163.com>
Authored: Wed Nov 14 22:33:29 2018 +0800
Committer: William Guo <gu...@apache.org>
Committed: Wed Nov 14 22:33:29 2018 +0800
----------------------------------------------------------------------
.../dsl/transform/TimelinessExpr2DQSteps.scala | 2 +-
.../griffin/measure/utils/ParamUtil.scala | 267 ++++++++++---------
.../griffin/measure/utils/ParamUtilTest.scala | 102 +++++++
3 files changed, 248 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/e16ce3c0/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/transform/TimelinessExpr2DQSteps.scala
----------------------------------------------------------------------
diff --git a/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/transform/TimelinessExpr2DQSteps.scala b/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/transform/TimelinessExpr2DQSteps.scala
index 3731da9..71eb452 100644
--- a/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/transform/TimelinessExpr2DQSteps.scala
+++ b/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/transform/TimelinessExpr2DQSteps.scala
@@ -231,7 +231,7 @@ case class TimelinessExpr2DQSteps(context: DQContext,
}
private def getPercentiles(details: Map[String, Any]): Seq[Double] = {
- details.getArr[Double](_percentileValues).filter(d => (d >= 0 && d <= 1))
+ details.getDoubleArr(_percentileValues).filter(d => (d >= 0 && d <= 1))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/e16ce3c0/measure/src/main/scala/org/apache/griffin/measure/utils/ParamUtil.scala
----------------------------------------------------------------------
diff --git a/measure/src/main/scala/org/apache/griffin/measure/utils/ParamUtil.scala b/measure/src/main/scala/org/apache/griffin/measure/utils/ParamUtil.scala
index c4420ef..283d7d1 100644
--- a/measure/src/main/scala/org/apache/griffin/measure/utils/ParamUtil.scala
+++ b/measure/src/main/scala/org/apache/griffin/measure/utils/ParamUtil.scala
@@ -19,193 +19,215 @@ under the License.
package org.apache.griffin.measure.utils
import scala.reflect.ClassTag
+import scala.util.Try
object ParamUtil {
- implicit class ParamMap(params: Map[String, Any]) {
- def getAny(key: String, defValue: Any): Any = {
- params.get(key) match {
- case Some(v) => v
- case _ => defValue
+ object TransUtil {
+ def toAny(value: Any): Option[Any] = Some(value)
+ def toAnyRef[T: ClassTag](value: Any): Option[T] = {
+ value match {
+ case v: T => Some(v)
+ case _ => None
}
}
-
- def getAnyRef[T: ClassTag](key: String, defValue: T): T = {
- params.get(key) match {
- case Some(v: T) => v
- case _ => defValue
+ def toStringOpt(value: Any): Option[String] = {
+ value match {
+ case v: String => Some(v)
+ case v => Some(v.toString)
}
}
-
- def getString(key: String, defValue: String): String = {
+ def toByte(value: Any): Option[Byte] = {
try {
- params.get(key) match {
- case Some(v: String) => v
- case Some(v) => v.toString
- case _ => defValue
+ value match {
+ case v: String => Some(v.toByte)
+ case v: Byte => Some(v.toByte)
+ case v: Short => Some(v.toByte)
+ case v: Int => Some(v.toByte)
+ case v: Long => Some(v.toByte)
+ case v: Float => Some(v.toByte)
+ case v: Double => Some(v.toByte)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: NumberFormatException => None
}
}
-
- def getLazyString(key: String, defValue: () => String): String = {
+ def toShort(value: Any): Option[Short] = {
try {
- params.get(key) match {
- case Some(v: String) => v
- case Some(v) => v.toString
- case _ => defValue()
+ value match {
+ case v: String => Some(v.toShort)
+ case v: Byte => Some(v.toShort)
+ case v: Short => Some(v.toShort)
+ case v: Int => Some(v.toShort)
+ case v: Long => Some(v.toShort)
+ case v: Float => Some(v.toShort)
+ case v: Double => Some(v.toShort)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue()
+ case _: NumberFormatException => None
}
}
-
- def getStringOrKey(key: String): String = getString(key, key)
-
- def getByte(key: String, defValue: Byte): Byte = {
+ def toInt(value: Any): Option[Int] = {
try {
- params.get(key) match {
- case Some(v: String) => v.toByte
- case Some(v: Byte) => v
- case Some(v: Short) => v.toByte
- case Some(v: Int) => v.toByte
- case Some(v: Long) => v.toByte
- case Some(v: Float) => v.toByte
- case Some(v: Double) => v.toByte
- case _ => defValue
+ value match {
+ case v: String => Some(v.toInt)
+ case v: Byte => Some(v.toInt)
+ case v: Short => Some(v.toInt)
+ case v: Int => Some(v.toInt)
+ case v: Long => Some(v.toInt)
+ case v: Float => Some(v.toInt)
+ case v: Double => Some(v.toInt)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: NumberFormatException => None
}
}
-
- def getShort(key: String, defValue: Short): Short = {
+ def toLong(value: Any): Option[Long] = {
try {
- params.get(key) match {
- case Some(v: String) => v.toShort
- case Some(v: Byte) => v.toShort
- case Some(v: Short) => v
- case Some(v: Int) => v.toShort
- case Some(v: Long) => v.toShort
- case Some(v: Float) => v.toShort
- case Some(v: Double) => v.toShort
- case _ => defValue
+ value match {
+ case v: String => Some(v.toLong)
+ case v: Byte => Some(v.toLong)
+ case v: Short => Some(v.toLong)
+ case v: Int => Some(v.toLong)
+ case v: Long => Some(v.toLong)
+ case v: Float => Some(v.toLong)
+ case v: Double => Some(v.toLong)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: NumberFormatException => None
}
}
-
- def getInt(key: String, defValue: Int): Int = {
+ def toFloat(value: Any): Option[Float] = {
try {
- params.get(key) match {
- case Some(v: String) => v.toInt
- case Some(v: Byte) => v.toInt
- case Some(v: Short) => v.toInt
- case Some(v: Int) => v
- case Some(v: Long) => v.toInt
- case Some(v: Float) => v.toInt
- case Some(v: Double) => v.toInt
- case _ => defValue
+ value match {
+ case v: String => Some(v.toFloat)
+ case v: Byte => Some(v.toFloat)
+ case v: Short => Some(v.toFloat)
+ case v: Int => Some(v.toFloat)
+ case v: Long => Some(v.toFloat)
+ case v: Float => Some(v.toFloat)
+ case v: Double => Some(v.toFloat)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: NumberFormatException => None
}
}
-
- def getLong(key: String, defValue: Long): Long = {
+ def toDouble(value: Any): Option[Double] = {
try {
- params.get(key) match {
- case Some(v: String) => v.toLong
- case Some(v: Byte) => v.toLong
- case Some(v: Short) => v.toLong
- case Some(v: Int) => v.toLong
- case Some(v: Long) => v
- case Some(v: Float) => v.toLong
- case Some(v: Double) => v.toLong
- case _ => defValue
+ value match {
+ case v: String => Some(v.toDouble)
+ case v: Byte => Some(v.toDouble)
+ case v: Short => Some(v.toDouble)
+ case v: Int => Some(v.toDouble)
+ case v: Long => Some(v.toDouble)
+ case v: Float => Some(v.toDouble)
+ case v: Double => Some(v.toDouble)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: NumberFormatException => None
}
}
-
- def getFloat(key: String, defValue: Float): Float = {
+ def toBoolean(value: Any): Option[Boolean] = {
try {
- params.get(key) match {
- case Some(v: String) => v.toFloat
- case Some(v: Byte) => v.toFloat
- case Some(v: Short) => v.toFloat
- case Some(v: Int) => v.toFloat
- case Some(v: Long) => v.toFloat
- case Some(v: Float) => v
- case Some(v: Double) => v.toFloat
- case _ => defValue
+ value match {
+ case v: String => Some(v.toBoolean)
+ case v: Boolean => Some(v)
+ case _ => None
}
} catch {
- case _: NumberFormatException => defValue
+ case _: IllegalArgumentException => None
}
}
+ }
+ import TransUtil._
+
+ implicit class ParamMap(params: Map[String, Any]) {
+ def getAny(key: String, defValue: Any): Any = {
+ params.get(key).flatMap(toAny).getOrElse(defValue)
+ }
+
+ def getAnyRef[T: ClassTag](key: String, defValue: T): T = {
+ params.get(key).flatMap(toAnyRef[T]).getOrElse(defValue)
+ }
+
+ def getString(key: String, defValue: String): String = {
+ params.get(key).flatMap(toStringOpt).getOrElse(defValue)
+ }
+
+ def getLazyString(key: String, defValue: () => String): String = {
+ params.get(key).flatMap(toStringOpt).getOrElse(defValue())
+ }
+
+ def getStringOrKey(key: String): String = getString(key, key)
+
+ def getByte(key: String, defValue: Byte): Byte = {
+ params.get(key).flatMap(toByte).getOrElse(defValue)
+ }
+
+ def getShort(key: String, defValue: Short): Short = {
+ params.get(key).flatMap(toShort).getOrElse(defValue)
+ }
+
+ def getInt(key: String, defValue: Int): Int = {
+ params.get(key).flatMap(toInt).getOrElse(defValue)
+ }
+
+ def getLong(key: String, defValue: Long): Long = {
+ params.get(key).flatMap(toLong).getOrElse(defValue)
+ }
+
+ def getFloat(key: String, defValue: Float): Float = {
+ params.get(key).flatMap(toFloat).getOrElse(defValue)
+ }
def getDouble(key: String, defValue: Double): Double = {
- try {
- params.get(key) match {
- case Some(v: String) => v.toDouble
- case Some(v: Byte) => v.toDouble
- case Some(v: Short) => v.toDouble
- case Some(v: Int) => v.toDouble
- case Some(v: Long) => v.toDouble
- case Some(v: Float) => v.toDouble
- case Some(v: Double) => v
- case _ => defValue
- }
- } catch {
- case _: NumberFormatException => defValue
- }
+ params.get(key).flatMap(toDouble).getOrElse(defValue)
}
def getBoolean(key: String, defValue: Boolean): Boolean = {
- try {
- params.get(key) match {
- case Some(v: String) => v.toBoolean
- case Some(v: Boolean) => v
- case _ => defValue
- }
- } catch {
- case _: NumberFormatException => defValue
- }
+ params.get(key).flatMap(toBoolean).getOrElse(defValue)
}
- case class StringAnyMap(values: Map[String, Any])
-
- def getParamMap(key: String, defValue: Map[String, Any]
+ def getParamAnyMap(key: String, defValue: Map[String, Any]
= Map[String, Any]()): Map[String, Any] = {
params.get(key) match {
- case Some(v: StringAnyMap) => v.values
+ case Some(v: Map[_, _]) => v.map(pair => (pair._1.toString, pair._2))
case _ => defValue
}
}
- def getParamMapOpt(key: String): Option[Map[String, Any]] = {
+ def getParamStringMap(key: String, defValue: Map[String, String]
+ = Map[String, String]()): Map[String, String] = {
params.get(key) match {
- case Some(v: StringAnyMap) => Some(v.values)
- case _ => None
+ case Some(v: Map[_, _]) => v.map(pair => (pair._1.toString, pair._2.toString))
+ case _ => defValue
}
}
- def getArr[T](key: String): Seq[T] = {
- case class TSeqs(values: Seq[T])
+ def getStringArr(key: String, defValue: Seq[String] = Nil): Seq[String] = {
params.get(key) match {
- case Some(seq: TSeqs) => seq.values
- case _ => Nil
+ case Some(seq: Seq[_]) => seq.flatMap(toStringOpt)
+ case _ => defValue
+ }
+ }
+
+ def getDoubleArr(key: String, defValue: Seq[Double] = Nil): Seq[Double] = {
+ params.get(key) match {
+ case Some(seq: Seq[_]) => seq.flatMap(toDouble)
+ case _ => defValue
}
}
def addIfNotExist(key: String, value: Any): Map[String, Any] = {
params.get(key) match {
- case Some(v) => params
- case _ => params + (key -> value)
+ case None => params + (key -> value)
+ case _ => params
}
}
@@ -215,3 +237,4 @@ object ParamUtil {
}
}
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/e16ce3c0/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala
----------------------------------------------------------------------
diff --git a/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala b/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala
new file mode 100644
index 0000000..54d193e
--- /dev/null
+++ b/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala
@@ -0,0 +1,102 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.griffin.measure.utils
+
+import org.scalatest._
+
+import org.apache.griffin.measure.utils.ParamUtil._
+
+class ParamUtilTest extends FlatSpec with Matchers with BeforeAndAfter {
+
+ val fruits = Map[String, Any]("A" -> "apple", "B" -> "banana", "O" -> "orange")
+ val numbers = Map[String, Any]("a" -> 1, "b" -> 5, "c" -> 3)
+ val ids = Seq[Any](2, 3, 5, 7)
+ val cities = Seq[Any]("LA", "NY", "SLC")
+ val percentiles = Seq[Any](.95, "0.4", ".3", 1, "static", "0.2")
+ var params: Map[String, Any] = _
+
+ before {
+ params = Map[String, Any](
+ "name" -> "alex",
+ "age" -> 34,
+ "fruits" -> fruits,
+ "numbers" -> numbers,
+ "ids" -> ids,
+ "cities" -> cities,
+ "percentiles" -> percentiles
+ )
+ }
+
+ "TransUtil" should "transform all basic data types" in {
+ import ParamUtil.TransUtil._
+ toAny("test") should be (Some("test"))
+ toAnyRef[Seq[_]]("test") should be (None)
+ toAnyRef[Seq[_]](Seq(1, 2)) should be (Some(Seq(1, 2)))
+ toStringOpt("test") should be (Some("test"))
+ toStringOpt(123) should be (Some("123"))
+ toByte(12) should be (Some(12))
+ toByte(123456) should not be (Some(123456))
+ toShort(12) should be (Some(12))
+ toShort(123456) should not be (Some(123456))
+ toInt(12) should be (Some(12))
+ toInt(1.8) should be (Some(1))
+ toInt("123456") should be (Some(123456))
+ toLong(123456) should be (Some(123456L))
+ toFloat(1.2) should be (Some(1.2f))
+ toDouble("1.21") should be (Some(1.21))
+ toBoolean(true) should be (Some(true))
+ toBoolean("false") should be (Some(false))
+ toBoolean("test") should be (None)
+ }
+
+ "params" should "extract string any map field" in {
+ params.getParamAnyMap("fruits") should be (fruits)
+ params.getParamAnyMap("numbers") should be (numbers)
+ params.getParamAnyMap("name") should be (Map.empty[String, Any])
+ }
+
+ "params" should "extract string string map field" in {
+ params.getParamStringMap("fruits") should be (fruits)
+ params.getParamStringMap("numbers") should be (Map[String, String]("a" -> "1", "b" -> "5", "c" -> "3"))
+ params.getParamStringMap("name") should be (Map.empty[String, String])
+ }
+
+ "params" should "extract array field" in {
+ params.getStringArr("ids") should be (Seq("2", "3", "5", "7"))
+ params.getStringArr("cities") should be (cities)
+ params.getStringArr("name") should be (Nil)
+ }
+
+ "params" should "get double array" in {
+ params.getDoubleArr("percentiles") should be (Seq(0.95, 0.4, 0.3, 1, 0.2))
+ }
+
+ "map" should "add if not exist" in {
+ val map = Map[String, Any]("a" -> 1, "b" -> 2)
+ map.addIfNotExist("a", 11) should be (Map[String, Any]("a" -> 1, "b" -> 2))
+ map.addIfNotExist("c", 11) should be (Map[String, Any]("a" -> 1, "b" -> 2, "c" -> 11))
+ }
+
+ "map" should "remove keys" in {
+ val map = Map[String, Any]("a" -> 1, "b" -> 2)
+ map.removeKeys(Seq("c", "d")) should be (Map[String, Any]("a" -> 1, "b" -> 2))
+ map.removeKeys(Seq("a")) should be (Map[String, Any]("b" -> 2))
+ }
+
+}