You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2016/10/10 08:38:51 UTC
[2/5] incubator-s2graph git commit: [S2GRAPH-117] migrated to play
2.4.8 (still emitting logback error)
[S2GRAPH-117] migrated to play 2.4.8 (still emitting logback error)
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c7967ff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c7967ff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c7967ff3
Branch: refs/heads/master
Commit: c7967ff346e40ed6c27fe570931cc6defd87ffe8
Parents: 1f5cfde
Author: Jong Wook Kim <jo...@nyu.edu>
Authored: Sun Oct 9 23:25:43 2016 -0400
Committer: Jong Wook Kim <jo...@nyu.edu>
Committed: Mon Oct 10 01:44:15 2016 -0400
----------------------------------------------------------------------
project/Common.scala | 5 +-
project/plugins.sbt | 2 +-
.../org/apache/s2graph/core/PostProcess.scala | 1 -
.../s2graph/core/rest/RequestParser.scala | 68 +++++++++++---------
.../counter/core/v2/ExactStorageGraph.scala | 4 +-
.../loader/core/CounterEtlFunctions.scala | 2 +-
.../counter/loader/core/CounterEtlItem.scala | 8 +--
.../counter/loader/core/CounterFunctions.scala | 2 +-
.../rest/play/controllers/EdgeController.scala | 4 +-
s2rest_play/build.sbt | 5 +-
10 files changed, 56 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index f0fb222..0e92f05 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -21,7 +21,7 @@ import sbt._
object Common {
val sparkVersion = "1.4.1"
- val playVersion = "2.5.9"
+ val playVersion = "2.4.8"
val hbaseVersion = "1.2.2"
val hadoopVersion = "2.7.3"
@@ -38,7 +38,8 @@ object Common {
val loggingExcludes = Seq(
ExclusionRule("commons-logging", "commons-logging"),
ExclusionRule("log4j", "log4j"),
- ExclusionRule("ch.qos.logback", "*"),
+ ExclusionRule("ch.qos.logback", "logback-classic"),
+ ExclusionRule("ch.qos.logback", "logback-core"),
ExclusionRule("org.slf4j", "jcl-over-slf4j"),
ExclusionRule("org.slf4j", "log4j-over-slf4j"),
ExclusionRule("org.slf4j", "slf4j-log4j12"),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index d98d1cb..3efaa7d 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -19,7 +19,7 @@
// use the Play sbt plugin for Play projects
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10")
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.8")
// http://www.scalastyle.org/sbt.html
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index caaa408..923eeef 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -186,7 +186,6 @@ object PostProcess extends JSONParser {
private def buildReplaceJson(jsValue: JsValue)(mapper: JsValue => JsValue): JsValue = {
def traverse(js: JsValue): JsValue = js match {
case JsNull => mapper(JsNull)
- case JsUndefined() => mapper(JsUndefined(""))
case JsNumber(v) => mapper(js)
case JsString(v) => mapper(js)
case JsBoolean(v) => mapper(js)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index 61db2ef..f67a89b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -29,6 +29,7 @@ import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.parsers.{Where, WhereParser}
import org.apache.s2graph.core.types._
import play.api.libs.json._
+import play.api.libs.json.Reads._
import scala.util.{Failure, Success, Try}
@@ -92,7 +93,7 @@ class RequestParser(config: Config) extends JSONParser {
private def extractScoring(labelId: Int, value: JsValue) = {
val ret = for {
- js <- parse[Option[JsObject]](value, "scoring")
+ js <- parseOption[JsObject](value, "scoring")
} yield {
for {
(k, v) <- js.fields
@@ -113,16 +114,16 @@ class RequestParser(config: Config) extends JSONParser {
val jsValue = Json.parse(replaced)
def extractKv(js: JsValue) = js match {
- case JsObject(obj) => obj
+ case JsObject(map) => map.toSeq
case JsArray(arr) => arr.flatMap {
- case JsObject(obj) => obj
- case _ => throw new RuntimeException(s"cannot support json type $js")
+ case JsObject(map) => map.toSeq
+ case _ => throw new RuntimeException(s"cannot support json type: $js")
}
case _ => throw new RuntimeException(s"cannot support json type: $js")
}
val ret = for {
- js <- parse[Option[JsObject]](jsValue, "interval")
+ js <- parseOption[JsObject](jsValue, "interval")
fromJs <- (js \ "from").asOpt[JsValue]
toJs <- (js \ "to").asOpt[JsValue]
} yield {
@@ -139,10 +140,10 @@ class RequestParser(config: Config) extends JSONParser {
val jsValue = Json.parse(replaced)
for {
- js <- parse[Option[JsObject]](jsValue, "duration")
+ js <- parseOption[JsObject](jsValue, "duration")
} yield {
- val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue)
- val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue)
+ val minTs = parseOption[Long](js, "from").getOrElse(Long.MaxValue)
+ val maxTs = parseOption[Long](js, "to").getOrElse(Long.MinValue)
if (minTs > maxTs) {
throw new BadQueryException("Duration error. Timestamp of From cannot be larger than To.")
@@ -154,7 +155,7 @@ class RequestParser(config: Config) extends JSONParser {
def extractHas(label: Label, jsValue: JsValue) = {
val ret = for {
- js <- parse[Option[JsObject]](jsValue, "has")
+ js <- parseOption[JsObject](jsValue, "has")
} yield {
for {
(k, v) <- js.fields
@@ -346,12 +347,12 @@ class RequestParser(config: Config) extends JSONParser {
private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = {
for {
- labelName <- parse[Option[String]](labelGroup, "label")
+ labelName <- parseOption[String](labelGroup, "label")
} yield {
val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found"))
- val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
+ val direction = parseOption[String](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
val limit = {
- parse[Option[Int]](labelGroup, "limit") match {
+ parseOption[Int](labelGroup, "limit") match {
case None => defaultLimit
case Some(l) if l < 0 => maxLimit
case Some(l) if l >= 0 =>
@@ -359,12 +360,12 @@ class RequestParser(config: Config) extends JSONParser {
Math.min(l, default)
}
}
- val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0)
+ val offset = parseOption[Int](labelGroup, "offset").getOrElse(0)
val interval = extractInterval(label, labelGroup)
val duration = extractDuration(label, labelGroup)
val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList
- val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false)
- val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false)
+ val exclude = parseOption[Boolean](labelGroup, "exclude").getOrElse(false)
+ val include = parseOption[Boolean](labelGroup, "include").getOrElse(false)
val hasFilter = extractHas(label, labelGroup)
val labelWithDir = LabelWithDirection(label.id.get, direction)
val indexNameOpt = (labelGroup \ "index").asOpt[String]
@@ -392,7 +393,7 @@ class RequestParser(config: Config) extends JSONParser {
}
val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
// TODO: refactor this. dirty
- val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
+ val duplicate = parseOption[String](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s)))
val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue]
@@ -430,16 +431,23 @@ class RequestParser(config: Config) extends JSONParser {
}
private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = {
- (js \ key).validate[R]
- .fold(
- errors => {
- val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg")
- val e = Json.obj("args" -> key, "error" -> msg)
- throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
- },
- r => {
- r
- })
+ (js \ key).validate[R] match {
+ case JsError(errors) =>
+ val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].flatMap(x => (x \ "msg").toOption)
+ val e = Json.obj("args" -> key, "error" -> msg)
+ throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
+ case JsSuccess(result, _) => result
+ }
+ }
+
+ private def parseOption[R](js: JsValue, key: String)(implicit read: Reads[R]): Option[R] = {
+ (js \ key).validateOpt[R] match {
+ case JsError(errors) =>
+ val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].flatMap(x => (x \ "msg").toOption)
+ val e = Json.obj("args" -> key, "error" -> msg)
+ throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
+ case JsSuccess(result, _) => result
+ }
}
def toJsValues(jsValue: JsValue): List[JsValue] = {
@@ -478,7 +486,7 @@ class RequestParser(config: Config) extends JSONParser {
val label = parse[String](jsValue, "label")
val timestamp = parse[Long](jsValue, "timestamp")
- val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
+ val direction = parseOption[String](jsValue, "direction").getOrElse("")
val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
for {
srcId <- srcIds
@@ -494,7 +502,7 @@ class RequestParser(config: Config) extends JSONParser {
def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = {
val id = parse[JsValue](jsValue, "id")
- val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
+ val ts = parseOption[Long](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get
val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
@@ -511,7 +519,7 @@ class RequestParser(config: Config) extends JSONParser {
Prop(propName, defaultValue, dataType)
}
- def toPropsElements(jsValue: JsValue): Seq[Prop] = for {
+ def toPropsElements(jsValue: JsLookupResult): Seq[Prop] = for {
jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil)
} yield {
val propName = (jsObj \ "name").as[String]
@@ -523,7 +531,7 @@ class RequestParser(config: Config) extends JSONParser {
Prop(propName, defaultValue, dataType)
}
- def toIndicesElements(jsValue: JsValue): Seq[Index] = for {
+ def toIndicesElements(jsValue: JsLookupResult): Seq[Index] = for {
jsObj <- jsValue.as[Seq[JsValue]]
indexName = (jsObj \ "name").as[String]
propNames = (jsObj \ "propNames").as[Seq[String]]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
index 3a196ef..18cfeb2 100644
--- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
@@ -209,10 +209,10 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
}
private def resultToExactKeyValues(policy: Counter, result: JsValue): (ExactKeyTrait, (core.ExactQualifier, Long)) = {
- val from = result \ "from" match {
+ val from = (result \ "from").get match {
case s: JsString => s.as[String]
case n: JsNumber => n.as[Long].toString
- case x: JsValue => throw new RuntimeException(s"$x's type must be string or number")
+ case x => throw new RuntimeException(s"$x's type must be string or number")
}
val dimension = (result \ "to").as[String]
val props = result \ "props"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
index 7dcf48a..247cd07 100644
--- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
@@ -69,7 +69,7 @@ object CounterEtlFunctions extends Logging {
} yield {
val jsValue = variable match {
case "_from" => JsString(srcId)
- case s => dimension \ s
+ case s => (dimension \ s).get
}
s"[[$variable]]" -> jsValue
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
index e3bb884..6eaa6e2 100644
--- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
@@ -30,10 +30,10 @@ case class CounterEtlItem(ts: Long, service: String, action: String, item: Strin
}
lazy val value = {
- property \ "value" match {
- case JsNumber(n) => n.longValue()
- case JsString(s) => s.toLong
- case _: JsUndefined => 1L
+ (property \ "value").toOption match {
+ case Some(JsNumber(n)) => n.longValue()
+ case Some(JsString(s)) => s.toLong
+ case None => 1L
case _ => throw new Exception("wrong type")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
index 05423b1..7b272be 100644
--- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
@@ -72,7 +72,7 @@ object CounterFunctions extends Logging with WithKafka {
Try {
for {
k <- keys
- jsValue = dimension \ k
+ jsValue <- (dimension \ k).toOption
} yield {
jsValue match {
case JsNumber(n) => n.toString()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
index 478df99..ec8324c 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
@@ -40,10 +40,12 @@ object EdgeController extends Controller {
private val s2: Graph = org.apache.s2graph.rest.play.Global.s2graph
private val requestParser: RequestParser = org.apache.s2graph.rest.play.Global.s2parser
+
private def jsToStr(js: JsValue): String = js match {
case JsString(s) => s
- case _ => js.toString()
+ case obj => obj.toString()
}
+ private def jsToStr(js: JsLookupResult): String = js.toOption.map(jsToStr).getOrElse("undefined")
def toTsv(jsValue: JsValue, op: String): String = {
val ts = jsToStr(jsValue \ "timestamp")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c7967ff3/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
index dfe7205..01f4c85 100644
--- a/s2rest_play/build.sbt
+++ b/s2rest_play/build.sbt
@@ -22,8 +22,9 @@ name := "s2rest_play"
scalacOptions in Test ++= Seq("-Yrangepos")
-libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, filters)).map(_.excludeLogging()) ++ Seq(
- "com.google.guava" % "guava" % "12.0.1" force() // use this old version of guava to avoid incompatibility
+libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, filters, specs2 % Test)).map(_.excludeLogging()) ++ Seq(
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
+ "org.specs2" %% "specs2-core" % "3.8.5" % "test"
)
enablePlugins(JavaServerAppPackaging)