You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by da...@apache.org on 2016/01/13 10:07:34 UTC
[1/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Repository: incubator-s2graph
Updated Branches:
refs/heads/master be304d6c9 -> f2311f25c
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/QuerySpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/QuerySpec.scala b/test/controllers/QuerySpec.scala
deleted file mode 100644
index f6381bc..0000000
--- a/test/controllers/QuerySpec.scala
+++ /dev/null
@@ -1,604 +0,0 @@
-package controllers
-
-import play.api.libs.json._
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
-import play.api.{Application => PlayApplication}
-
-import scala.concurrent.Await
-
-class QuerySpec extends SpecCommon with PlaySpecification {
-
- import Helper._
-
- implicit val app = FakeApplication()
-
- init()
-
- "query test" should {
- running(FakeApplication()) {
-
- // insert bulk and wait ..
- val bulkEdges: String = Seq(
- edge"1000 insert e 0 1 $testLabelName"($(weight = 40, is_hidden = true)),
- edge"2000 insert e 0 2 $testLabelName"($(weight = 30, is_hidden = false)),
- edge"3000 insert e 2 0 $testLabelName"($(weight = 20)),
- edge"4000 insert e 2 1 $testLabelName"($(weight = 10)),
- edge"3000 insert e 10 20 $testLabelName"($(weight = 20)),
- edge"4000 insert e 20 20 $testLabelName"($(weight = 10)),
- edge"1 insert e -1 1000 $testLabelName",
- edge"1 insert e -1 2000 $testLabelName",
- edge"1 insert e -1 3000 $testLabelName",
- edge"1 insert e 1000 10000 $testLabelName",
- edge"1 insert e 1000 11000 $testLabelName",
- edge"1 insert e 2000 11000 $testLabelName",
- edge"1 insert e 2000 12000 $testLabelName",
- edge"1 insert e 3000 12000 $testLabelName",
- edge"1 insert e 3000 13000 $testLabelName",
- edge"1 insert e 10000 100000 $testLabelName",
- edge"2 insert e 11000 200000 $testLabelName",
- edge"3 insert e 12000 300000 $testLabelName").mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
- }
-
- def queryParents(id: Long) = Json.parse( s"""
- {
- "returnTree": true,
- "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- }
- ],[{
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": -1
- }
- ]]
- }""".stripMargin)
-
- def queryExclude(id: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- },
- {
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": 2,
- "exclude": true
- }
- ]]
- }""")
-
- def queryTransform(id: Int, transforms: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "transform": $transforms
- }
- ]]
- }""")
-
- def queryWhere(id: Int, where: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "where": "${where}"
- }
- ]]
- }""")
-
- def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit,
- "_to": $to
- }
- ]]
- }
- """)
-
- def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit
- }
- ]]
- }
- """)
-
- def queryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- }
- ]
- }""")
-
-
- def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- },
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- }
- ]
- }""")
-
- def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 50,
- "sample": ${sample}
- },
- {
- "label": "${testLabelName2}",
- "direction": "out",
- "offset": 0,
- "limit": 50
- }]
- }
- ]
- }""")
-
- def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id)))
-
- def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
- Json.obj(
- "groupBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- def queryScore(id: Int, scoring: Map[String, Int]): JsValue = {
- val q = Json.obj(
- "srcVertices" -> Json.arr(
- Json.obj(
- "serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "id" -> id
- )
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- println(q)
- q
- }
-
- def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = {
- Json.obj(
- "orderBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- }
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- def queryIndex(ids: Seq[Int], indexName: String) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(label = testLabelName, index = indexName))
- val $steps = $a($(step = $step))
-
- val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson
- js
- }
-
- def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(
- label = testLabelName, direction = "out", offset = 0, limit = 100,
- duration = $(from = from, to = to)))
-
- val $steps = $a($(step = $step))
-
- $(srcVertices = $from, steps = $steps).toJson
- }
-
- "union query" in {
- running(FakeApplication()) {
- var result = getEdges(queryUnion(0, 2))
- result.as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryUnion(0, 3))
- result.as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryUnion(0, 4))
- result.as[List[JsValue]].size must equalTo(4)
-
- result = getEdges(queryUnion(0, 5))
- result.as[List[JsValue]].size must equalTo(5)
-
- val union = result.as[List[JsValue]].head
- val single = getEdges(querySingle(0))
-
- (union \\ "from").map(_.toString).sorted must equalTo((single \\ "from").map(_.toString).sorted)
- (union \\ "to").map(_.toString).sorted must equalTo((single \\ "to").map(_.toString).sorted)
- (union \\ "weight").map(_.toString).sorted must equalTo((single \\ "weight").map(_.toString).sorted)
- }
- }
-
- "get edge with where condition" in {
- running(FakeApplication()) {
- var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)"))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- result = getEdges(queryWhere(0, "_from=0"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryWhere(2, "_from=2 or weight in (-1)"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- }
- }
-
- "get edge exclude" in {
- running(FakeApplication()) {
- val result = getEdges(queryExclude(0))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- }
- }
-
- "get edge groupBy property" in {
- running(FakeApplication()) {
- val result = getEdges(queryGroupBy(0, Seq("weight")))
- (result \ "size").as[Int] must_== 2
- val weights = (result \\ "groupBy").map { js =>
- (js \ "weight").as[Int]
- }
- weights must contain(exactly(30, 40))
- weights must not contain (10)
- }
- }
-
- "edge transform " in {
- running(FakeApplication()) {
- var result = getEdges(queryTransform(0, "[[\"_to\"]]"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryTransform(0, "[[\"weight\"]]"))
- (result \\ "to").map(_.toString).sorted must equalTo((result \\ "weight").map(_.toString).sorted)
-
- result = getEdges(queryTransform(0, "[[\"_from\"]]"))
- val results = (result \ "results").as[JsValue]
- (result \\ "to").map(_.toString).sorted must equalTo((results \\ "from").map(_.toString).sorted)
- }
- }
-
- "index" in {
- running(FakeApplication()) {
- // weight order
- var result = getEdges(queryIndex(Seq(0), "idx_1"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(40))
-
- // timestamp order
- result = getEdges(queryIndex(Seq(0), "idx_2"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(30))
- }
- }
-
- "checkEdges" in {
- running(FakeApplication()) {
- val json = Json.parse( s"""
- [{"from": 0, "to": 1, "label": "$testLabelName"},
- {"from": 0, "to": 2, "label": "$testLabelName"}]
- """)
-
- def checkEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- val res = checkEdges(json)
- val typeRes = res.isInstanceOf[JsArray]
- typeRes must equalTo(true)
-
- val fst = res.as[Seq[JsValue]].head \ "to"
- fst.as[Int] must equalTo(1)
-
- val snd = res.as[Seq[JsValue]].last \ "to"
- snd.as[Int] must equalTo(2)
- }
- }
-
- "duration" in {
- running(FakeApplication()) {
- // get all
- var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size must equalTo(4)
-
- // inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
- // duration test after udpate
- // get all
- result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size must equalTo(4)
-
- // inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- true
- }
- }
-
- "returnTree" in {
- running(FakeApplication()) {
- val src = 100
- val tgt = 200
- val labelName = testLabelName
-
- val bulkEdges: String = Seq(
- edge"1001 insert e $src $tgt $labelName"
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- val result = getEdges(queryParents(src))
-
- val parents = (result \ "results").as[Seq[JsValue]]
- val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 }
- ret must equalTo(true)
- }
- }
-
- "pagination and _to" in {
- running(FakeApplication()) {
- val src = System.currentTimeMillis().toInt
- val labelName = testLabelName
- val bulkEdges: String = Seq(
- edge"1001 insert e $src 1 $labelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e $src 2 $labelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e $src 3 $labelName"($(weight = 30)),
- edge"4004 insert e $src 4 $labelName"($(weight = 40))
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- var result = getEdges(querySingle(src, offset = 0, limit = 2))
- println(result)
- var edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(2)
- (edges(0) \ "to").as[Long] must beEqualTo(4)
- (edges(1) \ "to").as[Long] must beEqualTo(3)
-
- result = getEdges(querySingle(src, offset = 1, limit = 2))
- println(result)
- edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(2)
- (edges(0) \ "to").as[Long] must beEqualTo(3)
- (edges(1) \ "to").as[Long] must beEqualTo(2)
-
- result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
- println(result)
- edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(1)
- }
- }
-
- "orderBy" >> {
- running(FakeApplication()) {
- // insert test set
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
- contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- // get edges
- val edges = getEdges(queryScore(0, Map("weight" -> 1)))
- val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
- val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
-
- println(edges)
- println(orderByScore)
- println(ascOrderByScore)
-
- val edgesTo = edges \ "results" \\ "to"
- val orderByTo = orderByScore \ "results" \\ "to"
- val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
-
- edgesTo must_== Seq(JsNumber(2), JsNumber(1))
- edgesTo must_== orderByTo
- ascOrderByTo must_== Seq(JsNumber(1), JsNumber(2))
- edgesTo.reverse must_== ascOrderByTo
- }
- }
-
- "query with sampling" in {
- running(FakeApplication()) {
- val sampleSize = 2
- val testId = 22
- val bulkEdges = Seq(
- edge"1442985659166 insert e $testId 122 $testLabelName",
- edge"1442985659166 insert e $testId 222 $testLabelName",
- edge"1442985659166 insert e $testId 322 $testLabelName",
-
- edge"1442985659166 insert e $testId 922 $testLabelName2",
- edge"1442985659166 insert e $testId 222 $testLabelName2",
- edge"1442985659166 insert e $testId 322 $testLabelName2",
-
- edge"1442985659166 insert e 122 1122 $testLabelName",
- edge"1442985659166 insert e 122 1222 $testLabelName",
- edge"1442985659166 insert e 122 1322 $testLabelName",
- edge"1442985659166 insert e 222 2122 $testLabelName",
- edge"1442985659166 insert e 222 2222 $testLabelName",
- edge"1442985659166 insert e 222 2322 $testLabelName",
- edge"1442985659166 insert e 322 3122 $testLabelName",
- edge"1442985659166 insert e 322 3222 $testLabelName",
- edge"1442985659166 insert e 322 3322 $testLabelName"
- )
-
- val req = FakeRequest(POST, "/graphs/edges/bulk").withBody(bulkEdges.mkString("\n"))
- Await.result(route(req).get, HTTP_REQ_WAITING_TIME)
-
- Thread.sleep(asyncFlushInterval)
-
-
- val result1 = getEdges(queryWithSampling(testId, sampleSize))
- println(Json.toJson(result1))
- (result1 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize, bulkEdges.size))
-
- val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result2))
- (result2 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
-
- val result3 = getEdges(twoQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result3))
- (result3 \ "results").as[List[JsValue]].size must equalTo(sampleSize + 3) // edges in testLabelName2 = 3
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/SpecCommon.scala
----------------------------------------------------------------------
diff --git a/test/controllers/SpecCommon.scala b/test/controllers/SpecCommon.scala
deleted file mode 100644
index 93648ce..0000000
--- a/test/controllers/SpecCommon.scala
+++ /dev/null
@@ -1,365 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import org.specs2.mutable.Specification
-import play.api.libs.json._
-import play.api.test.FakeApplication
-import play.api.test.Helpers._
-
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import scala.util.Random
-
-trait SpecCommon extends Specification {
- sequential
- object Helper {
-
- import org.json4s.native.Serialization
-
- type KV = Map[String, Any]
-
- import scala.language.dynamics
-
- def $aa[T](args: T*) = List($a(args: _ *))
-
- def $a[T](args: T*) = args.toList
-
- object $ extends Dynamic {
- def applyDynamicNamed(name: String)(args: (String, Any)*): Map[String, Any] = args.toMap
- }
-
- implicit class anyMapOps(map: Map[String, Any]) {
- def toJson: JsValue = {
- val js = Serialization.write(map)(org.json4s.DefaultFormats)
- Json.parse(js)
- }
- }
-
- implicit class S2Context(val sc: StringContext) {
- def edge(args: Any*)(implicit map: Map[String, Any] = Map.empty): String = {
- val parts = sc.s(args: _*).split("\\s")
- assert(parts.length == 6)
- (parts.toList :+ map.toJson.toString).mkString("\t")
- }
- }
-
- }
-
- val curTime = System.currentTimeMillis
- val t1 = curTime + 0
- val t2 = curTime + 1
- val t3 = curTime + 2
- val t4 = curTime + 3
- val t5 = curTime + 4
-
- protected val testServiceName = "s2graph"
- protected val testLabelName = "s2graph_label_test"
- protected val testLabelName2 = "s2graph_label_test_2"
- protected val testLabelNameV1 = "s2graph_label_test_v1"
- protected val testLabelNameWeak = "s2graph_label_test_weak"
- protected val testColumnName = "user_id_test"
- protected val testColumnType = "long"
- protected val testTgtColumnName = "item_id_test"
- protected val testHTableName = "test-htable"
- protected val newHTableName = "new-htable"
-
- val NUM_OF_EACH_TEST = 100
- val HTTP_REQ_WAITING_TIME = Duration(300, SECONDS)
- val asyncFlushInterval = 100
-
- val createService = s"""{"serviceName" : "$testServiceName"}"""
- val testLabelNameCreate =
- s"""
- {
- "label": "$testLabelName",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testColumnName",
- "tgtColumnType": "long",
- "indices": [
- {"name": "idx_1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
- {"name": "idx_2", "propNames": ["_timestamp"]}
- ],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "schemaVersion": "v2",
- "compressionAlgorithm": "gz",
- "hTableName": "$testHTableName"
- }"""
-
- val testLabelName2Create =
- s"""
- {
- "label": "$testLabelName2",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": false,
- "schemaVersion": "v3",
- "compressionAlgorithm": "gz"
- }"""
-
- val testLabelNameV1Create =
- s"""
- {
- "label": "$testLabelNameV1",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "${testTgtColumnName}_v1",
- "tgtColumnType": "string",
- "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": true,
- "schemaVersion": "v1",
- "compressionAlgorithm": "gz"
- }"""
- val testLabelNameWeakCreate =
- s"""
- {
- "label": "$testLabelNameWeak",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "weak",
- "isDirected": true,
- "compressionAlgorithm": "gz"
- }"""
-
- val vertexPropsKeys = List(
- ("age", "int")
- )
-
- val createVertex =
- s"""{
- "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "columnType": "$testColumnType",
- "props": [
- {"name": "is_active", "dataType": "boolean", "defaultValue": true},
- {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
- {"name": "nickname", "dataType": "string", "defaultValue": ".."},
- {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
- {"name": "age", "dataType": "integer", "defaultValue": 0}
- ]
- }"""
-
-
- val TS = System.currentTimeMillis()
-
- def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = {
- val s =
- s"""{
- "srcVertices": [
- {
- "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }
- ],
- "steps": [
- [
- {
- "label": "$labelName",
- "direction": "$dir",
- "offset": 0,
- "limit": 10,
- "cacheTTL": $cacheTTL
- }
- ]
- ]
- }"""
- println(s)
- Json.parse(s)
- }
-
- def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
- val arr = for {
- (label, dir, from, to) <- params
- } yield {
- Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
- }
-
- val s = Json.toJson(arr)
- println(s)
- s
- }
-
- def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
- Json.parse(
- s"""
- |[
- |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
- ]}
- |]
- """.stripMargin)
- }
-
- def randomProps() = {
- (for {
- (propKey, propType) <- vertexPropsKeys
- } yield {
- propKey -> Random.nextInt(100)
- }).toMap
- }
-
- def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
- ids.map { id =>
- Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
- }
- }
-
- def commonCheck(rslt: Future[play.api.mvc.Result]): JsValue = {
- status(rslt) must equalTo(OK)
- contentType(rslt) must beSome.which(_ == "application/json")
- val jsRslt = contentAsJson(rslt)
- println("======")
- println(jsRslt)
- println("======")
- jsRslt.as[JsObject].keys.contains("size") must equalTo(true)
- (jsRslt \ "size").as[Int] must greaterThan(0)
- jsRslt.as[JsObject].keys.contains("results") must equalTo(true)
- val jsRsltsObj = jsRslt \ "results"
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("from") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("to") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("_timestamp") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("props") must equalTo(true)
- jsRslt
- }
-
- def init() = {
- running(FakeApplication()) {
- println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- Management.deleteService(testServiceName)
-
- // 1. createService
- val result = AdminController.createServiceInner(Json.parse(createService))
- println(s">> Service created : $createService, $result")
-
- val labelNames = Map(testLabelName -> testLabelNameCreate,
- testLabelName2 -> testLabelName2Create,
- testLabelNameV1 -> testLabelNameV1Create,
- testLabelNameWeak -> testLabelNameWeakCreate)
-
- for {
- (labelName, create) <- labelNames
- } {
- Management.deleteLabel(labelName)
- Label.findByName(labelName, useCache = false) match {
- case None =>
- AdminController.createLabelInner(Json.parse(create))
- case Some(label) =>
- println(s">> Label already exist: $create, $label")
- }
- }
-
- // 5. create vertex
- vertexPropsKeys.map { case (key, keyType) =>
- Management.addVertexProp(testServiceName, testColumnName, key, keyType)
- }
-
- println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/StrongLabelDeleteSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/StrongLabelDeleteSpec.scala b/test/controllers/StrongLabelDeleteSpec.scala
deleted file mode 100644
index 04ccc79..0000000
--- a/test/controllers/StrongLabelDeleteSpec.scala
+++ /dev/null
@@ -1,345 +0,0 @@
-package controllers
-
-import java.util.concurrent.TimeUnit
-
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, Future}
-import scala.util.Random
-
-class StrongLabelDeleteSpec extends SpecCommon {
- init()
-// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
-
- def bulkEdges(startTs: Int = 0) = Seq(
- Seq(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""").mkString("\t")
- ).mkString("\n")
-
- def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
- labelName: String = testLabelName2, direction: String = "out") = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "$labelName",
- "direction": "${direction}",
- "offset": 0,
- "limit": -1,
- "duplicate": "raw"
- }
- ]]
- }""")
-
- def getEdges(queryJson: JsValue): JsValue = {
-// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
-
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- def getDegree(jsValue: JsValue): Long = {
- ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
- }
-
-
- "strong label delete test" should {
- running(FakeApplication()) {
- // insert bulk and wait ..
- val edges = bulkEdges()
- println(edges)
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
- }
-
- "test strong consistency select" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- result = getEdges(query(10))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- true
- }
- }
-
- "test strong consistency duration. insert -> delete -> insert" in {
- running(FakeApplication()) {
- val ts0 = 1
- val ts1 = 2
- val ts2 = 3
-
- val edges = Seq(
- Seq(5, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t"),
- Seq(10, "delete", "edge", "-10", "-20", testLabelName2).mkString("\t"),
- Seq(20, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t")
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
-
- val result = getEdges(query(-10))
-
- println(result)
-
- true
- }
- }
-
- "test strong consistency deleteAll" in {
- running(FakeApplication()) {
-
- val deletedAt = 100
- var result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- val json = Json.arr(Json.obj("label" -> testLabelName2,
- "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
- println(json)
- contentAsString(EdgeController.deleteAllInner(json, withWait = true))
-
- result = getEdges(query(11, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(12, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(10, direction = "out"))
- println(result)
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- (result \\ "to").size must equalTo(1)
- (result \\ "to").head.as[String] must equalTo("21")
-
- result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
-
- result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- true
-
- }
- }
- }
-
-
- "labelargeSet of contention" should {
- val labelName = testLabelName2
- val maxTgtId = 10
- val batchSize = 10
- val testNum = 3
- val numOfBatch = 10
-
- def testInner(startTs: Long, src: Long) = {
- val labelName = testLabelName2
- val lastOps = Array.fill(maxTgtId)("none")
- var currentTs = startTs
-
- val allRequests = for {
- ith <- (0 until numOfBatch)
- jth <- (0 until batchSize)
- } yield {
- currentTs += 1
-
- val tgt = Random.nextInt(maxTgtId)
- val op = if (Random.nextDouble() < 0.5) "delete" else "update"
-
- lastOps(tgt) = op
- Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
- }
-
-
- allRequests.foreach(println(_))
-// println(lastOps.count(op => op != "delete" && op != "none"))
-// println(lastOps)
-//
-// Thread.sleep(1000)
-
- val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequest =>
- val bulkEdge = bulkRequest.mkString("\n")
- EdgeController.mutateAndPublish(bulkEdge, withWait = true)
- }
-
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
- val queryJson = query(id = src)
- val result = getEdges(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- println(lastOps.toList)
- println(result)
-
- val ret = resultDegree == expectedDegree && resultSize == resultDegree
- if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
-
- (ret, currentTs)
- }
-
- "update delete" in {
- running(FakeApplication()) {
- val ret = for {
- i <- (0 until testNum)
- } yield {
- val src = System.currentTimeMillis()
-
- val (ret, last) = testInner(i, src)
- ret must beEqualTo(true)
- ret
- }
-
- ret.forall(identity)
- }
- }
-
- "update delete 2" in {
- running(FakeApplication()) {
-
- val src = System.currentTimeMillis()
- var ts = 0L
-
- val ret = for {
- i <- (0 until testNum)
- } yield {
- val (ret, lastTs) = testInner(ts, src)
- val deletedAt = lastTs + 1
- val deletedAt2 = lastTs + 2
- ts = deletedAt2 + 1 // nex start ts
-
- ret must beEqualTo(true)
-
-
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
- val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
-
- val deleteRet = EdgeController.deleteAllInner(deleteAllRequest, withWait = true)
- val deleteRet2 = EdgeController.deleteAllInner(deleteAllRequest2, withWait = true)
-
-
- Await.result(deleteRet, Duration(20, TimeUnit.MINUTES))
- Await.result(deleteRet2, Duration(20, TimeUnit.MINUTES))
-
- val result = getEdges(query(id = src))
- println(result)
-
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty must beEqualTo(true)
-
- val degreeAfterDeleteAll = getDegree(result)
- degreeAfterDeleteAll must beEqualTo(0)
- true
- }
-
- ret.forall(identity)
- }
- }
-
- /** this test stress out test on degree
- * when contention is low but number of adjacent edges are large */
- "large degrees" in {
- running(FakeApplication()) {
-
-
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- // val allRequests = insertRequests ++ deleteRequests
- val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
- EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
- }
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = insertRequests.size - deleteRequests.size
- val queryJson = query(id = src)
- val result = getEdges(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- // println(result)
-
- val ret = resultSize == expectedDegree && resultDegree == resultSize
- println(s"[MaxSize]: $maxSize")
- println(s"[DeleteSize]: $deleteSize")
- println(s"[ResultDegree]: $resultDegree")
- println(s"[ExpectedDegree]: $expectedDegree")
- println(s"[ResultSize]: $resultSize")
- ret must beEqualTo(true)
- }
- }
-
- "deleteAll" in {
- running(FakeApplication()) {
-
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- // val allRequests = insertRequests ++ deleteRequests
- val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
- EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
- }
- Await.result(Future.sequence(futures), Duration(10, TimeUnit.MINUTES))
-
- val deletedAt = System.currentTimeMillis()
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
-
- Await.result(EdgeController.deleteAllInner(deleteAllRequest, withWait = true), Duration(10, TimeUnit.MINUTES))
-
- val result = getEdges(query(id = src))
- println(result)
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty must beEqualTo(true)
-
- val degreeAfterDeleteAll = getDegree(result)
- degreeAfterDeleteAll must beEqualTo(0)
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/VertexSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/VertexSpec.scala b/test/controllers/VertexSpec.scala
deleted file mode 100644
index f427b57..0000000
--- a/test/controllers/VertexSpec.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-package controllers
-
-import play.api.libs.json._
-import play.api.test.FakeApplication
-import play.api.test.Helpers._
-
-class VertexSpec extends SpecCommon {
- // init()
-
- "vetex tc" should {
- "tc1" in {
-
- running(FakeApplication()) {
- val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
-
- val (serviceName, columnName) = (testServiceName, testColumnName)
-
- val data = vertexInsertsPayload(serviceName, columnName, ids)
- val payload = Json.parse(Json.toJson(data).toString)
- println(payload)
-
- val jsResult = contentAsString(VertexController.tryMutates(payload, "insert",
- Option(serviceName), Option(columnName), withWait = true))
-
- val query = vertexQueryJson(serviceName, columnName, ids)
- val ret = contentAsJson(QueryController.getVerticesInner(query))
- println(">>>", ret)
- val fetched = ret.as[Seq[JsValue]]
- for {
- (d, f) <- data.zip(fetched)
- } yield {
- (d \ "id") must beEqualTo((f \ "id"))
- ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age"))
- }
- }
- true
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/WeakLabelDeleteSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/WeakLabelDeleteSpec.scala b/test/controllers/WeakLabelDeleteSpec.scala
deleted file mode 100644
index 164c52b..0000000
--- a/test/controllers/WeakLabelDeleteSpec.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-package controllers
-
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-class WeakLabelDeleteSpec extends SpecCommon {
- init()
-
- def bulkEdges(startTs: Int = 0) = Seq(
- Seq(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""").mkString("\t")
- ).mkString("\n")
-
- "weak label delete test" should {
- running(FakeApplication()) {
- // insert bulk and wait ..
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(), withWait = true))
- }
-
-
- def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$columnName",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelNameWeak}",
- "direction": "${direction}",
- "offset": 0,
- "limit": 10,
- "duplicate": "raw"
- }
- ]]
- }""")
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- "test weak consistency select" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(4)
- result = getEdges(query(10))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- true
- }
- }
-
- "test weak consistency delete" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
-
- /** expect 4 edges */
- (result \ "results").as[List[JsValue]].size must equalTo(4)
- val edges = (result \ "results").as[List[JsObject]]
-
- contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "delete", withWait = true))
-
- /** expect noting */
- result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- /** insert should be ignored */
- contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "insert", withWait = true))
-
- result = getEdges(query(0))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
- }
- }
-
- "test weak consistency deleteAll" in {
- running(FakeApplication()) {
- val deletedAt = 100
- var result = getEdges(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
- "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
- println(json)
- contentAsString(EdgeController.deleteAllInner(json, withWait = true))
-
-
- result = getEdges(query(11, "out"))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(12, "out"))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(10, "out"))
-
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- (result \\ "to").size must equalTo(1)
- (result \\ "to").head.as[String] must equalTo("21")
-
- result = getEdges(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
-
- result = getEdges(query(20, "in", testTgtColumnName))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
- }
- }
- }
-
-}
-
[3/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/AdminController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/AdminController.scala b/s2rest_play/app/controllers/AdminController.scala
new file mode 100644
index 0000000..2eb248b
--- /dev/null
+++ b/s2rest_play/app/controllers/AdminController.scala
@@ -0,0 +1,419 @@
+package controllers
+
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.utils.logger
+import play.api.mvc
+import play.api.mvc.{Action, Controller}
+import play.api.libs.json._
+import play.api.libs.functional.syntax._
+
+import scala.util.{Failure, Success, Try}
+
+object AdminController extends Controller {
+
+ import ApplicationController._
+ private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
+
+ /**
+ * admin message formatter
+ * @tparam T
+ */
+ trait AdminMessageFormatter[T] {
+ def toJson(msg: T): JsValue
+ }
+
+ object AdminMessageFormatter {
+ implicit def jsValueToJson[T <: JsValue] = new AdminMessageFormatter[T] {
+ def toJson(js: T) = js
+ }
+
+ implicit val stringToJson = new AdminMessageFormatter[String] {
+ def toJson(js: String) = Json.obj("message" -> js)
+ }
+ }
+
+ def format[T: AdminMessageFormatter](f: JsValue => play.mvc.Result)(message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ f(formatter.toJson(message))
+ }
+
+ /**
+ * ok response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def ok[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ Ok(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ /**
+ * bad request response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def bad[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ BadRequest(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ /**
+ * not found response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def notFound[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ NotFound(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ private[AdminController] def tryResponse[T, R: AdminMessageFormatter](res: Try[T])(callback: T => R): mvc.Result = res match {
+ case Success(m) =>
+ val ret = callback(m)
+ logger.info(ret.toString)
+ ok(ret)
+ case Failure(error) =>
+ logger.error(error.getMessage, error)
+ error match {
+ case JsResultException(e) => bad(JsError.toFlatJson(e))
+ case _ => bad(error.getMessage)
+ }
+ }
+
+ def optionResponse[T, R: AdminMessageFormatter](res: Option[T])(callback: T => R): mvc.Result = res match {
+ case Some(m) => ok(callback(m))
+ case None => notFound("not found")
+ }
+
+ /**
+ * load all model cache
+ * @return
+ */
+ def loadCache() = Action { request =>
+ val startTs = System.currentTimeMillis()
+
+ if (!ApplicationController.isHealthy) {
+ loadCacheInner()
+ }
+
+ ok(s"${System.currentTimeMillis() - startTs}")
+ }
+
+ def loadCacheInner() = {
+ Service.findAll()
+ ServiceColumn.findAll()
+ Label.findAll()
+ LabelMeta.findAll()
+ LabelIndex.findAll()
+ ColumnMeta.findAll()
+ }
+
+ /**
+ * read
+ */
+
+ /**
+ * get service info
+ * @param serviceName
+ * @return
+ */
+ def getService(serviceName: String) = Action { request =>
+ val serviceOpt = Management.findService(serviceName)
+ optionResponse(serviceOpt)(_.toJson)
+ }
+
+ /**
+ * get label info
+ * @param labelName
+ * @return
+ */
+ def getLabel(labelName: String) = Action { request =>
+ val labelOpt = Management.findLabel(labelName)
+ optionResponse(labelOpt)(_.toJson)
+ }
+
+ /**
+ * get all labels of service
+ * @param serviceName
+ * @return
+ */
+ def getLabels(serviceName: String) = Action { request =>
+ Service.findByName(serviceName) match {
+ case None => notFound(s"Service $serviceName not found")
+ case Some(service) =>
+ val src = Label.findBySrcServiceId(service.id.get)
+ val tgt = Label.findByTgtServiceId(service.id.get)
+
+ ok(Json.obj("from" -> src.map(_.toJson), "to" -> tgt.map(_.toJson)))
+ }
+ }
+
+ /**
+ * get service columns
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def getServiceColumn(serviceName: String, columnName: String) = Action { request =>
+ val serviceColumnOpt = for {
+ service <- Service.findByName(serviceName)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName, useCache = false)
+ } yield serviceColumn
+
+ optionResponse(serviceColumnOpt)(_.toJson)
+ }
+
+ /**
+ * create
+ */
+
+ /**
+ * create service
+ * @return
+ */
+ def createService() = Action(parse.json) { request =>
+ val serviceTry = createServiceInner(request.body)
+ tryResponse(serviceTry)(_.toJson)
+ }
+
+ def createServiceInner(jsValue: JsValue) = {
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = requestParser.toServiceElements(jsValue)
+ Management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ }
+
+ /**
+ * create label
+ * @return
+ */
+ def createLabel() = Action(parse.json) { request =>
+ val ret = createLabelInner(request.body)
+ tryResponse(ret)(_.toJson)
+ }
+
+ def createLabelInner(json: JsValue) = for {
+ labelArgs <- requestParser.toLabelElements(json)
+ label <- (Management.createLabel _).tupled(labelArgs)
+ } yield label
+
+ /**
+ * add index
+ * @return
+ */
+ def addIndex() = Action(parse.json) { request =>
+ val ret = addIndexInner(request.body)
+ tryResponse(ret)(_.label + " is updated")
+ }
+
+ def addIndexInner(json: JsValue) = for {
+ (labelName, indices) <- requestParser.toIndexElements(json)
+ label <- Management.addIndex(labelName, indices)
+ } yield label
+
+ /**
+ * create service column
+ * @return
+ */
+ def createServiceColumn() = Action(parse.json) { request =>
+ val serviceColumnTry = createServiceColumnInner(request.body)
+ tryResponse(serviceColumnTry) { (columns: Seq[ColumnMeta]) => Json.obj("metas" -> columns.map(_.toJson)) }
+ }
+
+ def createServiceColumnInner(jsValue: JsValue) = for {
+ (serviceName, columnName, columnType, props) <- requestParser.toServiceColumnElements(jsValue)
+ serviceColumn <- Management.createServiceColumn(serviceName, columnName, columnType, props)
+ } yield serviceColumn
+
+ /**
+ * delete
+ */
+
+ /**
+ * delete label
+ * @param labelName
+ * @return
+ */
+ def deleteLabel(labelName: String) = Action { request =>
+ val deleteLabelTry = deleteLabelInner(labelName)
+ tryResponse(deleteLabelTry)(labelName => labelName + " is deleted")
+ }
+
+ def deleteLabelInner(labelName: String) = Management.deleteLabel(labelName)
+
+ /**
+ * delete servieColumn
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def deleteServiceColumn(serviceName: String, columnName: String) = Action { request =>
+ val serviceColumnTry = deleteServiceColumnInner(serviceName, columnName)
+ tryResponse(serviceColumnTry)(columnName => columnName + " is deleted")
+ }
+
+ def deleteServiceColumnInner(serviceName: String, columnName: String) =
+ Management.deleteColumn(serviceName, columnName)
+
+ /**
+ * update
+ */
+
+ /**
+ * add Prop to label
+ * @param labelName
+ * @return
+ */
+ def addProp(labelName: String) = Action(parse.json) { request =>
+ val labelMetaTry = addPropInner(labelName, request.body)
+ tryResponse(labelMetaTry)(_.toJson)
+ }
+
+ def addPropInner(labelName: String, js: JsValue) = for {
+ prop <- requestParser.toPropElements(js)
+ labelMeta <- Management.addProp(labelName, prop)
+ } yield labelMeta
+
+ /**
+ * add prop to serviceColumn
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def addServiceColumnProp(serviceName: String, columnName: String) = Action(parse.json) { request =>
+ addServiceColumnPropInner(serviceName, columnName)(request.body) match {
+ case None => bad(s"can`t find service with $serviceName or can`t find serviceColumn with $columnName")
+ case Some(m) => Ok(m.toJson).as(applicationJsonHeader)
+ }
+ }
+
+ def addServiceColumnPropInner(serviceName: String, columnName: String)(js: JsValue) = {
+ for {
+ service <- Service.findByName(serviceName)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName)
+ prop <- requestParser.toPropElements(js).toOption
+ } yield {
+ ColumnMeta.findOrInsert(serviceColumn.id.get, prop.name, prop.defaultValue)
+ }
+ }
+
+ /**
+ * add props to serviceColumn
+ * @param serviecName
+ * @param columnName
+ * @return
+ */
+ def addServiceColumnProps(serviecName: String, columnName: String) = Action(parse.json) { request =>
+ val jsObjs = request.body.asOpt[List[JsObject]].getOrElse(List.empty[JsObject])
+ val newProps = for {
+ js <- jsObjs
+ newProp <- addServiceColumnPropInner(serviecName, columnName)(js)
+ } yield newProp
+ ok(s"${newProps.size} is added.")
+ }
+
+ /**
+ * copy label
+ * @param oldLabelName
+ * @param newLabelName
+ * @return
+ */
+ def copyLabel(oldLabelName: String, newLabelName: String) = Action { request =>
+ val copyTry = Management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
+ tryResponse(copyTry)(_.label + "created")
+ }
+
+ /**
+ * rename label
+ * @param oldLabelName
+ * @param newLabelName
+ * @return
+ */
+ def renameLabel(oldLabelName: String, newLabelName: String) = Action { request =>
+ Label.findByName(oldLabelName) match {
+ case None => NotFound.as(applicationJsonHeader)
+ case Some(label) =>
+ Management.updateLabelName(oldLabelName, newLabelName)
+ ok(s"Label was updated")
+ }
+ }
+
+ /**
+ * update HTable for a label
+ * @param labelName
+ * @param newHTableName
+ * @return
+ */
+ def updateHTable(labelName: String, newHTableName: String) = Action { request =>
+ val updateTry = Management.updateHTable(labelName, newHTableName)
+ tryResponse(updateTry)(_.toString + " label(s) updated.")
+ }
+
+
+ case class HTableParams(cluster: String, hTableName: String,
+ preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
+
+ override def toString(): String = {
+ s"""HtableParams
+ |-- cluster : $cluster
+ |-- hTableName : $hTableName
+ |-- preSplitSize : $preSplitSize
+ |-- hTableTTL : $hTableTTL
+ |-- compressionAlgorithm : $compressionAlgorithm
+ |""".stripMargin
+ }
+ }
+
+ implicit object HTableParamsJsonConverter extends Format[HTableParams] {
+ def reads(json: JsValue): JsResult[HTableParams] = (
+ (__ \ "cluster").read[String] and
+ (__ \ "hTableName").read[String] and
+ (__ \ "preSplitSize").read[Int] and
+ (__ \ "hTableTTL").readNullable[Int] and
+ (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
+
+ def writes(o: HTableParams): JsValue = Json.obj(
+ "cluster" -> o.cluster,
+ "hTableName" -> o.hTableName,
+ "preSplitSize" -> o.preSplitSize,
+ "hTableTTL" -> o.hTableTTL,
+ "compressionAlgorithm" -> o.compressionAlgorithm
+ )
+ }
+
+ implicit object JsErrorJsonWriter extends Writes[JsError] {
+ def writes(o: JsError): JsValue = Json.obj(
+ "errors" -> JsArray(
+ o.errors.map {
+ case (path, validationErrors) => Json.obj(
+ "path" -> Json.toJson(path.toString()),
+ "validationErrors" -> JsArray(validationErrors.map(validationError => Json.obj(
+ "message" -> JsString(validationError.message),
+ "args" -> JsArray(validationError.args.map(_ match {
+ case x: Int => JsNumber(x)
+ case x => JsString(x.toString)
+ }))
+ )))
+ )
+ }
+ )
+ )
+ }
+
+ def createHTable() = Action { request =>
+
+ // Management.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
+ request.body.asJson.map(_.validate[HTableParams] match {
+ case JsSuccess(hTableParams, _) => {
+ Management.createTable(hTableParams.cluster, hTableParams.hTableName, List("e", "v"), hTableParams.preSplitSize, hTableParams.hTableTTL, hTableParams.compressionAlgorithm.getOrElse(Management.defaultCompressionAlgorithm))
+ logger.info(hTableParams.toString())
+ ok(s"HTable was created.")
+ }
+ case err@JsError(_) => bad(Json.toJson(err))
+ }).getOrElse(bad("Invalid Json."))
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ApplicationController.scala b/s2rest_play/app/controllers/ApplicationController.scala
new file mode 100644
index 0000000..fefe93e
--- /dev/null
+++ b/s2rest_play/app/controllers/ApplicationController.scala
@@ -0,0 +1,83 @@
+package controllers
+
+import com.kakao.s2graph.core.utils.logger
+import play.api.libs.iteratee.Enumerator
+import play.api.libs.json.{JsString, JsValue}
+import play.api.mvc._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object ApplicationController extends Controller {
+
+ var isHealthy = true
+ var deployInfo = ""
+ val applicationJsonHeader = "application/json"
+
+ val jsonParser: BodyParser[JsValue] = controllers.s2parse.json
+
+ def updateHealthCheck(isHealthy: Boolean) = Action { request =>
+ this.isHealthy = isHealthy
+ Ok(this.isHealthy + "\n")
+ }
+
+ def healthCheck() = withHeader(parse.anyContent) { request =>
+ if (isHealthy) Ok(deployInfo)
+ else NotFound
+ }
+
+ def jsonResponse(json: JsValue, headers: (String, String)*) =
+ if (ApplicationController.isHealthy) {
+ Ok(json).as(applicationJsonHeader).withHeaders(headers: _*)
+ } else {
+ Result(
+ header = ResponseHeader(OK),
+ body = Enumerator(json.toString.getBytes()),
+ connection = HttpConnection.Close
+ ).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*)
+ }
+
+ def toLogMessage[A](request: Request[A], result: Result)(startedAt: Long): String = {
+ val duration = System.currentTimeMillis() - startedAt
+ val isQueryRequest = result.header.headers.contains("result_size")
+ val resultSize = result.header.headers.getOrElse("result_size", "-1")
+
+ try {
+ val body = request.body match {
+ case AnyContentAsJson(jsValue) => jsValue match {
+ case JsString(str) => str
+ case _ => jsValue.toString
+ }
+ case _ => request.body.toString
+ }
+
+ val str =
+ if (isQueryRequest)
+ s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
+ else
+ s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
+
+ logger.info(s"${request.method} ${request.uri} result_size: $resultSize")
+
+ str
+ } finally {
+ /* pass */
+ }
+ }
+
+ def withHeaderAsync[A](bodyParser: BodyParser[A])(block: Request[A] => Future[Result])(implicit ex: ExecutionContext) =
+ Action.async(bodyParser) { request =>
+ val startedAt = System.currentTimeMillis()
+ block(request).map { r =>
+ logger.info(toLogMessage(request, r)(startedAt))
+ r
+ }
+ }
+
+ def withHeader[A](bodyParser: BodyParser[A])(block: Request[A] => Result) =
+ Action(bodyParser) { request: Request[A] =>
+ val startedAt = System.currentTimeMillis()
+ val r = block(request)
+ logger.info(toLogMessage(request, r)(startedAt))
+ r
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/CounterController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/CounterController.scala b/s2rest_play/app/controllers/CounterController.scala
new file mode 100644
index 0000000..c2b4dc2
--- /dev/null
+++ b/s2rest_play/app/controllers/CounterController.scala
@@ -0,0 +1,747 @@
+package controllers
+
+import com.kakao.s2graph.core.ExceptionHandler
+import com.kakao.s2graph.core.ExceptionHandler.KafkaMessage
+import com.kakao.s2graph.core.mysqls.Label
+import config.CounterConfig
+import models._
+import org.apache.kafka.clients.producer.ProducerRecord
+import play.api.Play
+import play.api.libs.json.Reads._
+import play.api.libs.json._
+import play.api.mvc.{Action, Controller, Request}
+import s2.config.S2CounterConfig
+import s2.counter.core.TimedQualifier.IntervalUnit
+import s2.counter.core._
+import s2.counter.core.v1.{ExactStorageAsyncHBase, RankingStorageRedis}
+import s2.counter.core.v2.{ExactStorageGraph, RankingStorageGraph}
+import s2.models.Counter.ItemType
+import s2.models.{Counter, CounterModel}
+import s2.util.{CartesianProduct, ReduceMapValue, UnitConverter}
+
+import scala.concurrent.Future
+import scala.util.{Failure, Success, Try}
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 5. 22..
+ */
+object CounterController extends Controller {
+ import play.api.libs.concurrent.Execution.Implicits.defaultContext
+
+ val config = Play.current.configuration.underlying
+ val s2config = new S2CounterConfig(config)
+
+ private val exactCounterMap = Map(
+ s2.counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
+ s2.counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
+ )
+ private val rankingCounterMap = Map(
+ s2.counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
+ s2.counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
+ )
+
+ private val tablePrefixMap = Map (
+ s2.counter.VERSION_1 -> "s2counter",
+ s2.counter.VERSION_2 -> "s2counter_v2"
+ )
+
+ private def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
+ private def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
+
+ lazy val counterModel = new CounterModel(config)
+
+ def getQueryString[T](key: String, default: String)(implicit request: Request[T]): String = {
+ request.getQueryString(key).getOrElse(default)
+ }
+
+ implicit val counterWrites = new Writes[Counter] {
+ override def writes(o: Counter): JsValue = Json.obj(
+ "version" -> o.version.toInt,
+ "autoComb" -> o.autoComb,
+ "dimension" -> o.dimension,
+ "useProfile" -> o.useProfile,
+ "bucketImpId" -> o.bucketImpId,
+ "useRank" -> o.useRank,
+ "intervalUnit" -> o.intervalUnit,
+ "ttl" -> o.ttl,
+ "dailyTtl" -> o.dailyTtl,
+ "rateAction" -> o.rateActionId.flatMap { actionId =>
+ counterModel.findById(actionId, useCache = false).map { actionPolicy =>
+ Json.obj("service" -> actionPolicy.service, "action" -> actionPolicy.action)
+ }
+ },
+ "rateBase" -> o.rateBaseId.flatMap { baseId =>
+ counterModel.findById(baseId, useCache = false).map { basePolicy =>
+ Json.obj("service" -> basePolicy.service, "action" -> basePolicy.action)
+ }
+ },
+ "rateThreshold" -> o.rateThreshold
+ )
+ }
+
+ def createAction(service: String, action: String) = Action(s2parse.json) { implicit request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case None =>
+ val body = request.body
+ val version = (body \ "version").asOpt[Int].map(_.toByte).getOrElse(s2.counter.VERSION_2)
+ val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(true)
+ val dimension = (body \ "dimension").asOpt[String].getOrElse("")
+ val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(false)
+ val bucketImpId = (body \ "bucketImpId").asOpt[String]
+
+ val useExact = (body \ "useExact").asOpt[Boolean].getOrElse(true)
+ val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(true)
+
+ val intervalUnit = (body \ "intervalUnit").asOpt[String]
+ // 2 day
+ val ttl = (body \ "ttl").asOpt[Int].getOrElse(2 * 24 * 60 * 60)
+ val dailyTtl = (body \ "dailyTtl").asOpt[Int]
+ val regionMultiplier = (body \ "regionMultiplier").asOpt[Int].getOrElse(1)
+
+ val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
+ val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
+ val rateThreshold = (body \ "rateThreshold").asOpt[Int]
+
+ val rateActionId = {
+ for {
+ actionMap <- rateAction
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action)
+ } yield {
+ policy.id
+ }
+ }
+ val rateBaseId = {
+ for {
+ actionMap <- rateBase
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action)
+ } yield {
+ policy.id
+ }
+ }
+
+ val hbaseTable = {
+ Seq(tablePrefixMap(version), service, ttl) ++ dailyTtl mkString "_"
+ }
+
+ // find label
+ val itemType = Label.findByName(action, useCache = false) match {
+ case Some(label) =>
+ ItemType.withName(label.tgtColumnType.toUpperCase)
+ case None =>
+ val strItemType = (body \ "itemType").asOpt[String].getOrElse("STRING")
+ ItemType.withName(strItemType.toUpperCase)
+ }
+ val policy = Counter(useFlag = true, version, service, action, itemType, autoComb = autoComb, dimension,
+ useProfile = useProfile, bucketImpId, useRank = useRank, ttl, dailyTtl, Some(hbaseTable), intervalUnit,
+ rateActionId, rateBaseId, rateThreshold)
+
+ // prepare exact storage
+ exactCounter(version).prepare(policy)
+ if (useRank) {
+ // prepare ranking storage
+ rankingCounter(version).prepare(policy)
+ }
+ counterModel.createServiceAction(policy)
+ Ok(Json.toJson(Map("msg" -> s"created $service/$action")))
+ case Some(policy) =>
+ Ok(Json.toJson(Map("msg" -> s"already exist $service/$action")))
+ }
+ }
+
+ def getAction(service: String, action: String) = Action { request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(policy) =>
+ Ok(Json.toJson(policy))
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def updateAction(service: String, action: String) = Action(s2parse.json) { request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(oldPolicy) =>
+ val body = request.body
+ val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(oldPolicy.autoComb)
+ val dimension = (body \ "dimension").asOpt[String].getOrElse(oldPolicy.dimension)
+ val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(oldPolicy.useProfile)
+ val bucketImpId = (body \ "bucketImpId").asOpt[String] match {
+ case Some(s) => Some(s)
+ case None => oldPolicy.bucketImpId
+ }
+
+ val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(oldPolicy.useRank)
+
+ val intervalUnit = (body \ "intervalUnit").asOpt[String] match {
+ case Some(s) => Some(s)
+ case None => oldPolicy.intervalUnit
+ }
+
+ val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
+ val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
+ val rateThreshold = (body \ "rateThreshold").asOpt[Int] match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateThreshold
+ }
+
+ val rateActionId = {
+ for {
+ actionMap <- rateAction
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ policy.id
+ }
+ } match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateActionId
+ }
+ val rateBaseId = {
+ for {
+ actionMap <- rateBase
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ policy.id
+ }
+ } match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateBaseId
+ }
+
+ // new counter
+ val policy = Counter(id = oldPolicy.id, useFlag = oldPolicy.useFlag, oldPolicy.version, service, action, oldPolicy.itemType, autoComb = autoComb, dimension,
+ useProfile = useProfile, bucketImpId, useRank = useRank, oldPolicy.ttl, oldPolicy.dailyTtl, oldPolicy.hbaseTable, intervalUnit,
+ rateActionId, rateBaseId, rateThreshold)
+
+ counterModel.updateServiceAction(policy)
+ Ok(Json.toJson(Map("msg" -> s"updated $service/$action")))
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def prepareAction(service: String, action: String) = Action(s2parse.json) { request =>
+ // for data migration
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(policy) =>
+ val body = request.body
+ val version = (body \ "version").as[Int].toByte
+ if (version != policy.version) {
+ // change table name
+ val newTableName = Seq(tablePrefixMap(version), service, policy.ttl) ++ policy.dailyTtl mkString "_"
+ val newPolicy = policy.copy(version = version, hbaseTable = Some(newTableName))
+ exactCounter(version).prepare(newPolicy)
+ if (newPolicy.useRank) {
+ rankingCounter(version).prepare(newPolicy)
+ }
+ Ok(Json.toJson(Map("msg" -> s"prepare storage v$version $service/$action")))
+ } else {
+ Ok(Json.toJson(Map("msg" -> s"already prepared storage v$version $service/$action")))
+ }
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def deleteAction(service: String, action: String) = Action.apply {
+ {
+ for {
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ Try {
+ exactCounter(policy.version).destroy(policy)
+ if (policy.useRank) {
+ rankingCounter(policy.version).destroy(policy)
+ }
+ counterModel.deleteServiceAction(policy)
+ } match {
+ case Success(v) =>
+ Ok(Json.toJson(Map("msg" -> s"deleted $service/$action")))
+ case Failure(ex) =>
+ throw ex
+ }
+ }
+ }.getOrElse(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+
+ def getExactCountAsync(service: String, action: String, item: String) = Action.async { implicit request =>
+ val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ val limit = getQueryString("limit", "1").toInt
+
+ val qsSum = request.getQueryString("sum")
+
+ val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
+ val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
+
+ val limitOpt = (optFrom, optTo) match {
+ case (Some(_), Some(_)) =>
+ None
+ case _ =>
+ Some(limit)
+ }
+
+ // find dimension
+ lazy val dimQueryValues = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').filter(_.nonEmpty).toSet)
+ }
+// Logger.warn(s"$dimQueryValues")
+
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
+ val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ try {
+// Logger.warn(s"$tqs $qsSum")
+ if (tqs.head.length > 1 && qsSum.isDefined) {
+ getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum).map { jsVal =>
+ Ok(jsVal)
+ }
+ } else {
+ getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues).map { jsVal =>
+ Ok(jsVal)
+ }
+ }
+ } catch {
+ case e: Exception =>
+ throw e
+// Future.successful(BadRequest(s"$service, $action, $item"))
+ }
+ case None =>
+ Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+ }
+
+ /**
+ * [{
+ * "service": , "action", "itemIds": [], "interval": string, "limit": int, "from": ts, "to": ts,
+ * "dimensions": [{"key": list[String]}]
+ * }]
+ * @return
+ */
+ private def parseExactCountParam(jsValue: JsValue) = {
+ val service = (jsValue \ "service").as[String]
+ val action = (jsValue \ "action").as[String]
+ val itemIds = (jsValue \ "itemIds").as[Seq[String]]
+ val intervals = (jsValue \ "intervals").asOpt[Seq[String]].getOrElse(Seq("t")).distinct.map(IntervalUnit.withName)
+ val limit = (jsValue \ "limit").asOpt[Int].getOrElse(1)
+ val from = (jsValue \ "from").asOpt[Long]
+ val to = (jsValue \ "to").asOpt[Long]
+ val sum = (jsValue \ "sum").asOpt[String]
+ val dimensions = {
+ for {
+ dimension <- (jsValue \ "dimensions").asOpt[Seq[JsObject]].getOrElse(Nil)
+ (k, vs) <- dimension.fields
+ } yield {
+ k -> vs.as[Seq[String]].toSet
+ }
+ }.toMap
+ (service, action, itemIds, intervals, limit, from, to, dimensions, sum)
+ }
+
+ def getExactCountAsyncMulti = Action.async(s2parse.json) { implicit request =>
+ val jsValue = request.body
+ try {
+ val futures = {
+ for {
+ jsObject <- jsValue.asOpt[List[JsObject]].getOrElse(Nil)
+ (service, action, itemIds, intervalUnits, limit, from, to, dimQueryValues, qsSum) = parseExactCountParam(jsObject)
+ optFrom = from.map(UnitConverter.toMillis)
+ optTo = to.map(UnitConverter.toMillis)
+ timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ policy <- counterModel.findByServiceAction(service, action).toSeq
+ item <- itemIds
+ } yield {
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
+ val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ val limitOpt = (optFrom, optTo) match {
+ case (Some(_), Some(_)) =>
+ None
+ case _ =>
+ Some(limit)
+ }
+
+// Logger.warn(s"$item, $limit, $optFrom, $optTo, $qsSum, $tqs")
+
+ if (tqs.head.length > 1 && qsSum.isDefined) {
+ getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum)
+ } else {
+ getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues)
+ }
+ }
+ }
+ Future.sequence(futures).map { rets =>
+ Ok(Json.toJson(rets))
+ }
+ } catch {
+ case e: Exception =>
+ throw e
+// Future.successful(BadRequest(s"$jsValue"))
+ }
+ }
+
+ private [controllers] def fetchedToResult(fetchedCounts: FetchedCountsGrouped, limitOpt: Option[Int]): Seq[ExactCounterIntervalItem] = {
+ for {
+ ((interval, dimKeyValues), values) <- fetchedCounts.intervalWithCountMap
+ } yield {
+ val counterItems = {
+ val sortedItems = values.toSeq.sortBy { case (eq, v) => -eq.tq.ts }
+ val limited = limitOpt match {
+ case Some(limit) => sortedItems.take(limit)
+ case None => sortedItems
+ }
+ for {
+ (eq, value) <- limited
+ } yield {
+ ExactCounterItem(eq.tq.ts, value, value.toDouble)
+ }
+ }
+ ExactCounterIntervalItem(interval.toString, dimKeyValues, counterItems)
+ }
+ }.toSeq
+
+ private def decayedToResult(decayedCounts: DecayedCounts): Seq[ExactCounterIntervalItem] = {
+ for {
+ (eq, score) <- decayedCounts.qualifierWithCountMap
+ } yield {
+ ExactCounterIntervalItem(eq.tq.q.toString, eq.dimKeyValues, Seq(ExactCounterItem(eq.tq.ts, score.toLong, score)))
+ }
+ }.toSeq
+
+ private def getExactCountToJs(policy: Counter,
+ item: String,
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ limitOpt: Option[Int],
+ dimQueryValues: Map[String, Set[String]]): Future[JsValue] = {
+ exactCounter(policy.version).getCountsAsync(policy, Seq(item), timeRange, dimQueryValues).map { seq =>
+ val items = {
+ for {
+ fetched <- seq
+ } yield {
+ fetchedToResult(fetched, limitOpt)
+ }
+ }.flatten
+ Json.toJson(ExactCounterResult(ExactCounterResultMeta(policy.service, policy.action, item), items))
+ }
+ }
+
+ private def getDecayedCountToJs(policy: Counter,
+ item: String,
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQueryValues: Map[String, Set[String]],
+ qsSum: Option[String]): Future[JsValue] = {
+ exactCounter(policy.version).getDecayedCountsAsync(policy, Seq(item), timeRange, dimQueryValues, qsSum).map { seq =>
+ val decayedCounts = seq.head
+ val meta = ExactCounterResultMeta(policy.service, policy.action, decayedCounts.exactKey.itemKey)
+ val intervalItems = decayedToResult(decayedCounts)
+ Json.toJson(ExactCounterResult(meta, intervalItems))
+ }
+ }
+
+ def getRankingCountAsync(service: String, action: String) = Action.async { implicit request =>
+ lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ lazy val limit = getQueryString("limit", "1").toInt
+ lazy val kValue = getQueryString("k", "10").toInt
+
+ lazy val qsSum = request.getQueryString("sum")
+
+ lazy val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
+ lazy val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
+
+ // find dimension
+ lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').toList)
+ }
+
+ val dimensions = {
+ for {
+ values <- CartesianProduct(dimensionMap.values.toList).toSeq
+ } yield {
+ dimensionMap.keys.zip(values).toMap
+ }
+ }
+
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optTo)
+ val dimKeys = {
+ for {
+ dimension <- dimensions
+ } yield {
+ dimension -> tqs.map(tq => RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension)))
+ }
+ }
+
+ // if tqs has only 1 tq, do not apply sum function
+ try {
+ val rankResult = {
+ if (tqs.length > 1 && qsSum.isDefined) {
+ getSumRankCounterResultAsync(policy, dimKeys, kValue, qsSum)
+ } else {
+ // no summary
+ Future.successful(getRankCounterResult(policy, dimKeys, kValue))
+ }
+ }
+
+ rankResult.map { result =>
+ Ok(Json.toJson(result))
+ }
+ } catch {
+ case e: UnsupportedOperationException =>
+ Future.successful(NotImplemented(Json.toJson(
+ Map("msg" -> e.getMessage)
+ )))
+ case e: Throwable =>
+ throw e
+ }
+ case None =>
+ Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+ }
+
+ def deleteRankingCount(service: String, action: String) = Action.async { implicit request =>
+ lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ lazy val limit = getQueryString("limit", "1").toInt
+
+ // find dimension
+ lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').toList)
+ }
+
+ val dimensions = {
+ for {
+ values <- CartesianProduct(dimensionMap.values.toList).toSeq
+ } yield {
+ dimensionMap.keys.zip(values).toMap
+ }
+ }
+
+ Future {
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit)
+ val keys = {
+ for {
+ dimension <- dimensions
+ tq <- tqs
+ } yield {
+ RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension))
+ }
+ }
+
+ for {
+ key <- keys
+ } {
+ rankingCounter(policy.version).delete(key)
+ }
+
+ Ok(JsObject(
+ Seq(
+ ("msg", Json.toJson(s"delete ranking in $service.$action")),
+ ("items", Json.toJson({
+ for {
+ key <- keys
+ } yield {
+ s"${key.eq.tq.q}.${key.eq.tq.ts}.${key.eq.dimension}"
+ }
+ }))
+ )
+ ))
+ case None =>
+ NotFound(Json.toJson(
+ Map("msg" -> s"$service.$action not found")
+ ))
+ }
+ }
+ }
+
+ val reduceRateRankingValue = new ReduceMapValue[ExactKeyTrait, RateRankingValue](RateRankingValue.reduce, RateRankingValue(-1, -1))
+
+ // change format
+ private def getDecayedCountsAsync(policy: Counter,
+ items: Seq[String],
+ timeRange: (TimedQualifier, TimedQualifier),
+ dimension: Map[String, String],
+ qsSum: Option[String]): Future[Seq[(ExactKeyTrait, Double)]] = {
+ exactCounter(policy.version).getDecayedCountsAsync(policy, items, Seq(timeRange), dimension.mapValues(s => Set(s)), qsSum).map { seq =>
+ for {
+ DecayedCounts(exactKey, qcMap) <- seq
+ value <- qcMap.values
+ } yield {
+ exactKey -> value
+ }
+ }
+ }
+
+ def getSumRankCounterResultAsync(policy: Counter,
+ dimKeys: Seq[(Map[String, String], Seq[RankingKey])],
+ kValue: Int,
+ qsSum: Option[String]): Future[RankCounterResult] = {
+ val futures = {
+ for {
+ (dimension, keys) <- dimKeys
+ } yield {
+ val tqs = keys.map(rk => rk.eq.tq)
+ val (tqFrom, tqTo) = (tqs.last, tqs.head)
+ val items = rankingCounter(policy.version).getAllItems(keys, kValue)
+// Logger.warn(s"item count: ${items.length}")
+ val future = {
+ if (policy.isRateCounter) {
+ val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
+ val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
+
+ val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
+ }.toMap
+ }
+ val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
+ }.toMap
+ }
+ futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
+ reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
+// Logger.warn(s"$k -> $rrv")
+ k -> rrv.rankingValue.score
+ }.toSeq
+ }
+ }
+ else if (policy.isTrendCounter) {
+ val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
+ val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
+
+ val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
+ }.toMap
+ }
+ val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom.add(-1), tqTo.add(-1)), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
+ }.toMap
+ }
+ futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
+ reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
+// Logger.warn(s"$k -> $rrv")
+ k -> rrv.rankingValue.score
+ }.toSeq
+ }
+ }
+ else {
+ getDecayedCountsAsync(policy, items, (tqFrom, tqTo), dimension, qsSum)
+ }
+ }
+ future.map { keyWithScore =>
+ val ranking = keyWithScore.sortBy(-_._2).take(kValue)
+ val rankCounterItems = {
+ for {
+ idx <- ranking.indices
+ (exactKey, score) = ranking(idx)
+ } yield {
+ val realId = policy.itemType match {
+ case ItemType.BLOB => exactCounter(policy.version).getBlobValue(policy, exactKey.itemKey)
+ .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} ${exactKey.itemKey}"))
+ case _ => exactKey.itemKey
+ }
+ RankCounterItem(idx + 1, realId, score)
+ }
+ }
+
+ val eq = ExactQualifier(tqFrom, dimension)
+ RankCounterDimensionItem(eq.tq.q.toString, eq.tq.ts, eq.dimension, -1, rankCounterItems)
+ }
+ }
+ }
+
+ Future.sequence(futures).map { dimensionResultList =>
+ RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
+ }
+ }
+
+ def getRankCounterResult(policy: Counter, dimKeys: Seq[(Map[String, String], Seq[RankingKey])], kValue: Int): RankCounterResult = {
+ val dimensionResultList = {
+ for {
+ (dimension, keys) <- dimKeys
+ key <- keys
+ } yield {
+ val rankingValue = rankingCounter(policy.version).getTopK(key, kValue)
+ val ranks = {
+ for {
+ rValue <- rankingValue.toSeq
+ idx <- rValue.values.indices
+ rank = idx + 1
+ } yield {
+ val (id, score) = rValue.values(idx)
+ val realId = policy.itemType match {
+ case ItemType.BLOB =>
+ exactCounter(policy.version)
+ .getBlobValue(policy, id)
+ .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} $id"))
+ case _ => id
+ }
+ RankCounterItem(rank, realId, score)
+ }
+ }
+ val eq = key.eq
+ val tq = eq.tq
+ RankCounterDimensionItem(tq.q.toString, tq.ts, eq.dimension, rankingValue.map(v => v.totalScore).getOrElse(0d), ranks)
+ }
+ }
+
+ RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
+ }
+
+ type Record = ProducerRecord[String, String]
+
+ def incrementCount(service: String, action: String, item: String) = Action.async(s2parse.json) { request =>
+ Future {
+ /**
+ * {
+ * timestamp: Long
+ * property: {}
+ * value: Int
+ * }
+ */
+ lazy val metaMap = Map("service" -> service, "action" -> action, "item" -> item)
+ counterModel.findByServiceAction(service, action).map { policy =>
+ val body = request.body
+ try {
+ val ts = (body \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis()).toString
+ val dimension = (body \ "dimension").asOpt[JsValue].getOrElse(Json.obj())
+ val property = (body \ "property").asOpt[JsValue].getOrElse(Json.obj())
+
+ val msg = List(ts, service, action, item, dimension, property).mkString("\t")
+
+ // produce to kafka
+ // hash partitioner by key
+ ExceptionHandler.enqueue(KafkaMessage(new Record(CounterConfig.KAFKA_TOPIC_COUNTER, s"$ts.$item", msg)))
+
+ Ok(Json.toJson(
+ Map(
+ "meta" -> metaMap
+ )
+ ))
+ }
+ catch {
+ case e: JsResultException =>
+ BadRequest(Json.toJson(
+ Map("msg" -> s"need timestamp.")
+ ))
+ }
+ }.getOrElse {
+ NotFound(Json.toJson(
+ Map("msg" -> s"$service.$action not found")
+ ))
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/EdgeController.scala b/s2rest_play/app/controllers/EdgeController.scala
new file mode 100644
index 0000000..32af6c3
--- /dev/null
+++ b/s2rest_play/app/controllers/EdgeController.scala
@@ -0,0 +1,222 @@
+package controllers
+
+import actors.QueueActor
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.{LabelMeta, Label}
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.types.LabelWithDirection
+import com.kakao.s2graph.core.utils.logger
+import config.Config
+import org.apache.kafka.clients.producer.ProducerRecord
+import play.api.libs.json._
+import play.api.mvc.{Controller, Result}
+
+import scala.collection.Seq
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
+
+object EdgeController extends Controller {
+
+ import ExceptionHandler._
+ import controllers.ApplicationController._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
+ private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
+
+ def tryMutates(jsValue: JsValue, operation: String, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+
+ else {
+ try {
+ logger.debug(s"$jsValue")
+ val edges = requestParser.toEdges(jsValue, operation)
+ for (edge <- edges) {
+ if (edge.isAsync)
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, edge, None))
+ else
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, edge, None))
+ }
+
+ val edgesToStore = edges.filterNot(e => e.isAsync)
+
+ if (withWait) {
+ val rets = s2.mutateEdges(edgesToStore, withWait = true)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = edgesToStore.map { edge => QueueActor.router ! edge; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
+ case e: Exception =>
+ logger.error(s"mutateAndPublish: $e", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+ }
+
+ def mutateAndPublish(str: String, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+
+ logger.debug(s"$str")
+ val edgeStrs = str.split("\\n")
+
+ var vertexCnt = 0L
+ var edgeCnt = 0L
+ try {
+ val elements =
+ for (edgeStr <- edgeStrs; str <- GraphUtil.parseString(edgeStr); element <- Graph.toGraphElement(str)) yield {
+ element match {
+ case v: Vertex => vertexCnt += 1
+ case e: Edge => edgeCnt += 1
+ }
+ if (element.isAsync) {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, element, Some(str)))
+ } else {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, element, Some(str)))
+ }
+ element
+ }
+
+ //FIXME:
+ val elementsToStore = elements.filterNot(e => e.isAsync)
+ if (withWait) {
+ val rets = s2.mutateElements(elementsToStore, withWait)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = elementsToStore.map { element => QueueActor.router ! element; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+
+
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
+ case e: Throwable =>
+ logger.error(s"mutateAndPublish: $e", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+
+ def mutateBulk() = withHeaderAsync(parse.text) { request =>
+ mutateAndPublish(request.body)
+ }
+
+ def inserts() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert")
+ }
+
+ def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", withWait = true)
+ }
+
+ def insertsBulk() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insertBulk")
+ }
+
+ def deletes() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete")
+ }
+
+ def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", withWait = true)
+ }
+
+ def updates() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "update")
+ }
+
+ def updatesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "update", withWait = true)
+ }
+
+ def increments() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "increment")
+ }
+
+ def incrementCounts() = withHeaderAsync(jsonParser) { request =>
+ val jsValue = request.body
+ val edges = requestParser.toEdges(jsValue, "incrementCount")
+ s2.incrementCounts(edges).map { results =>
+ val json = results.map { case (isSuccess, resultCount) =>
+ Json.obj("success" -> isSuccess, "result" -> resultCount)
+ }
+
+ jsonResponse(Json.toJson(json))
+ }
+ }
+
+ def deleteAll() = withHeaderAsync(jsonParser) { request =>
+ deleteAllInner(request.body, withWait = false)
+ }
+
+ def deleteAllInner(jsValue: JsValue, withWait: Boolean) = {
+ val deleteResults = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
+
+ val labelName = (json \ "label").as[String]
+ val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil)
+ val direction = (json \ "direction").asOpt[String].getOrElse("out")
+
+ val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
+ val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
+ val vertices = requestParser.toVertices(labelName, direction, ids)
+
+ /** logging for delete all request */
+
+ val kafkaMessages = for {
+ id <- ids
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
+ val topic = if (label.isAsync) Config.KAFKA_LOG_TOPIC_ASYNC else Config.KAFKA_LOG_TOPIC
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+ ExceptionHandler.enqueues(kafkaMessages)
+
+ val future = s2.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
+ future.onFailure { case ex: Exception =>
+ logger.error(s"[Error]: deleteAllInner failed.", ex)
+ val kafkaMessages = for {
+ id <- ids
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
+ val topic = failTopic
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+ ExceptionHandler.enqueues(kafkaMessages)
+ throw ex
+ }
+ if (withWait) {
+ future.map { ret =>
+ if (!ret) {
+ logger.error(s"[Error]: deleteAllInner failed.")
+ val kafkaMessages = for {
+ id <- ids
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
+ val topic = failTopic
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+ ExceptionHandler.enqueues(kafkaMessages)
+ false
+ } else {
+ true
+ }
+ }
+ } else {
+ Future.successful(true)
+ }
+ })
+
+ deleteResults.map { rst =>
+ logger.debug(s"deleteAllInner: $rst")
+ Ok(s"deleted... ${rst.toString()}")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/ExperimentController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ExperimentController.scala b/s2rest_play/app/controllers/ExperimentController.scala
new file mode 100644
index 0000000..5ae2379
--- /dev/null
+++ b/s2rest_play/app/controllers/ExperimentController.scala
@@ -0,0 +1,114 @@
+package controllers
+
+
+import java.net.URL
+
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.utils.logger
+import play.api.Play.current
+import play.api.libs.json.{JsObject, JsString, JsValue, Json}
+import play.api.libs.ws.WS
+import play.api.mvc._
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+object ExperimentController extends Controller {
+ val impressionKey = "S2-Impression-Id"
+
+ import ApplicationController._
+
+ def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(parse.anyContent) { request =>
+ val bucketOpt = for {
+ service <- Service.findByAccessToken(accessToken)
+ experiment <- Experiment.findBy(service.id.get, experimentName)
+ bucket <- experiment.findBucket(uuid)
+ } yield bucket
+
+ bucketOpt match {
+ case None => Future.successful(NotFound("bucket is not found."))
+ case Some(bucket) =>
+ try {
+ if (bucket.isGraphQuery) buildRequestInner(request, bucket, uuid)
+ else buildRequest(request, bucket, uuid)
+ } catch {
+ case e: Exception =>
+ logger.error(e.toString())
+ Future.successful(BadRequest(s"wrong or missing template parameter: ${e.getMessage}"))
+ }
+ }
+ }
+
+ def makeRequestJson(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): JsValue = {
+ var body = bucket.requestBody.replace("#uuid", uuid)
+
+ for {
+ requestKeyJson <- requestKeyJsonOpt
+ jsObj <- requestKeyJson.asOpt[JsObject]
+ (key, value) <- jsObj.fieldSet
+ } {
+ val replacement = value match {
+ case JsString(s) => s
+ case _ => value.toString
+ }
+ body = body.replace(key, replacement)
+ }
+
+ Json.parse(body)
+ }
+
+ private def buildRequestInner(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = {
+ if (bucket.isEmpty) Future.successful(Ok(Json.obj("isEmpty" -> true)).withHeaders(impressionKey -> bucket.impressionId))
+ else {
+ val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid)
+ val url = new URL(bucket.apiPath)
+ val path = url.getPath()
+
+ // dummy log for sampling
+ val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody"
+ logger.info(experimentLog)
+
+ val response = path match {
+ case "/graphs/getEdges" => controllers.QueryController.getEdgesInner(jsonBody)
+ case "/graphs/getEdges/grouped" => controllers.QueryController.getEdgesWithGroupingInner(jsonBody)
+ case "/graphs/getEdgesExcluded" => controllers.QueryController.getEdgesExcludedInner(jsonBody)
+ case "/graphs/getEdgesExcluded/grouped" => controllers.QueryController.getEdgesExcludedWithGroupingInner(jsonBody)
+ case "/graphs/checkEdges" => controllers.QueryController.checkEdgesInner(jsonBody)
+ case "/graphs/getEdgesGrouped" => controllers.QueryController.getEdgesGroupedInner(jsonBody)
+ case "/graphs/getEdgesGroupedExcluded" => controllers.QueryController.getEdgesGroupedExcludedInner(jsonBody)
+ case "/graphs/getEdgesGroupedExcludedFormatted" => controllers.QueryController.getEdgesGroupedExcludedFormattedInner(jsonBody)
+ }
+ response.map { r => r.withHeaders(impressionKey -> bucket.impressionId) }
+ }
+ }
+
+ private def toSimpleMap(map: Map[String, Seq[String]]): Map[String, String] = {
+ for {
+ (k, vs) <- map
+ headVal <- vs.headOption
+ } yield {
+ k -> headVal
+ }
+ }
+
+ private def buildRequest(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = {
+ val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid)
+
+ val url = bucket.apiPath
+ val headers = request.headers.toSimpleMap.toSeq
+ val verb = bucket.httpVerb.toUpperCase
+ val qs = toSimpleMap(request.queryString).toSeq
+
+ val ws = WS.url(url)
+ .withMethod(verb)
+ .withBody(jsonBody)
+ .withHeaders(headers: _*)
+ .withQueryString(qs: _*)
+
+ ws.stream().map {
+ case (proxyResponse, proxyBody) =>
+ Result(ResponseHeader(proxyResponse.status, proxyResponse.headers.mapValues(_.toList.head)), proxyBody).withHeaders(impressionKey -> bucket.impressionId)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/JsonBodyParser.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/JsonBodyParser.scala b/s2rest_play/app/controllers/JsonBodyParser.scala
new file mode 100644
index 0000000..3e3d40c
--- /dev/null
+++ b/s2rest_play/app/controllers/JsonBodyParser.scala
@@ -0,0 +1,73 @@
+package controllers
+
+import com.kakao.s2graph.core.utils.logger
+import play.api.Play
+import play.api.libs.iteratee.Iteratee
+import play.api.libs.json.{JsValue, Json}
+import play.api.mvc._
+
+import scala.concurrent.Future
+import scala.util.control.NonFatal
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 9. 1..
+ */
+
+object s2parse extends BodyParsers {
+
+ import parse._
+
+ val defaultMaxTextLength = 1024 * 512
+ val defaultMaxJsonLength = 1024 * 512
+
+// def json: BodyParser[JsValue] = json(DEFAULT_MAX_TEXT_LENGTH)
+ def json: BodyParser[JsValue] = json(defaultMaxTextLength)
+
+ def json(maxLength: Int): BodyParser[JsValue] = when(
+ _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
+ tolerantJson(maxLength),
+ createBadResult("Expecting text/json or application/json body")
+ )
+
+ def tolerantJson(maxLength: Int): BodyParser[JsValue] =
+ tolerantBodyParser[JsValue]("json", maxLength, "Invalid Json") { (request, bytes) =>
+ // Encoding notes: RFC 4627 requires that JSON be encoded in Unicode, and states that whether that's
+ // UTF-8, UTF-16 or UTF-32 can be auto detected by reading the first two bytes. So we ignore the declared
+ // charset and don't decode, we passing the byte array as is because Jackson supports auto detection.
+ Json.parse(bytes)
+ }
+
+ private def tolerantBodyParser[A](name: String, maxLength: Int, errorMessage: String)(parser: (RequestHeader, Array[Byte]) => A): BodyParser[A] =
+ BodyParser(name + ", maxLength=" + maxLength) { request =>
+ import play.api.libs.iteratee.Execution.Implicits.trampoline
+ import play.api.libs.iteratee.Traversable
+
+ import scala.util.control.Exception._
+
+ val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] =
+ Traversable.takeUpTo[Array[Byte]](maxLength).transform(
+ Iteratee.consume[Array[Byte]]().map { bytes =>
+ allCatch[A].either {
+ parser(request, bytes)
+ }.left.map {
+ case NonFatal(e) =>
+ val txt = new String(bytes)
+ logger.error(s"$errorMessage: $txt", e)
+ createBadResult(s"$errorMessage: $e")(request)
+ case t => throw t
+ }
+ }
+ ).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
+
+ bodyParser.mapM {
+ case Left(tooLarge) => Future.successful(Left(tooLarge))
+ case Right(Left(badResult)) => badResult.map(Left.apply)
+ case Right(Right(body)) => Future.successful(Right(body))
+ }
+ }
+
+ private def createBadResult(msg: String): RequestHeader => Future[Result] = { request =>
+ Play.maybeApplication.map(_.global.onBadRequest(request, msg))
+ .getOrElse(Future.successful(Results.BadRequest))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/PublishController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/PublishController.scala b/s2rest_play/app/controllers/PublishController.scala
new file mode 100644
index 0000000..b1495d5
--- /dev/null
+++ b/s2rest_play/app/controllers/PublishController.scala
@@ -0,0 +1,54 @@
+package controllers
+
+import com.kakao.s2graph.core.ExceptionHandler
+import config.Config
+import org.apache.kafka.clients.producer.ProducerRecord
+import play.api.mvc._
+
+import scala.concurrent.Future
+
+object PublishController extends Controller {
+
+ import ApplicationController._
+ import ExceptionHandler._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ /**
+ * never check validation on string. just redirect strings to kafka.
+ */
+ val serviceNotExistException = new RuntimeException(s"service is not created in s2graph. create service first.")
+
+ // private def toService(topic: String): String = {
+ // Service.findByName(topic).map(service => s"${service.serviceName}-${Config.PHASE}").getOrElse(throw serviceNotExistException)
+ // }
+ def publishOnly(topic: String) = withHeaderAsync(parse.text) { request =>
+ if (!Config.IS_WRITE_SERVER) Future.successful(UNAUTHORIZED)
+ // val kafkaTopic = toService(topic)
+ val strs = request.body.split("\n")
+ strs.foreach(str => {
+ val keyedMessage = new ProducerRecord[Key, Val](Config.KAFKA_LOG_TOPIC, str)
+ // val keyedMessage = new ProducerRecord[Key, Val](kafkaTopic, s"$str")
+ // logger.debug(s"$kafkaTopic, $str")
+ ExceptionHandler.enqueue(KafkaMessage(keyedMessage))
+ })
+ Future.successful(
+ Ok("publish success.\n").withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
+ )
+ // try {
+ //
+ // } catch {
+ // case e: Exception => Future.successful(BadRequest(e.getMessage))
+ // }
+ }
+
+ def publish(topic: String) = publishOnly(topic)
+
+ // def mutateBulk(topic: String) = Action.async(parse.text) { request =>
+ // EdgeController.mutateAndPublish(Config.KAFKA_LOG_TOPIC, Config.KAFKA_FAIL_TOPIC, request.body).map { result =>
+ // result.withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
+ // }
+ // }
+ def mutateBulk(topic: String) = withHeaderAsync(parse.text) { request =>
+ EdgeController.mutateAndPublish(request.body)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/QueryController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/QueryController.scala b/s2rest_play/app/controllers/QueryController.scala
new file mode 100644
index 0000000..480c142
--- /dev/null
+++ b/s2rest_play/app/controllers/QueryController.scala
@@ -0,0 +1,311 @@
+package controllers
+
+
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.types.{LabelWithDirection, VertexId}
+import com.kakao.s2graph.core.utils.logger
+import config.Config
+import play.api.libs.json.{JsArray, JsObject, JsValue, Json}
+import play.api.mvc.{Action, Controller, Result}
+
+import scala.concurrent._
+import scala.language.postfixOps
+import scala.util.Try
+
+object QueryController extends Controller {
+
+ import ApplicationController._
+ import play.api.libs.concurrent.Execution.Implicits.defaultContext
+
+ private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
+ private val requestParser = com.kakao.s2graph.rest.Global.s2parser
+
+ private def badQueryExceptionResults(ex: Exception) = Future.successful(BadRequest(Json.obj("message" -> ex.getMessage)).as(applicationJsonHeader))
+
+ private def errorResults = Future.successful(Ok(PostProcess.timeoutResults).as(applicationJsonHeader))
+
+ def getEdges() = withHeaderAsync(jsonParser) { request =>
+ getEdgesInner(request.body)
+ }
+
+ def getEdgesExcluded = withHeaderAsync(jsonParser) { request =>
+ getEdgesExcludedInner(request.body)
+ }
+
+ private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = {
+ val filterOutQueryResultsLs = q.filterOutQuery match {
+ case Some(filterOutQuery) => s2.getEdges(filterOutQuery)
+ case None => Future.successful(Seq.empty)
+ }
+
+ for {
+ queryResultsLs <- s2.getEdges(q)
+ filterOutResultsLs <- filterOutQueryResultsLs
+ } yield {
+ val json = post(queryResultsLs, filterOutResultsLs)
+ json
+ }
+ }
+
+ private def calcSize(js: JsValue): Int = js match {
+ case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0)
+ case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum
+ case _ => 0
+ }
+
+ private def getEdgesAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = {
+ if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
+ val fetch = eachQuery(post) _
+// logger.info(jsonQuery)
+
+ Try {
+ val future = jsonQuery match {
+ case JsArray(arr) => Future.traverse(arr.map(requestParser.toQuery(_)))(fetch).map(JsArray)
+ case obj@JsObject(_) => fetch(requestParser.toQuery(obj))
+ case _ => throw BadQueryException("Cannot support")
+ }
+
+ future map { json => jsonResponse(json, "result_size" -> calcSize(json).toString) }
+
+ } recover {
+ case e: BadQueryException =>
+ logger.error(s"$jsonQuery, $e", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"$jsonQuery, $e", e)
+ errorResults
+ } get
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ private def getEdgesExcludedAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = {
+
+ if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
+
+ Try {
+ val q = requestParser.toQuery(jsonQuery)
+ val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
+
+ val fetchFuture = s2.getEdges(q)
+ val excludeFuture = s2.getEdges(filterOutQuery)
+
+ for {
+ queryResultLs <- fetchFuture
+ exclude <- excludeFuture
+ } yield {
+ val json = post(queryResultLs, exclude)
+ jsonResponse(json, "result_size" -> calcSize(json).toString)
+ }
+ } recover {
+ case e: BadQueryException =>
+ logger.error(s"$jsonQuery, $e", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"$jsonQuery, $e", e)
+ errorResults
+ } get
+ }
+
+ def getEdgesInner(jsonQuery: JsValue) = {
+ getEdgesAsync(jsonQuery)(PostProcess.toSimpleVertexArrJson)
+ }
+
+ def getEdgesExcludedInner(jsValue: JsValue) = {
+ getEdgesExcludedAsync(jsValue)(PostProcess.toSimpleVertexArrJson)
+ }
+
+ def getEdgesWithGrouping() = withHeaderAsync(jsonParser) { request =>
+ getEdgesWithGroupingInner(request.body)
+ }
+
+ def getEdgesWithGroupingInner(jsonQuery: JsValue) = {
+ getEdgesAsync(jsonQuery)(PostProcess.summarizeWithListFormatted)
+ }
+
+ def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonParser) { request =>
+ getEdgesExcludedWithGroupingInner(request.body)
+ }
+
+ def getEdgesExcludedWithGroupingInner(jsonQuery: JsValue) = {
+ getEdgesExcludedAsync(jsonQuery)(PostProcess.summarizeWithListExcludeFormatted)
+ }
+
+ def getEdgesGroupedInner(jsonQuery: JsValue) = {
+ getEdgesAsync(jsonQuery)(PostProcess.summarizeWithList)
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def getEdgesGrouped() = withHeaderAsync(jsonParser) { request =>
+ getEdgesGroupedInner(request.body)
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def getEdgesGroupedExcluded() = withHeaderAsync(jsonParser) { request =>
+ getEdgesGroupedExcludedInner(request.body)
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def getEdgesGroupedExcludedInner(jsonQuery: JsValue): Future[Result] = {
+ if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
+
+ Try {
+ val q = requestParser.toQuery(jsonQuery)
+ val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
+
+ val fetchFuture = s2.getEdges(q)
+ val excludeFuture = s2.getEdges(filterOutQuery)
+
+ for {
+ queryResultLs <- fetchFuture
+ exclude <- excludeFuture
+ } yield {
+ val json = PostProcess.summarizeWithListExclude(queryResultLs, exclude)
+ jsonResponse(json, "result_size" -> calcSize(json).toString)
+ }
+ } recover {
+ case e: BadQueryException =>
+ logger.error(s"$jsonQuery, $e", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"$jsonQuery, $e", e)
+ errorResults
+ } get
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def getEdgesGroupedExcludedFormatted = withHeaderAsync(jsonParser) { request =>
+ getEdgesGroupedExcludedFormattedInner(request.body)
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def getEdgesGroupedExcludedFormattedInner(jsonQuery: JsValue): Future[Result] = {
+ if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
+
+ Try {
+ val q = requestParser.toQuery(jsonQuery)
+ val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
+
+ val fetchFuture = s2.getEdges(q)
+ val excludeFuture = s2.getEdges(filterOutQuery)
+
+ for {
+ queryResultLs <- fetchFuture
+ exclude <- excludeFuture
+ } yield {
+ val json = PostProcess.summarizeWithListExcludeFormatted(queryResultLs, exclude)
+ jsonResponse(json, "result_size" -> calcSize(json).toString)
+ }
+ } recover {
+ case e: BadQueryException =>
+ logger.error(s"$jsonQuery, $e", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"$jsonQuery, $e", e)
+ errorResults
+ } get
+ }
+
+ def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) = Action.async { request =>
+ if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized)
+ val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
+ checkEdgesInner(params)
+ }
+
+ /**
+ * Vertex
+ */
+
+ def checkEdgesInner(jsValue: JsValue) = {
+ try {
+ val params = jsValue.as[List[JsValue]]
+ var isReverted = false
+ val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]()
+ val quads = for {
+ param <- params
+ labelName <- (param \ "label").asOpt[String]
+ direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out"))
+ label <- Label.findByName(labelName)
+ srcId <- requestParser.jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ tgtId <- requestParser.jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ } yield {
+ val labelWithDir = LabelWithDirection(label.id.get, direction)
+ labelWithDirs += labelWithDir
+ val (src, tgt, dir) = if (direction == 1) {
+ isReverted = true
+ (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)),
+ Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0)
+ } else {
+ (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)),
+ Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0)
+ }
+
+ // logger.debug(s"SrcVertex: $src")
+ // logger.debug(s"TgtVertex: $tgt")
+ // logger.debug(s"direction: $dir")
+ (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir)))
+ }
+
+ s2.checkEdges(quads).map { case queryRequestWithResultLs =>
+ val edgeJsons = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ convertedEdge = if (isReverted) edge.duplicateEdge else edge
+ edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam)
+ } yield Json.toJson(edgeJson)
+
+ val json = Json.toJson(edgeJsons)
+ jsonResponse(json, "result_size" -> edgeJsons.size.toString)
+ }
+ } catch {
+ case e: Exception =>
+ logger.error(s"$jsValue, $e", e)
+ errorResults
+ }
+ }
+
+ def checkEdges() = withHeaderAsync(jsonParser) { request =>
+ if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized)
+
+ checkEdgesInner(request.body)
+ }
+
+ def getVertices() = withHeaderAsync(jsonParser) { request =>
+ getVerticesInner(request.body)
+ }
+
+ def getVerticesInner(jsValue: JsValue) = {
+ if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
+
+ val jsonQuery = jsValue
+ val ts = System.currentTimeMillis()
+ val props = "{}"
+
+ Try {
+ val vertices = jsonQuery.as[List[JsValue]].flatMap { js =>
+ val serviceName = (js \ "serviceName").as[String]
+ val columnName = (js \ "columnName").as[String]
+ for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield {
+ Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props)
+ }
+ }
+
+ s2.getVertices(vertices) map { vertices =>
+ val json = PostProcess.verticesToJson(vertices)
+ jsonResponse(json, "result_size" -> calcSize(json).toString)
+ }
+ } recover {
+ case e: play.api.libs.json.JsResultException =>
+ logger.error(s"$jsonQuery, $e", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"$jsonQuery, $e", e)
+ errorResults
+ } get
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/TestController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/TestController.scala b/s2rest_play/app/controllers/TestController.scala
new file mode 100644
index 0000000..8558ae5
--- /dev/null
+++ b/s2rest_play/app/controllers/TestController.scala
@@ -0,0 +1,24 @@
+package controllers
+
+import play.api.mvc.{Action, Controller}
+import util.TestDataLoader
+
+import scala.concurrent.Future
+
+
+object TestController extends Controller {
+ import ApplicationController._
+
+ def getRandomId() = withHeader(parse.anyContent) { request =>
+ val id = TestDataLoader.randomId
+ Ok(s"${id}")
+ }
+
+ def pingAsync() = Action.async(parse.json) { request =>
+ Future.successful(Ok("Pong\n"))
+ }
+
+ def ping() = Action(parse.json) { request =>
+ Ok("Pong\n")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/controllers/VertexController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/VertexController.scala b/s2rest_play/app/controllers/VertexController.scala
new file mode 100644
index 0000000..977c2fc
--- /dev/null
+++ b/s2rest_play/app/controllers/VertexController.scala
@@ -0,0 +1,86 @@
+package controllers
+
+
+import actors.QueueActor
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.utils.logger
+import com.kakao.s2graph.core.{ExceptionHandler, Graph, GraphExceptions}
+import config.Config
+import play.api.libs.json.{JsValue, Json}
+import play.api.mvc.{Controller, Result}
+
+import scala.concurrent.Future
+
+object VertexController extends Controller {
+ private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
+ private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
+
+ import ExceptionHandler._
+ import controllers.ApplicationController._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ def tryMutates(jsValue: JsValue, operation: String, serviceNameOpt: Option[String] = None, columnNameOpt: Option[String] = None, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+ else {
+ try {
+ val vertices = requestParser.toVertices(jsValue, operation, serviceNameOpt, columnNameOpt)
+
+ for (vertex <- vertices) {
+ if (vertex.isAsync)
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, vertex, None))
+ else
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, vertex, None))
+ }
+
+ //FIXME:
+ val verticesToStore = vertices.filterNot(v => v.isAsync)
+
+ if (withWait) {
+ val rets = s2.mutateVertices(verticesToStore, withWait = true)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = verticesToStore.map { vertex => QueueActor.router ! vertex; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"e"))
+ case e: Exception =>
+ logger.error(s"[Failed] tryMutates", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+ }
+
+ def inserts() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert")
+ }
+
+ def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", withWait = true)
+ }
+
+ def insertsSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", Some(serviceName), Some(columnName))
+ }
+
+ def deletes() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete")
+ }
+
+ def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", withWait = true)
+ }
+
+ def deletesSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", Some(serviceName), Some(columnName))
+ }
+
+ def deletesAll() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "deleteAll")
+ }
+
+ def deletesAllSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "deleteAll", Some(serviceName), Some(columnName))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/models/ExactCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/ExactCounterItem.scala b/s2rest_play/app/models/ExactCounterItem.scala
new file mode 100644
index 0000000..244c046
--- /dev/null
+++ b/s2rest_play/app/models/ExactCounterItem.scala
@@ -0,0 +1,38 @@
+package models
+
+import play.api.libs.json.{Json, Writes}
+
+/**
+ * Created by alec on 15. 4. 15..
+ */
+case class ExactCounterItem(ts: Long, count: Long, score: Double)
+
+case class ExactCounterIntervalItem(interval: String, dimension: Map[String, String], counter: Seq[ExactCounterItem])
+
+case class ExactCounterResultMeta(service: String, action: String, item: String)
+
+case class ExactCounterResult(meta: ExactCounterResultMeta, data: Seq[ExactCounterIntervalItem])
+
+object ExactCounterItem {
+ implicit val writes = new Writes[ExactCounterItem] {
+ def writes(item: ExactCounterItem) = Json.obj(
+ "ts" -> item.ts,
+ "time" -> tsFormat.format(item.ts),
+ "count" -> item.count,
+ "score" -> item.score
+ )
+ }
+ implicit val reads = Json.reads[ExactCounterItem]
+}
+
+object ExactCounterIntervalItem {
+ implicit val format = Json.format[ExactCounterIntervalItem]
+}
+
+object ExactCounterResultMeta {
+ implicit val format = Json.format[ExactCounterResultMeta]
+}
+
+object ExactCounterResult {
+ implicit val formats = Json.format[ExactCounterResult]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/models/RankCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/RankCounterItem.scala b/s2rest_play/app/models/RankCounterItem.scala
new file mode 100644
index 0000000..aaa7df7
--- /dev/null
+++ b/s2rest_play/app/models/RankCounterItem.scala
@@ -0,0 +1,40 @@
+package models
+
+import play.api.libs.json.{Json, Writes}
+
+/**
+ * Created by alec on 15. 4. 15..
+ */
+case class RankCounterItem(rank: Int, id: String, score: Double)
+
+case class RankCounterDimensionItem(interval: String, ts: Long, dimension: String, total: Double, ranks: Seq[RankCounterItem])
+
+case class RankCounterResultMeta(service: String, action: String)
+
+case class RankCounterResult(meta: RankCounterResultMeta, data: Seq[RankCounterDimensionItem])
+
+object RankCounterItem {
+ implicit val format = Json.format[RankCounterItem]
+}
+
+object RankCounterDimensionItem {
+ implicit val writes = new Writes[RankCounterDimensionItem] {
+ def writes(item: RankCounterDimensionItem) = Json.obj(
+ "interval" -> item.interval,
+ "ts" -> item.ts,
+ "time" -> tsFormat.format(item.ts),
+ "dimension" -> item.dimension,
+ "total" -> item.total,
+ "ranks" -> item.ranks
+ )
+ }
+ implicit val reads = Json.reads[RankCounterDimensionItem]
+}
+
+object RankCounterResultMeta {
+ implicit val format = Json.format[RankCounterResultMeta]
+}
+
+object RankCounterResult {
+ implicit val format = Json.format[RankCounterResult]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/models/package.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/package.scala b/s2rest_play/app/models/package.scala
new file mode 100644
index 0000000..17fa8e1
--- /dev/null
+++ b/s2rest_play/app/models/package.scala
@@ -0,0 +1,8 @@
+import java.text.SimpleDateFormat
+
+/**
+ * Created by alec on 15. 4. 20..
+ */
+package object models {
+ def tsFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/util/TestDataLoader.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/util/TestDataLoader.scala b/s2rest_play/app/util/TestDataLoader.scala
new file mode 100644
index 0000000..45a9b61
--- /dev/null
+++ b/s2rest_play/app/util/TestDataLoader.scala
@@ -0,0 +1,70 @@
+package util
+
+import java.io.File
+
+import scala.collection.mutable.{ArrayBuffer, HashMap, ListBuffer}
+import scala.io.Source
+import scala.util.Random
+
+
+object TestDataLoader {
+ val step = 100
+ val prob = 1.0
+ val (testIds, testIdsHist, testIdsHistCnt) = loadSeeds("./talk_vertices.txt")
+ val maxId = testIds.length
+ // val randoms = (0 until 100).map{ i => new SecureRandom }
+ // val idx = new AtomicInteger(0)
+ // def randomId() = {
+ // val r = randoms(idx.getAndIncrement() % randoms.size)
+ // testAccountIds(r.nextInt(maxId))
+ // }
+ def randomId(histStep: Int) = {
+ for {
+ maxId <- testIdsHistCnt.get(histStep)
+ rIdx = Random.nextInt(maxId.toInt)
+ hist <- testIdsHist.get(histStep)
+ id = hist(rIdx)
+ } yield {
+// logger.debug(s"randomId: $histStep = $id[$rIdx / $maxId]")
+ id
+ }
+ }
+ def randomId() = {
+ val id = testIds(Random.nextInt(maxId))
+ // logger.debug(s"$id")
+ id
+ }
+ private def loadSeeds(filePath: String) = {
+ val histogram = new HashMap[Long, ListBuffer[Long]]
+ val histogramCnt = new HashMap[Long, Long]
+ val ids = new ArrayBuffer[Long]
+
+ var idx = 0
+// logger.debug(s"$filePath start to load file.")
+ for (line <- Source.fromFile(new File(filePath)).getLines) {
+ // testAccountIds(idx) = line.toLong
+// if (idx % 10000 == 0) logger.debug(s"$idx")
+ idx += 1
+
+ val parts = line.split("\\t")
+ val id = parts.head.toLong
+ val count = parts.last.toLong / step
+ if (count > 1 && Random.nextDouble < prob) {
+ histogram.get(count) match {
+ case None =>
+ histogram.put(count, new ListBuffer[Long])
+ histogram.get(count).get += id
+ histogramCnt.put(count, 1)
+ case Some(existed) =>
+ existed += id
+ histogramCnt.put(count, histogramCnt.getOrElse(count, 0L) + 1L)
+ }
+ ids += id
+ }
+
+ }
+// logger.debug(s"upload $filePath finished.")
+// logger.debug(s"${histogram.size}")
+ (ids, histogram.map(t => (t._1 -> t._2.toArray[Long])), histogramCnt)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
new file mode 100755
index 0000000..6e0f0bd
--- /dev/null
+++ b/s2rest_play/build.sbt
@@ -0,0 +1,12 @@
+name := "s2rest_play"
+
+scalacOptions in Test ++= Seq("-Yrangepos")
+
+libraryDependencies ++= Seq(
+ ws,
+ filters,
+ "com.github.danielwegener" % "logback-kafka-appender" % "0.0.3"
+)
+
+enablePlugins(JavaServerAppPackaging)
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/conf/application.conf
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/application.conf b/s2rest_play/conf/application.conf
new file mode 100644
index 0000000..e69de29
[4/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
new file mode 100644
index 0000000..5b0dfbd
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
@@ -0,0 +1,528 @@
+package com.kakao.s2graph.core.Integrate
+
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json._
+
+class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import TestUtil._
+
+ val insert = "insert"
+ val e = "e"
+ val weight = "weight"
+ val is_hidden = "is_hidden"
+
+ test("interval") {
+ def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "index": "$index",
+ "interval": {
+ "from": [ { "$prop": $fromVal } ],
+ "to": [ { "$prop": $toVal } ]
+ }
+ }
+ ]]
+ }
+ """)
+
+ var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
+ (edges \ "size").toString should be("2")
+
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
+ (edges \ "size").toString should be("2")
+ }
+
+ test("get edge with where condition") {
+ def queryWhere(id: Int, where: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "where": "${where}"
+ }
+ ]]
+ }""")
+
+ var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdgesSync(queryWhere(0, "_from=0"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("get edge exclude") {
+ def queryExclude(id: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ },
+ {
+ "label": "${testLabelName}",
+ "direction": "in",
+ "offset": 0,
+ "limit": 2,
+ "exclude": true
+ }
+ ]]
+ }""")
+
+ val result = getEdgesSync(queryExclude(0))
+ (result \ "results").as[List[JsValue]].size should be(1)
+ }
+
+ test("get edge groupBy property") {
+ def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
+ Json.obj(
+ "groupBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+ }
+
+ val result = getEdgesSync(queryGroupBy(0, Seq("weight")))
+ (result \ "size").as[Int] should be(2)
+ val weights = (result \\ "groupBy").map { js =>
+ (js \ "weight").as[Int]
+ }
+ weights should contain(30)
+ weights should contain(40)
+
+ weights should not contain (10)
+ }
+
+ test("edge transform") {
+ def queryTransform(id: Int, transforms: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "transform": $transforms
+ }
+ ]]
+ }""")
+
+ var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryTransform(0, "[[\"weight\"]]"))
+ (result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted)
+
+ result = getEdgesSync(queryTransform(0, "[[\"_from\"]]"))
+ val results = (result \ "results").as[JsValue]
+ (result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted)
+ }
+
+ test("index") {
+ def queryIndex(ids: Seq[Int], indexName: String) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName))
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps)
+ js
+ }
+
+ // weight order
+ var result = getEdgesSync(queryIndex(Seq(0), "idx_1"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40))
+
+ // timestamp order
+ result = getEdgesSync(queryIndex(Seq(0), "idx_2"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30))
+ }
+
+ // "checkEdges" in {
+ // running(FakeApplication()) {
+ // val json = Json.parse( s"""
+ // [{"from": 0, "to": 1, "label": "$testLabelName"},
+ // {"from": 0, "to": 2, "label": "$testLabelName"}]
+ // """)
+ //
+ // def checkEdges(queryJson: JsValue): JsValue = {
+ // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
+ // contentAsJson(ret)
+ // }
+ //
+ // val res = checkEdges(json)
+ // val typeRes = res.isInstanceOf[JsArray]
+ // typeRes must equalTo(true)
+ //
+ // val fst = res.as[Seq[JsValue]].head \ "to"
+ // fst.as[Int] must equalTo(1)
+ //
+ // val snd = res.as[Seq[JsValue]].last \ "to"
+ // snd.as[Int] must equalTo(2)
+ // }
+ // }
+
+ test("return tree") {
+ def queryParents(id: Long) = Json.parse(
+ s"""
+ {
+ "returnTree": true,
+ "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ }
+ ],[{
+ "label": "$testLabelName",
+ "direction": "in",
+ "offset": 0,
+ "limit": -1
+ }
+ ]]
+ }""".stripMargin)
+
+ val src = 100
+ val tgt = 200
+
+ insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
+
+ val result = TestUtil.getEdgesSync(queryParents(src))
+ val parents = (result \ "results").as[Seq[JsValue]]
+ val ret = parents.forall {
+ edge => (edge \ "parents").as[Seq[JsValue]].size == 1
+ }
+
+ ret should be(true)
+ }
+
+
+
+ test("pagination and _to") {
+ def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": $offset,
+ "limit": $limit,
+ "_to": $to
+ }
+ ]]
+ }
+ """)
+
+ val src = System.currentTimeMillis().toInt
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
+ )
+ insertEdgesSync(bulkEdges: _*)
+
+ var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
+ var edges = (result \ "results").as[List[JsValue]]
+
+ edges.size should be(2)
+ (edges(0) \ "to").as[Long] should be(4)
+ (edges(1) \ "to").as[Long] should be(3)
+
+ result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
+
+ edges = (result \ "results").as[List[JsValue]]
+ edges.size should be(2)
+ (edges(0) \ "to").as[Long] should be(3)
+ (edges(1) \ "to").as[Long] should be(2)
+
+ result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+ edges = (result \ "results").as[List[JsValue]]
+ edges.size should be(1)
+ }
+ test("order by") {
+ def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj(
+ "srcVertices" -> Json.arr(
+ Json.obj(
+ "serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "id" -> id
+ )
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+ def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj(
+ "orderBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ // get edges
+ val edges = getEdgesSync(queryScore(0, Map("weight" -> 1)))
+ val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
+ val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
+
+ val edgesTo = edges \ "results" \\ "to"
+ val orderByTo = orderByScore \ "results" \\ "to"
+ val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
+
+ edgesTo should be(Seq(JsNumber(2), JsNumber(1)))
+ edgesTo should be(orderByTo)
+ ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2)))
+ edgesTo.reverse should be(ascOrderByTo)
+ }
+
+ test("query with sampling") {
+ def queryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ },
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50,
+ "sample": $sample
+ },
+ {
+ "label": "$testLabelName2",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50
+ }]
+ }
+ ]
+ }""")
+
+ val sampleSize = 2
+ val ts = "1442985659166"
+ val testId = 22
+
+ val bulkEdges = Seq(
+ toEdge(ts, insert, e, testId, 122, testLabelName),
+ toEdge(ts, insert, e, testId, 222, testLabelName),
+ toEdge(ts, insert, e, testId, 322, testLabelName),
+
+ toEdge(ts, insert, e, testId, 922, testLabelName2),
+ toEdge(ts, insert, e, testId, 222, testLabelName2),
+ toEdge(ts, insert, e, testId, 322, testLabelName2),
+
+ toEdge(ts, insert, e, 122, 1122, testLabelName),
+ toEdge(ts, insert, e, 122, 1222, testLabelName),
+ toEdge(ts, insert, e, 122, 1322, testLabelName),
+ toEdge(ts, insert, e, 222, 2122, testLabelName),
+ toEdge(ts, insert, e, 222, 2222, testLabelName),
+ toEdge(ts, insert, e, 222, 2322, testLabelName),
+ toEdge(ts, insert, e, 322, 3122, testLabelName),
+ toEdge(ts, insert, e, 322, 3222, testLabelName),
+ toEdge(ts, insert, e, 322, 3322, testLabelName)
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val result1 = getEdgesSync(queryWithSampling(testId, sampleSize))
+ (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
+
+ val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
+ (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
+
+ val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
+ (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
+ }
+
+ def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": $offset,
+ "limit": $limit
+ }
+ ]]
+ }
+ """)
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+ override def initTestData(): Unit = {
+ super.initTestData()
+
+ insertEdgesSync(
+ toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
+ toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
+ toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, -1, 1000, testLabelName),
+ toEdge(1, insert, e, -1, 2000, testLabelName),
+ toEdge(1, insert, e, -1, 3000, testLabelName),
+ toEdge(1, insert, e, 1000, 10000, testLabelName),
+ toEdge(1, insert, e, 1000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 13000, testLabelName),
+ toEdge(1, insert, e, 10000, 100000, testLabelName),
+ toEdge(2, insert, e, 11000, 200000, testLabelName),
+ toEdge(3, insert, e, 12000, 300000, testLabelName)
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
new file mode 100644
index 0000000..69a49b4
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
@@ -0,0 +1,282 @@
+package com.kakao.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, Future}
+import scala.util.Random
+
+class StrongLabelDeleteTest extends IntegrateCommon {
+
+ import StrongDeleteUtil._
+ import TestUtil._
+
+ test("Strong consistency select") {
+ insertEdgesSync(bulkEdges(): _*)
+
+ var result = getEdgesSync(query(0))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ result = getEdgesSync(query(10))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("Strong consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val deleteParam = Json.arr(
+ Json.obj("label" -> testLabelName2,
+ "direction" -> "in",
+ "ids" -> Json.arr("20"),
+ "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteParam)
+
+ result = getEdgesSync(query(11, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, direction = "out"))
+ println(result)
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ test("update delete") {
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val src = System.currentTimeMillis()
+
+ val (ret, last) = testInner(i, src)
+ ret should be(true)
+ ret
+ }
+
+ ret.forall(identity)
+ }
+
+ test("update delete 2") {
+ val src = System.currentTimeMillis()
+ var ts = 0L
+
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val (ret, lastTs) = testInner(ts, src)
+ val deletedAt = lastTs + 1
+ val deletedAt2 = lastTs + 2
+ ts = deletedAt2 + 1 // nex start ts
+
+ ret should be(true)
+
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+ val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
+
+ val deleteRet = deleteAllSync(deleteAllRequest)
+ val deleteRet2 = deleteAllSync(deleteAllRequest2)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+
+ degreeAfterDeleteAll should be(0)
+ degreeAfterDeleteAll === (0)
+ }
+
+ ret.forall(identity)
+ }
+
+ /** This test stress out test on degree
+ * when contention is low but number of adjacent edges are large
+ * Large set of contention test
+ */
+ test("large degrees") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ // val allRequests = insertRequests ++ deleteRequests
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = insertRequests.size - deleteRequests.size
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ // println(result)
+
+ val ret = resultSize == expectedDegree && resultDegree == resultSize
+ println(s"[MaxSize]: $maxSize")
+ println(s"[DeleteSize]: $deleteSize")
+ println(s"[ResultDegree]: $resultDegree")
+ println(s"[ExpectedDegree]: $expectedDegree")
+ println(s"[ResultSize]: $resultSize")
+ ret should be(true)
+ }
+
+ test("deleteAll") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val deletedAt = System.currentTimeMillis()
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteAllRequest)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+ degreeAfterDeleteAll should be(0)
+ }
+
+ object StrongDeleteUtil {
+
+ val labelName = testLabelName2
+ val maxTgtId = 10
+ val batchSize = 10
+ val testNum = 3
+ val numOfBatch = 10
+
+ def testInner(startTs: Long, src: Long) = {
+ val labelName = testLabelName2
+ val lastOps = Array.fill(maxTgtId)("none")
+ var currentTs = startTs
+
+ val allRequests = for {
+ ith <- 0 until numOfBatch
+ jth <- 0 until batchSize
+ } yield {
+ currentTs += 1
+
+ val tgt = Random.nextInt(maxTgtId)
+ val op = if (Random.nextDouble() < 0.5) "delete" else "update"
+
+ lastOps(tgt) = op
+ Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
+ }
+
+ allRequests.foreach(println(_))
+
+ val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ println(lastOps.toList)
+ println(result)
+
+ val ret = resultDegree == expectedDegree && resultSize == resultDegree
+ if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
+
+ (ret, currentTs)
+ }
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""")
+ )
+
+ def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
+ labelName: String = testLabelName2, direction: String = "out") = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$labelName",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": -1,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+
+ def getDegree(jsValue: JsValue): Long = {
+ ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
new file mode 100644
index 0000000..ffbec11
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
@@ -0,0 +1,71 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core.PostProcess
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.Await
+import scala.util.Random
+
+
+class VertexTestHelper extends IntegrateCommon {
+
+ import TestUtil._
+ import VertexTestHelper._
+
+ test("vertex") {
+ val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
+ val (serviceName, columnName) = (testServiceName, testColumnName)
+
+ val data = vertexInsertsPayload(serviceName, columnName, ids)
+ val payload = Json.parse(Json.toJson(data).toString)
+ println(payload)
+
+ val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName))
+ Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime)
+
+ val res = graph.getVertices(vertices).map { vertices =>
+ PostProcess.verticesToJson(vertices)
+ }
+
+ val ret = Await.result(res, HttpRequestWaitingTime)
+ val fetched = ret.as[Seq[JsValue]]
+ for {
+ (d, f) <- data.zip(fetched)
+ } yield {
+ (d \ "id") should be(f \ "id")
+ ((d \ "props") \ "age") should be((f \ "props") \ "age")
+ }
+ }
+
+ object VertexTestHelper {
+ def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
+ Json.parse(
+ s"""
+ |[
+ |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
+ ]}
+ |]
+ """.stripMargin)
+ }
+
+ def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
+ ids.map { id =>
+ Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
+ }
+ }
+
+ val vertexPropsKeys = List(
+ ("age", "int")
+ )
+
+ def randomProps() = {
+ (for {
+ (propKey, propType) <- vertexPropsKeys
+ } yield {
+ propKey -> Random.nextInt(100)
+ }).toMap
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
new file mode 100644
index 0000000..b80d9c7
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
@@ -0,0 +1,129 @@
+package com.kakao.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json.{JsObject, JsValue, Json}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import TestUtil._
+ import WeakLabelDeleteHelper._
+
+ test("test weak consistency select") {
+ var result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(4)
+ result = getEdgesSync(query(10))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("test weak consistency delete") {
+ var result = getEdgesSync(query(0))
+ println(result)
+
+ /** expect 4 edges */
+ (result \ "results").as[List[JsValue]].size should be(4)
+ val edges = (result \ "results").as[List[JsObject]]
+ val edgesToStore = parser.toEdges(Json.toJson(edges), "delete")
+ val rets = graph.mutateEdges(edgesToStore, withWait = true)
+ Await.result(rets, Duration(20, TimeUnit.MINUTES))
+
+ /** expect noting */
+ result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ /** insert should be ignored */
+ val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert")
+ val rets2 = graph.mutateEdges(edgesToStore2, withWait = true)
+ Await.result(rets2, Duration(20, TimeUnit.MINUTES))
+
+ result = getEdgesSync(query(0))
+ (result \ "results").as[List[JsValue]].size should be(0)
+ }
+
+
+ test("test weak consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
+ "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
+ println(json)
+ deleteAllSync(json)
+
+ result = getEdgesSync(query(11, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, "out"))
+
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+
+ override def initTestData(): Unit = {
+ super.initTestData()
+ insertEdgesSync(bulkEdges(): _*)
+ }
+
+ object WeakLabelDeleteHelper {
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""")
+ )
+
+ def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$columnName",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelNameWeak}",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
index 84349cb..127f2fe 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
@@ -3,9 +3,6 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.types.{InnerValLike, InnerVal}
import org.scalatest.{Matchers, FunSuite}
-/**
- * Created by shon on 5/30/15.
- */
class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser {
import types.HBaseType._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
index f6f1520..61818fc 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
@@ -1,12 +1,9 @@
package com.kakao.s2graph.core
-import com.kakao.s2graph.core.OrderingUtil.MultiValueOrdering
+import com.kakao.s2graph.core.OrderingUtil._
import org.scalatest.{FunSuite, Matchers}
import play.api.libs.json.JsString
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 5..
- */
class OrderingUtilTest extends FunSuite with Matchers {
test("test SeqMultiOrdering") {
val jsLs: Seq[Seq[Any]] = Seq(
@@ -50,7 +47,7 @@ class OrderingUtilTest extends FunSuite with Matchers {
)
val ascendingLs: Seq[Boolean] = Seq(false)
- val resultJsLs = jsLs.sorted(new TupleMultiOrdering[Any](ascendingLs))
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
resultJsLs.toString() should equal(sortedJsLs.toString())
}
@@ -74,7 +71,7 @@ class OrderingUtilTest extends FunSuite with Matchers {
)
val ascendingLs: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new TupleMultiOrdering[Any](ascendingLs))
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
resultJsLs.toString() should equal(sortedJsLs.toString())
}
@@ -99,7 +96,7 @@ class OrderingUtilTest extends FunSuite with Matchers {
)
val ascendingLs: Seq[Boolean] = Seq(true, true, false)
- val resultJsLs = jsLs.sorted(new TupleMultiOrdering[Any](ascendingLs))
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
resultJsLs.toString() should equal(sortedJsLs.toString())
}
@@ -126,7 +123,7 @@ class OrderingUtilTest extends FunSuite with Matchers {
)
val ascendingLs: Seq[Boolean] = Seq(true, true, true, false)
- val resultJsLs = jsLs.sorted(new TupleMultiOrdering[Any](ascendingLs))
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
resultJsLs.toString() should equal(sortedJsLs.toString())
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
index fae3d06..ed9aaa5 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
@@ -1,6 +1,6 @@
package com.kakao.s2graph.core
- import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.mysqls._
//import com.kakao.s2graph.core.models._
@@ -9,13 +9,7 @@ import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.{PutRequest, KeyValue}
-/**
- * Created by shon on 6/1/15.
- */
trait TestCommon {
-
-
-
val ts = System.currentTimeMillis()
val testServiceId = 1
val testColumnId = 1
@@ -104,95 +98,95 @@ trait TestCommon {
val idxPropsWithTsLsV2 = idxPropsLsV2.map { idxProps =>
idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }
}
-//
-// def testOrder(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]],
-// innerVals: Iterable[InnerValLike], skipHashBytes: Boolean = false)
-// (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
-// fromBytesFunc: Array[Byte] => HBaseSerializable) = {
-// /** check if increasing target vertex id is ordered properly with same indexProps */
-// val rets = for {
-// idxProps <- idxPropsLs
-// } yield {
-// val head = createFunc(idxProps, innerVals.head)
-// val start = head
-// var prev = head
-// val rets = for {
-// innerVal <- innerVals.tail
-// } yield {
-// val current = createFunc(idxProps, innerVal)
-// val bytes = current.bytes
-// val decoded = fromBytesFunc(bytes)
-// println(s"current: $current")
-// println(s"decoded: $decoded")
-//
-// val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
-// val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
-// val (isSame, orderPreserved) = (current, decoded) match {
-// case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
-// /** _to is used in indexProps */
-// (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
-// case _ =>
-// (current == decoded, lessThan(currentBytes, prevBytes))
-// }
-//
-// println(s"$current ${bytes.toList}")
-// println(s"$prev ${prev.bytes.toList}")
-// println(s"SerDe[$isSame], Order[$orderPreserved]")
-// prev = current
-// isSame && orderPreserved
-// }
-// rets.forall(x => x)
-// }
-// rets.forall(x => x)
-// }
-// def testOrderReverse(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]], innerVals: Iterable[InnerValLike],
-// skipHashBytes: Boolean = false)
-// (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
-// fromBytesFunc: Array[Byte] => HBaseSerializable) = {
-// /** check if increasing target vertex id is ordered properly with same indexProps */
-// val rets = for {
-// innerVal <- innerVals
-// } yield {
-// val head = createFunc(idxPropsLs.head, innerVal)
-// val start = head
-// var prev = head
-// val rets = for {
-// idxProps <- idxPropsLs.tail
-// } yield {
-// val current = createFunc(idxProps, innerVal)
-// val bytes = current.bytes
-// val decoded = fromBytesFunc(bytes)
-// println(s"current: $current")
-// println(s"decoded: $decoded")
-//
-// val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
-// val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
-// val (isSame, orderPreserved) = (current, decoded) match {
-// case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
-// /** _to is used in indexProps */
-// (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
-// case _ =>
-// (current == decoded, lessThan(currentBytes, prevBytes))
-// }
-//
-// println(s"$current ${bytes.toList}")
-// println(s"$prev ${prev.bytes.toList}")
-// println(s"SerDe[$isSame], Order[$orderPreserved]")
-// prev = current
-// isSame && orderPreserved
-// }
-//
-// rets.forall(x => x)
-// }
-//
-// rets.forall(x => x)
-// }
-//
-//
-// def putToKeyValues(put: PutRequest) = {
-// val ts = put.timestamp()
-// for ((q, v) <- put.qualifiers().zip(put.values)) yield {
-// new KeyValue(put.key(), put.family(), q, ts, v)
-// }
-// }
+ //
+ // def testOrder(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]],
+ // innerVals: Iterable[InnerValLike], skipHashBytes: Boolean = false)
+ // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
+ // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
+ // /** check if increasing target vertex id is ordered properly with same indexProps */
+ // val rets = for {
+ // idxProps <- idxPropsLs
+ // } yield {
+ // val head = createFunc(idxProps, innerVals.head)
+ // val start = head
+ // var prev = head
+ // val rets = for {
+ // innerVal <- innerVals.tail
+ // } yield {
+ // val current = createFunc(idxProps, innerVal)
+ // val bytes = current.bytes
+ // val decoded = fromBytesFunc(bytes)
+ // println(s"current: $current")
+ // println(s"decoded: $decoded")
+ //
+ // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
+ // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
+ // val (isSame, orderPreserved) = (current, decoded) match {
+ // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
+ // /** _to is used in indexProps */
+ // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
+ // case _ =>
+ // (current == decoded, lessThan(currentBytes, prevBytes))
+ // }
+ //
+ // println(s"$current ${bytes.toList}")
+ // println(s"$prev ${prev.bytes.toList}")
+ // println(s"SerDe[$isSame], Order[$orderPreserved]")
+ // prev = current
+ // isSame && orderPreserved
+ // }
+ // rets.forall(x => x)
+ // }
+ // rets.forall(x => x)
+ // }
+ // def testOrderReverse(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]], innerVals: Iterable[InnerValLike],
+ // skipHashBytes: Boolean = false)
+ // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
+ // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
+ // /** check if increasing target vertex id is ordered properly with same indexProps */
+ // val rets = for {
+ // innerVal <- innerVals
+ // } yield {
+ // val head = createFunc(idxPropsLs.head, innerVal)
+ // val start = head
+ // var prev = head
+ // val rets = for {
+ // idxProps <- idxPropsLs.tail
+ // } yield {
+ // val current = createFunc(idxProps, innerVal)
+ // val bytes = current.bytes
+ // val decoded = fromBytesFunc(bytes)
+ // println(s"current: $current")
+ // println(s"decoded: $decoded")
+ //
+ // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
+ // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
+ // val (isSame, orderPreserved) = (current, decoded) match {
+ // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
+ // /** _to is used in indexProps */
+ // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
+ // case _ =>
+ // (current == decoded, lessThan(currentBytes, prevBytes))
+ // }
+ //
+ // println(s"$current ${bytes.toList}")
+ // println(s"$prev ${prev.bytes.toList}")
+ // println(s"SerDe[$isSame], Order[$orderPreserved]")
+ // prev = current
+ // isSame && orderPreserved
+ // }
+ //
+ // rets.forall(x => x)
+ // }
+ //
+ // rets.forall(x => x)
+ // }
+ //
+ //
+ // def putToKeyValues(put: PutRequest) = {
+ // val ts = put.timestamp()
+ // for ((q, v) <- put.qualifiers().zip(put.values)) yield {
+ // new KeyValue(put.key(), put.family(), q, ts, v)
+ // }
+ // }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
index fca7dc0..e2a3363 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
@@ -2,12 +2,12 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
import com.kakao.s2graph.core.mysqls._
+import scalikejdbc.AutoSession
//import com.kakao.s2graph.core.models._
-
import com.kakao.s2graph.core.types.{InnerVal, LabelWithDirection}
-import com.typesafe.config.ConfigFactory
+import com.typesafe.config.{Config, ConfigFactory}
import scala.concurrent.ExecutionContext
@@ -16,9 +16,28 @@ trait TestCommonWithModels {
import InnerVal._
import types.HBaseType._
- val config = ConfigFactory.load()
+ var graph: Graph = _
+ var config: Config = _
+
+ def initTests() = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+
+ implicit val session = AutoSession
+
+ deleteTestLabel()
+ deleteTestService()
+
+ createTestService()
+ createTestLabel()
+ }
+
+ def zkQuorum = config.getString("hbase.zookeeper.quorum")
+
+ def cluster = config.getString("hbase.zookeeper.quorum")
+
+ implicit val session = AutoSession
- val zkQuorum = config.getString("hbase.zookeeper.quorum")
val serviceName = "_test_service"
val serviceNameV2 = "_test_service_v2"
val columnName = "user_id"
@@ -31,7 +50,6 @@ trait TestCommonWithModels {
val tgtColumnType = "string"
val tgtColumnTypeV2 = "string"
- val cluster = config.getString("hbase.zookeeper.quorum")
val hTableName = "_test_cases"
val preSplitSize = 0
val labelName = "_test_label"
@@ -54,37 +72,29 @@ trait TestCommonWithModels {
val consistencyLevel = "strong"
val hTableTTL = None
- val graph = new Graph(config)(ExecutionContext.Implicits.global)
-
- def initTests() = {
- deleteTestLabel()
- deleteTestService()
-
- Thread.sleep(1000)
-
- createTestService()
- createTestLabel()
- }
def createTestService() = {
+ implicit val session = AutoSession
Management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
Management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
}
def deleteTestService() = {
+ implicit val session = AutoSession
Management.deleteService(serviceName)
Management.deleteService(serviceNameV2)
}
def deleteTestLabel() = {
+ implicit val session = AutoSession
Management.deleteLabel(labelName)
Management.deleteLabel(labelNameV2)
Management.deleteLabel(undirectedLabelName)
Management.deleteLabel(undirectedLabelNameV2)
}
-
def createTestLabel() = {
+ implicit val session = AutoSession
Management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4")
@@ -98,31 +108,37 @@ trait TestCommonWithModels {
isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
}
- /** */
- initTests()
+ def service = Service.findByName(serviceName, useCache = false).get
+
+ def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
+
+ def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
+
+ def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
+
+ def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
+
+ def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
+
+ def label = Label.findByName(labelName, useCache = false).get
+
+ def labelV2 = Label.findByName(labelNameV2, useCache = false).get
+
+ def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
- lazy val service = Service.findByName(serviceName, useCache = false).get
- lazy val serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
+ def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
- lazy val column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
- lazy val columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
+ def dir = GraphUtil.directions("out")
- lazy val tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
- lazy val tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
+ def op = GraphUtil.operations("insert")
- lazy val label = Label.findByName(labelName, useCache = false).get
- lazy val labelV2 = Label.findByName(labelNameV2, useCache = false).get
+ def labelOrderSeq = LabelIndex.DefaultSeq
- lazy val undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
- lazy val undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
+ def labelWithDir = LabelWithDirection(label.id.get, dir)
- lazy val dir = GraphUtil.directions("out")
- lazy val op = GraphUtil.operations("insert")
- lazy val labelOrderSeq = LabelIndex.DefaultSeq
+ def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
- lazy val labelWithDir = LabelWithDirection(label.id.get, dir)
- lazy val labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
+ def queryParam = QueryParam(labelWithDir)
- lazy val queryParam = QueryParam(labelWithDir)
- lazy val queryParamV2 = QueryParam(labelWithDirV2)
+ def queryParamV2 = QueryParam(labelWithDirV2)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
index 478da12..f66eabf 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
@@ -5,43 +5,47 @@ import java.util.concurrent.ExecutorService
import com.kakao.s2graph.core.mysqls.{Label, Model}
import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon, Graph}
import com.typesafe.config.ConfigFactory
-import org.scalatest.{FunSuite, Matchers}
+import org.scalatest.{BeforeAndAfterAll, Sequential, FunSuite, Matchers}
import scala.concurrent.ExecutionContext
-/**
- * Created by shon on 5/12/15.
- */
-class ModelTest extends FunSuite with Matchers with TestCommonWithModels {
+class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll {
+ override def beforeAll(): Unit = {
+ initTests()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
-// val serviceName = "testService"
-// val newServiceName = "newTestService"
-// val cluster = "localhost"
-// val hbaseTableName = "s2graph-dev"
-// val columnName = "user_id"
-// val columnType = "long"
-// val labelName = "model_test_label"
-// val newLabelName = "new_model_test_label"
-// val columnMetaName = "is_valid_user"
-// val labelMetaName = "is_hidden"
-// val hbaseTableTTL = -1
-// val id = 1
-//
-// val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster,
-// "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1))
-// val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get,
-// "columnName" -> columnName, "columnType" -> columnType))
-// val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte))
-// val label = HLabel(Map("id" -> id, "label" -> labelName,
-// "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType,
-// "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType,
-// "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get,
-// "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1
-// ))
-// val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte,
-// "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false))
-// val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte,
-// "metaSeqs" -> "0", "formular" -> "none"))
+ // val serviceName = "testService"
+ // val newServiceName = "newTestService"
+ // val cluster = "localhost"
+ // val hbaseTableName = "s2graph-dev"
+ // val columnName = "user_id"
+ // val columnType = "long"
+ // val labelName = "model_test_label"
+ // val newLabelName = "new_model_test_label"
+ // val columnMetaName = "is_valid_user"
+ // val labelMetaName = "is_hidden"
+ // val hbaseTableTTL = -1
+ // val id = 1
+ //
+ // val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster,
+ // "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1))
+ // val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get,
+ // "columnName" -> columnName, "columnType" -> columnType))
+ // val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte))
+ // val label = HLabel(Map("id" -> id, "label" -> labelName,
+ // "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType,
+ // "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType,
+ // "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get,
+ // "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1
+ // ))
+ // val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte,
+ // "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false))
+ // val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte,
+ // "metaSeqs" -> "0", "formular" -> "none"))
test("test Label.findByName") {
val labelOpt = Label.findByName(labelName, useCache = false)
println(labelOpt)
@@ -66,66 +70,66 @@ class ModelTest extends FunSuite with Matchers with TestCommonWithModels {
val tgtColumn = labelOpt.get.tgtService
println(tgtColumn)
}
-// test("test create") {
-// service.create()
-// HService.findByName(serviceName, useCache = false) == Some(service)
-//
-// serviceColumn.create()
-// HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn)
-//
-// columnMeta.create()
-// HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta)
-//
-// label.create()
-// HLabel.findByName(labelName, useCache = false) == Some(label)
-//
-// labelMeta.create()
-// HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta)
-//
-// labelIndex.create()
-// HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex)
-// }
-//
-// test("test update") {
-// service.update("cluster", "...")
-// HService.findById(service.id.get, useCache = false).cluster == "..."
-//
-// service.update("serviceName", newServiceName)
-// assert(HService.findByName(serviceName, useCache = false) == None)
-// HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get}
-//
-// label.update("label", newLabelName)
-// HLabel.findById(label.id.get, useCache = false).label == "newLabelName"
-//
-// label.update("consistencyLevel", "strong")
-// HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" &&
-// HLabel.findByName(newLabelName).isDefined &&
-// HLabel.findByName(labelName) == None
-//
-// }
-// test("test read by index") {
-// val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false)
-// val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false)
-// labels.length == 1 &&
-// labels.head == label
-// idxs.length == 1 &&
-// idxs.head == labelIndex
-// }
-// test("test delete") {
-//// HLabel.findByName(labelName).foreach { label =>
-//// label.deleteAll()
-//// }
-// HLabel.findByName(newLabelName).foreach { label =>
-// label.deleteAll()
-// }
-// HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty &&
-// HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty
-//
-// service.deleteAll()
-// }
+ // test("test create") {
+ // service.create()
+ // HService.findByName(serviceName, useCache = false) == Some(service)
+ //
+ // serviceColumn.create()
+ // HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn)
+ //
+ // columnMeta.create()
+ // HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta)
+ //
+ // label.create()
+ // HLabel.findByName(labelName, useCache = false) == Some(label)
+ //
+ // labelMeta.create()
+ // HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta)
+ //
+ // labelIndex.create()
+ // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex)
+ // }
+ //
+ // test("test update") {
+ // service.update("cluster", "...")
+ // HService.findById(service.id.get, useCache = false).cluster == "..."
+ //
+ // service.update("serviceName", newServiceName)
+ // assert(HService.findByName(serviceName, useCache = false) == None)
+ // HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get}
+ //
+ // label.update("label", newLabelName)
+ // HLabel.findById(label.id.get, useCache = false).label == "newLabelName"
+ //
+ // label.update("consistencyLevel", "strong")
+ // HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" &&
+ // HLabel.findByName(newLabelName).isDefined &&
+ // HLabel.findByName(labelName) == None
+ //
+ // }
+ // test("test read by index") {
+ // val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false)
+ // val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false)
+ // labels.length == 1 &&
+ // labels.head == label
+ // idxs.length == 1 &&
+ // idxs.head == labelIndex
+ // }
+ // test("test delete") {
+ //// HLabel.findByName(labelName).foreach { label =>
+ //// label.deleteAll()
+ //// }
+ // HLabel.findByName(newLabelName).foreach { label =>
+ // label.deleteAll()
+ // }
+ // HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty &&
+ // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty
+ //
+ // service.deleteAll()
+ // }
-// test("test labelIndex") {
-// println(HLabelIndex.findByLabelIdAll(1))
-// }
+ // test("test labelIndex") {
+ // println(HLabelIndex.findByLabelIdAll(1))
+ // }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
index 2bc764d..078b5a9 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
@@ -6,9 +6,6 @@ import com.typesafe.config.ConfigFactory
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
import scalikejdbc._
-/**
- * Created by hsleep on 2015. 11. 30..
- */
class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
val Ttl = 2
override def beforeAll(): Unit = {
@@ -19,37 +16,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
val props = new Properties()
props.setProperty("cache.ttl.seconds", Ttl.toString)
Model.apply(ConfigFactory.load(ConfigFactory.parseProperties(props)))
- /*
-CREATE TABLE `experiments` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `service_id` integer NOT NULL,
- `service_name` varchar(128) NOT NULL,
- `name` varchar(64) NOT NULL,
- `description` varchar(255) NOT NULL,
- `experiment_type` varchar(8) NOT NULL DEFAULT 'u',
- `total_modular` int NOT NULL DEFAULT 100,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_service_id_name` (`service_id`, `name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-CREATE TABLE `buckets` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `experiment_id` integer NOT NULL,
- `uuid_mods` varchar(64) NOT NULL,
- `traffic_ratios` varchar(64) NOT NULL,
- `http_verb` varchar(8) NOT NULL,
- `api_path` text NOT NULL,
- `uuid_key` varchar(128),
- `uuid_placeholder` varchar(64),
- `request_body` text NOT NULL,
- `timeout` int NOT NULL DEFAULT 1000,
- `impression_id` varchar(64) NOT NULL,
- `is_graph_query` tinyint NOT NULL DEFAULT 1,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_impression_id` (`impression_id`),
- INDEX `idx_experiment_id` (`experiment_id`),
- INDEX `idx_impression_id` (`impression_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
- */
+
implicit val session = AutoSession
sql"""DELETE FROM buckets""".update().apply()
sql"""DELETE FROM experiments""".update().apply()
@@ -58,6 +25,7 @@ CREATE TABLE `buckets` (
sql"""INSERT INTO
buckets(experiment_id, modular, http_verb, api_path, request_body, impression_id)
VALUES($expId, "1~100", "POST", "/a/b/c", "None", "imp1")""".update().apply()
+
}
"Experiment" should "find bucket list" in {
@@ -70,7 +38,7 @@ CREATE TABLE `buckets` (
}
it should "update bucket list after cache ttl time" in {
- Experiment.findBy(1, "exp1").foreach { exp =>
+ Experiment.findBy(1, "exp1").foreach { exp =>
val bucket = exp.buckets.head
bucket.impressionId should equal("imp1")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
index 73d0d76..393d8b1 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
@@ -8,7 +8,6 @@ import play.api.libs.json.Json
class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
// dummy data for dummy edge
-
import HBaseType.{VERSION1, VERSION2}
val ts = System.currentTimeMillis()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
index b404559..38b5b99 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
@@ -1,15 +1,16 @@
package com.kakao.s2graph.core.storage.hbase
import com.kakao.s2graph.core.Graph
+import com.typesafe.config.ConfigFactory
+
import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.GetRequest
import org.scalatest.{FunSuite, Matchers}
-import scala.concurrent.ExecutionContext.Implicits.global
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 9..
- */
+import scala.concurrent.ExecutionContext
+
class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
+
val dummyRequests = {
for {
id <- 0 until 1000
@@ -18,7 +19,8 @@ class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
}
}
- val config = Graph.DefaultConfig
+ implicit val ec = ExecutionContext.Implicits.global
+ val config = ConfigFactory.load()
val graph = new Graph(config)
val qb = new AsynchbaseQueryBuilder(graph.storage.asInstanceOf[AsynchbaseStorage])
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
index c0c242b..69a299d 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
@@ -1,14 +1,14 @@
package com.kakao.s2graph.core.types
+import com.kakao.s2graph.core.TestCommonWithModels
import com.kakao.s2graph.core.types._
import org.apache.hadoop.hbase.util.{Bytes, OrderedBytes, SimplePositionedByteRange}
import org.scalatest.{Matchers, FunSuite}
import play.api.libs.json.Json
-/**
- * Created by shon on 5/28/15.
- */
-class InnerValTest extends FunSuite with Matchers {
+class InnerValTest extends FunSuite with Matchers with TestCommonWithModels {
+ initTests()
+
import HBaseType.{VERSION2, VERSION1}
val decimals = List(
BigDecimal(Long.MinValue),
@@ -44,52 +44,52 @@ class InnerValTest extends FunSuite with Matchers {
for {
innerVal <- ranges
} {
- val bytes = innerVal.bytes
- val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version)
- innerVal == decoded shouldBe true
- bytes.length == numOfBytesUsed shouldBe true
- }
+ val bytes = innerVal.bytes
+ val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version)
+ innerVal == decoded shouldBe true
+ bytes.length == numOfBytesUsed shouldBe true
+ }
}
-// test("big decimal") {
-// for {
-// version <- List(VERSION2, VERSION1)
-// } {
-// val innerVals = decimals.map { num => InnerVal.withNumber(num, version)}
-// testEncodeDecode(innerVals, version)
-// }
-// }
-// test("text") {
-// for {
-// version <- List(VERSION2)
-// } {
-// val innerVals = texts.map { t => InnerVal.withStr(t, version) }
-// testEncodeDecode(innerVals, version)
-// }
-// }
-// test("string") {
-// for {
-// version <- List(VERSION2, VERSION1)
-// } {
-// val innerVals = strings.map { t => InnerVal.withStr(t, version) }
-// testEncodeDecode(innerVals, version)
-// }
-// }
-// test("blob") {
-// for {
-// version <- List(VERSION2)
-// } {
-// val innerVals = blobs.map { t => InnerVal.withBlob(t, version) }
-// testEncodeDecode(innerVals, version)
-// }
-// }
-// test("boolean") {
-// for {
-// version <- List(VERSION2, VERSION1)
-// } {
-// val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) }
-// testEncodeDecode(innerVals, version)
-// }
-// }
+ // test("big decimal") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)}
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("text") {
+ // for {
+ // version <- List(VERSION2)
+ // } {
+ // val innerVals = texts.map { t => InnerVal.withStr(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("string") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = strings.map { t => InnerVal.withStr(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("blob") {
+ // for {
+ // version <- List(VERSION2)
+ // } {
+ // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("boolean") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
test("korean") {
val small = InnerVal.withStr("ê°€", VERSION2)
val large = InnerVal.withStr("나", VERSION2)
@@ -99,32 +99,32 @@ class InnerValTest extends FunSuite with Matchers {
println (Bytes.compareTo(smallBytes, largeBytes))
true
}
-// test("innerVal") {
-// val srcVal = InnerVal.withLong(44391298, VERSION2)
-// val srcValV1 = InnerVal.withLong(44391298, VERSION1)
-// val tgtVal = InnerVal.withLong(7295564, VERSION2)
-//
-// val a = VertexId(0, srcVal)
-// val b = SourceVertexId(0, srcVal)
-// val c = TargetVertexId(0, srcVal)
-// val aa = VertexId(0, srcValV1)
-// val bb = SourceVertexId(0, srcValV1)
-// val cc = TargetVertexId(0, srcValV1)
-// println(a.bytes.toList)
-// println(b.bytes.toList)
-// println(c.bytes.toList)
-//
-// println(aa.bytes.toList)
-// println(bb.bytes.toList)
-// println(cc.bytes.toList)
-// }
-// test("aa") {
-// val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes
-// val pbr = new SimplePositionedByteRange(bytes)
-// pbr.setOffset(1)
-// println(pbr.getPosition)
-// val num = OrderedBytes.decodeNumericAsBigDecimal(pbr)
-// println(pbr.getPosition)
-// true
-// }
+ // test("innerVal") {
+ // val srcVal = InnerVal.withLong(44391298, VERSION2)
+ // val srcValV1 = InnerVal.withLong(44391298, VERSION1)
+ // val tgtVal = InnerVal.withLong(7295564, VERSION2)
+ //
+ // val a = VertexId(0, srcVal)
+ // val b = SourceVertexId(0, srcVal)
+ // val c = TargetVertexId(0, srcVal)
+ // val aa = VertexId(0, srcValV1)
+ // val bb = SourceVertexId(0, srcValV1)
+ // val cc = TargetVertexId(0, srcValV1)
+ // println(a.bytes.toList)
+ // println(b.bytes.toList)
+ // println(c.bytes.toList)
+ //
+ // println(aa.bytes.toList)
+ // println(bb.bytes.toList)
+ // println(cc.bytes.toList)
+ // }
+ // test("aa") {
+ // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes
+ // val pbr = new SimplePositionedByteRange(bytes)
+ // pbr.setOffset(1)
+ // println(pbr.getPosition)
+ // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr)
+ // println(pbr.getPosition)
+ // true
+ // }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/Bootstrap.scala b/s2rest_play/app/Bootstrap.scala
new file mode 100644
index 0000000..a661ac3
--- /dev/null
+++ b/s2rest_play/app/Bootstrap.scala
@@ -0,0 +1,77 @@
+package com.kakao.s2graph.rest
+
+import java.util.concurrent.Executors
+
+import actors.QueueActor
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.utils.logger
+import com.kakao.s2graph.core.{ExceptionHandler, Graph}
+import config.Config
+import controllers.{AdminController, ApplicationController}
+import play.api.Application
+import play.api.mvc.{WithFilters, _}
+import play.filters.gzip.GzipFilter
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.io.Source
+import scala.util.Try
+
+object Global extends WithFilters(new GzipFilter()) {
+ var s2graph: Graph = _
+ var s2parser: RequestParser = _
+
+ // Application entry point
+ override def onStart(app: Application) {
+ ApplicationController.isHealthy = false
+
+ val numOfThread = Runtime.getRuntime.availableProcessors()
+ val threadPool = Executors.newFixedThreadPool(numOfThread)
+ val ec = ExecutionContext.fromExecutor(threadPool)
+
+ val config = Config.conf.underlying
+
+ // init s2graph with config
+ s2graph = new Graph(config)(ec)
+ s2parser = new RequestParser(s2graph.config) // merged config
+
+ QueueActor.init(s2graph)
+
+ if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ ExceptionHandler.apply(config)
+ }
+
+ val defaultHealthOn = Config.conf.getBoolean("app.health.on").getOrElse(true)
+ ApplicationController.deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
+
+ ApplicationController.isHealthy = defaultHealthOn
+ logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
+ }
+
+ override def onStop(app: Application) {
+ QueueActor.shutdown()
+
+ if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ ExceptionHandler.shutdown()
+ }
+
+ /**
+ * shutdown hbase client for flush buffers.
+ */
+ s2graph.shutdown()
+ }
+
+ override def onError(request: RequestHeader, ex: Throwable): Future[Result] = {
+ logger.error(s"onError => ip:${request.remoteAddress}, request:${request}", ex)
+ Future.successful(Results.InternalServerError)
+ }
+
+ override def onHandlerNotFound(request: RequestHeader): Future[Result] = {
+ logger.error(s"onHandlerNotFound => ip:${request.remoteAddress}, request:${request}")
+ Future.successful(Results.NotFound)
+ }
+
+ override def onBadRequest(request: RequestHeader, error: String): Future[Result] = {
+ logger.error(s"onBadRequest => ip:${request.remoteAddress}, request:$request, error:$error")
+ Future.successful(Results.BadRequest(error))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/actors/QueueActor.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/actors/QueueActor.scala b/s2rest_play/app/actors/QueueActor.scala
new file mode 100644
index 0000000..74bc65d
--- /dev/null
+++ b/s2rest_play/app/actors/QueueActor.scala
@@ -0,0 +1,92 @@
+package actors
+
+import java.util.concurrent.TimeUnit
+
+import actors.Protocol.FlushAll
+import akka.actor._
+import com.kakao.s2graph.core.ExceptionHandler._
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.utils.logger
+import config.Config
+import play.api.Play.current
+import play.api.libs.concurrent.Akka
+
+import scala.collection.mutable
+import scala.concurrent.duration.Duration
+
+/**
+ * Created by shon on 9/2/15.
+ */
+object Protocol {
+
+ case object Flush
+
+ case object FlushAll
+
+}
+
+object QueueActor {
+ /** we are throttling down here so fixed number of actor to constant */
+ var router: ActorRef = _
+
+ // Akka.system.actorOf(props(), name = "queueActor")
+ def init(s2: Graph) = {
+ router = Akka.system.actorOf(props(s2))
+ }
+
+ def shutdown() = {
+ router ! FlushAll
+ Akka.system.shutdown()
+ Thread.sleep(Config.ASYNC_HBASE_CLIENT_FLUSH_INTERVAL * 2)
+ }
+
+ def props(s2: Graph): Props = Props(classOf[QueueActor], s2)
+}
+
+class QueueActor(s2: Graph) extends Actor with ActorLogging {
+
+ import Protocol._
+
+ implicit val ec = context.system.dispatcher
+ // logger.error(s"QueueActor: $self")
+ val queue = mutable.Queue.empty[GraphElement]
+ var queueSize = 0L
+ val maxQueueSize = Config.LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE
+ val timeUnitInMillis = 10
+ val rateLimitTimeStep = 1000 / timeUnitInMillis
+ val rateLimit = Config.LOCAL_QUEUE_ACTOR_RATE_LIMIT / rateLimitTimeStep
+
+
+ context.system.scheduler.schedule(Duration.Zero, Duration(timeUnitInMillis, TimeUnit.MILLISECONDS), self, Flush)
+
+ override def receive: Receive = {
+ case element: GraphElement =>
+
+ if (queueSize > maxQueueSize) {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_FAIL_TOPIC, element, None))
+ } else {
+ queueSize += 1L
+ queue.enqueue(element)
+ }
+
+ case Flush =>
+ val elementsToFlush =
+ if (queue.size < rateLimit) queue.dequeueAll(_ => true)
+ else (0 until rateLimit).map(_ => queue.dequeue())
+
+ val flushSize = elementsToFlush.size
+
+ queueSize -= elementsToFlush.length
+ s2.mutateElements(elementsToFlush)
+
+ if (flushSize > 0) {
+ logger.info(s"flush: $flushSize, $queueSize")
+ }
+
+ case FlushAll =>
+ s2.mutateElements(queue)
+ context.stop(self)
+
+ case _ => logger.error("unknown protocol")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/config/Config.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/config/Config.scala b/s2rest_play/app/config/Config.scala
new file mode 100644
index 0000000..98b87c5
--- /dev/null
+++ b/s2rest_play/app/config/Config.scala
@@ -0,0 +1,41 @@
+package config
+
+import play.api.Play
+
+object Config {
+ // HBASE
+ lazy val HBASE_ZOOKEEPER_QUORUM = conf.getString("hbase.zookeeper.quorum").getOrElse("localhost")
+
+
+ // HBASE CLIENT
+ lazy val ASYNC_HBASE_CLIENT_FLUSH_INTERVAL = conf.getInt("async.hbase.client.flush.interval").getOrElse(1000).toShort
+ lazy val RPC_TIMEOUT = conf.getInt("hbase.client.operation.timeout").getOrElse(1000)
+ lazy val MAX_ATTEMPT = conf.getInt("hbase.client.operation.maxAttempt").getOrElse(3)
+
+ // PHASE
+ lazy val PHASE = conf.getString("phase").getOrElse("dev")
+ lazy val conf = Play.current.configuration
+
+ // CACHE
+ lazy val CACHE_TTL_SECONDS = conf.getInt("cache.ttl.seconds").getOrElse(600)
+ lazy val CACHE_MAX_SIZE = conf.getInt("cache.max.size").getOrElse(10000)
+
+ //KAFKA
+ lazy val KAFKA_METADATA_BROKER_LIST = conf.getString("kafka.metadata.broker.list").getOrElse("localhost")
+ lazy val KAFKA_PRODUCER_POOL_SIZE = conf.getInt("kafka.producer.pool.size").getOrElse(0)
+ lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}"
+ lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async"
+ lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"
+
+ // is query or write
+ lazy val IS_QUERY_SERVER = conf.getBoolean("is.query.server").getOrElse(true)
+ lazy val IS_WRITE_SERVER = conf.getBoolean("is.write.server").getOrElse(true)
+
+
+ // query limit per step
+ lazy val QUERY_HARD_LIMIT = conf.getInt("query.hard.limit").getOrElse(300)
+
+ // local queue actor
+ lazy val LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE = conf.getInt("local.queue.actor.max.queue.size").getOrElse(10000)
+ lazy val LOCAL_QUEUE_ACTOR_RATE_LIMIT = conf.getInt("local.queue.actor.rate.limit").getOrElse(1000)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/app/config/CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/config/CounterConfig.scala b/s2rest_play/app/config/CounterConfig.scala
new file mode 100644
index 0000000..2569d55
--- /dev/null
+++ b/s2rest_play/app/config/CounterConfig.scala
@@ -0,0 +1,10 @@
+package config
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 9. 3..
+ */
+object CounterConfig {
+ // kafka
+ lazy val KAFKA_TOPIC_COUNTER = s"s2counter-${Config.PHASE}"
+ lazy val KAFKA_TOPIC_COUNTER_TRX = s"s2counter-trx-${Config.PHASE}"
+}
[7/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
[S2GRAPH-7] Abstract common codes for rest project into s2core
Move PostProcess.scala to s2core from root project
Move RequestParser from Root to s2core
Make RestCaller for abstract over http layer
Move root project to s2rest_play
Move Test from root to s2core
JIRA:
[S2GRAPH-7] https://issues.apache.org/jira/browse/S2GRAPH-7
Pull Request:
Closes #4
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f2311f25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f2311f25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f2311f25
Branch: refs/heads/master
Commit: f2311f25c5edf46104d149e91ff746ef808ded55
Parents: be304d6
Author: daewon <da...@apache.org>
Authored: Wed Jan 13 17:50:47 2016 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Jan 13 18:05:01 2016 +0900
----------------------------------------------------------------------
CHANGES | 6 +-
app/Bootstrap.scala | 75 --
app/actors/QueueActor.scala | 92 ---
app/config/Config.scala | 41 -
app/config/CounterConfig.scala | 10 -
app/controllers/AdminController.scala | 417 -----------
app/controllers/ApplicationController.scala | 83 ---
app/controllers/CounterController.scala | 747 -------------------
app/controllers/EdgeController.scala | 220 ------
app/controllers/ExperimentController.scala | 113 ---
app/controllers/JsonBodyParser.scala | 73 --
app/controllers/PostProcess.scala | 432 -----------
app/controllers/PublishController.scala | 54 --
app/controllers/QueryController.scala | 310 --------
app/controllers/RequestParser.scala | 460 ------------
app/controllers/TestController.scala | 24 -
app/controllers/VertexController.scala | 84 ---
app/models/ExactCounterItem.scala | 38 -
app/models/RankCounterItem.scala | 40 -
app/models/package.scala | 8 -
app/util/TestDataLoader.scala | 70 --
build.sbt | 23 +-
conf/logger.xml | 83 ---
conf/reference.conf | 129 ----
conf/routes | 124 ---
conf/test.conf | 2 -
project/plugins.sbt | 8 +-
s2core/src/main/resources/logback.xml | 29 +-
.../scala/com/kakao/s2graph/core/Graph.scala | 13 +-
.../com/kakao/s2graph/core/OrderingUtil.scala | 134 ++--
.../com/kakao/s2graph/core/PostProcess.scala | 439 +++++++++++
.../com/kakao/s2graph/core/mysqls/Model.scala | 13 +
.../kakao/s2graph/core/rest/RequestParser.scala | 509 +++++++++++++
.../kakao/s2graph/core/rest/RestCaller.scala | 183 +++++
.../storage/hbase/AsynchbaseQueryBuilder.scala | 2 +-
.../com/kakao/s2graph/core/utils/Logger.scala | 6 +-
.../scala/com/kakao/s2graph/core/EdgeTest.scala | 29 +
.../com/kakao/s2graph/core/GraphTest.scala | 8 -
.../kakao/s2graph/core/Integrate/CrudTest.scala | 226 ++++++
.../core/Integrate/IntegrateCommon.scala | 311 ++++++++
.../s2graph/core/Integrate/QueryTest.scala | 528 +++++++++++++
.../core/Integrate/StrongLabelDeleteTest.scala | 282 +++++++
.../core/Integrate/VertexTestHelper.scala | 71 ++
.../core/Integrate/WeakLabelDeleteTest.scala | 129 ++++
.../com/kakao/s2graph/core/JsonParserTest.scala | 3 -
.../kakao/s2graph/core/OrderingUtilTest.scala | 13 +-
.../com/kakao/s2graph/core/TestCommon.scala | 190 +++--
.../s2graph/core/TestCommonWithModels.scala | 88 ++-
.../kakao/s2graph/core/models/ModelTest.scala | 190 ++---
.../s2graph/core/mysqls/ExperimentSpec.scala | 38 +-
.../s2graph/core/parsers/WhereParserTest.scala | 1 -
.../hbase/AsynchbaseQueryBuilderTest.scala | 12 +-
.../kakao/s2graph/core/types/InnerValTest.scala | 154 ++--
s2rest_play/app/Bootstrap.scala | 77 ++
s2rest_play/app/actors/QueueActor.scala | 92 +++
s2rest_play/app/config/Config.scala | 41 +
s2rest_play/app/config/CounterConfig.scala | 10 +
.../app/controllers/AdminController.scala | 419 +++++++++++
.../app/controllers/ApplicationController.scala | 83 +++
.../app/controllers/CounterController.scala | 747 +++++++++++++++++++
.../app/controllers/EdgeController.scala | 222 ++++++
.../app/controllers/ExperimentController.scala | 114 +++
.../app/controllers/JsonBodyParser.scala | 73 ++
.../app/controllers/PublishController.scala | 54 ++
.../app/controllers/QueryController.scala | 311 ++++++++
.../app/controllers/TestController.scala | 24 +
.../app/controllers/VertexController.scala | 86 +++
s2rest_play/app/models/ExactCounterItem.scala | 38 +
s2rest_play/app/models/RankCounterItem.scala | 40 +
s2rest_play/app/models/package.scala | 8 +
s2rest_play/app/util/TestDataLoader.scala | 70 ++
s2rest_play/build.sbt | 12 +
s2rest_play/conf/application.conf | 0
s2rest_play/conf/logger.xml | 83 +++
s2rest_play/conf/reference.conf | 131 ++++
s2rest_play/conf/routes | 124 +++
s2rest_play/conf/test.conf | 2 +
.../test/benchmark/BenchmarkCommon.scala | 15 +
s2rest_play/test/benchmark/GraphUtilSpec.scala | 125 ++++
.../test/benchmark/JsonBenchmarkSpec.scala | 46 ++
.../benchmark/OrderingUtilBenchmarkSpec.scala | 103 +++
.../test/benchmark/SamplingBenchmarkSpec.scala | 85 +++
.../test/controllers/PostProcessSpec.scala | 112 +++
test/benchmark/BenchmarkCommon.scala | 18 -
test/benchmark/GraphUtilSpec.scala | 125 ----
test/benchmark/JsonBenchmarkSpec.scala | 46 --
test/benchmark/OrderingUtilBenchmarkSpec.scala | 103 ---
test/benchmark/PostProcessBenchmarkSpec.scala | 238 ------
test/benchmark/SamplingBenchmarkSpec.scala | 90 ---
test/controllers/AdminControllerSpec.scala | 27 -
test/controllers/BasicCrudSpec.scala | 251 -------
test/controllers/EdgeControllerSpec.scala | 22 -
test/controllers/PostProcessSpec.scala | 115 ---
test/controllers/QueryCacheSpec.scala | 89 ---
test/controllers/QuerySpec.scala | 604 ---------------
test/controllers/SpecCommon.scala | 365 ---------
test/controllers/StrongLabelDeleteSpec.scala | 345 ---------
test/controllers/VertexSpec.scala | 39 -
test/controllers/WeakLabelDeleteSpec.scala | 126 ----
99 files changed, 6537 insertions(+), 6780 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 11a40e3..3575cd8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,4 @@
-
- Change Log
+ Change Log
Release 0.12.1 - unreleased
@@ -10,8 +9,9 @@ Release 0.12.1 - unreleased
BUG FIXES
TASKS
-
+
S2GRAPH-5: Add Apache RAT to valid LICENSE errors. (Committed by DOYUNG YOON).
SUB TASKS
+ S2GRAPH-7: Abstract common codes for rest project into s2core. (Committed by daewon).
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/app/Bootstrap.scala b/app/Bootstrap.scala
deleted file mode 100644
index 1879d90..0000000
--- a/app/Bootstrap.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-package com.kakao.s2graph.rest
-
-import java.util.concurrent.Executors
-
-import actors.QueueActor
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{ExceptionHandler, Graph}
-import config.Config
-import controllers.{AdminController, ApplicationController}
-import play.api.Application
-import play.api.mvc.{WithFilters, _}
-import play.filters.gzip.GzipFilter
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.io.Source
-import scala.util.Try
-
-object Global extends WithFilters(new GzipFilter()) {
- var s2graph: Graph = _
-
- // Application entry point
- override def onStart(app: Application) {
- ApplicationController.isHealthy = false
-
- val numOfThread = Runtime.getRuntime.availableProcessors()
- val threadPool = Executors.newFixedThreadPool(numOfThread)
- val ec = ExecutionContext.fromExecutor(threadPool)
-
- val config = Config.conf.underlying
-
- // init s2graph with config
- s2graph = new Graph(config)(ec)
-
- QueueActor.init(s2graph)
-
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
- ExceptionHandler.apply(config)
- }
-
- val defaultHealthOn = Config.conf.getBoolean("app.health.on").getOrElse(true)
- ApplicationController.deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
-
- AdminController.loadCacheInner()
- ApplicationController.isHealthy = defaultHealthOn
- logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
- }
-
- override def onStop(app: Application) {
- QueueActor.shutdown()
-
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
- ExceptionHandler.shutdown()
- }
-
- /**
- * shutdown hbase client for flush buffers.
- */
- s2graph.shutdown()
- }
-
- override def onError(request: RequestHeader, ex: Throwable): Future[Result] = {
- logger.error(s"onError => ip:${request.remoteAddress}, request:${request}", ex)
- Future.successful(Results.InternalServerError)
- }
-
- override def onHandlerNotFound(request: RequestHeader): Future[Result] = {
- logger.error(s"onHandlerNotFound => ip:${request.remoteAddress}, request:${request}")
- Future.successful(Results.NotFound)
- }
-
- override def onBadRequest(request: RequestHeader, error: String): Future[Result] = {
- logger.error(s"onBadRequest => ip:${request.remoteAddress}, request:$request, error:$error")
- Future.successful(Results.BadRequest(error))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/actors/QueueActor.scala
----------------------------------------------------------------------
diff --git a/app/actors/QueueActor.scala b/app/actors/QueueActor.scala
deleted file mode 100644
index 74bc65d..0000000
--- a/app/actors/QueueActor.scala
+++ /dev/null
@@ -1,92 +0,0 @@
-package actors
-
-import java.util.concurrent.TimeUnit
-
-import actors.Protocol.FlushAll
-import akka.actor._
-import com.kakao.s2graph.core.ExceptionHandler._
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.utils.logger
-import config.Config
-import play.api.Play.current
-import play.api.libs.concurrent.Akka
-
-import scala.collection.mutable
-import scala.concurrent.duration.Duration
-
-/**
- * Created by shon on 9/2/15.
- */
-object Protocol {
-
- case object Flush
-
- case object FlushAll
-
-}
-
-object QueueActor {
- /** we are throttling down here so fixed number of actor to constant */
- var router: ActorRef = _
-
- // Akka.system.actorOf(props(), name = "queueActor")
- def init(s2: Graph) = {
- router = Akka.system.actorOf(props(s2))
- }
-
- def shutdown() = {
- router ! FlushAll
- Akka.system.shutdown()
- Thread.sleep(Config.ASYNC_HBASE_CLIENT_FLUSH_INTERVAL * 2)
- }
-
- def props(s2: Graph): Props = Props(classOf[QueueActor], s2)
-}
-
-class QueueActor(s2: Graph) extends Actor with ActorLogging {
-
- import Protocol._
-
- implicit val ec = context.system.dispatcher
- // logger.error(s"QueueActor: $self")
- val queue = mutable.Queue.empty[GraphElement]
- var queueSize = 0L
- val maxQueueSize = Config.LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE
- val timeUnitInMillis = 10
- val rateLimitTimeStep = 1000 / timeUnitInMillis
- val rateLimit = Config.LOCAL_QUEUE_ACTOR_RATE_LIMIT / rateLimitTimeStep
-
-
- context.system.scheduler.schedule(Duration.Zero, Duration(timeUnitInMillis, TimeUnit.MILLISECONDS), self, Flush)
-
- override def receive: Receive = {
- case element: GraphElement =>
-
- if (queueSize > maxQueueSize) {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_FAIL_TOPIC, element, None))
- } else {
- queueSize += 1L
- queue.enqueue(element)
- }
-
- case Flush =>
- val elementsToFlush =
- if (queue.size < rateLimit) queue.dequeueAll(_ => true)
- else (0 until rateLimit).map(_ => queue.dequeue())
-
- val flushSize = elementsToFlush.size
-
- queueSize -= elementsToFlush.length
- s2.mutateElements(elementsToFlush)
-
- if (flushSize > 0) {
- logger.info(s"flush: $flushSize, $queueSize")
- }
-
- case FlushAll =>
- s2.mutateElements(queue)
- context.stop(self)
-
- case _ => logger.error("unknown protocol")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/config/Config.scala
----------------------------------------------------------------------
diff --git a/app/config/Config.scala b/app/config/Config.scala
deleted file mode 100644
index 98b87c5..0000000
--- a/app/config/Config.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-package config
-
-import play.api.Play
-
-object Config {
- // HBASE
- lazy val HBASE_ZOOKEEPER_QUORUM = conf.getString("hbase.zookeeper.quorum").getOrElse("localhost")
-
-
- // HBASE CLIENT
- lazy val ASYNC_HBASE_CLIENT_FLUSH_INTERVAL = conf.getInt("async.hbase.client.flush.interval").getOrElse(1000).toShort
- lazy val RPC_TIMEOUT = conf.getInt("hbase.client.operation.timeout").getOrElse(1000)
- lazy val MAX_ATTEMPT = conf.getInt("hbase.client.operation.maxAttempt").getOrElse(3)
-
- // PHASE
- lazy val PHASE = conf.getString("phase").getOrElse("dev")
- lazy val conf = Play.current.configuration
-
- // CACHE
- lazy val CACHE_TTL_SECONDS = conf.getInt("cache.ttl.seconds").getOrElse(600)
- lazy val CACHE_MAX_SIZE = conf.getInt("cache.max.size").getOrElse(10000)
-
- //KAFKA
- lazy val KAFKA_METADATA_BROKER_LIST = conf.getString("kafka.metadata.broker.list").getOrElse("localhost")
- lazy val KAFKA_PRODUCER_POOL_SIZE = conf.getInt("kafka.producer.pool.size").getOrElse(0)
- lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}"
- lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async"
- lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"
-
- // is query or write
- lazy val IS_QUERY_SERVER = conf.getBoolean("is.query.server").getOrElse(true)
- lazy val IS_WRITE_SERVER = conf.getBoolean("is.write.server").getOrElse(true)
-
-
- // query limit per step
- lazy val QUERY_HARD_LIMIT = conf.getInt("query.hard.limit").getOrElse(300)
-
- // local queue actor
- lazy val LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE = conf.getInt("local.queue.actor.max.queue.size").getOrElse(10000)
- lazy val LOCAL_QUEUE_ACTOR_RATE_LIMIT = conf.getInt("local.queue.actor.rate.limit").getOrElse(1000)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/config/CounterConfig.scala
----------------------------------------------------------------------
diff --git a/app/config/CounterConfig.scala b/app/config/CounterConfig.scala
deleted file mode 100644
index 2569d55..0000000
--- a/app/config/CounterConfig.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package config
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 9. 3..
- */
-object CounterConfig {
- // kafka
- lazy val KAFKA_TOPIC_COUNTER = s"s2counter-${Config.PHASE}"
- lazy val KAFKA_TOPIC_COUNTER_TRX = s"s2counter-trx-${Config.PHASE}"
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/AdminController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/AdminController.scala b/app/controllers/AdminController.scala
deleted file mode 100644
index b31f01f..0000000
--- a/app/controllers/AdminController.scala
+++ /dev/null
@@ -1,417 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.utils.logger
-import play.api.mvc
-import play.api.mvc.{Action, Controller}
-import play.api.libs.json._
-import play.api.libs.functional.syntax._
-
-import scala.util.{Failure, Success, Try}
-
-object AdminController extends Controller with RequestParser {
-
- import ApplicationController._
-
- /**
- * admin message formatter
- * @tparam T
- */
- trait AdminMessageFormatter[T] {
- def toJson(msg: T): JsValue
- }
-
- object AdminMessageFormatter {
- implicit def jsValueToJson[T <: JsValue] = new AdminMessageFormatter[T] {
- def toJson(js: T) = js
- }
-
- implicit val stringToJson = new AdminMessageFormatter[String] {
- def toJson(js: String) = Json.obj("message" -> js)
- }
- }
-
- def format[T: AdminMessageFormatter](f: JsValue => play.mvc.Result)(message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- f(formatter.toJson(message))
- }
-
- /**
- * ok response
- * @param message
- * @tparam T
- * @return
- */
- def ok[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- Ok(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- /**
- * bad request response
- * @param message
- * @tparam T
- * @return
- */
- def bad[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- BadRequest(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- /**
- * not found response
- * @param message
- * @tparam T
- * @return
- */
- def notFound[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- NotFound(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- private[AdminController] def tryResponse[T, R: AdminMessageFormatter](res: Try[T])(callback: T => R): mvc.Result = res match {
- case Success(m) =>
- val ret = callback(m)
- logger.info(ret.toString)
- ok(ret)
- case Failure(error) =>
- logger.error(error.getMessage, error)
- error match {
- case JsResultException(e) => bad(JsError.toFlatJson(e))
- case _ => bad(error.getMessage)
- }
- }
-
- def optionResponse[T, R: AdminMessageFormatter](res: Option[T])(callback: T => R): mvc.Result = res match {
- case Some(m) => ok(callback(m))
- case None => notFound("not found")
- }
-
- /**
- * load all model cache
- * @return
- */
- def loadCache() = Action { request =>
- val startTs = System.currentTimeMillis()
-
- if (!ApplicationController.isHealthy) {
- loadCacheInner()
- }
-
- ok(s"${System.currentTimeMillis() - startTs}")
- }
-
- def loadCacheInner() = {
- Service.findAll()
- ServiceColumn.findAll()
- Label.findAll()
- LabelMeta.findAll()
- LabelIndex.findAll()
- ColumnMeta.findAll()
- }
-
- /**
- * read
- */
-
- /**
- * get service info
- * @param serviceName
- * @return
- */
- def getService(serviceName: String) = Action { request =>
- val serviceOpt = Management.findService(serviceName)
- optionResponse(serviceOpt)(_.toJson)
- }
-
- /**
- * get label info
- * @param labelName
- * @return
- */
- def getLabel(labelName: String) = Action { request =>
- val labelOpt = Management.findLabel(labelName)
- optionResponse(labelOpt)(_.toJson)
- }
-
- /**
- * get all labels of service
- * @param serviceName
- * @return
- */
- def getLabels(serviceName: String) = Action { request =>
- Service.findByName(serviceName) match {
- case None => notFound(s"Service $serviceName not found")
- case Some(service) =>
- val src = Label.findBySrcServiceId(service.id.get)
- val tgt = Label.findByTgtServiceId(service.id.get)
-
- ok(Json.obj("from" -> src.map(_.toJson), "to" -> tgt.map(_.toJson)))
- }
- }
-
- /**
- * get service columns
- * @param serviceName
- * @param columnName
- * @return
- */
- def getServiceColumn(serviceName: String, columnName: String) = Action { request =>
- val serviceColumnOpt = for {
- service <- Service.findByName(serviceName)
- serviceColumn <- ServiceColumn.find(service.id.get, columnName, useCache = false)
- } yield serviceColumn
-
- optionResponse(serviceColumnOpt)(_.toJson)
- }
-
- /**
- * create
- */
-
- /**
- * create service
- * @return
- */
- def createService() = Action(parse.json) { request =>
- val serviceTry = createServiceInner(request.body)
- tryResponse(serviceTry)(_.toJson)
- }
-
- def createServiceInner(jsValue: JsValue) = {
- val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = toServiceElements(jsValue)
- Management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
- }
-
- /**
- * create label
- * @return
- */
- def createLabel() = Action(parse.json) { request =>
- val ret = createLabelInner(request.body)
- tryResponse(ret)(_.toJson)
- }
-
- def createLabelInner(json: JsValue) = for {
- labelArgs <- toLabelElements(json)
- label <- (Management.createLabel _).tupled(labelArgs)
- } yield label
-
- /**
- * add index
- * @return
- */
- def addIndex() = Action(parse.json) { request =>
- val ret = addIndexInner(request.body)
- tryResponse(ret)(_.label + " is updated")
- }
-
- def addIndexInner(json: JsValue) = for {
- (labelName, indices) <- toIndexElements(json)
- label <- Management.addIndex(labelName, indices)
- } yield label
-
- /**
- * create service column
- * @return
- */
- def createServiceColumn() = Action(parse.json) { request =>
- val serviceColumnTry = createServiceColumnInner(request.body)
- tryResponse(serviceColumnTry) { (columns: Seq[ColumnMeta]) => Json.obj("metas" -> columns.map(_.toJson)) }
- }
-
- def createServiceColumnInner(jsValue: JsValue) = for {
- (serviceName, columnName, columnType, props) <- toServiceColumnElements(jsValue)
- serviceColumn <- Management.createServiceColumn(serviceName, columnName, columnType, props)
- } yield serviceColumn
-
- /**
- * delete
- */
-
- /**
- * delete label
- * @param labelName
- * @return
- */
- def deleteLabel(labelName: String) = Action { request =>
- val deleteLabelTry = deleteLabelInner(labelName)
- tryResponse(deleteLabelTry)(labelName => labelName + " is deleted")
- }
-
- def deleteLabelInner(labelName: String) = Management.deleteLabel(labelName)
-
- /**
- * delete servieColumn
- * @param serviceName
- * @param columnName
- * @return
- */
- def deleteServiceColumn(serviceName: String, columnName: String) = Action { request =>
- val serviceColumnTry = deleteServiceColumnInner(serviceName, columnName)
- tryResponse(serviceColumnTry)(columnName => columnName + " is deleted")
- }
-
- def deleteServiceColumnInner(serviceName: String, columnName: String) =
- Management.deleteColumn(serviceName, columnName)
-
- /**
- * update
- */
-
- /**
- * add Prop to label
- * @param labelName
- * @return
- */
- def addProp(labelName: String) = Action(parse.json) { request =>
- val labelMetaTry = addPropInner(labelName, request.body)
- tryResponse(labelMetaTry)(_.toJson)
- }
-
- def addPropInner(labelName: String, js: JsValue) = for {
- prop <- toPropElements(js)
- labelMeta <- Management.addProp(labelName, prop)
- } yield labelMeta
-
- /**
- * add prop to serviceColumn
- * @param serviceName
- * @param columnName
- * @return
- */
- def addServiceColumnProp(serviceName: String, columnName: String) = Action(parse.json) { request =>
- addServiceColumnPropInner(serviceName, columnName)(request.body) match {
- case None => bad(s"can`t find service with $serviceName or can`t find serviceColumn with $columnName")
- case Some(m) => Ok(m.toJson).as(applicationJsonHeader)
- }
- }
-
- def addServiceColumnPropInner(serviceName: String, columnName: String)(js: JsValue) = {
- for {
- service <- Service.findByName(serviceName)
- serviceColumn <- ServiceColumn.find(service.id.get, columnName)
- prop <- toPropElements(js).toOption
- } yield {
- ColumnMeta.findOrInsert(serviceColumn.id.get, prop.name, prop.defaultValue)
- }
- }
-
- /**
- * add props to serviceColumn
- * @param serviecName
- * @param columnName
- * @return
- */
- def addServiceColumnProps(serviecName: String, columnName: String) = Action(parse.json) { request =>
- val jsObjs = request.body.asOpt[List[JsObject]].getOrElse(List.empty[JsObject])
- val newProps = for {
- js <- jsObjs
- newProp <- addServiceColumnPropInner(serviecName, columnName)(js)
- } yield newProp
- ok(s"${newProps.size} is added.")
- }
-
- /**
- * copy label
- * @param oldLabelName
- * @param newLabelName
- * @return
- */
- def copyLabel(oldLabelName: String, newLabelName: String) = Action { request =>
- val copyTry = Management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
- tryResponse(copyTry)(_.label + "created")
- }
-
- /**
- * rename label
- * @param oldLabelName
- * @param newLabelName
- * @return
- */
- def renameLabel(oldLabelName: String, newLabelName: String) = Action { request =>
- Label.findByName(oldLabelName) match {
- case None => NotFound.as(applicationJsonHeader)
- case Some(label) =>
- Management.updateLabelName(oldLabelName, newLabelName)
- ok(s"Label was updated")
- }
- }
-
- /**
- * update HTable for a label
- * @param labelName
- * @param newHTableName
- * @return
- */
- def updateHTable(labelName: String, newHTableName: String) = Action { request =>
- val updateTry = Management.updateHTable(labelName, newHTableName)
- tryResponse(updateTry)(_.toString + " label(s) updated.")
- }
-
-
- case class HTableParams(cluster: String, hTableName: String,
- preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
-
- override def toString(): String = {
- s"""HtableParams
- |-- cluster : $cluster
- |-- hTableName : $hTableName
- |-- preSplitSize : $preSplitSize
- |-- hTableTTL : $hTableTTL
- |-- compressionAlgorithm : $compressionAlgorithm
- |""".stripMargin
- }
- }
-
- implicit object HTableParamsJsonConverter extends Format[HTableParams] {
- def reads(json: JsValue): JsResult[HTableParams] = (
- (__ \ "cluster").read[String] and
- (__ \ "hTableName").read[String] and
- (__ \ "preSplitSize").read[Int] and
- (__ \ "hTableTTL").readNullable[Int] and
- (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
-
- def writes(o: HTableParams): JsValue = Json.obj(
- "cluster" -> o.cluster,
- "hTableName" -> o.hTableName,
- "preSplitSize" -> o.preSplitSize,
- "hTableTTL" -> o.hTableTTL,
- "compressionAlgorithm" -> o.compressionAlgorithm
- )
- }
-
- implicit object JsErrorJsonWriter extends Writes[JsError] {
- def writes(o: JsError): JsValue = Json.obj(
- "errors" -> JsArray(
- o.errors.map {
- case (path, validationErrors) => Json.obj(
- "path" -> Json.toJson(path.toString()),
- "validationErrors" -> JsArray(validationErrors.map(validationError => Json.obj(
- "message" -> JsString(validationError.message),
- "args" -> JsArray(validationError.args.map(_ match {
- case x: Int => JsNumber(x)
- case x => JsString(x.toString)
- }))
- )))
- )
- }
- )
- )
- }
-
- def createHTable() = Action { request =>
-
- // Management.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
- request.body.asJson.map(_.validate[HTableParams] match {
- case JsSuccess(hTableParams, _) => {
- Management.createTable(hTableParams.cluster, hTableParams.hTableName, List("e", "v"), hTableParams.preSplitSize, hTableParams.hTableTTL, hTableParams.compressionAlgorithm.getOrElse(Management.defaultCompressionAlgorithm))
- logger.info(hTableParams.toString())
- ok(s"HTable was created.")
- }
- case err@JsError(_) => bad(Json.toJson(err))
- }).getOrElse(bad("Invalid Json."))
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/ApplicationController.scala b/app/controllers/ApplicationController.scala
deleted file mode 100644
index fefe93e..0000000
--- a/app/controllers/ApplicationController.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.iteratee.Enumerator
-import play.api.libs.json.{JsString, JsValue}
-import play.api.mvc._
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object ApplicationController extends Controller {
-
- var isHealthy = true
- var deployInfo = ""
- val applicationJsonHeader = "application/json"
-
- val jsonParser: BodyParser[JsValue] = controllers.s2parse.json
-
- def updateHealthCheck(isHealthy: Boolean) = Action { request =>
- this.isHealthy = isHealthy
- Ok(this.isHealthy + "\n")
- }
-
- def healthCheck() = withHeader(parse.anyContent) { request =>
- if (isHealthy) Ok(deployInfo)
- else NotFound
- }
-
- def jsonResponse(json: JsValue, headers: (String, String)*) =
- if (ApplicationController.isHealthy) {
- Ok(json).as(applicationJsonHeader).withHeaders(headers: _*)
- } else {
- Result(
- header = ResponseHeader(OK),
- body = Enumerator(json.toString.getBytes()),
- connection = HttpConnection.Close
- ).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*)
- }
-
- def toLogMessage[A](request: Request[A], result: Result)(startedAt: Long): String = {
- val duration = System.currentTimeMillis() - startedAt
- val isQueryRequest = result.header.headers.contains("result_size")
- val resultSize = result.header.headers.getOrElse("result_size", "-1")
-
- try {
- val body = request.body match {
- case AnyContentAsJson(jsValue) => jsValue match {
- case JsString(str) => str
- case _ => jsValue.toString
- }
- case _ => request.body.toString
- }
-
- val str =
- if (isQueryRequest)
- s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
- else
- s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
-
- logger.info(s"${request.method} ${request.uri} result_size: $resultSize")
-
- str
- } finally {
- /* pass */
- }
- }
-
- def withHeaderAsync[A](bodyParser: BodyParser[A])(block: Request[A] => Future[Result])(implicit ex: ExecutionContext) =
- Action.async(bodyParser) { request =>
- val startedAt = System.currentTimeMillis()
- block(request).map { r =>
- logger.info(toLogMessage(request, r)(startedAt))
- r
- }
- }
-
- def withHeader[A](bodyParser: BodyParser[A])(block: Request[A] => Result) =
- Action(bodyParser) { request: Request[A] =>
- val startedAt = System.currentTimeMillis()
- val r = block(request)
- logger.info(toLogMessage(request, r)(startedAt))
- r
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/CounterController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/CounterController.scala b/app/controllers/CounterController.scala
deleted file mode 100644
index c2b4dc2..0000000
--- a/app/controllers/CounterController.scala
+++ /dev/null
@@ -1,747 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.ExceptionHandler
-import com.kakao.s2graph.core.ExceptionHandler.KafkaMessage
-import com.kakao.s2graph.core.mysqls.Label
-import config.CounterConfig
-import models._
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.Play
-import play.api.libs.json.Reads._
-import play.api.libs.json._
-import play.api.mvc.{Action, Controller, Request}
-import s2.config.S2CounterConfig
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.counter.core.v1.{ExactStorageAsyncHBase, RankingStorageRedis}
-import s2.counter.core.v2.{ExactStorageGraph, RankingStorageGraph}
-import s2.models.Counter.ItemType
-import s2.models.{Counter, CounterModel}
-import s2.util.{CartesianProduct, ReduceMapValue, UnitConverter}
-
-import scala.concurrent.Future
-import scala.util.{Failure, Success, Try}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 22..
- */
-object CounterController extends Controller {
- import play.api.libs.concurrent.Execution.Implicits.defaultContext
-
- val config = Play.current.configuration.underlying
- val s2config = new S2CounterConfig(config)
-
- private val exactCounterMap = Map(
- s2.counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
- s2.counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
- )
- private val rankingCounterMap = Map(
- s2.counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
- s2.counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
- )
-
- private val tablePrefixMap = Map (
- s2.counter.VERSION_1 -> "s2counter",
- s2.counter.VERSION_2 -> "s2counter_v2"
- )
-
- private def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
- private def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
-
- lazy val counterModel = new CounterModel(config)
-
- def getQueryString[T](key: String, default: String)(implicit request: Request[T]): String = {
- request.getQueryString(key).getOrElse(default)
- }
-
- implicit val counterWrites = new Writes[Counter] {
- override def writes(o: Counter): JsValue = Json.obj(
- "version" -> o.version.toInt,
- "autoComb" -> o.autoComb,
- "dimension" -> o.dimension,
- "useProfile" -> o.useProfile,
- "bucketImpId" -> o.bucketImpId,
- "useRank" -> o.useRank,
- "intervalUnit" -> o.intervalUnit,
- "ttl" -> o.ttl,
- "dailyTtl" -> o.dailyTtl,
- "rateAction" -> o.rateActionId.flatMap { actionId =>
- counterModel.findById(actionId, useCache = false).map { actionPolicy =>
- Json.obj("service" -> actionPolicy.service, "action" -> actionPolicy.action)
- }
- },
- "rateBase" -> o.rateBaseId.flatMap { baseId =>
- counterModel.findById(baseId, useCache = false).map { basePolicy =>
- Json.obj("service" -> basePolicy.service, "action" -> basePolicy.action)
- }
- },
- "rateThreshold" -> o.rateThreshold
- )
- }
-
- def createAction(service: String, action: String) = Action(s2parse.json) { implicit request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case None =>
- val body = request.body
- val version = (body \ "version").asOpt[Int].map(_.toByte).getOrElse(s2.counter.VERSION_2)
- val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(true)
- val dimension = (body \ "dimension").asOpt[String].getOrElse("")
- val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(false)
- val bucketImpId = (body \ "bucketImpId").asOpt[String]
-
- val useExact = (body \ "useExact").asOpt[Boolean].getOrElse(true)
- val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(true)
-
- val intervalUnit = (body \ "intervalUnit").asOpt[String]
- // 2 day
- val ttl = (body \ "ttl").asOpt[Int].getOrElse(2 * 24 * 60 * 60)
- val dailyTtl = (body \ "dailyTtl").asOpt[Int]
- val regionMultiplier = (body \ "regionMultiplier").asOpt[Int].getOrElse(1)
-
- val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
- val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
- val rateThreshold = (body \ "rateThreshold").asOpt[Int]
-
- val rateActionId = {
- for {
- actionMap <- rateAction
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action)
- } yield {
- policy.id
- }
- }
- val rateBaseId = {
- for {
- actionMap <- rateBase
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action)
- } yield {
- policy.id
- }
- }
-
- val hbaseTable = {
- Seq(tablePrefixMap(version), service, ttl) ++ dailyTtl mkString "_"
- }
-
- // find label
- val itemType = Label.findByName(action, useCache = false) match {
- case Some(label) =>
- ItemType.withName(label.tgtColumnType.toUpperCase)
- case None =>
- val strItemType = (body \ "itemType").asOpt[String].getOrElse("STRING")
- ItemType.withName(strItemType.toUpperCase)
- }
- val policy = Counter(useFlag = true, version, service, action, itemType, autoComb = autoComb, dimension,
- useProfile = useProfile, bucketImpId, useRank = useRank, ttl, dailyTtl, Some(hbaseTable), intervalUnit,
- rateActionId, rateBaseId, rateThreshold)
-
- // prepare exact storage
- exactCounter(version).prepare(policy)
- if (useRank) {
- // prepare ranking storage
- rankingCounter(version).prepare(policy)
- }
- counterModel.createServiceAction(policy)
- Ok(Json.toJson(Map("msg" -> s"created $service/$action")))
- case Some(policy) =>
- Ok(Json.toJson(Map("msg" -> s"already exist $service/$action")))
- }
- }
-
- def getAction(service: String, action: String) = Action { request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(policy) =>
- Ok(Json.toJson(policy))
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def updateAction(service: String, action: String) = Action(s2parse.json) { request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(oldPolicy) =>
- val body = request.body
- val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(oldPolicy.autoComb)
- val dimension = (body \ "dimension").asOpt[String].getOrElse(oldPolicy.dimension)
- val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(oldPolicy.useProfile)
- val bucketImpId = (body \ "bucketImpId").asOpt[String] match {
- case Some(s) => Some(s)
- case None => oldPolicy.bucketImpId
- }
-
- val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(oldPolicy.useRank)
-
- val intervalUnit = (body \ "intervalUnit").asOpt[String] match {
- case Some(s) => Some(s)
- case None => oldPolicy.intervalUnit
- }
-
- val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
- val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
- val rateThreshold = (body \ "rateThreshold").asOpt[Int] match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateThreshold
- }
-
- val rateActionId = {
- for {
- actionMap <- rateAction
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- policy.id
- }
- } match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateActionId
- }
- val rateBaseId = {
- for {
- actionMap <- rateBase
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- policy.id
- }
- } match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateBaseId
- }
-
- // new counter
- val policy = Counter(id = oldPolicy.id, useFlag = oldPolicy.useFlag, oldPolicy.version, service, action, oldPolicy.itemType, autoComb = autoComb, dimension,
- useProfile = useProfile, bucketImpId, useRank = useRank, oldPolicy.ttl, oldPolicy.dailyTtl, oldPolicy.hbaseTable, intervalUnit,
- rateActionId, rateBaseId, rateThreshold)
-
- counterModel.updateServiceAction(policy)
- Ok(Json.toJson(Map("msg" -> s"updated $service/$action")))
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def prepareAction(service: String, action: String) = Action(s2parse.json) { request =>
- // for data migration
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(policy) =>
- val body = request.body
- val version = (body \ "version").as[Int].toByte
- if (version != policy.version) {
- // change table name
- val newTableName = Seq(tablePrefixMap(version), service, policy.ttl) ++ policy.dailyTtl mkString "_"
- val newPolicy = policy.copy(version = version, hbaseTable = Some(newTableName))
- exactCounter(version).prepare(newPolicy)
- if (newPolicy.useRank) {
- rankingCounter(version).prepare(newPolicy)
- }
- Ok(Json.toJson(Map("msg" -> s"prepare storage v$version $service/$action")))
- } else {
- Ok(Json.toJson(Map("msg" -> s"already prepared storage v$version $service/$action")))
- }
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def deleteAction(service: String, action: String) = Action.apply {
- {
- for {
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- Try {
- exactCounter(policy.version).destroy(policy)
- if (policy.useRank) {
- rankingCounter(policy.version).destroy(policy)
- }
- counterModel.deleteServiceAction(policy)
- } match {
- case Success(v) =>
- Ok(Json.toJson(Map("msg" -> s"deleted $service/$action")))
- case Failure(ex) =>
- throw ex
- }
- }
- }.getOrElse(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
-
- def getExactCountAsync(service: String, action: String, item: String) = Action.async { implicit request =>
- val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- val limit = getQueryString("limit", "1").toInt
-
- val qsSum = request.getQueryString("sum")
-
- val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
- val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
-
- val limitOpt = (optFrom, optTo) match {
- case (Some(_), Some(_)) =>
- None
- case _ =>
- Some(limit)
- }
-
- // find dimension
- lazy val dimQueryValues = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').filter(_.nonEmpty).toSet)
- }
-// Logger.warn(s"$dimQueryValues")
-
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
- val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- try {
-// Logger.warn(s"$tqs $qsSum")
- if (tqs.head.length > 1 && qsSum.isDefined) {
- getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum).map { jsVal =>
- Ok(jsVal)
- }
- } else {
- getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues).map { jsVal =>
- Ok(jsVal)
- }
- }
- } catch {
- case e: Exception =>
- throw e
-// Future.successful(BadRequest(s"$service, $action, $item"))
- }
- case None =>
- Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
- }
-
- /**
- * [{
- * "service": , "action", "itemIds": [], "interval": string, "limit": int, "from": ts, "to": ts,
- * "dimensions": [{"key": list[String]}]
- * }]
- * @return
- */
- private def parseExactCountParam(jsValue: JsValue) = {
- val service = (jsValue \ "service").as[String]
- val action = (jsValue \ "action").as[String]
- val itemIds = (jsValue \ "itemIds").as[Seq[String]]
- val intervals = (jsValue \ "intervals").asOpt[Seq[String]].getOrElse(Seq("t")).distinct.map(IntervalUnit.withName)
- val limit = (jsValue \ "limit").asOpt[Int].getOrElse(1)
- val from = (jsValue \ "from").asOpt[Long]
- val to = (jsValue \ "to").asOpt[Long]
- val sum = (jsValue \ "sum").asOpt[String]
- val dimensions = {
- for {
- dimension <- (jsValue \ "dimensions").asOpt[Seq[JsObject]].getOrElse(Nil)
- (k, vs) <- dimension.fields
- } yield {
- k -> vs.as[Seq[String]].toSet
- }
- }.toMap
- (service, action, itemIds, intervals, limit, from, to, dimensions, sum)
- }
-
- def getExactCountAsyncMulti = Action.async(s2parse.json) { implicit request =>
- val jsValue = request.body
- try {
- val futures = {
- for {
- jsObject <- jsValue.asOpt[List[JsObject]].getOrElse(Nil)
- (service, action, itemIds, intervalUnits, limit, from, to, dimQueryValues, qsSum) = parseExactCountParam(jsObject)
- optFrom = from.map(UnitConverter.toMillis)
- optTo = to.map(UnitConverter.toMillis)
- timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- policy <- counterModel.findByServiceAction(service, action).toSeq
- item <- itemIds
- } yield {
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
- val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- val limitOpt = (optFrom, optTo) match {
- case (Some(_), Some(_)) =>
- None
- case _ =>
- Some(limit)
- }
-
-// Logger.warn(s"$item, $limit, $optFrom, $optTo, $qsSum, $tqs")
-
- if (tqs.head.length > 1 && qsSum.isDefined) {
- getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum)
- } else {
- getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues)
- }
- }
- }
- Future.sequence(futures).map { rets =>
- Ok(Json.toJson(rets))
- }
- } catch {
- case e: Exception =>
- throw e
-// Future.successful(BadRequest(s"$jsValue"))
- }
- }
-
- private [controllers] def fetchedToResult(fetchedCounts: FetchedCountsGrouped, limitOpt: Option[Int]): Seq[ExactCounterIntervalItem] = {
- for {
- ((interval, dimKeyValues), values) <- fetchedCounts.intervalWithCountMap
- } yield {
- val counterItems = {
- val sortedItems = values.toSeq.sortBy { case (eq, v) => -eq.tq.ts }
- val limited = limitOpt match {
- case Some(limit) => sortedItems.take(limit)
- case None => sortedItems
- }
- for {
- (eq, value) <- limited
- } yield {
- ExactCounterItem(eq.tq.ts, value, value.toDouble)
- }
- }
- ExactCounterIntervalItem(interval.toString, dimKeyValues, counterItems)
- }
- }.toSeq
-
- private def decayedToResult(decayedCounts: DecayedCounts): Seq[ExactCounterIntervalItem] = {
- for {
- (eq, score) <- decayedCounts.qualifierWithCountMap
- } yield {
- ExactCounterIntervalItem(eq.tq.q.toString, eq.dimKeyValues, Seq(ExactCounterItem(eq.tq.ts, score.toLong, score)))
- }
- }.toSeq
-
- private def getExactCountToJs(policy: Counter,
- item: String,
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- limitOpt: Option[Int],
- dimQueryValues: Map[String, Set[String]]): Future[JsValue] = {
- exactCounter(policy.version).getCountsAsync(policy, Seq(item), timeRange, dimQueryValues).map { seq =>
- val items = {
- for {
- fetched <- seq
- } yield {
- fetchedToResult(fetched, limitOpt)
- }
- }.flatten
- Json.toJson(ExactCounterResult(ExactCounterResultMeta(policy.service, policy.action, item), items))
- }
- }
-
- private def getDecayedCountToJs(policy: Counter,
- item: String,
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQueryValues: Map[String, Set[String]],
- qsSum: Option[String]): Future[JsValue] = {
- exactCounter(policy.version).getDecayedCountsAsync(policy, Seq(item), timeRange, dimQueryValues, qsSum).map { seq =>
- val decayedCounts = seq.head
- val meta = ExactCounterResultMeta(policy.service, policy.action, decayedCounts.exactKey.itemKey)
- val intervalItems = decayedToResult(decayedCounts)
- Json.toJson(ExactCounterResult(meta, intervalItems))
- }
- }
-
- def getRankingCountAsync(service: String, action: String) = Action.async { implicit request =>
- lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- lazy val limit = getQueryString("limit", "1").toInt
- lazy val kValue = getQueryString("k", "10").toInt
-
- lazy val qsSum = request.getQueryString("sum")
-
- lazy val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
- lazy val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
-
- // find dimension
- lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').toList)
- }
-
- val dimensions = {
- for {
- values <- CartesianProduct(dimensionMap.values.toList).toSeq
- } yield {
- dimensionMap.keys.zip(values).toMap
- }
- }
-
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optTo)
- val dimKeys = {
- for {
- dimension <- dimensions
- } yield {
- dimension -> tqs.map(tq => RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension)))
- }
- }
-
- // if tqs has only 1 tq, do not apply sum function
- try {
- val rankResult = {
- if (tqs.length > 1 && qsSum.isDefined) {
- getSumRankCounterResultAsync(policy, dimKeys, kValue, qsSum)
- } else {
- // no summary
- Future.successful(getRankCounterResult(policy, dimKeys, kValue))
- }
- }
-
- rankResult.map { result =>
- Ok(Json.toJson(result))
- }
- } catch {
- case e: UnsupportedOperationException =>
- Future.successful(NotImplemented(Json.toJson(
- Map("msg" -> e.getMessage)
- )))
- case e: Throwable =>
- throw e
- }
- case None =>
- Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
- }
-
- def deleteRankingCount(service: String, action: String) = Action.async { implicit request =>
- lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- lazy val limit = getQueryString("limit", "1").toInt
-
- // find dimension
- lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').toList)
- }
-
- val dimensions = {
- for {
- values <- CartesianProduct(dimensionMap.values.toList).toSeq
- } yield {
- dimensionMap.keys.zip(values).toMap
- }
- }
-
- Future {
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit)
- val keys = {
- for {
- dimension <- dimensions
- tq <- tqs
- } yield {
- RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension))
- }
- }
-
- for {
- key <- keys
- } {
- rankingCounter(policy.version).delete(key)
- }
-
- Ok(JsObject(
- Seq(
- ("msg", Json.toJson(s"delete ranking in $service.$action")),
- ("items", Json.toJson({
- for {
- key <- keys
- } yield {
- s"${key.eq.tq.q}.${key.eq.tq.ts}.${key.eq.dimension}"
- }
- }))
- )
- ))
- case None =>
- NotFound(Json.toJson(
- Map("msg" -> s"$service.$action not found")
- ))
- }
- }
- }
-
- val reduceRateRankingValue = new ReduceMapValue[ExactKeyTrait, RateRankingValue](RateRankingValue.reduce, RateRankingValue(-1, -1))
-
- // change format
- private def getDecayedCountsAsync(policy: Counter,
- items: Seq[String],
- timeRange: (TimedQualifier, TimedQualifier),
- dimension: Map[String, String],
- qsSum: Option[String]): Future[Seq[(ExactKeyTrait, Double)]] = {
- exactCounter(policy.version).getDecayedCountsAsync(policy, items, Seq(timeRange), dimension.mapValues(s => Set(s)), qsSum).map { seq =>
- for {
- DecayedCounts(exactKey, qcMap) <- seq
- value <- qcMap.values
- } yield {
- exactKey -> value
- }
- }
- }
-
- def getSumRankCounterResultAsync(policy: Counter,
- dimKeys: Seq[(Map[String, String], Seq[RankingKey])],
- kValue: Int,
- qsSum: Option[String]): Future[RankCounterResult] = {
- val futures = {
- for {
- (dimension, keys) <- dimKeys
- } yield {
- val tqs = keys.map(rk => rk.eq.tq)
- val (tqFrom, tqTo) = (tqs.last, tqs.head)
- val items = rankingCounter(policy.version).getAllItems(keys, kValue)
-// Logger.warn(s"item count: ${items.length}")
- val future = {
- if (policy.isRateCounter) {
- val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
- val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
-
- val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
- }.toMap
- }
- val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
- }.toMap
- }
- futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
- reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
-// Logger.warn(s"$k -> $rrv")
- k -> rrv.rankingValue.score
- }.toSeq
- }
- }
- else if (policy.isTrendCounter) {
- val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
- val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
-
- val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
- }.toMap
- }
- val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom.add(-1), tqTo.add(-1)), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
- }.toMap
- }
- futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
- reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
-// Logger.warn(s"$k -> $rrv")
- k -> rrv.rankingValue.score
- }.toSeq
- }
- }
- else {
- getDecayedCountsAsync(policy, items, (tqFrom, tqTo), dimension, qsSum)
- }
- }
- future.map { keyWithScore =>
- val ranking = keyWithScore.sortBy(-_._2).take(kValue)
- val rankCounterItems = {
- for {
- idx <- ranking.indices
- (exactKey, score) = ranking(idx)
- } yield {
- val realId = policy.itemType match {
- case ItemType.BLOB => exactCounter(policy.version).getBlobValue(policy, exactKey.itemKey)
- .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} ${exactKey.itemKey}"))
- case _ => exactKey.itemKey
- }
- RankCounterItem(idx + 1, realId, score)
- }
- }
-
- val eq = ExactQualifier(tqFrom, dimension)
- RankCounterDimensionItem(eq.tq.q.toString, eq.tq.ts, eq.dimension, -1, rankCounterItems)
- }
- }
- }
-
- Future.sequence(futures).map { dimensionResultList =>
- RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
- }
- }
-
- def getRankCounterResult(policy: Counter, dimKeys: Seq[(Map[String, String], Seq[RankingKey])], kValue: Int): RankCounterResult = {
- val dimensionResultList = {
- for {
- (dimension, keys) <- dimKeys
- key <- keys
- } yield {
- val rankingValue = rankingCounter(policy.version).getTopK(key, kValue)
- val ranks = {
- for {
- rValue <- rankingValue.toSeq
- idx <- rValue.values.indices
- rank = idx + 1
- } yield {
- val (id, score) = rValue.values(idx)
- val realId = policy.itemType match {
- case ItemType.BLOB =>
- exactCounter(policy.version)
- .getBlobValue(policy, id)
- .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} $id"))
- case _ => id
- }
- RankCounterItem(rank, realId, score)
- }
- }
- val eq = key.eq
- val tq = eq.tq
- RankCounterDimensionItem(tq.q.toString, tq.ts, eq.dimension, rankingValue.map(v => v.totalScore).getOrElse(0d), ranks)
- }
- }
-
- RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
- }
-
- type Record = ProducerRecord[String, String]
-
- def incrementCount(service: String, action: String, item: String) = Action.async(s2parse.json) { request =>
- Future {
- /**
- * {
- * timestamp: Long
- * property: {}
- * value: Int
- * }
- */
- lazy val metaMap = Map("service" -> service, "action" -> action, "item" -> item)
- counterModel.findByServiceAction(service, action).map { policy =>
- val body = request.body
- try {
- val ts = (body \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis()).toString
- val dimension = (body \ "dimension").asOpt[JsValue].getOrElse(Json.obj())
- val property = (body \ "property").asOpt[JsValue].getOrElse(Json.obj())
-
- val msg = List(ts, service, action, item, dimension, property).mkString("\t")
-
- // produce to kafka
- // hash partitioner by key
- ExceptionHandler.enqueue(KafkaMessage(new Record(CounterConfig.KAFKA_TOPIC_COUNTER, s"$ts.$item", msg)))
-
- Ok(Json.toJson(
- Map(
- "meta" -> metaMap
- )
- ))
- }
- catch {
- case e: JsResultException =>
- BadRequest(Json.toJson(
- Map("msg" -> s"need timestamp.")
- ))
- }
- }.getOrElse {
- NotFound(Json.toJson(
- Map("msg" -> s"$service.$action not found")
- ))
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/EdgeController.scala b/app/controllers/EdgeController.scala
deleted file mode 100644
index 6279b2c..0000000
--- a/app/controllers/EdgeController.scala
+++ /dev/null
@@ -1,220 +0,0 @@
-package controllers
-
-import actors.QueueActor
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{LabelMeta, Label}
-import com.kakao.s2graph.core.types.LabelWithDirection
-import com.kakao.s2graph.core.utils.logger
-import config.Config
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.libs.json._
-import play.api.mvc.{Controller, Result}
-
-import scala.collection.Seq
-import scala.concurrent.Future
-import scala.util.{Failure, Success}
-
-object EdgeController extends Controller with RequestParser {
-
- import ExceptionHandler._
- import controllers.ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits._
-
- private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
-
- def tryMutates(jsValue: JsValue, operation: String, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
-
- else {
- try {
- logger.debug(s"$jsValue")
- val edges = toEdges(jsValue, operation)
- for (edge <- edges) {
- if (edge.isAsync)
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, edge, None))
- else
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, edge, None))
- }
-
- val edgesToStore = edges.filterNot(e => e.isAsync)
-
- if (withWait) {
- val rets = s2.mutateEdges(edgesToStore, withWait = true)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = edgesToStore.map { edge => QueueActor.router ! edge; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
- case e: Exception =>
- logger.error(s"mutateAndPublish: $e", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
- }
-
- def mutateAndPublish(str: String, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
-
- logger.debug(s"$str")
- val edgeStrs = str.split("\\n")
-
- var vertexCnt = 0L
- var edgeCnt = 0L
- try {
- val elements =
- for (edgeStr <- edgeStrs; str <- GraphUtil.parseString(edgeStr); element <- Graph.toGraphElement(str)) yield {
- element match {
- case v: Vertex => vertexCnt += 1
- case e: Edge => edgeCnt += 1
- }
- if (element.isAsync) {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, element, Some(str)))
- } else {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, element, Some(str)))
- }
- element
- }
-
- //FIXME:
- val elementsToStore = elements.filterNot(e => e.isAsync)
- if (withWait) {
- val rets = s2.mutateElements(elementsToStore, withWait)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = elementsToStore.map { element => QueueActor.router ! element; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
-
-
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
- case e: Throwable =>
- logger.error(s"mutateAndPublish: $e", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
-
- def mutateBulk() = withHeaderAsync(parse.text) { request =>
- mutateAndPublish(request.body)
- }
-
- def inserts() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert")
- }
-
- def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", withWait = true)
- }
-
- def insertsBulk() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insertBulk")
- }
-
- def deletes() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete")
- }
-
- def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", withWait = true)
- }
-
- def updates() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "update")
- }
-
- def updatesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "update", withWait = true)
- }
-
- def increments() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "increment")
- }
-
- def incrementCounts() = withHeaderAsync(jsonParser) { request =>
- val jsValue = request.body
- val edges = toEdges(jsValue, "incrementCount")
- s2.incrementCounts(edges).map { results =>
- val json = results.map { case (isSuccess, resultCount) =>
- Json.obj("success" -> isSuccess, "result" -> resultCount)
- }
-
- jsonResponse(Json.toJson(json))
- }
- }
-
- def deleteAll() = withHeaderAsync(jsonParser) { request =>
- deleteAllInner(request.body, withWait = false)
- }
-
- def deleteAllInner(jsValue: JsValue, withWait: Boolean) = {
- val deleteResults = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
-
- val labelName = (json \ "label").as[String]
- val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil)
- val direction = (json \ "direction").asOpt[String].getOrElse("out")
-
- val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
- val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
- val vertices = toVertices(labelName, direction, ids)
-
- /** logging for delete all request */
-
- val kafkaMessages = for {
- id <- ids
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
- val topic = if (label.isAsync) Config.KAFKA_LOG_TOPIC_ASYNC else Config.KAFKA_LOG_TOPIC
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
- ExceptionHandler.enqueues(kafkaMessages)
-
- val future = s2.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
- future.onFailure { case ex: Exception =>
- logger.error(s"[Error]: deleteAllInner failed.", ex)
- val kafkaMessages = for {
- id <- ids
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
- val topic = failTopic
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
- ExceptionHandler.enqueues(kafkaMessages)
- throw ex
- }
- if (withWait) {
- future.map { ret =>
- if (!ret) {
- logger.error(s"[Error]: deleteAllInner failed.")
- val kafkaMessages = for {
- id <- ids
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", direction).mkString("\t")
- val topic = failTopic
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
- ExceptionHandler.enqueues(kafkaMessages)
- false
- } else {
- true
- }
- }
- } else {
- Future.successful(true)
- }
- })
-
- deleteResults.map { rst =>
- logger.debug(s"deleteAllInner: $rst")
- Ok(s"deleted... ${rst.toString()}")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/ExperimentController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/ExperimentController.scala b/app/controllers/ExperimentController.scala
deleted file mode 100644
index 41614ef..0000000
--- a/app/controllers/ExperimentController.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-package controllers
-
-
-import java.net.URL
-
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.utils.logger
-import play.api.Play.current
-import play.api.libs.json.{JsObject, JsString, JsValue, Json}
-import play.api.libs.ws.WS
-import play.api.mvc._
-
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
-
-object ExperimentController extends Controller with RequestParser {
- val impressionKey = "S2-Impression-Id"
-
- import ApplicationController._
-
- def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(parse.anyContent) { request =>
- val bucketOpt = for {
- service <- Service.findByAccessToken(accessToken)
- experiment <- Experiment.findBy(service.id.get, experimentName)
- bucket <- experiment.findBucket(uuid)
- } yield bucket
-
- bucketOpt match {
- case None => Future.successful(NotFound("bucket is not found."))
- case Some(bucket) =>
- try {
- if (bucket.isGraphQuery) buildRequestInner(request, bucket, uuid)
- else buildRequest(request, bucket, uuid)
- } catch {
- case e: Exception =>
- logger.error(e.toString())
- Future.successful(BadRequest(s"wrong or missing template parameter: ${e.getMessage}"))
- }
- }
- }
-
- def makeRequestJson(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): JsValue = {
- var body = bucket.requestBody.replace("#uuid", uuid)
-
- for {
- requestKeyJson <- requestKeyJsonOpt
- jsObj <- requestKeyJson.asOpt[JsObject]
- (key, value) <- jsObj.fieldSet
- } {
- val replacement = value match {
- case JsString(s) => s
- case _ => value.toString
- }
- body = body.replace(key, replacement)
- }
-
- Json.parse(body)
- }
-
- private def buildRequestInner(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = {
- if (bucket.isEmpty) Future.successful(Ok(Json.obj("isEmpty" -> true)).withHeaders(impressionKey -> bucket.impressionId))
- else {
- val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid)
- val url = new URL(bucket.apiPath)
- val path = url.getPath()
-
- // dummy log for sampling
- val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody"
- logger.info(experimentLog)
-
- val response = path match {
- case "/graphs/getEdges" => controllers.QueryController.getEdgesInner(jsonBody)
- case "/graphs/getEdges/grouped" => controllers.QueryController.getEdgesWithGroupingInner(jsonBody)
- case "/graphs/getEdgesExcluded" => controllers.QueryController.getEdgesExcludedInner(jsonBody)
- case "/graphs/getEdgesExcluded/grouped" => controllers.QueryController.getEdgesExcludedWithGroupingInner(jsonBody)
- case "/graphs/checkEdges" => controllers.QueryController.checkEdgesInner(jsonBody)
- case "/graphs/getEdgesGrouped" => controllers.QueryController.getEdgesGroupedInner(jsonBody)
- case "/graphs/getEdgesGroupedExcluded" => controllers.QueryController.getEdgesGroupedExcludedInner(jsonBody)
- case "/graphs/getEdgesGroupedExcludedFormatted" => controllers.QueryController.getEdgesGroupedExcludedFormattedInner(jsonBody)
- }
- response.map { r => r.withHeaders(impressionKey -> bucket.impressionId) }
- }
- }
-
- private def toSimpleMap(map: Map[String, Seq[String]]): Map[String, String] = {
- for {
- (k, vs) <- map
- headVal <- vs.headOption
- } yield {
- k -> headVal
- }
- }
-
- private def buildRequest(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = {
- val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid)
-
- val url = bucket.apiPath
- val headers = request.headers.toSimpleMap.toSeq
- val verb = bucket.httpVerb.toUpperCase
- val qs = toSimpleMap(request.queryString).toSeq
-
- val ws = WS.url(url)
- .withMethod(verb)
- .withBody(jsonBody)
- .withHeaders(headers: _*)
- .withQueryString(qs: _*)
-
- ws.stream().map {
- case (proxyResponse, proxyBody) =>
- Result(ResponseHeader(proxyResponse.status, proxyResponse.headers.mapValues(_.toList.head)), proxyBody).withHeaders(impressionKey -> bucket.impressionId)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/JsonBodyParser.scala
----------------------------------------------------------------------
diff --git a/app/controllers/JsonBodyParser.scala b/app/controllers/JsonBodyParser.scala
deleted file mode 100644
index 3e3d40c..0000000
--- a/app/controllers/JsonBodyParser.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.Play
-import play.api.libs.iteratee.Iteratee
-import play.api.libs.json.{JsValue, Json}
-import play.api.mvc._
-
-import scala.concurrent.Future
-import scala.util.control.NonFatal
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 9. 1..
- */
-
-object s2parse extends BodyParsers {
-
- import parse._
-
- val defaultMaxTextLength = 1024 * 512
- val defaultMaxJsonLength = 1024 * 512
-
-// def json: BodyParser[JsValue] = json(DEFAULT_MAX_TEXT_LENGTH)
- def json: BodyParser[JsValue] = json(defaultMaxTextLength)
-
- def json(maxLength: Int): BodyParser[JsValue] = when(
- _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
- tolerantJson(maxLength),
- createBadResult("Expecting text/json or application/json body")
- )
-
- def tolerantJson(maxLength: Int): BodyParser[JsValue] =
- tolerantBodyParser[JsValue]("json", maxLength, "Invalid Json") { (request, bytes) =>
- // Encoding notes: RFC 4627 requires that JSON be encoded in Unicode, and states that whether that's
- // UTF-8, UTF-16 or UTF-32 can be auto detected by reading the first two bytes. So we ignore the declared
- // charset and don't decode, we passing the byte array as is because Jackson supports auto detection.
- Json.parse(bytes)
- }
-
- private def tolerantBodyParser[A](name: String, maxLength: Int, errorMessage: String)(parser: (RequestHeader, Array[Byte]) => A): BodyParser[A] =
- BodyParser(name + ", maxLength=" + maxLength) { request =>
- import play.api.libs.iteratee.Execution.Implicits.trampoline
- import play.api.libs.iteratee.Traversable
-
- import scala.util.control.Exception._
-
- val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] =
- Traversable.takeUpTo[Array[Byte]](maxLength).transform(
- Iteratee.consume[Array[Byte]]().map { bytes =>
- allCatch[A].either {
- parser(request, bytes)
- }.left.map {
- case NonFatal(e) =>
- val txt = new String(bytes)
- logger.error(s"$errorMessage: $txt", e)
- createBadResult(s"$errorMessage: $e")(request)
- case t => throw t
- }
- }
- ).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
-
- bodyParser.mapM {
- case Left(tooLarge) => Future.successful(Left(tooLarge))
- case Right(Left(badResult)) => badResult.map(Left.apply)
- case Right(Right(body)) => Future.successful(Right(body))
- }
- }
-
- private def createBadResult(msg: String): RequestHeader => Future[Result] = { request =>
- Play.maybeApplication.map(_.global.onBadRequest(request, msg))
- .getOrElse(Future.successful(Results.BadRequest))
- }
-}
[2/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/conf/logger.xml
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/logger.xml b/s2rest_play/conf/logger.xml
new file mode 100644
index 0000000..2d767c2
--- /dev/null
+++ b/s2rest_play/conf/logger.xml
@@ -0,0 +1,83 @@
+<configuration>
+
+ <conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel"/>
+
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/application.log</file>
+
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>logs/application.%i.log</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>500MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ </appender>
+
+
+ <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/error.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>logs/error.log.%d.%i</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>500MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>3</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%coloredLevel %logger{15} - %message%n%xException%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="ACTOR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/actor.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>logs/actor.log.%d.%i</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>200MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>7</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="play" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="application" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="error" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="ERROR"/>
+ </logger>
+
+ <logger name="actor" level="INFO">
+ <appender-ref ref="ACTOR"/>
+ </logger>
+
+ <logger name="akka" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/conf/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/reference.conf b/s2rest_play/conf/reference.conf
new file mode 100644
index 0000000..a992e5d
--- /dev/null
+++ b/s2rest_play/conf/reference.conf
@@ -0,0 +1,131 @@
+# This is the main configuration file for the application.
+# ~~~~~
+
+# Secret key
+# ~~~~~
+# The secret key is used to secure cryptographics functions.
+#
+# This must be changed for production, but we recommend not changing it in this file.
+#
+# See http://www.playframework.com/documentation/latest/ApplicationSecret for more details.
+application.secret="/`==g^yr2DNnZGK_L^rguLZeR`60uLOVgY@OhyTv:maatl:Tl>9or/d1xME3b/Pi"
+
+# The application languages
+# ~~~~~
+application.langs="en"
+
+# Global object class
+# ~~~~~
+# Define the Global object class for this application.
+# Default to Global in the root package.
+# application.global=Global
+
+# Router
+# ~~~~~
+# Define the Router object to use for this application.
+# This router will be looked up first when the application is starting up,
+# so make sure this is the entry point.
+# Furthermore, it's assumed your route file is named properly.
+# So for an application router like `my.application.Router`,
+# you may need to define a router file `conf/my.application.routes`.
+# Default to Routes in the root package (and conf/routes)
+# application.router=my.application.Routes
+
+# Database configuration
+# ~~~~~
+# You can declare as many datasources as you want.
+# By convention, the default datasource is named `default`
+#
+# db.default.driver=org.h2.Driver
+# db.default.url="jdbc:h2:mem:play"
+# db.default.user=sa
+# db.default.password=""
+
+# Evolutions
+# ~~~~~
+# You can disable evolutions if needed
+# evolutionplugin=disabled
+
+# Logger
+# ~~~~~
+# You can also configure logback (http://logback.qos.ch/),
+# by providing an application-logger.xml file in the conf directory.
+
+# Root logger:
+logger.root=ERROR
+
+# Logger used by the framework:
+logger.play=INFO
+
+# Logger provided to your application:
+logger.application=DEBUG
+
+# APP PHASE
+phase=dev
+host=localhost
+
+# DB
+s2graph.models.table.name="models-dev"
+hbase.zookeeper.quorum=${host}
+db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
+# Query server
+is.query.server=true
+is.write.server=true
+query.hard.limit=100000
+
+# Local Cache
+cache.ttl.seconds=60
+cache.max.size=100000
+
+# HBASE
+#hbase.client.operation.timeout=1000
+#async.hbase.client.flush.interval=100
+hbase.table.compression.algorithm="gz"
+
+# Asynchbase
+hbase.client.retries.number=100
+hbase.rpcs.buffered_flush_interval=100
+hbase.rpc.timeout=0
+#hbase.nsre.high_watermark=1000000
+#hbase.timer.tick=5
+#hbase.timer.ticks_per_wheel=5
+
+# Kafka
+kafka.metadata.broker.list=${host}
+kafka.producer.pool.size=0
+
+# HTTP
+parsers.text.maxLength=512K
+parsers.json.maxLength=512K
+trustxforwarded=false
+
+# Local Queue Actor
+local.queue.actor.max.queue.size=100000
+local.queue.actor.rate.limit=1000000
+
+# local retry number
+max.retry.number=100
+max.back.off=50
+delete.all.fetch.size=10000
+hbase.fail.prob=-1.0
+
+# max allowd edges for deleteAll is multiply of above two configuration.
+
+# set global obejct package, TODO: remove global
+application.global=com.kakao.s2graph.rest.Global
+
+akka {
+ loggers = ["akka.event.slf4j.Slf4jLogger"]
+ loglevel = "DEBUG"
+}
+
+
+# Future cache.
+future.cache.max.size=100000
+future.cache.expire.after.write=10000
+future.cache.expire.after.access=5000
+
+
+# Counter
+redis.instances = [${host}]
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/conf/routes
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/routes b/s2rest_play/conf/routes
new file mode 100644
index 0000000..df4a1ee
--- /dev/null
+++ b/s2rest_play/conf/routes
@@ -0,0 +1,124 @@
+# Routes
+# This file defines all application routes (Higher priority routes first)
+# ~~~~
+
+
+# publish
+#POST /publish/:topic controllers.PublishController.publish(topic)
+POST /publish/:topic controllers.PublishController.mutateBulk(topic)
+POST /publishOnly/:topic controllers.PublishController.publishOnly(topic)
+
+#### Health Check
+#GET /health_check.html controllers.Assets.at(path="/public", file="health_check.html")
+GET /health_check.html controllers.ApplicationController.healthCheck()
+PUT /health_check/:isHealthy controllers.ApplicationController.updateHealthCheck(isHealthy: Boolean)
+
+## Edge
+POST /graphs/edges/insert controllers.EdgeController.inserts()
+POST /graphs/edges/insertWithWait controllers.EdgeController.insertsWithWait()
+POST /graphs/edges/insertBulk controllers.EdgeController.insertsBulk()
+POST /graphs/edges/delete controllers.EdgeController.deletes()
+POST /graphs/edges/deleteWithWait controllers.EdgeController.deletesWithWait()
+POST /graphs/edges/deleteAll controllers.EdgeController.deleteAll()
+POST /graphs/edges/update controllers.EdgeController.updates()
+POST /graphs/edges/updateWithWait controllers.EdgeController.updatesWithWait()
+POST /graphs/edges/increment controllers.EdgeController.increments()
+POST /graphs/edges/incrementCount controllers.EdgeController.incrementCounts()
+POST /graphs/edges/bulk controllers.EdgeController.mutateBulk()
+
+## Vertex
+POST /graphs/vertices/insert controllers.VertexController.inserts()
+POST /graphs/vertices/insertWithWait controllers.VertexController.insertsWithWait()
+POST /graphs/vertices/insert/:serviceName/:columnName controllers.VertexController.insertsSimple(serviceName, columnName)
+POST /graphs/vertices/delete controllers.VertexController.deletes()
+POST /graphs/vertices/deleteWithWait controllers.VertexController.deletesWithWait()
+POST /graphs/vertices/delete/:serviceName/:columnName controllers.VertexController.deletesSimple(serviceName, columnName)
+POST /graphs/vertices/deleteAll controllers.VertexController.deletesAll()
+POST /graphs/vertices/deleteAll/:serviceName/:columnName controllers.VertexController.deletesAllSimple(serviceName, columnName)
+
+
+### SELECT Edges
+POST /graphs/getEdges controllers.QueryController.getEdges()
+POST /graphs/getEdges/grouped controllers.QueryController.getEdgesWithGrouping()
+POST /graphs/getEdgesExcluded controllers.QueryController.getEdgesExcluded()
+POST /graphs/getEdgesExcluded/grouped controllers.QueryController.getEdgesExcludedWithGrouping()
+POST /graphs/checkEdges controllers.QueryController.checkEdges()
+
+### this will be deprecated
+POST /graphs/getEdgesGrouped controllers.QueryController.getEdgesGrouped()
+POST /graphs/getEdgesGroupedExcluded controllers.QueryController.getEdgesGroupedExcluded()
+POST /graphs/getEdgesGroupedExcludedFormatted controllers.QueryController.getEdgesGroupedExcludedFormatted()
+GET /graphs/getEdge/:srcId/:tgtId/:labelName/:direction controllers.QueryController.getEdge(srcId, tgtId, labelName, direction)
+
+
+### SELECT Vertices
+#POST /graphs/getVertex controllers.QueryController.getVertex()
+POST /graphs/getVertices controllers.QueryController.getVertices()
+
+
+#### ADMIN
+POST /graphs/createService controllers.AdminController.createService()
+GET /graphs/getService/:serviceName controllers.AdminController.getService(serviceName)
+GET /graphs/getLabels/:serviceName controllers.AdminController.getLabels(serviceName)
+POST /graphs/createLabel controllers.AdminController.createLabel()
+POST /graphs/addIndex controllers.AdminController.addIndex()
+GET /graphs/getLabel/:labelName controllers.AdminController.getLabel(labelName)
+PUT /graphs/deleteLabel/:labelName controllers.AdminController.deleteLabel(labelName)
+
+POST /graphs/addProp/:labelName controllers.AdminController.addProp(labelName)
+POST /graphs/createServiceColumn controllers.AdminController.createServiceColumn()
+PUT /graphs/deleteServiceColumn/:serviceName/:columnName controllers.AdminController.deleteServiceColumn(serviceName, columnName)
+POST /graphs/addServiceColumnProp/:serviceName/:columnName controllers.AdminController.addServiceColumnProp(serviceName, columnName)
+POST /graphs/addServiceColumnProps/:serviceName/:columnName controllers.AdminController.addServiceColumnProps(serviceName, columnName)
+GET /graphs/getServiceColumn/:serviceName/:columnName controllers.AdminController.getServiceColumn(serviceName, columnName)
+POST /graphs/createHTable controllers.AdminController.createHTable()
+
+
+
+
+#### TEST
+#GET /graphs/testGetEdges/:label/:limit/:friendCntStep controllers.QueryController.testGetEdges(label, limit: Int, friendCntStep: Int)
+#GET /graphs/testGetEdges2/:label1/:limit1/:label2/:limit2 controllers.QueryController.testGetEdges2(label1, limit1: Int, label2, limit2: Int)
+#GET /graphs/testGetEdges3/:label1/:limit1/:label2/:limit2/:label3/:limit3 controllers.QueryController.testGetEdges3(label1, limit1: Int, label2, limit2: Int, label3, limit3: Int)
+POST /ping controllers.TestController.ping()
+POST /pingAsync controllers.TestController.pingAsync()
+GET /graphs/testId controllers.TestController.getRandomId()
+
+# Map static resources from the /public folder to the /assets URL path
+GET /images/*file controllers.Assets.at(path="/public/images", file)
+GET /javascripts/*file controllers.Assets.at(path="/public/javascripts", file)
+GET /stylesheets/*file controllers.Assets.at(path="/public/stylesheets", file)
+GET /font-awesome-4.1.0/*file controllers.Assets.at(path="/public/font-awesome-4.1.0", file)
+GET /swagger/*file controllers.Assets.at(path="/public/swagger-ui", file)
+
+
+# AdminController API
+#GET /admin/services controllers.AdminController.allServices
+GET /admin/labels/:serviceName controllers.AdminController.getLabels(serviceName)
+#POST /admin/labels/delete/:zkAddr/:tableName/:labelIds/:minTs/:maxTs controllers.AdminController.deleteEdges(zkAddr, tableName, labelIds, minTs: Long, maxTs: Long)
+#POST /admin/labels/deleteAll/:zkAddr/:tableName/:minTs/:maxTs controllers.AdminController.deleteAllEdges(zkAddr, tableName, minTs: Long, maxTs: Long)
+#POST /admin/swapLabel/:oldLabelName/:newLabelName controllers.AdminController.swapLabel(oldLabelName, newLabelName)
+#GET /admin/reloadLabel/:labelName controllers.AdminController.reloadLabel(labelName)
+#POST /admin/getEdges controllers.AdminController.getEdges()
+POST /graphs/copyLabel/:oldLabelName/:newLabelName controllers.AdminController.copyLabel(oldLabelName, newLabelName)
+POST /graphs/renameLabel/:oldLabelName/:newLabelName controllers.AdminController.renameLabel(oldLabelName, newLabelName)
+POST /graphs/updateHTable/:labelName/:newHTableName controllers.AdminController.updateHTable(labelName, newHTableName)
+PUT /graphs/loadCache controllers.AdminController.loadCache()
+
+
+# Counter Admin API
+POST /counter/v1/:service/:action controllers.CounterController.createAction(service, action)
+GET /counter/v1/:service/:action controllers.CounterController.getAction(service, action)
+PUT /counter/v1/:service/:action controllers.CounterController.updateAction(service, action)
+PUT /counter/v1/:service/:action/prepare controllers.CounterController.prepareAction(service, action)
+DELETE /counter/v1/:service/:action controllers.CounterController.deleteAction(service, action)
+
+# Counter API
+GET /counter/v1/:service/:action/ranking controllers.CounterController.getRankingCountAsync(service, action)
+DELETE /counter/v1/:service/:action/ranking controllers.CounterController.deleteRankingCount(service, action)
+GET /counter/v1/:service/:action/:item controllers.CounterController.getExactCountAsync(service, action, item)
+PUT /counter/v1/:service/:action/:item controllers.CounterController.incrementCount(service, action, item)
+POST /counter/v1/mget controllers.CounterController.getExactCountAsyncMulti()
+
+# Experiment API
+POST /graphs/experiment/:accessToken/:experimentName/:uuid controllers.ExperimentController.experiment(accessToken, experimentName, uuid)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/conf/test.conf
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/test.conf b/s2rest_play/conf/test.conf
new file mode 100644
index 0000000..c51baef
--- /dev/null
+++ b/s2rest_play/conf/test.conf
@@ -0,0 +1,2 @@
+max.retry.number=10000
+hbase.fail.prob=0.1
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/benchmark/BenchmarkCommon.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/BenchmarkCommon.scala b/s2rest_play/test/benchmark/BenchmarkCommon.scala
new file mode 100644
index 0000000..9dd69ec
--- /dev/null
+++ b/s2rest_play/test/benchmark/BenchmarkCommon.scala
@@ -0,0 +1,15 @@
+package benchmark
+
+import org.specs2.mutable.Specification
+
+trait BenchmarkCommon extends Specification {
+ val wrapStr = s"\n=================================================="
+
+ def duration[T](prefix: String = "")(block: => T) = {
+ val startTs = System.currentTimeMillis()
+ val ret = block
+ val endTs = System.currentTimeMillis()
+ println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr")
+ ret
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/benchmark/GraphUtilSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/GraphUtilSpec.scala b/s2rest_play/test/benchmark/GraphUtilSpec.scala
new file mode 100644
index 0000000..b5ce93a
--- /dev/null
+++ b/s2rest_play/test/benchmark/GraphUtilSpec.scala
@@ -0,0 +1,125 @@
+package benchmark
+
+import com.kakao.s2graph.core.{Management, GraphUtil}
+import com.kakao.s2graph.core.types.{SourceVertexId, HBaseType, InnerVal, VertexId}
+import org.apache.hadoop.hbase.util.Bytes
+import play.api.test.{FakeApplication, PlaySpecification}
+
+import scala.collection.mutable
+import scala.collection.mutable.ListBuffer
+import scala.util.Random
+
+class GraphUtilSpec extends BenchmarkCommon with PlaySpecification {
+
+ def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean =
+ Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0
+
+ def betweenShort(value: Short, start: Short, end: Short): Boolean =
+ start <= value && value <= end
+
+
+ "GraphUtil" should {
+ "test murmur3 hash function distribution" in {
+ val testNum = 1000000
+ val bucketSize = Short.MaxValue / 40
+ val countsNew = new mutable.HashMap[Int, Int]()
+ val counts = new mutable.HashMap[Int, Int]()
+ for {
+ i <- (0 until testNum)
+ } {
+ val h = GraphUtil.murmur3(i.toString) / bucketSize
+ val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize
+ counts += (h -> (counts.getOrElse(h, 0) + 1))
+ countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1))
+ }
+ val all = counts.toList.sortBy { case (bucket, count) => count }.reverse
+ val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse
+ val top = all.take(10)
+ val bottom = all.takeRight(10)
+ val topNew = allNew.take(10)
+ val bottomNew = allNew.takeRight(10)
+ println(s"Top: $top")
+ println(s"Bottom: $bottom")
+ println("-" * 50)
+ println(s"TopNew: $topNew")
+ println(s"Bottom: $bottomNew")
+ true
+ }
+
+ "test murmur hash skew2" in {
+ running(FakeApplication()) {
+ import HBaseType._
+ val testNum = 1000000L
+ val regionCount = 40
+ val window = Int.MaxValue / regionCount
+ val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]()
+ for {
+ i <- (0 until regionCount)
+ } yield {
+ val startKey = Bytes.toBytes(i * window)
+ val endKey = Bytes.toBytes((i + 1) * window)
+ rangeBytes += (startKey.toList -> endKey.toList)
+ }
+
+
+
+ val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]()
+ val counts = new collection.mutable.HashMap[Short, Long]()
+ stats += (0 -> (rangeBytes.head -> 0L))
+
+ for (i <- (0L until testNum)) {
+ val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION))
+ val bytes = vertexId.bytes
+ val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString())
+ val shortVal = counts.getOrElse(shortKey, 0L) + 1L
+ counts += (shortKey -> shortVal)
+ var j = 0
+ var found = false
+ while (j < rangeBytes.size && !found) {
+ val (start, end) = rangeBytes(j)
+ if (between(bytes, start.toArray, end.toArray)) {
+ found = true
+ }
+ j += 1
+ }
+ val head = rangeBytes(j - 1)
+ val key = j - 1
+ val value = stats.get(key) match {
+ case None => 0L
+ case Some(v) => v._2 + 1
+ }
+ stats += (key -> (head, value))
+ }
+ val sorted = stats.toList.sortBy(kv => kv._2._2).reverse
+ println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
+ sorted.foreach { case (idx, ((start, end), cnt)) =>
+ val startShort = Bytes.toShort(start.take(2).toArray)
+ val endShort = Bytes.toShort(end.take(2).toArray)
+ val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
+ println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
+
+ }
+ println("\n" * 10)
+ println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
+ stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) =>
+ val startShort = Bytes.toShort(start.take(2).toArray)
+ val endShort = Bytes.toShort(end.take(2).toArray)
+ val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
+ println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
+
+ }
+ }
+ true
+ }
+
+ "Bytes compareTo" in {
+ val x = Array[Byte](11, -12, -26, -14, -23)
+ val startKey = Array[Byte](0, 0, 0, 0)
+ val endKey = Array[Byte](12, -52, -52, -52)
+ println(Bytes.compareTo(startKey, x))
+ println(Bytes.compareTo(endKey, x))
+ true
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
new file mode 100644
index 0000000..bf24ed7
--- /dev/null
+++ b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
@@ -0,0 +1,46 @@
+package benchmark
+
+import play.api.libs.json.JsNumber
+import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
+import play.libs.Json
+
+class JsonBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
+ "to json" should {
+ implicit val app = FakeApplication()
+
+ "json benchmark" in new WithApplication(app) {
+
+ duration("map to json") {
+ (0 to 100) foreach { n =>
+ val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ Json.toJson(numberMaps)
+ }
+ }
+
+ duration("directMakeJson") {
+ (0 to 100) foreach { n =>
+ var jsObj = play.api.libs.json.Json.obj()
+ (0 to 100).foreach { n =>
+ jsObj += (n.toString -> JsNumber(n * n))
+ }
+ }
+ }
+
+ duration("map to json 2") {
+ (0 to 500) foreach { n =>
+ val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ Json.toJson(numberMaps)
+ }
+ }
+
+ duration("directMakeJson 2") {
+ (0 to 500) foreach { n =>
+ var jsObj = play.api.libs.json.Json.obj()
+ (0 to 100).foreach { n =>
+ jsObj += (n.toString -> JsNumber(n * n))
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
new file mode 100644
index 0000000..d38ff5e
--- /dev/null
+++ b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
@@ -0,0 +1,103 @@
+package benchmark
+
+import com.kakao.s2graph.core.OrderingUtil._
+import com.kakao.s2graph.core.SeqMultiOrdering
+import play.api.libs.json.{JsNumber, JsValue}
+import play.api.test.PlaySpecification
+import play.api.{Application => PlayApplication}
+
+import scala.util.Random
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 9..
+ */
+class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
+ "OrderingUtilBenchmarkSpec" should {
+
+ "performance MultiOrdering any" >> {
+ val tupLs = (0 until 10) map { i =>
+ Random.nextDouble() -> Random.nextLong()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(tup._1, tup._2)
+ }
+
+ val sorted1 = duration("TupleOrdering double,long") {
+ (0 until 10000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }.map { x => x._1 }
+
+ val sorted2 = duration("MultiOrdering double,long") {
+ (0 until 10000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
+ }
+ seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
+ }.map { x => x.head }
+
+ sorted1.toString() must_== sorted2.toString()
+ }
+
+ "performance MultiOrdering double" >> {
+ val tupLs = (0 until 500) map { i =>
+ Random.nextDouble() -> Random.nextDouble()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(tup._1, tup._2)
+ }
+
+ duration("MultiOrdering double") {
+ (0 until 10000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
+ }
+ }
+
+ duration("TupleOrdering double") {
+ (0 until 10000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ }
+
+ 1 must_== 1
+ }
+
+ "performance MultiOrdering jsvalue" >> {
+ val tupLs = (0 until 500) map { i =>
+ Random.nextDouble() -> Random.nextLong()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(JsNumber(tup._1), JsNumber(tup._2))
+ }
+
+ val sorted1 = duration("TupleOrdering double,long") {
+ (0 until 10000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+
+ val sorted2 = duration("MultiOrdering jsvalue") {
+ (0 until 10000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
+ }
+ seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
+ }
+
+ 1 must_== 1
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
new file mode 100644
index 0000000..0c27a2a
--- /dev/null
+++ b/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
@@ -0,0 +1,85 @@
+package benchmark
+import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
+import scala.annotation.tailrec
+import scala.util.Random
+
+class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
+ "sample" should {
+ implicit val app = FakeApplication()
+
+ "sample benchmark" in new WithApplication(app) {
+ @tailrec
+ def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (set.size == n) set
+ else randomInt(n, range, set + Random.nextInt(range))
+ }
+
+ // sample using random array
+ def randomArraySample[T](num: Int, ls: List[T]): List[T] = {
+ val randomNum = randomInt(num, ls.size)
+ var sample = List.empty[T]
+ var idx = 0
+ ls.foreach { e =>
+ if (randomNum.contains(idx)) sample = e :: sample
+ idx += 1
+ }
+ sample
+ }
+
+ // sample using shuffle
+ def shuffleSample[T](num: Int, ls: List[T]): List[T] = {
+ Random.shuffle(ls).take(num)
+ }
+
+ // sample using random number generation
+ def rngSample[T](num: Int, ls: List[T]): List[T] = {
+ var sampled = List.empty[T]
+ val N = ls.size // population
+ var t = 0 // total input records dealt with
+ var m = 0 // number of items selected so far
+
+ while (m < num) {
+ val u = Random.nextDouble()
+ if ( (N - t)*u < num - m) {
+ sampled = ls(t) :: sampled
+ m += 1
+ }
+ t += 1
+ }
+ sampled
+ }
+
+ // test data
+ val testLimit = 500000
+ val testNum = 10
+ val testData = (0 to 1000).toList
+
+ // dummy for warm-up
+ (0 to testLimit) foreach { n =>
+ randomArraySample(testNum, testData)
+ shuffleSample(testNum, testData)
+ rngSample(testNum, testData)
+ }
+
+ duration("Random Array Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = randomArraySample(testNum, testData)
+ }
+ }
+
+ duration("Shuffle Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = shuffleSample(testNum, testData)
+ }
+ }
+
+ duration("RNG Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = rngSample(testNum, testData)
+ }
+ }
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2rest_play/test/controllers/PostProcessSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/PostProcessSpec.scala b/s2rest_play/test/controllers/PostProcessSpec.scala
new file mode 100644
index 0000000..cea132a
--- /dev/null
+++ b/s2rest_play/test/controllers/PostProcessSpec.scala
@@ -0,0 +1,112 @@
+package controllers
+
+import com.kakao.s2graph.core.{OrderingUtil, SeqMultiOrdering}
+import play.api.libs.json.{JsNumber, JsString, JsValue}
+import play.api.test.PlaySpecification
+
+class PostProcessSpec extends PlaySpecification {
+ import OrderingUtil._
+
+ "test order by json" >> {
+ val jsLs: Seq[Seq[JsValue]] = Seq(
+ Seq(JsNumber(0), JsString("a")),
+ Seq(JsNumber(0), JsString("b")),
+ Seq(JsNumber(1), JsString("a")),
+ Seq(JsNumber(1), JsString("b")),
+ Seq(JsNumber(2), JsString("c"))
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[Seq[JsValue]] = Seq(
+ Seq(JsNumber(2), JsString("c")),
+ Seq(JsNumber(1), JsString("a")),
+ Seq(JsNumber(1), JsString("b")),
+ Seq(JsNumber(0), JsString("a")),
+ Seq(JsNumber(0), JsString("b"))
+ )
+
+ val orderParam: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] {
+ override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = {
+ val xe = x.iterator
+ val ye = y.iterator
+ val oe = orderParam.iterator
+
+ while (xe.hasNext && ye.hasNext && oe.hasNext) {
+ val (xev, yev) = oe.next() match {
+ case true => xe.next() -> ye.next()
+ case false => ye.next() -> xe.next()
+ }
+ val res = (xev, yev) match {
+ case (JsNumber(xv), JsNumber(yv)) =>
+ Ordering[BigDecimal].compare(xv, yv)
+ case (JsString(xv), JsString(yv)) =>
+ Ordering[String].compare(xv, yv)
+ case _ => throw new Exception("type mismatch")
+ }
+ if (res != 0) return res
+ }
+
+ Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
+ }
+ })
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+
+ "test order by primitive type" >> {
+ val jsLs: Seq[Seq[Any]] = Seq(
+ Seq(0, "a"),
+ Seq(0, "b"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(2, "c")
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[Seq[Any]] = Seq(
+ Seq(2, "c"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(0, "a"),
+ Seq(0, "b")
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+
+ "test order by primitive type with short ascending list" >> {
+ val jsLs: Seq[Seq[Any]] = Seq(
+ Seq(0, "a"),
+ Seq(1, "b"),
+ Seq(0, "b"),
+ Seq(1, "a"),
+ Seq(2, "c"),
+ Seq(1, "c"),
+ Seq(1, "d"),
+ Seq(1, "f"),
+ Seq(1, "e")
+ )
+
+ // number descending, string ascending(default)
+ val sortedJsLs: Seq[Seq[Any]] = Seq(
+ Seq(2, "c"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(1, "c"),
+ Seq(1, "d"),
+ Seq(1, "e"),
+ Seq(1, "f"),
+ Seq(0, "a"),
+ Seq(0, "b")
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false)
+ val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/BenchmarkCommon.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/BenchmarkCommon.scala b/test/benchmark/BenchmarkCommon.scala
deleted file mode 100644
index cd465e8..0000000
--- a/test/benchmark/BenchmarkCommon.scala
+++ /dev/null
@@ -1,18 +0,0 @@
-package benchmark
-
-import org.specs2.mutable.Specification
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6..
- */
-trait BenchmarkCommon extends Specification {
- val wrapStr = s"\n=================================================="
-
- def duration[T](prefix: String = "")(block: => T) = {
- val startTs = System.currentTimeMillis()
- val ret = block
- val endTs = System.currentTimeMillis()
- println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr")
- ret
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/GraphUtilSpec.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/GraphUtilSpec.scala b/test/benchmark/GraphUtilSpec.scala
deleted file mode 100644
index b5ce93a..0000000
--- a/test/benchmark/GraphUtilSpec.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-package benchmark
-
-import com.kakao.s2graph.core.{Management, GraphUtil}
-import com.kakao.s2graph.core.types.{SourceVertexId, HBaseType, InnerVal, VertexId}
-import org.apache.hadoop.hbase.util.Bytes
-import play.api.test.{FakeApplication, PlaySpecification}
-
-import scala.collection.mutable
-import scala.collection.mutable.ListBuffer
-import scala.util.Random
-
-class GraphUtilSpec extends BenchmarkCommon with PlaySpecification {
-
- def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean =
- Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0
-
- def betweenShort(value: Short, start: Short, end: Short): Boolean =
- start <= value && value <= end
-
-
- "GraphUtil" should {
- "test murmur3 hash function distribution" in {
- val testNum = 1000000
- val bucketSize = Short.MaxValue / 40
- val countsNew = new mutable.HashMap[Int, Int]()
- val counts = new mutable.HashMap[Int, Int]()
- for {
- i <- (0 until testNum)
- } {
- val h = GraphUtil.murmur3(i.toString) / bucketSize
- val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize
- counts += (h -> (counts.getOrElse(h, 0) + 1))
- countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1))
- }
- val all = counts.toList.sortBy { case (bucket, count) => count }.reverse
- val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse
- val top = all.take(10)
- val bottom = all.takeRight(10)
- val topNew = allNew.take(10)
- val bottomNew = allNew.takeRight(10)
- println(s"Top: $top")
- println(s"Bottom: $bottom")
- println("-" * 50)
- println(s"TopNew: $topNew")
- println(s"Bottom: $bottomNew")
- true
- }
-
- "test murmur hash skew2" in {
- running(FakeApplication()) {
- import HBaseType._
- val testNum = 1000000L
- val regionCount = 40
- val window = Int.MaxValue / regionCount
- val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]()
- for {
- i <- (0 until regionCount)
- } yield {
- val startKey = Bytes.toBytes(i * window)
- val endKey = Bytes.toBytes((i + 1) * window)
- rangeBytes += (startKey.toList -> endKey.toList)
- }
-
-
-
- val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]()
- val counts = new collection.mutable.HashMap[Short, Long]()
- stats += (0 -> (rangeBytes.head -> 0L))
-
- for (i <- (0L until testNum)) {
- val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION))
- val bytes = vertexId.bytes
- val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString())
- val shortVal = counts.getOrElse(shortKey, 0L) + 1L
- counts += (shortKey -> shortVal)
- var j = 0
- var found = false
- while (j < rangeBytes.size && !found) {
- val (start, end) = rangeBytes(j)
- if (between(bytes, start.toArray, end.toArray)) {
- found = true
- }
- j += 1
- }
- val head = rangeBytes(j - 1)
- val key = j - 1
- val value = stats.get(key) match {
- case None => 0L
- case Some(v) => v._2 + 1
- }
- stats += (key -> (head, value))
- }
- val sorted = stats.toList.sortBy(kv => kv._2._2).reverse
- println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
- sorted.foreach { case (idx, ((start, end), cnt)) =>
- val startShort = Bytes.toShort(start.take(2).toArray)
- val endShort = Bytes.toShort(end.take(2).toArray)
- val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
- println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
-
- }
- println("\n" * 10)
- println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
- stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) =>
- val startShort = Bytes.toShort(start.take(2).toArray)
- val endShort = Bytes.toShort(end.take(2).toArray)
- val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
- println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
-
- }
- }
- true
- }
-
- "Bytes compareTo" in {
- val x = Array[Byte](11, -12, -26, -14, -23)
- val startKey = Array[Byte](0, 0, 0, 0)
- val endKey = Array[Byte](12, -52, -52, -52)
- println(Bytes.compareTo(startKey, x))
- println(Bytes.compareTo(endKey, x))
- true
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/JsonBenchmarkSpec.scala b/test/benchmark/JsonBenchmarkSpec.scala
deleted file mode 100644
index bf24ed7..0000000
--- a/test/benchmark/JsonBenchmarkSpec.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-package benchmark
-
-import play.api.libs.json.JsNumber
-import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
-import play.libs.Json
-
-class JsonBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
- "to json" should {
- implicit val app = FakeApplication()
-
- "json benchmark" in new WithApplication(app) {
-
- duration("map to json") {
- (0 to 100) foreach { n =>
- val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
- Json.toJson(numberMaps)
- }
- }
-
- duration("directMakeJson") {
- (0 to 100) foreach { n =>
- var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
- jsObj += (n.toString -> JsNumber(n * n))
- }
- }
- }
-
- duration("map to json 2") {
- (0 to 500) foreach { n =>
- val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
- Json.toJson(numberMaps)
- }
- }
-
- duration("directMakeJson 2") {
- (0 to 500) foreach { n =>
- var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
- jsObj += (n.toString -> JsNumber(n * n))
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/OrderingUtilBenchmarkSpec.scala b/test/benchmark/OrderingUtilBenchmarkSpec.scala
deleted file mode 100644
index d38ff5e..0000000
--- a/test/benchmark/OrderingUtilBenchmarkSpec.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-package benchmark
-
-import com.kakao.s2graph.core.OrderingUtil._
-import com.kakao.s2graph.core.SeqMultiOrdering
-import play.api.libs.json.{JsNumber, JsValue}
-import play.api.test.PlaySpecification
-import play.api.{Application => PlayApplication}
-
-import scala.util.Random
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 9..
- */
-class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
- "OrderingUtilBenchmarkSpec" should {
-
- "performance MultiOrdering any" >> {
- val tupLs = (0 until 10) map { i =>
- Random.nextDouble() -> Random.nextLong()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(tup._1, tup._2)
- }
-
- val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }.map { x => x._1 }
-
- val sorted2 = duration("MultiOrdering double,long") {
- (0 until 10000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
- }
- seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
- }.map { x => x.head }
-
- sorted1.toString() must_== sorted2.toString()
- }
-
- "performance MultiOrdering double" >> {
- val tupLs = (0 until 500) map { i =>
- Random.nextDouble() -> Random.nextDouble()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(tup._1, tup._2)
- }
-
- duration("MultiOrdering double") {
- (0 until 10000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
- }
- }
-
- duration("TupleOrdering double") {
- (0 until 10000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- }
-
- 1 must_== 1
- }
-
- "performance MultiOrdering jsvalue" >> {
- val tupLs = (0 until 500) map { i =>
- Random.nextDouble() -> Random.nextLong()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(JsNumber(tup._1), JsNumber(tup._2))
- }
-
- val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
-
- val sorted2 = duration("MultiOrdering jsvalue") {
- (0 until 10000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
- }
- seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
- }
-
- 1 must_== 1
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/PostProcessBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/PostProcessBenchmarkSpec.scala b/test/benchmark/PostProcessBenchmarkSpec.scala
deleted file mode 100644
index 70644d6..0000000
--- a/test/benchmark/PostProcessBenchmarkSpec.scala
+++ /dev/null
@@ -1,238 +0,0 @@
-package benchmark
-
-import com.kakao.s2graph.core.mysqls.Label
-import com.kakao.s2graph.core.{Graph, Management}
-import controllers._
-import play.api.libs.json.{JsValue, Json}
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6..
- */
-class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification {
- sequential
-
- import Helper._
-
- init()
-
- override def init() = {
- running(FakeApplication()) {
- println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- Management.deleteService(testServiceName)
-
- // 1. createService
- val result = AdminController.createServiceInner(Json.parse(createService))
- println(s">> Service created : $createService, $result")
-
- val labelNames = Map(
- testLabelNameWeak -> testLabelNameWeakCreate
- )
-
- for {
- (labelName, create) <- labelNames
- } {
- Management.deleteLabel(labelName)
- Label.findByName(labelName, useCache = false) match {
- case None =>
- AdminController.createLabelInner(Json.parse(create))
- case Some(label) =>
- println(s">> Label already exist: $create, $label")
- }
- }
-
- // create edges
- val bulkEdges: String = (0 until 500).map { i =>
- edge"${System.currentTimeMillis()} insert e 0 $i $testLabelNameWeak"($(weight=i))
- }.mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- }
- }
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
-
-// "test performance of getEdges orderBy" >> {
-// running(FakeApplication()) {
-// val strJs =
-// s"""
-// |{
-// | "orderBy": [
-// | {"score": "DESC"},
-// | {"timestamp": "DESC"}
-// | ],
-// | "srcVertices": [
-// | {
-// | "serviceName": "$testServiceName",
-// | "columnName": "$testColumnName",
-// | "ids": [0]
-// | }
-// | ],
-// | "steps": [
-// | {
-// | "step": [
-// | {
-// | "cacheTTL": 60000,
-// | "label": "$testLabelNameWeak",
-// | "offset": 0,
-// | "limit": -1,
-// | "direction": "out",
-// | "scoring": [
-// | {"weight": 1}
-// | ]
-// | }
-// | ]
-// | }
-// | ]
-// |}
-// """.stripMargin
-//
-// object Parser extends RequestParser
-//
-// val js = Json.parse(strJs)
-//
-// val q = Parser.toQuery(js)
-//
-// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
-//
-// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
-//
-// (resultJs \ "size").as[Int] must_== 500
-//
-// (0 to 5) foreach { _ =>
-// duration("toSimpleVertexArrJson new orderBy") {
-// (0 to 1000) foreach { _ =>
-// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
-// }
-// }
-// }
-//
-// (resultJs \ "size").as[Int] must_== 500
-// }
-// }
-
- "test performance of getEdges" >> {
- running(FakeApplication()) {
- val strJs =
- s"""
- |{
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "ids": [0]
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "cacheTTL": 60000,
- | "label": "$testLabelNameWeak",
- | "offset": 0,
- | "limit": -1,
- | "direction": "out",
- | "scoring": [
- | {"weight": 1}
- | ]
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
-
- object Parser extends RequestParser
-
- val js = Json.parse(strJs)
-
- val q = Parser.toQuery(js)
-
- val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
-
- val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
-
- (0 to 5) foreach { _ =>
- duration("toSimpleVertexArrJson new") {
- (0 to 1000) foreach { _ =>
- PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
- }
- }
- }
-
- (resultJs \ "size").as[Int] must_== 500
- }
- }
-
-// "test performance of getEdges withScore=false" >> {
-// running(FakeApplication()) {
-// val strJs =
-// s"""
-// |{
-// | "withScore": false,
-// | "srcVertices": [
-// | {
-// | "serviceName": "$testServiceName",
-// | "columnName": "$testColumnName",
-// | "ids": [0]
-// | }
-// | ],
-// | "steps": [
-// | {
-// | "step": [
-// | {
-// | "cacheTTL": 60000,
-// | "label": "$testLabelNameWeak",
-// | "offset": 0,
-// | "limit": -1,
-// | "direction": "out",
-// | "scoring": [
-// | {"weight": 1}
-// | ]
-// | }
-// | ]
-// | }
-// | ]
-// |}
-// """.stripMargin
-//
-// object Parser extends RequestParser
-//
-// val js = Json.parse(strJs)
-//
-// val q = Parser.toQuery(js)
-//
-// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
-//
-// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
-//
-// (resultJs \ "size").as[Int] must_== 500
-//
-// (0 to 5) foreach { _ =>
-// duration("toSimpleVertexArrJson withScore=false org") {
-// (0 to 1000) foreach { _ =>
-// PostProcess.toSimpleVertexArrJsonOrg(queryResultLs, Nil)
-// }
-// }
-//
-// duration("toSimpleVertexArrJson withScore=false new") {
-// (0 to 1000) foreach { _ =>
-// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
-// }
-// }
-// }
-//
-// (resultJs \ "size").as[Int] must_== 500
-// }
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/benchmark/SamplingBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/test/benchmark/SamplingBenchmarkSpec.scala b/test/benchmark/SamplingBenchmarkSpec.scala
deleted file mode 100644
index 8943a62..0000000
--- a/test/benchmark/SamplingBenchmarkSpec.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-package benchmark
-
-/**
- * Created by jojo on 12/1/15.
- */
-
-import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
-import scala.annotation.tailrec
-import scala.util.Random
-
-class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
- "to json" should {
- implicit val app = FakeApplication()
-
- "json benchmark" in new WithApplication(app) {
- @tailrec
- def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (set.size == n) set
- else randomInt(n, range, set + Random.nextInt(range))
- }
-
- // sample using random array
- def randomArraySample[T](num: Int, ls: List[T]): List[T] = {
- val randomNum = randomInt(num, ls.size)
- var sample = List.empty[T]
- var idx = 0
- ls.foreach { e =>
- if (randomNum.contains(idx)) sample = e :: sample
- idx += 1
- }
- sample
- }
-
- // sample using shuffle
- def shuffleSample[T](num: Int, ls: List[T]): List[T] = {
- Random.shuffle(ls).take(num)
- }
-
- // sample using random number generation
- def rngSample[T](num: Int, ls: List[T]): List[T] = {
- var sampled = List.empty[T]
- val N = ls.size // population
- var t = 0 // total input records dealt with
- var m = 0 // number of items selected so far
-
- while (m < num) {
- val u = Random.nextDouble()
- if ( (N - t)*u < num - m) {
- sampled = ls(t) :: sampled
- m += 1
- }
- t += 1
- }
- sampled
- }
-
- // test data
- val testLimit = 500000
- val testNum = 10
- val testData = (0 to 1000).toList
-
- // dummy for warm-up
- (0 to testLimit) foreach { n =>
- randomArraySample(testNum, testData)
- shuffleSample(testNum, testData)
- rngSample(testNum, testData)
- }
-
- duration("Random Array Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = randomArraySample(testNum, testData)
- }
- }
-
- duration("Shuffle Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = shuffleSample(testNum, testData)
- }
- }
-
- duration("RNG Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = rngSample(testNum, testData)
- }
- }
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/AdminControllerSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/AdminControllerSpec.scala b/test/controllers/AdminControllerSpec.scala
deleted file mode 100644
index e41fccb..0000000
--- a/test/controllers/AdminControllerSpec.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.mysqls.Label
-import play.api.http.HeaderNames
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-import scala.concurrent.Await
-
-/**
- * Created by mojo22jojo(hyunsung.jo@gmail.com) on 15. 10. 13..
- */
-class AdminControllerSpec extends SpecCommon {
- init()
- "EdgeControllerSpec" should {
- "update htable" in {
- running(FakeApplication()) {
- val insertUrl = s"/graphs/updateHTable/$testLabelName/$newHTableName"
-
- val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "text/plain")
-
- Await.result(route(req).get, HTTP_REQ_WAITING_TIME)
- Label.findByName(testLabelName, useCache = true).get.hTableName mustEqual newHTableName
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/BasicCrudSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/BasicCrudSpec.scala b/test/controllers/BasicCrudSpec.scala
deleted file mode 100644
index f4f11b4..0000000
--- a/test/controllers/BasicCrudSpec.scala
+++ /dev/null
@@ -1,251 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.Management
-import com.kakao.s2graph.core.mysqls._
-
-//import com.kakao.s2graph.core.models._
-
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-import scala.concurrent.Await
-
-
-class BasicCrudSpec extends SpecCommon {
- sequential
-
- var seed = 0
- def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
- for {
- labelName <- List(testLabelName, testLabelName2)
- i <- 0 until NUM_OF_EACH_TEST
- } {
- seed += 1
-// val srcId = ((tcNum * 1000) + i).toString
-// val tgtId = if (labelName == testLabelName) s"${srcId + 1000 + i}" else s"${srcId + 1000 + i}abc"
- val srcId = seed.toString
- val tgtId = srcId
-
- val maxTs = opWithProps.map(t => t._1).max
-
- /** insert edges */
- println(s"---- TC${tcNum}_init ----")
- val bulkEdge = (for ((ts, op, props) <- opWithProps) yield {
- List(ts, op, "e", srcId, tgtId, labelName, props).mkString("\t")
- }).mkString("\n")
-
- val req = EdgeController.mutateAndPublish(bulkEdge, withWait = true)
- val res = Await.result(req, HTTP_REQ_WAITING_TIME)
-
- res.header.status must equalTo(200)
-
- println(s"---- TC${tcNum}_init ----")
-// Thread.sleep(100)
-
- for {
- label <- Label.findByName(labelName)
- direction <- List("out", "in")
- cacheTTL <- List(-1L)
- } {
- val (serviceName, columnName, id, otherId) = direction match {
- case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
- case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
- }
- val qId = if (labelName == testLabelName) id else "\"" + id + "\""
- val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(query)).get
- val jsResult = commonCheck(ret)
-
- val results = jsResult \ "results"
- val deegrees = (jsResult \ "degrees").as[List[JsObject]]
- val propsLs = (results \\ "props").seq
- (deegrees.head \ LabelMeta.degree.name).as[Int] must equalTo(1)
-
- val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
- val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
-
- from must equalTo(id.toString)
- to must equalTo(otherId.toString)
-// (results \\ "_timestamp").seq.last.as[Long] must equalTo(maxTs)
- for ((key, expectedVal) <- expected) {
- propsLs.last.as[JsObject].keys.contains(key) must equalTo(true)
- (propsLs.last \ key).toString must equalTo(expectedVal)
- }
- Await.result(ret, HTTP_REQ_WAITING_TIME)
- }
- }
- }
-
- init()
- "Basic Crud " should {
- "tc1" in {
- running(FakeApplication()) {
-
- var tcNum = 0
- var tcString = ""
- var bulkQueries = List.empty[(Long, String, String)]
- var expected = Map.empty[String, String]
-
- tcNum = 7
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 8
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 9
- tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 10
- tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 11
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 12
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 13
- tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 14
- tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 15
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 16
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 17
- tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 18
- tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 19
- tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
- bulkQueries = List(
- (t5, "update", "{\"is_blocked\": true}"),
- (t1, "insert", "{\"is_hidden\": false}"),
- (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
- (t2, "delete", ""),
- (t4, "update", "{\"time\": 1, \"weight\": -10}"))
- expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- true
- }
- }
- }
-
- "toLogString" in {
- running(FakeApplication()) {
- val bulkQueries = List(
- ("1445240543366", "update", "{\"is_blocked\":true}"),
- ("1445240543362", "insert", "{\"is_hidden\":false}"),
- ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
- ("1445240543363", "delete", "{}"),
- ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
-
- val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
-
- val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
- Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
- }).mkString("\n")
-
- val expected = Seq(
- Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"),
- Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"),
- Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"),
- Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"),
- Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}")
- ).map(_.mkString("\t")).mkString("\n")
-
- bulkEdge must equalTo(expected)
-
- true
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/EdgeControllerSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/EdgeControllerSpec.scala b/test/controllers/EdgeControllerSpec.scala
deleted file mode 100644
index e76404a..0000000
--- a/test/controllers/EdgeControllerSpec.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-package controllers
-
-import play.api.http.HeaderNames
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification, WithApplication}
-import play.api.{Application => PlayApplication}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 9. 1..
- */
-class EdgeControllerSpec extends PlaySpecification {
-// "EdgeControllerSpec" should {
-// implicit val app = FakeApplication()
-//
-// "bad request invalid json" in new WithApplication(app) {
-// val insertUrl = "http://localhost:9000/graphs/edges/insert"
-// val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-// val result = EdgeController.inserts().apply(req).run
-//
-// status(result) must_== BAD_REQUEST
-// }
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/PostProcessSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/PostProcessSpec.scala b/test/controllers/PostProcessSpec.scala
deleted file mode 100644
index 9573791..0000000
--- a/test/controllers/PostProcessSpec.scala
+++ /dev/null
@@ -1,115 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.SeqMultiOrdering
-import com.kakao.s2graph.core.OrderingUtil._
-import play.api.libs.json.{JsNumber, JsString, JsValue}
-import play.api.test.PlaySpecification
-
-/**
- * Created by hsleep on 2015. 11. 4..
- */
-class PostProcessSpec extends SpecCommon with PlaySpecification {
-
- "test order by json" >> {
- val jsLs: Seq[Seq[JsValue]] = Seq(
- Seq(JsNumber(0), JsString("a")),
- Seq(JsNumber(0), JsString("b")),
- Seq(JsNumber(1), JsString("a")),
- Seq(JsNumber(1), JsString("b")),
- Seq(JsNumber(2), JsString("c"))
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[Seq[JsValue]] = Seq(
- Seq(JsNumber(2), JsString("c")),
- Seq(JsNumber(1), JsString("a")),
- Seq(JsNumber(1), JsString("b")),
- Seq(JsNumber(0), JsString("a")),
- Seq(JsNumber(0), JsString("b"))
- )
-
- val orderParam: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] {
- override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = {
- val xe = x.iterator
- val ye = y.iterator
- val oe = orderParam.iterator
-
- while (xe.hasNext && ye.hasNext && oe.hasNext) {
- val (xev, yev) = oe.next() match {
- case true => xe.next() -> ye.next()
- case false => ye.next() -> xe.next()
- }
- val res = (xev, yev) match {
- case (JsNumber(xv), JsNumber(yv)) =>
- Ordering[BigDecimal].compare(xv, yv)
- case (JsString(xv), JsString(yv)) =>
- Ordering[String].compare(xv, yv)
- case _ => throw new Exception("type mismatch")
- }
- if (res != 0) return res
- }
-
- Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
- }
- })
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-
- "test order by primitive type" >> {
- val jsLs: Seq[Seq[Any]] = Seq(
- Seq(0, "a"),
- Seq(0, "b"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(2, "c")
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[Seq[Any]] = Seq(
- Seq(2, "c"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(0, "a"),
- Seq(0, "b")
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-
- "test order by primitive type with short ascending list" >> {
- val jsLs: Seq[Seq[Any]] = Seq(
- Seq(0, "a"),
- Seq(1, "b"),
- Seq(0, "b"),
- Seq(1, "a"),
- Seq(2, "c"),
- Seq(1, "c"),
- Seq(1, "d"),
- Seq(1, "f"),
- Seq(1, "e")
- )
-
- // number descending, string ascending(default)
- val sortedJsLs: Seq[Seq[Any]] = Seq(
- Seq(2, "c"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(1, "c"),
- Seq(1, "d"),
- Seq(1, "e"),
- Seq(1, "f"),
- Seq(0, "a"),
- Seq(0, "b")
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false)
- val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/test/controllers/QueryCacheSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/QueryCacheSpec.scala b/test/controllers/QueryCacheSpec.scala
deleted file mode 100644
index 7d91aa9..0000000
--- a/test/controllers/QueryCacheSpec.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-//package test.controllers
-//
-////import com.kakao.s2graph.core.models._
-//
-//import controllers.EdgeController
-//import play.api.libs.json._
-//import play.api.test.Helpers._
-//import play.api.test.{FakeApplication, FakeRequest}
-//
-//class QueryCacheSpec extends SpecCommon {
-// init()
-//
-// "cache test" should {
-// def queryWithTTL(id: Int, cacheTTL: Long) = Json.parse( s"""
-// { "srcVertices": [
-// { "serviceName": "${testServiceName}",
-// "columnName": "${testColumnName}",
-// "id": ${id}
-// }],
-// "steps": [[ {
-// "label": "${testLabelName}",
-// "direction": "out",
-// "offset": 0,
-// "limit": 10,
-// "cacheTTL": ${cacheTTL},
-// "scoring": {"weight": 1} }]]
-// }""")
-//
-// def getEdges(queryJson: JsValue): JsValue = {
-// var ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
-// contentAsJson(ret)
-// }
-//
-// // init
-// running(FakeApplication()) {
-// // insert bulk and wait ..
-// val bulkEdges: String = Seq(
-// Seq("1", "insert", "e", "0", "2", "s2graph_label_test", "{}").mkString("\t"),
-// Seq("1", "insert", "e", "1", "2", "s2graph_label_test", "{}").mkString("\t")
-// ).mkString("\n")
-//
-// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-// Thread.sleep(asyncFlushInterval)
-// }
-//
-// "tc1: query with {id: 0, ttl: 1000}" in {
-// running(FakeApplication()) {
-// var jsRslt = getEdges(queryWithTTL(0, 1000))
-// var cacheRemain = (jsRslt \\ "cacheRemain").head
-// cacheRemain.as[Int] must greaterThan(500)
-//
-// // get edges from cache after wait 500ms
-// Thread.sleep(500)
-// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryWithTTL(0, 1000))).get
-// jsRslt = contentAsJson(ret)
-// cacheRemain = (jsRslt \\ "cacheRemain").head
-// cacheRemain.as[Int] must lessThan(500)
-// }
-// }
-//
-// "tc2: query with {id: 1, ttl: 3000}" in {
-// running(FakeApplication()) {
-// var jsRslt = getEdges(queryWithTTL(1, 3000))
-// var cacheRemain = (jsRslt \\ "cacheRemain").head
-// // before update: is_blocked is false
-// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false))
-//
-// val bulkEdges = Seq(
-// Seq("2", "update", "e", "0", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t"),
-// Seq("2", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t")
-// ).mkString("\n")
-//
-// // update edges with {is_blocked: true}
-// jsRslt = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-//
-// Thread.sleep(asyncFlushInterval)
-//
-// // prop 'is_blocked' still false, cause queryResult on cache
-// jsRslt = getEdges(queryWithTTL(1, 3000))
-// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false))
-//
-// // after wait 3000ms prop 'is_blocked' is updated to true, cache cleared
-// Thread.sleep(3000)
-// jsRslt = getEdges(queryWithTTL(1, 3000))
-// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(true))
-// }
-// }
-// }
-//}
[5/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala b/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
index 217b6eb..33ec7d9 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
@@ -3,9 +3,6 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.types.InnerValLike
import play.api.libs.json.{JsNumber, JsString, JsValue}
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 5..
- */
object OrderingUtil {
implicit object JsValueOrdering extends Ordering[JsValue] {
@@ -38,9 +35,54 @@ object OrderingUtil {
}
}
}
+
+ def TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]): Ordering[(T, T, T, T)] = {
+ new Ordering[(T, T, T, T)] {
+ override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
+ val len = ascendingLs.length
+ val it = ascendingLs.iterator
+ if (len >= 1) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare1 = ord.compare(x._1, y._1)
+ if (compare1 != 0) return compare1
+ }
+
+ if (len >= 2) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare2 = ord.compare(x._2, y._2)
+ if (compare2 != 0) return compare2
+ }
+
+ if (len >= 3) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare3 = ord.compare(x._3, y._3)
+ if (compare3 != 0) return compare3
+ }
+
+ if (len >= 4) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare4 = ord.compare(x._4, y._4)
+ if (compare4 != 0) return compare4
+ }
+ 0
+ }
+ }
+ }
}
-class SeqMultiOrdering[T: Ordering](ascendingLs: Seq[Boolean], defaultAscending: Boolean = true)(implicit ord: Ordering[T]) extends Ordering[Seq[T]] {
+class SeqMultiOrdering[T](ascendingLs: Seq[Boolean], defaultAscending: Boolean = true)(implicit ord: Ordering[T]) extends Ordering[Seq[T]] {
override def compare(x: Seq[T], y: Seq[T]): Int = {
val xe = x.iterator
val ye = y.iterator
@@ -60,45 +102,45 @@ class SeqMultiOrdering[T: Ordering](ascendingLs: Seq[Boolean], defaultAscending:
}
}
-class TupleMultiOrdering[T: Ordering](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]) extends Ordering[(T, T, T, T)] {
- override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
- val len = ascendingLs.length
- val it = ascendingLs.iterator
- if (len >= 1) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare1 = ord.compare(x._1, y._1)
- if (compare1 != 0) return compare1
- }
-
- if (len >= 2) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare2 = ord.compare(x._2, y._2)
- if (compare2 != 0) return compare2
- }
-
- if (len >= 3) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare3 = ord.compare(x._3, y._3)
- if (compare3 != 0) return compare3
- }
-
- if (len >= 4) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare4 = ord.compare(x._4, y._4)
- if (compare4 != 0) return compare4
- }
- 0
- }
-}
+//class TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]) extends Ordering[(T, T, T, T)] {
+// override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
+// val len = ascendingLs.length
+// val it = ascendingLs.iterator
+// if (len >= 1) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare1 = ord.compare(x._1, y._1)
+// if (compare1 != 0) return compare1
+// }
+//
+// if (len >= 2) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare2 = ord.compare(x._2, y._2)
+// if (compare2 != 0) return compare2
+// }
+//
+// if (len >= 3) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare3 = ord.compare(x._3, y._3)
+// if (compare3 != 0) return compare3
+// }
+//
+// if (len >= 4) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare4 = ord.compare(x._4, y._4)
+// if (compare4 != 0) return compare4
+// }
+// 0
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
new file mode 100644
index 0000000..0a26d26
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
@@ -0,0 +1,439 @@
+package com.kakao.s2graph.core
+
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.types.{InnerVal, InnerValLike}
+import play.api.libs.json.{Json, _}
+
+import scala.collection.mutable.ListBuffer
+
+object PostProcess extends JSONParser {
+ /**
+ * Result Entity score field name
+ */
+ val timeoutResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isTimeout" -> true)
+ val emptyResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isEmpty" -> true)
+ def badRequestResults(ex: => Exception) = ex match {
+ case ex: BadQueryException => Json.obj("message" -> ex.msg)
+ case _ => Json.obj("message" -> ex.getMessage)
+ }
+
+ val SCORE_FIELD_NAME = "scoreSum"
+ val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
+
+ def groupEdgeResult(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+ // filterNot {case (edge, score) => edge.props.contains(LabelMeta.degreeSeq)}
+ val groupedEdgesWithRank = (for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
+ } yield {
+ (queryRequest.queryParam, edge, score)
+ }).groupBy {
+ case (queryParam, edge, rank) if edge.labelWithDir.dir == GraphUtil.directions("in") =>
+ (queryParam.label.srcColumn, queryParam.label.label, queryParam.label.tgtColumn, edge.tgtVertex.innerId, edge.isDegree)
+ case (queryParam, edge, rank) =>
+ (queryParam.label.tgtColumn, queryParam.label.label, queryParam.label.srcColumn, edge.tgtVertex.innerId, edge.isDegree)
+ }
+
+ val ret = for {
+ ((tgtColumn, labelName, srcColumn, target, isDegreeEdge), edgesAndRanks) <- groupedEdgesWithRank if !isDegreeEdge
+ edgesWithRanks = edgesAndRanks.groupBy(x => x._2.srcVertex).map(_._2.head)
+ id <- innerValToJsValue(target, tgtColumn.columnType)
+ } yield {
+ Json.obj("name" -> tgtColumn.columnName, "id" -> id,
+ SCORE_FIELD_NAME -> edgesWithRanks.map(_._3).sum,
+ "label" -> labelName,
+ "aggr" -> Json.obj(
+ "name" -> srcColumn.columnName,
+ "ids" -> edgesWithRanks.flatMap { case (queryParam, edge, rank) =>
+ innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)
+ },
+ "edges" -> edgesWithRanks.map { case (queryParam, edge, rank) =>
+ Json.obj("id" -> innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType),
+ "props" -> propsToJson(edge),
+ "score" -> rank
+ )
+ }
+ )
+ )
+ }
+
+ ret.toList
+ }
+
+ def sortWithFormatted(jsons: Seq[JsObject],
+ scoreField: String = "scoreSum",
+ queryRequestWithResultLs: Seq[QueryRequestWithResult],
+ decrease: Boolean = true): JsObject = {
+ val ordering = if (decrease) -1 else 1
+ val sortedJsons = jsons.sortBy { jsObject => (jsObject \ scoreField).as[Double] * ordering }
+
+ if (queryRequestWithResultLs.isEmpty) Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
+ else Json.obj(
+ "size" -> sortedJsons.size,
+ "results" -> sortedJsons,
+ "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId()
+ )
+ }
+
+ private def toHashKey(edge: Edge, queryParam: QueryParam, fields: Seq[String], delimiter: String = ","): Int = {
+ val ls = for {
+ field <- fields
+ } yield {
+ field match {
+ case "from" | "_from" => edge.srcVertex.innerId
+ case "to" | "_to" => edge.tgtVertex.innerId
+ case "label" => edge.labelWithDir.labelId
+ case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
+ case "_timestamp" | "timestamp" => edge.ts
+ case _ =>
+ queryParam.label.metaPropsInvMap.get(field) match {
+ case None => throw new RuntimeException(s"unknow column: $field")
+ case Some(labelMeta) => edge.propsWithTs.get(labelMeta.seq) match {
+ case None => labelMeta.defaultValue
+ case Some(propVal) => propVal
+ }
+ }
+ }
+ }
+ val ret = ls.hashCode()
+ ret
+ }
+
+ def resultInnerIds(queryRequestWithResultLs: Seq[QueryRequestWithResult], isSrcVertex: Boolean = false): Seq[Int] = {
+ for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ q = queryRequest.query
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ } yield toHashKey(edge, queryRequest.queryParam, q.filterOutFields)
+ }
+
+ def summarizeWithListExcludeFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs, decrease = true)
+ }
+
+ def summarizeWithList(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
+ }
+
+ def summarizeWithListFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
+ }
+
+ def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult]): JsValue = {
+ toSimpleVertexArrJson(queryRequestWithResultLs, Seq.empty[QueryRequestWithResult])
+ }
+
+ private def orderBy(q: Query,
+ orderByColumns: Seq[(String, Boolean)],
+ rawEdges: ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))])
+ : ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))] = {
+ import com.kakao.s2graph.core.OrderingUtil._
+
+ if (q.withScore && orderByColumns.nonEmpty) {
+ val ascendingLs = orderByColumns.map(_._2)
+ rawEdges.sortBy(_._3)(OrderingUtil.TupleMultiOrdering[Any](ascendingLs))
+ } else {
+ rawEdges
+ }
+ }
+
+ private def getColumnValue(keyWithJs: Map[String, JsValue], score: Double, edge: Edge, column: String): Any = {
+ column match {
+ case "score" => score
+ case "timestamp" | "_timestamp" => edge.ts
+ case _ =>
+ keyWithJs.get(column) match {
+ case None => keyWithJs.get("props").map { js => (js \ column).as[JsValue] }.get
+ case Some(x) => x
+ }
+ }
+ }
+
+ def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsValue = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+
+ val degrees = ListBuffer[JsValue]()
+ val rawEdges = ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))]()
+
+ if (queryRequestWithResultLs.isEmpty) {
+ Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr())
+ } else {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs.head).get
+ val query = queryRequest.query
+ val queryParam = queryRequest.queryParam
+
+ val orderByColumns = query.orderByColumns.filter { case (column, _) =>
+ column match {
+ case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true
+ case _ =>
+ queryParam.label.metaPropNames.contains(column)
+ }
+ }
+
+ /** build result jsons */
+ for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ queryParam = queryRequest.queryParam
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, query.filterOutFields))
+ } {
+ // edge to json
+ val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
+ val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
+ if (edge.isDegree && fromOpt.isDefined) {
+ degrees += Json.obj(
+ "from" -> fromOpt.get,
+ "label" -> queryRequest.queryParam.label.label,
+ "direction" -> GraphUtil.fromDirection(edge.labelWithDir.dir),
+ LabelMeta.degree.name -> innerValToJsValue(edge.propsWithTs(LabelMeta.degreeSeq).innerVal, InnerVal.LONG)
+ )
+ } else {
+ val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam)
+ val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match {
+ case 0 =>
+ (None, None, None, None)
+ case 1 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, None, None, None)
+ case 2 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, None, None)
+ case 3 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, v3, None)
+ case _ =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, v3, v4)
+ }
+
+ val currentEdge = (keyWithJs, score, orderByValues)
+ rawEdges += currentEdge
+ }
+ }
+
+ if (query.groupByColumns.isEmpty) {
+ // ordering
+ val edges = orderBy(query, orderByColumns, rawEdges).map(_._1)
+
+ Json.obj(
+ "size" -> edges.size,
+ "degrees" -> degrees,
+ "results" -> edges,
+ "impressionId" -> query.impressionId()
+ )
+ } else {
+ val grouped = rawEdges.groupBy { case (keyWithJs, _, _) =>
+ val props = keyWithJs.get("props")
+
+ for {
+ column <- query.groupByColumns
+ value <- keyWithJs.get(column) match {
+ case None => props.flatMap { js => (js \ column).asOpt[JsValue] }
+ case Some(x) => Some(x)
+ }
+ } yield column -> value
+ }
+
+ val groupedEdges = {
+ for {
+ (groupByKeyVals, groupedRawEdges) <- grouped
+ } yield {
+ val scoreSum = groupedRawEdges.map(x => x._2).sum
+ // ordering
+ val edges = orderBy(query, orderByColumns, groupedRawEdges).map(_._1)
+ Json.obj(
+ "groupBy" -> Json.toJson(groupByKeyVals.toMap),
+ "scoreSum" -> scoreSum,
+ "agg" -> edges
+ )
+ }
+ }
+
+ val groupedSortedJsons = groupedEdges.toList.sortBy { jsVal => -1 * (jsVal \ "scoreSum").as[Double] }
+ Json.obj(
+ "size" -> groupedEdges.size,
+ "degrees" -> degrees,
+ "results" -> groupedSortedJsons,
+ "impressionId" -> query.impressionId()
+ )
+ }
+ }
+ }
+
+ def verticesToJson(vertices: Iterable[Vertex]) = {
+ Json.toJson(vertices.flatMap { v => vertexToJson(v) })
+ }
+
+ def propsToJson(edge: Edge, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ for {
+ (seq, innerValWithTs) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
+ labelMeta <- queryParam.label.metaPropsMap.get(seq)
+ jsValue <- innerValToJsValue(innerValWithTs.innerVal, labelMeta.dataType)
+ } yield labelMeta.name -> jsValue
+ }
+
+ private def edgeParent(parentEdges: Seq[EdgeWithScore], q: Query, queryParam: QueryParam): JsValue = {
+ if (parentEdges.isEmpty) {
+ JsNull
+ } else {
+ val parents = for {
+ parent <- parentEdges
+ (parentEdge, parentScore) = EdgeWithScore.unapply(parent).get
+ parentQueryParam = QueryParam(parentEdge.labelWithDir)
+ parents = edgeParent(parentEdge.parentEdges, q, parentQueryParam) if parents != JsNull
+ } yield {
+ val originalEdge = parentEdge.originalEdgeOpt.getOrElse(parentEdge)
+ val edgeJson = edgeToJsonInner(originalEdge, parentScore, q, parentQueryParam) + ("parents" -> parents)
+ Json.toJson(edgeJson)
+ }
+
+ Json.toJson(parents)
+ }
+ }
+
+ /** TODO */
+ def edgeToJsonInner(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ val (srcColumn, tgtColumn) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
+
+ val kvMapOpt = for {
+ from <- innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
+ to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType)
+ } yield {
+ val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props"
+
+ val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam)
+ val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap
+
+ val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) =>
+ val jsValue = column match {
+ case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp))
+ case "from" => from
+ case "to" => to
+ case "label" => JsString(queryParam.label.label)
+ case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
+ case "_timestamp" | "timestamp" => JsNumber(edge.ts)
+ case "score" => JsNumber(score)
+ case "props" if propsMap.nonEmpty => Json.toJson(propsMap)
+ case _ => JsNull
+ }
+
+ if (jsValue == JsNull) map else map + (column -> jsValue)
+ }
+ kvMap
+ }
+
+ kvMapOpt.getOrElse(Map.empty)
+ }
+
+ def edgeToJson(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ val kvs = edgeToJsonInner(edge, score, q, queryParam)
+ if (kvs.nonEmpty && q.returnTree) kvs + ("parents" -> Json.toJson(edgeParent(edge.parentEdges, q, queryParam)))
+ else kvs
+ }
+
+ def vertexToJson(vertex: Vertex): Option[JsObject] = {
+ val serviceColumn = ServiceColumn.findById(vertex.id.colId)
+
+ for {
+ id <- innerValToJsValue(vertex.innerId, serviceColumn.columnType)
+ } yield {
+ Json.obj("serviceName" -> serviceColumn.service.serviceName,
+ "columnName" -> serviceColumn.columnName,
+ "id" -> id, "props" -> propsToJson(vertex),
+ "timestamp" -> vertex.ts,
+ // "belongsTo" -> vertex.belongLabelIds)
+ "belongsTo" -> vertex.belongLabelIds.flatMap(Label.findByIdOpt(_).map(_.label)))
+ }
+ }
+
+ private def keysToName(seqsToNames: Map[Int, String], props: Map[Int, InnerValLike]) = {
+ for {
+ (seq, value) <- props
+ name <- seqsToNames.get(seq)
+ } yield (name, value)
+ }
+
+ private def propsToJson(vertex: Vertex) = {
+ val serviceColumn = vertex.serviceColumn
+ val props = for {
+ (propKey, innerVal) <- vertex.props
+ columnMeta <- ColumnMeta.findByIdAndSeq(serviceColumn.id.get, propKey.toByte, useCache = true)
+ jsValue <- innerValToJsValue(innerVal, columnMeta.dataType)
+ } yield {
+ (columnMeta.name -> jsValue)
+ }
+ props.toMap
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def propsToJson(edge: Edge) = {
+ for {
+ (seq, v) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
+ metaProp <- edge.label.metaPropsMap.get(seq)
+ jsValue <- innerValToJsValue(v.innerVal, metaProp.dataType)
+ } yield {
+ (metaProp.name, jsValue)
+ }
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def summarizeWithListExclude(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsObject = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+
+
+ val groupedEdgesWithRank = (for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
+ } yield {
+ (edge, score)
+ }).groupBy { case (edge, score) =>
+ (edge.label.tgtColumn, edge.label.srcColumn, edge.tgtVertex.innerId)
+ }
+
+ val jsons = for {
+ ((tgtColumn, srcColumn, target), edgesAndRanks) <- groupedEdgesWithRank
+ (edges, ranks) = edgesAndRanks.groupBy(x => x._1.srcVertex).map(_._2.head).unzip
+ tgtId <- innerValToJsValue(target, tgtColumn.columnType)
+ } yield {
+ Json.obj(tgtColumn.columnName -> tgtId,
+ s"${srcColumn.columnName}s" ->
+ edges.flatMap(edge => innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)), "scoreSum" -> ranks.sum)
+ }
+ val sortedJsons = jsons.toList.sortBy { jsObj => (jsObj \ "scoreSum").as[Double] }.reverse
+ if (queryRequestWithResultLs.isEmpty) {
+ Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
+ } else {
+ Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons,
+ "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId())
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
index f40afab..d41e9fa 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
@@ -54,6 +54,19 @@ object Model {
res
}
}
+
+ def shutdown() = {
+ ConnectionPool.closeAll()
+ }
+
+ def loadCache() = {
+ Service.findAll()
+ ServiceColumn.findAll()
+ Label.findAll()
+ LabelMeta.findAll()
+ LabelIndex.findAll()
+ ColumnMeta.findAll()
+ }
}
trait Model[V] extends SQLSyntaxSupport[V] {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
new file mode 100644
index 0000000..98cc68b
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -0,0 +1,509 @@
+package com.kakao.s2graph.core.rest
+
+import com.kakao.s2graph.core.GraphExceptions.{BadQueryException, ModelNotFoundException}
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.parsers.WhereParser
+import com.kakao.s2graph.core.types._
+import com.typesafe.config.Config
+import play.api.libs.json._
+
+import scala.util.{Failure, Success, Try}
+
+class RequestParser(config: Config) extends JSONParser {
+
+ import Management.JsonModel._
+
+ val hardLimit = 100000
+ val defaultLimit = 100
+ val DefaultRpcTimeout = config.getInt("hbase.rpc.timeout")
+ val DefaultMaxAttempt = config.getInt("hbase.client.retries.number")
+ val DefaultCluster = config.getString("hbase.zookeeper.quorum")
+ val DefaultCompressionAlgorithm = config.getString("hbase.table.compression.algorithm")
+ val DefaultPhase = config.getString("phase")
+
+ private def extractScoring(labelId: Int, value: JsValue) = {
+ val ret = for {
+ js <- parse[Option[JsObject]](value, "scoring")
+ } yield {
+ for {
+ (k, v) <- js.fields
+ labelOrderType <- LabelMeta.findByName(labelId, k)
+ } yield {
+ val value = v match {
+ case n: JsNumber => n.as[Double]
+ case _ => throw new Exception("scoring weight should be double.")
+ }
+ (labelOrderType.seq, value)
+ }
+ }
+ ret
+ }
+
+ def extractInterval(label: Label, jsValue: JsValue) = {
+ def extractKv(js: JsValue) = js match {
+ case JsObject(obj) => obj
+ case JsArray(arr) => arr.flatMap {
+ case JsObject(obj) => obj
+ case _ => throw new RuntimeException(s"cannot support json type $js")
+ }
+ case _ => throw new RuntimeException(s"cannot support json type: $js")
+ }
+
+ val ret = for {
+ js <- parse[Option[JsObject]](jsValue, "interval")
+ fromJs <- (js \ "from").asOpt[JsValue]
+ toJs <- (js \ "to").asOpt[JsValue]
+ } yield {
+ val from = Management.toProps(label, extractKv(fromJs))
+ val to = Management.toProps(label, extractKv(toJs))
+ (from, to)
+ }
+
+ ret
+ }
+
+ def extractDuration(label: Label, jsValue: JsValue) = {
+ for {
+ js <- parse[Option[JsObject]](jsValue, "duration")
+ } yield {
+ val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue)
+ val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue)
+
+ (minTs, maxTs)
+ }
+ }
+
+ def extractHas(label: Label, jsValue: JsValue) = {
+ val ret = for {
+ js <- parse[Option[JsObject]](jsValue, "has")
+ } yield {
+ for {
+ (k, v) <- js.fields
+ labelMeta <- LabelMeta.findByName(label.id.get, k)
+ value <- jsValueToInnerVal(v, labelMeta.dataType, label.schemaVersion)
+ } yield {
+ labelMeta.seq -> value
+ }
+ }
+ ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
+ }
+
+ def extractWhere(labelMap: Map[String, Label], jsValue: JsValue) = {
+ (jsValue \ "where").asOpt[String] match {
+ case None => Success(WhereParser.success)
+ case Some(where) =>
+ WhereParser(labelMap).parse(where) match {
+ case s@Success(_) => s
+ case Failure(ex) => throw BadQueryException(ex.getMessage, ex)
+ }
+ }
+ }
+
+ def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[Vertex] = {
+ val vertices = for {
+ label <- Label.findByName(labelName).toSeq
+ serviceColumn = if (direction == "out") label.srcColumn else label.tgtColumn
+ id <- ids
+ innerId <- jsValueToInnerVal(id, serviceColumn.columnType, label.schemaVersion)
+ } yield {
+ Vertex(SourceVertexId(serviceColumn.id.get, innerId), System.currentTimeMillis())
+ }
+ vertices.toSeq
+ }
+
+ def toQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): Query = {
+ try {
+ val vertices =
+ (for {
+ value <- parse[List[JsValue]](jsValue, "srcVertices")
+ serviceName = parse[String](value, "serviceName")
+ column = parse[String](value, "columnName")
+ } yield {
+ val service = Service.findByName(serviceName).getOrElse(throw BadQueryException("service not found"))
+ val col = ServiceColumn.find(service.id.get, column).getOrElse(throw BadQueryException("bad column name"))
+ val (idOpt, idsOpt) = ((value \ "id").asOpt[JsValue], (value \ "ids").asOpt[List[JsValue]])
+ for {
+ idVal <- idOpt ++ idsOpt.toSeq.flatten
+
+ /** bug, need to use labels schemaVersion */
+ innerVal <- jsValueToInnerVal(idVal, col.columnType, col.schemaVersion)
+ } yield {
+ Vertex(SourceVertexId(col.id.get, innerVal), System.currentTimeMillis())
+ }
+ }).flatten
+
+ if (vertices.isEmpty) throw BadQueryException("srcVertices`s id is empty")
+
+ val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name))
+ val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q => q.copy(filterOutFields = filterOutFields) }
+ val steps = parse[Vector[JsValue]](jsValue, "steps")
+ val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true)
+ val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty)
+ val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty)
+ val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs =>
+ for {
+ js <- jsLs
+ (column, orderJs) <- js.fields
+ } yield {
+ val ascending = orderJs.as[String].toUpperCase match {
+ case "ASC" => true
+ case "DESC" => false
+ }
+ column -> ascending
+ }
+ }.getOrElse(List("score" -> false, "timestamp" -> false))
+ val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true)
+ val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false)
+
+ // TODO: throw exception, when label dosn't exist
+ val labelMap = (for {
+ js <- jsValue \\ "label"
+ labelName <- js.asOpt[String]
+ label <- Label.findByName(labelName)
+ } yield (labelName, label)).toMap
+
+ val querySteps =
+ steps.zipWithIndex.map { case (step, stepIdx) =>
+ val labelWeights = step match {
+ case obj: JsObject =>
+ val converted = for {
+ (k, v) <- (obj \ "weights").asOpt[JsObject].getOrElse(Json.obj()).fields
+ l <- Label.findByName(k)
+ } yield {
+ l.id.get -> v.toString().toDouble
+ }
+ converted.toMap
+ case _ => Map.empty[Int, Double]
+ }
+ val queryParamJsVals = step match {
+ case arr: JsArray => arr.as[List[JsValue]]
+ case obj: JsObject => (obj \ "step").as[List[JsValue]]
+ case _ => List.empty[JsValue]
+ }
+ val nextStepScoreThreshold = step match {
+ case obj: JsObject => (obj \ "nextStepThreshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
+ case _ => QueryParam.DefaultThreshold
+ }
+ val nextStepLimit = step match {
+ case obj: JsObject => (obj \ "nextStepLimit").asOpt[Int].getOrElse(-1)
+ case _ => -1
+ }
+ val cacheTTL = step match {
+ case obj: JsObject => (obj \ "cacheTTL").asOpt[Int].getOrElse(-1)
+ case _ => -1
+ }
+ val queryParams =
+ for {
+ labelGroup <- queryParamJsVals
+ queryParam <- parseQueryParam(labelMap, labelGroup)
+ } yield {
+ val (_, columnName) =
+ if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) {
+ (queryParam.label.srcService.serviceName, queryParam.label.srcColumnName)
+ } else {
+ (queryParam.label.tgtService.serviceName, queryParam.label.tgtColumnName)
+ }
+ //FIXME:
+ if (stepIdx == 0 && vertices.nonEmpty && !vertices.exists(v => v.serviceColumn.columnName == columnName)) {
+ throw BadQueryException("srcVertices contains incompatiable serviceName or columnName with first step.")
+ }
+
+ queryParam
+ }
+ Step(queryParams.toList, labelWeights = labelWeights,
+ // scoreThreshold = stepThreshold,
+ nextStepScoreThreshold = nextStepScoreThreshold,
+ nextStepLimit = nextStepLimit,
+ cacheTTL = cacheTTL)
+
+ }
+
+ val ret = Query(
+ vertices,
+ querySteps,
+ removeCycle = removeCycle,
+ selectColumns = selectColumns,
+ groupByColumns = groupByColumns,
+ orderByColumns = orderByColumns,
+ filterOutQuery = filterOutQuery,
+ filterOutFields = filterOutFields,
+ withScore = withScore,
+ returnTree = returnTree
+ )
+ // logger.debug(ret.toString)
+ ret
+ } catch {
+ case e: BadQueryException =>
+ throw e
+ case e: ModelNotFoundException =>
+ throw BadQueryException(e.getMessage, e)
+ case e: Exception =>
+ throw BadQueryException(s"$jsValue, $e", e)
+ }
+ }
+
+ private def parseQueryParam(labelMap: Map[String, Label], labelGroup: JsValue): Option[QueryParam] = {
+ for {
+ labelName <- parse[Option[String]](labelGroup, "label")
+ } yield {
+ val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found"))
+ val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
+ val limit = {
+ parse[Option[Int]](labelGroup, "limit") match {
+ case None => defaultLimit
+ case Some(l) if l < 0 => Int.MaxValue
+ case Some(l) if l >= 0 =>
+ val default = hardLimit
+ Math.min(l, default)
+ }
+ }
+ val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0)
+ val interval = extractInterval(label, labelGroup)
+ val duration = extractDuration(label, labelGroup)
+ val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList
+ val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false)
+ val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false)
+ val hasFilter = extractHas(label, labelGroup)
+ val labelWithDir = LabelWithDirection(label.id.get, direction)
+ val indexNameOpt = (labelGroup \ "index").asOpt[String]
+ val indexSeq = indexNameOpt match {
+ case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq)
+ case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
+ }
+ val where = extractWhere(labelMap, labelGroup)
+ val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
+ val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout)
+ val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt)
+ val tgtVertexInnerIdOpt = (labelGroup \ "_to").asOpt[JsValue].flatMap { jsVal =>
+ jsValueToInnerVal(jsVal, label.tgtColumnWithDir(direction).columnType, label.schemaVersion)
+ }
+ val cacheTTL = (labelGroup \ "cacheTTL").asOpt[Long].getOrElse(-1L)
+ val timeDecayFactor = (labelGroup \ "timeDecay").asOpt[JsObject].map { jsVal =>
+ val initial = (jsVal \ "initial").asOpt[Double].getOrElse(1.0)
+ val decayRate = (jsVal \ "decayRate").asOpt[Double].getOrElse(0.1)
+ if (decayRate >= 1.0 || decayRate <= 0.0) throw new BadQueryException("decay rate should be 0.0 ~ 1.0")
+ val timeUnit = (jsVal \ "timeUnit").asOpt[Double].getOrElse(60 * 60 * 24.0)
+ TimeDecay(initial, decayRate, timeUnit)
+ }
+ val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
+ // TODO: refactor this. dirty
+ val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
+
+ val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s)))
+ val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue]
+ val scorePropagateOp = (labelGroup \ "scorePropagateOp").asOpt[String].getOrElse("multiply")
+ val sample = (labelGroup \ "sample").asOpt[Int].getOrElse(-1)
+
+ // FIXME: Order of command matter
+ QueryParam(labelWithDir)
+ .sample(sample)
+ .limit(offset, limit)
+ .rank(RankParam(label.id.get, scoring))
+ .exclude(exclude)
+ .include(include)
+ .interval(interval)
+ .duration(duration)
+ .has(hasFilter)
+ .labelOrderSeq(indexSeq)
+ .interval(interval)
+ .where(where)
+ .duplicatePolicy(duplicate)
+ .includeDegree(includeDegree)
+ .rpcTimeout(rpcTimeout)
+ .maxAttempt(maxAttempt)
+ .tgtVertexInnerIdOpt(tgtVertexInnerIdOpt)
+ .cacheTTLInMillis(cacheTTL)
+ .timeDecay(timeDecayFactor)
+ .threshold(threshold)
+ .transformer(transformer)
+ .scorePropagateOp(scorePropagateOp)
+ }
+ }
+
+ private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = {
+ (js \ key).validate[R]
+ .fold(
+ errors => {
+ val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg")
+ val e = Json.obj("args" -> key, "error" -> msg)
+ throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
+ },
+ r => {
+ r
+ })
+ }
+
+ def toJsValues(jsValue: JsValue): List[JsValue] = {
+ jsValue match {
+ case obj: JsObject => List(obj)
+ case arr: JsArray => arr.as[List[JsValue]]
+ case _ => List.empty[JsValue]
+ }
+
+ }
+
+ def toEdges(jsValue: JsValue, operation: String): List[Edge] = {
+ toJsValues(jsValue).map(toEdge(_, operation))
+ }
+
+ def toEdge(jsValue: JsValue, operation: String) = {
+
+ val srcId = parse[JsValue](jsValue, "from") match {
+ case s: JsString => s.as[String]
+ case o@_ => s"${o}"
+ }
+ val tgtId = parse[JsValue](jsValue, "to") match {
+ case s: JsString => s.as[String]
+ case o@_ => s"${o}"
+ }
+ val label = parse[String](jsValue, "label")
+ val timestamp = parse[Long](jsValue, "timestamp")
+ val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
+ val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
+ Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
+
+ }
+
+ def toVertices(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None) = {
+ toJsValues(jsValue).map(toVertex(_, operation, serviceName, columnName))
+ }
+
+ def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = {
+ val id = parse[JsValue](jsValue, "id")
+ val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
+ val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
+ val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get
+ val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
+ Management.toVertex(ts, operation, id.toString, sName, cName, props.toString)
+ }
+
+ def toPropElements(jsObj: JsValue) = Try {
+ val propName = (jsObj \ "name").as[String]
+ val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
+ val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
+ case JsString(s) => s
+ case _@js => js.toString
+ }
+ Prop(propName, defaultValue, dataType)
+ }
+
+ def toPropsElements(jsValue: JsValue): Seq[Prop] = for {
+ jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil)
+ } yield {
+ val propName = (jsObj \ "name").as[String]
+ val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
+ val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
+ case JsString(s) => s
+ case _@js => js.toString
+ }
+ Prop(propName, defaultValue, dataType)
+ }
+
+ def toIndicesElements(jsValue: JsValue): Seq[Index] = for {
+ jsObj <- jsValue.as[Seq[JsValue]]
+ indexName = (jsObj \ "name").as[String]
+ propNames = (jsObj \ "propNames").as[Seq[String]]
+ } yield Index(indexName, propNames)
+
+ def toLabelElements(jsValue: JsValue) = Try {
+ val labelName = parse[String](jsValue, "label")
+ val srcServiceName = parse[String](jsValue, "srcServiceName")
+ val tgtServiceName = parse[String](jsValue, "tgtServiceName")
+ val srcColumnName = parse[String](jsValue, "srcColumnName")
+ val tgtColumnName = parse[String](jsValue, "tgtColumnName")
+ val srcColumnType = parse[String](jsValue, "srcColumnType")
+ val tgtColumnType = parse[String](jsValue, "tgtColumnType")
+ val serviceName = (jsValue \ "serviceName").asOpt[String].getOrElse(tgtServiceName)
+ val isDirected = (jsValue \ "isDirected").asOpt[Boolean].getOrElse(true)
+
+ val allProps = toPropsElements(jsValue \ "props")
+ val indices = toIndicesElements(jsValue \ "indices")
+
+ val consistencyLevel = (jsValue \ "consistencyLevel").asOpt[String].getOrElse("weak")
+
+ // expect new label don`t provide hTableName
+ val hTableName = (jsValue \ "hTableName").asOpt[String]
+ val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
+ val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(HBaseType.DEFAULT_VERSION)
+ val isAsync = (jsValue \ "isAsync").asOpt[Boolean].getOrElse(false)
+ val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
+
+ (labelName, srcServiceName, srcColumnName, srcColumnType,
+ tgtServiceName, tgtColumnName, tgtColumnType, isDirected, serviceName,
+ indices, allProps, consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
+ }
+
+ def toIndexElements(jsValue: JsValue) = Try {
+ val labelName = parse[String](jsValue, "label")
+ val indices = toIndicesElements(jsValue \ "indices")
+ (labelName, indices)
+ }
+
+ def toServiceElements(jsValue: JsValue) = {
+ val serviceName = parse[String](jsValue, "serviceName")
+ val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(DefaultCluster)
+ val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${DefaultPhase}")
+ val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1)
+ val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
+ val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
+ (serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
+ }
+
+ def toServiceColumnElements(jsValue: JsValue) = Try {
+ val serviceName = parse[String](jsValue, "serviceName")
+ val columnName = parse[String](jsValue, "columnName")
+ val columnType = parse[String](jsValue, "columnType")
+ val props = toPropsElements(jsValue \ "props")
+ (serviceName, columnName, columnType, props)
+ }
+
+ def toCheckEdgeParam(jsValue: JsValue) = {
+ val params = jsValue.as[List[JsValue]]
+ var isReverted = false
+ val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]()
+ val quads = for {
+ param <- params
+ labelName <- (param \ "label").asOpt[String]
+ direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out"))
+ label <- Label.findByName(labelName)
+ srcId <- jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ tgtId <- jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ } yield {
+ val labelWithDir = LabelWithDirection(label.id.get, direction)
+ labelWithDirs += labelWithDir
+ val (src, tgt, dir) = if (direction == 1) {
+ isReverted = true
+ (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)),
+ Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0)
+ } else {
+ (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)),
+ Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0)
+ }
+ (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir)))
+ }
+
+ (quads, isReverted)
+ }
+
+ def toGraphElements(str: String): Seq[GraphElement] = {
+ val edgeStrs = str.split("\\n")
+
+ for {
+ edgeStr <- edgeStrs
+ str <- GraphUtil.parseString(edgeStr)
+ element <- Graph.toGraphElement(str)
+ } yield element
+ }
+
+ def toDeleteParam(json: JsValue) = {
+ val labelName = (json \ "label").as[String]
+ val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil)
+ val direction = (json \ "direction").asOpt[String].getOrElse("out")
+
+ val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
+ val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
+ val vertices = toVertices(labelName, direction, ids)
+ (labels, direction, ids, ts, vertices)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
new file mode 100644
index 0000000..bef1dec
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
@@ -0,0 +1,183 @@
+package com.kakao.s2graph.core.rest
+
+import java.net.URL
+
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.{Bucket, Experiment, Service}
+import com.kakao.s2graph.core.utils.logger
+import play.api.libs.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Try
+
+/**
+ * Public API only return Future.successful or Future.failed
+ * Don't throw exception
+ */
+class RestCaller(graph: Graph)(implicit ec: ExecutionContext) {
+ val s2Parser = new RequestParser(graph.config)
+
+ /**
+ * Public APIS
+ */
+ def experiment(contentsBody: JsValue, accessToken: String, experimentName: String, uuid: String): Future[(JsValue, String)] = {
+ try {
+ val bucketOpt = for {
+ service <- Service.findByAccessToken(accessToken)
+ experiment <- Experiment.findBy(service.id.get, experimentName)
+ bucket <- experiment.findBucket(uuid)
+ } yield bucket
+
+ val bucket = bucketOpt.getOrElse(throw new RuntimeException("bucket is not found"))
+ if (bucket.isGraphQuery) buildRequestInner(contentsBody, bucket, uuid).map(_ -> bucket.impressionId)
+ else throw new RuntimeException("not supported yet")
+ } catch {
+ case e: Exception => Future.failed(e)
+ }
+ }
+
+ def uriMatch(uri: String, jsQuery: JsValue): Future[JsValue] = {
+ try {
+ uri match {
+ case "/graphs/getEdges" => getEdgesAsync(jsQuery)(PostProcess.toSimpleVertexArrJson)
+ case "/graphs/getEdges/grouped" => getEdgesAsync(jsQuery)(PostProcess.summarizeWithListFormatted)
+ case "/graphs/getEdgesExcluded" => getEdgesExcludedAsync(jsQuery)(PostProcess.toSimpleVertexArrJson)
+ case "/graphs/getEdgesExcluded/grouped" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted)
+ case "/graphs/checkEdges" => checkEdges(jsQuery)
+ case "/graphs/getEdgesGrouped" => getEdgesAsync(jsQuery)(PostProcess.summarizeWithList)
+ case "/graphs/getEdgesGroupedExcluded" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExclude)
+ case "/graphs/getEdgesGroupedExcludedFormatted" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted)
+ case "/graphs/getVertices" => getVertices(jsQuery)
+ case _ => throw new RuntimeException("route is not found")
+ }
+ } catch {
+ case e: Exception => Future.failed(e)
+ }
+ }
+
+ def checkEdges(jsValue: JsValue): Future[JsValue] = {
+ try {
+ val (quads, isReverted) = s2Parser.toCheckEdgeParam(jsValue)
+
+ graph.checkEdges(quads).map { case queryRequestWithResultLs =>
+ val edgeJsons = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ convertedEdge = if (isReverted) edge.duplicateEdge else edge
+ edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam)
+ } yield Json.toJson(edgeJson)
+
+ Json.toJson(edgeJsons)
+ }
+ } catch {
+ case e: Exception => Future.failed(e)
+ }
+ }
+
+ /**
+ * Private APIS
+ */
+ private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = {
+ val filterOutQueryResultsLs = q.filterOutQuery match {
+ case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
+ case None => Future.successful(Seq.empty)
+ }
+
+ for {
+ queryResultsLs <- graph.getEdges(q)
+ filterOutResultsLs <- filterOutQueryResultsLs
+ } yield {
+ val json = post(queryResultsLs, filterOutResultsLs)
+ json
+ }
+ }
+
+ private def getEdgesAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
+
+ val fetch = eachQuery(post) _
+ jsonQuery match {
+ case JsArray(arr) => Future.traverse(arr.map(s2Parser.toQuery(_)))(fetch).map(JsArray)
+ case obj@JsObject(_) => fetch(s2Parser.toQuery(obj))
+ case _ => throw BadQueryException("Cannot support")
+ }
+ }
+
+ private def getEdgesExcludedAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
+ val q = s2Parser.toQuery(jsonQuery)
+ val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
+
+ val fetchFuture = graph.getEdges(q)
+ val excludeFuture = graph.getEdges(filterOutQuery)
+
+ for {
+ queryResultLs <- fetchFuture
+ exclude <- excludeFuture
+ } yield {
+ post(queryResultLs, exclude)
+ }
+ }
+
+ private def getVertices(jsValue: JsValue) = {
+ val jsonQuery = jsValue
+ val ts = System.currentTimeMillis()
+ val props = "{}"
+
+ val vertices = jsonQuery.as[List[JsValue]].flatMap { js =>
+ val serviceName = (js \ "serviceName").as[String]
+ val columnName = (js \ "columnName").as[String]
+ for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield {
+ Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props)
+ }
+ }
+
+ graph.getVertices(vertices) map { vertices => PostProcess.verticesToJson(vertices) }
+ }
+
+
+ private def makeRequestJson(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): JsValue = {
+ var body = bucket.requestBody.replace("#uuid", uuid)
+ for {
+ requestKeyJson <- requestKeyJsonOpt
+ jsObj <- requestKeyJson.asOpt[JsObject]
+ (key, value) <- jsObj.fieldSet
+ } {
+ val replacement = value match {
+ case JsString(s) => s
+ case _ => value.toString
+ }
+ body = body.replace(key, replacement)
+ }
+
+ Try(Json.parse(body)).recover {
+ case e: Exception =>
+ throw new BadQueryException(s"wrong or missing template parameter: ${e.getMessage.takeWhile(_ != '\n')}")
+ } get
+ }
+
+ private def buildRequestInner(contentsBody: JsValue, bucket: Bucket, uuid: String): Future[JsValue] = {
+ if (bucket.isEmpty) Future.successful(PostProcess.emptyResults)
+ else {
+ val jsonBody = makeRequestJson(Option(contentsBody), bucket, uuid)
+ val url = new URL(bucket.apiPath)
+ val path = url.getPath()
+
+ // dummy log for sampling
+ val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody"
+
+ logger.info(experimentLog)
+
+ uriMatch(path, jsonBody)
+ }
+ }
+
+ def calcSize(js: JsValue): Int = js match {
+ case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0)
+ case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum
+ case _ => 0
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
index 824bb0d..717a5ac 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
@@ -90,7 +90,7 @@ class AsynchbaseQueryBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionC
}
val maxSize = storage.config.getInt("future.cache.max.size")
- val futureCacheTTL = storage.config.getInt("future.cache.max.idle.ttl")
+ val futureCacheTTL = storage.config.getInt("future.cache.expire.after.access")
val futureCache = CacheBuilder.newBuilder()
.initialCapacity(maxSize)
.concurrencyLevel(Runtime.getRuntime.availableProcessors())
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
index 4f2ba9f..d281017 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
@@ -1,7 +1,7 @@
package com.kakao.s2graph.core.utils
-import play.api.Logger
import play.api.libs.json.JsValue
+import org.slf4j.LoggerFactory
import scala.language.{higherKinds, implicitConversions}
@@ -29,8 +29,8 @@ object logger {
}
}
- private val logger = Logger("application")
- private val errorLogger = Logger("error")
+ private val logger = LoggerFactory.getLogger("application")
+ private val errorLogger = LoggerFactory.getLogger("error")
def info[T: Loggable](msg: => T) = logger.info(implicitly[Loggable[T]].toLogMessage(msg))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
index 6374d43..2aad32f 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
@@ -4,8 +4,37 @@ import com.kakao.s2graph.core.mysqls.LabelMeta
import com.kakao.s2graph.core.types.{InnerVal, InnerValLikeWithTs, VertexId}
import com.kakao.s2graph.core.utils.logger
import org.scalatest.FunSuite
+import org.scalatest.matchers.Matcher
class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
+ initTests()
+
+ test("toLogString") {
+ val testLabelName = labelNameV2
+ val bulkQueries = List(
+ ("1445240543366", "update", "{\"is_blocked\":true}"),
+ ("1445240543362", "insert", "{\"is_hidden\":false}"),
+ ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
+ ("1445240543363", "delete", "{}"),
+ ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
+
+ val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
+
+ val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
+ Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
+ }).mkString("\n")
+
+ val expected = Seq(
+ Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"),
+ Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"),
+ Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"),
+ Seq("1445240543363", "delete", "e", "1", "2", testLabelName),
+ Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}")
+ ).map(_.mkString("\t")).mkString("\n")
+
+ assert(bulkEdge === expected)
+ }
+
test("buildOperation") {
val schemaVersion = "v2"
val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
deleted file mode 100644
index 9a58bd0..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.kakao.s2graph.core
-
-/**
- * Created by shon on 5/29/15.
- */
-class GraphTest {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
new file mode 100644
index 0000000..1c09778
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -0,0 +1,226 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core.mysqls._
+import play.api.libs.json.{JsObject, Json}
+
+class CrudTest extends IntegrateCommon {
+ import CrudHelper._
+ import TestUtil._
+
+ test("test CRUD") {
+ var tcNum = 0
+ var tcString = ""
+ var bulkQueries = List.empty[(Long, String, String)]
+ var expected = Map.empty[String, String]
+
+ val curTime = System.currentTimeMillis
+ val t1 = curTime + 0
+ val t2 = curTime + 1
+ val t3 = curTime + 2
+ val t4 = curTime + 3
+ val t5 = curTime + 4
+
+ val tcRunner = new CrudTestRunner()
+ tcNum = 1
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 2
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 3
+ tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 4
+ tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 5
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 6
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 7
+ tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 8
+ tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 9
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 10
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 11
+ tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 12
+ tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 13
+ tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
+ bulkQueries = List(
+ (t5, "update", "{\"is_blocked\": true}"),
+ (t1, "insert", "{\"is_hidden\": false}"),
+ (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
+ (t2, "delete", ""),
+ (t4, "update", "{\"time\": 1, \"weight\": -10}"))
+ expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ }
+
+
+ object CrudHelper {
+
+ class CrudTestRunner {
+ var seed = 0
+
+ def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
+ for {
+ labelName <- List(testLabelName, testLabelName2)
+ i <- 0 until NumOfEachTest
+ } {
+ seed += 1
+ val srcId = seed.toString
+ val tgtId = srcId
+
+ val maxTs = opWithProps.map(t => t._1).max
+
+ /** insert edges */
+ println(s"---- TC${tcNum}_init ----")
+ val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
+ TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
+ })
+
+ TestUtil.insertEdgesSync(bulkEdges: _*)
+
+ for {
+ label <- Label.findByName(labelName)
+ direction <- List("out", "in")
+ cacheTTL <- List(-1L)
+ } {
+ val (serviceName, columnName, id, otherId) = direction match {
+ case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
+ case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
+ }
+
+ val qId = if (labelName == testLabelName) id else "\"" + id + "\""
+ val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
+
+ val jsResult = TestUtil.getEdgesSync(query)
+
+ val results = jsResult \ "results"
+ val deegrees = (jsResult \ "degrees").as[List[JsObject]]
+ val propsLs = (results \\ "props").seq
+ (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
+
+ val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
+ val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
+
+ from should be(id.toString)
+ to should be(otherId.toString)
+ (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
+
+ for ((key, expectedVal) <- expected) {
+ propsLs.last.as[JsObject].keys.contains(key) should be(true)
+ (propsLs.last \ key).toString should be(expectedVal)
+ }
+ }
+ }
+ }
+
+ def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = Json.parse(
+ s""" { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id } ],
+ "steps": [ [ {
+ "label": "$labelName",
+ "direction": "$dir",
+ "offset": 0,
+ "limit": 10,
+ "cacheTTL": $cacheTTL }]]}""")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
new file mode 100644
index 0000000..0ced48f
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -0,0 +1,311 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.Label
+import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.utils.logger
+import com.typesafe.config._
+import org.scalatest._
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext, Future}
+
+trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
+
+ import TestUtil._
+
+ var graph: Graph = _
+ var parser: RequestParser = _
+ var config: Config = _
+
+ override def beforeAll = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+ parser = new RequestParser(graph.config)
+ initTestData()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
+
+ /**
+ * Make Service, Label, Vertex for integrate test
+ */
+ def initTestData() = {
+ println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ Management.deleteService(testServiceName)
+
+ // 1. createService
+ val jsValue = Json.parse(createService)
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) =
+ parser.toServiceElements(jsValue)
+
+ val tryRes =
+ Management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ println(s">> Service created : $createService, $tryRes")
+
+ val labelNames = Map(testLabelName -> testLabelNameCreate,
+ testLabelName2 -> testLabelName2Create,
+ testLabelNameV1 -> testLabelNameV1Create,
+ testLabelNameWeak -> testLabelNameWeakCreate)
+
+ for {
+ (labelName, create) <- labelNames
+ } {
+ Management.deleteLabel(labelName)
+ Label.findByName(labelName, useCache = false) match {
+ case None =>
+ val json = Json.parse(create)
+ val tryRes = for {
+ labelArgs <- parser.toLabelElements(json)
+ label <- (Management.createLabel _).tupled(labelArgs)
+ } yield label
+
+ tryRes.get
+ case Some(label) =>
+ println(s">> Label already exist: $create, $label")
+ }
+ }
+
+ val vertexPropsKeys = List("age" -> "int")
+
+ vertexPropsKeys.map { case (key, keyType) =>
+ Management.addVertexProp(testServiceName, testColumnName, key, keyType)
+ }
+
+ println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ }
+
+
+ /**
+ * Test Helpers
+ */
+ object TestUtil {
+ implicit def ec = scala.concurrent.ExecutionContext.global
+
+ // def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
+ // val arr = for {
+ // (label, dir, from, to) <- params
+ // } yield {
+ // Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
+ // }
+ //
+ // val s = Json.toJson(arr)
+ // s
+ // }
+
+ def deleteAllSync(jsValue: JsValue) = {
+ val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
+ val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json)
+ val future = graph.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
+
+ future
+ })
+
+ Await.result(future, HttpRequestWaitingTime)
+ }
+
+ def getEdgesSync(queryJson: JsValue): JsValue = {
+ logger.info(Json.prettyPrint(queryJson))
+
+ val ret = graph.getEdges(parser.toQuery(queryJson))
+ val result = Await.result(ret, HttpRequestWaitingTime)
+ val jsResult = PostProcess.toSimpleVertexArrJson(result)
+
+ jsResult
+ }
+
+ def insertEdgesSync(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ val jsResult = Await.result(req, HttpRequestWaitingTime)
+
+ jsResult
+ }
+
+ def insertEdgesAsync(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ req
+ }
+
+ def toEdge(elems: Any*): String = elems.mkString("\t")
+
+ // common tables
+ val testServiceName = "s2graph"
+ val testLabelName = "s2graph_label_test"
+ val testLabelName2 = "s2graph_label_test_2"
+ val testLabelNameV1 = "s2graph_label_test_v1"
+ val testLabelNameWeak = "s2graph_label_test_weak"
+ val testColumnName = "user_id_test"
+ val testColumnType = "long"
+ val testTgtColumnName = "item_id_test"
+ val testHTableName = "test-htable"
+ val newHTableName = "new-htable"
+ val index1 = "idx_1"
+ val index2 = "idx_2"
+
+ val NumOfEachTest = 30
+ val HttpRequestWaitingTime = Duration("60 seconds")
+
+ val createService = s"""{"serviceName" : "$testServiceName"}"""
+
+ val testLabelNameCreate =
+ s"""
+ {
+ "label": "$testLabelName",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testColumnName",
+ "tgtColumnType": "long",
+ "indices": [
+ {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
+ {"name": "$index2", "propNames": ["_timestamp"]}
+ ],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "schemaVersion": "v2",
+ "compressionAlgorithm": "gz",
+ "hTableName": "$testHTableName"
+ }"""
+
+ val testLabelName2Create =
+ s"""
+ {
+ "label": "$testLabelName2",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": false,
+ "schemaVersion": "v3",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameV1Create =
+ s"""
+ {
+ "label": "$testLabelNameV1",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "${testTgtColumnName}_v1",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": true,
+ "schemaVersion": "v1",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameWeakCreate =
+ s"""
+ {
+ "label": "$testLabelNameWeak",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "weak",
+ "isDirected": true,
+ "compressionAlgorithm": "gz"
+ }"""
+ }
+}
[6/7] incubator-s2graph git commit: [S2GRAPH-7] Abstract common codes
for rest project into s2core
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/PostProcess.scala
----------------------------------------------------------------------
diff --git a/app/controllers/PostProcess.scala b/app/controllers/PostProcess.scala
deleted file mode 100644
index bbc7266..0000000
--- a/app/controllers/PostProcess.scala
+++ /dev/null
@@ -1,432 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLike}
-import play.api.libs.json.{Json, _}
-
-import scala.collection.mutable.ListBuffer
-
-object PostProcess extends JSONParser {
- /**
- * Result Entity score field name
- */
- val SCORE_FIELD_NAME = "scoreSum"
- val timeoutResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isTimeout" -> true)
- val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
-
- def groupEdgeResult(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
- // filterNot {case (edge, score) => edge.props.contains(LabelMeta.degreeSeq)}
- val groupedEdgesWithRank = (for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
- } yield {
- (queryRequest.queryParam, edge, score)
- }).groupBy {
- case (queryParam, edge, rank) if edge.labelWithDir.dir == GraphUtil.directions("in") =>
- (queryParam.label.srcColumn, queryParam.label.label, queryParam.label.tgtColumn, edge.tgtVertex.innerId, edge.isDegree)
- case (queryParam, edge, rank) =>
- (queryParam.label.tgtColumn, queryParam.label.label, queryParam.label.srcColumn, edge.tgtVertex.innerId, edge.isDegree)
- }
-
- val ret = for {
- ((tgtColumn, labelName, srcColumn, target, isDegreeEdge), edgesAndRanks) <- groupedEdgesWithRank if !isDegreeEdge
- edgesWithRanks = edgesAndRanks.groupBy(x => x._2.srcVertex).map(_._2.head)
- id <- innerValToJsValue(target, tgtColumn.columnType)
- } yield {
- Json.obj("name" -> tgtColumn.columnName, "id" -> id,
- SCORE_FIELD_NAME -> edgesWithRanks.map(_._3).sum,
- "label" -> labelName,
- "aggr" -> Json.obj(
- "name" -> srcColumn.columnName,
- "ids" -> edgesWithRanks.flatMap { case (queryParam, edge, rank) =>
- innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)
- },
- "edges" -> edgesWithRanks.map { case (queryParam, edge, rank) =>
- Json.obj("id" -> innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType),
- "props" -> propsToJson(edge),
- "score" -> rank
- )
- }
- )
- )
- }
-
- ret.toList
- }
-
- def sortWithFormatted(jsons: Seq[JsObject],
- scoreField: String = "scoreSum",
- queryRequestWithResultLs: Seq[QueryRequestWithResult],
- decrease: Boolean = true): JsObject = {
- val ordering = if (decrease) -1 else 1
- val sortedJsons = jsons.sortBy { jsObject => (jsObject \ scoreField).as[Double] * ordering }
-
- if (queryRequestWithResultLs.isEmpty) Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
- else Json.obj(
- "size" -> sortedJsons.size,
- "results" -> sortedJsons,
- "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId()
- )
- }
-
- private def toHashKey(edge: Edge, queryParam: QueryParam, fields: Seq[String], delimiter: String = ","): Int = {
- val ls = for {
- field <- fields
- } yield {
- field match {
- case "from" | "_from" => edge.srcVertex.innerId
- case "to" | "_to" => edge.tgtVertex.innerId
- case "label" => edge.labelWithDir.labelId
- case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
- case "_timestamp" | "timestamp" => edge.ts
- case _ =>
- queryParam.label.metaPropsInvMap.get(field) match {
- case None => throw new RuntimeException(s"unknow column: $field")
- case Some(labelMeta) => edge.propsWithTs.get(labelMeta.seq) match {
- case None => labelMeta.defaultValue
- case Some(propVal) => propVal
- }
- }
- }
- }
- val ret = ls.hashCode()
- ret
- }
-
- def resultInnerIds(queryRequestWithResultLs: Seq[QueryRequestWithResult], isSrcVertex: Boolean = false): Seq[Int] = {
- for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- q = queryRequest.query
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- } yield toHashKey(edge, queryRequest.queryParam, q.filterOutFields)
- }
-
- def summarizeWithListExcludeFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs, decrease = true)
- }
-
- def summarizeWithList(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
- }
-
- def summarizeWithListFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
- }
-
- def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult]): JsValue = {
- toSimpleVertexArrJson(queryRequestWithResultLs, Seq.empty[QueryRequestWithResult])
- }
-
- private def orderBy(q: Query,
- orderByColumns: Seq[(String, Boolean)],
- rawEdges: ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))])
- : ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))] = {
- import com.kakao.s2graph.core.OrderingUtil._
-
- if (q.withScore && orderByColumns.nonEmpty) {
- val ascendingLs = orderByColumns.map(_._2)
- rawEdges.sortBy(_._3)(new TupleMultiOrdering[Any](ascendingLs))
- } else {
- rawEdges
- }
- }
-
- private def getColumnValue(keyWithJs: Map[String, JsValue], score: Double, edge: Edge, column: String): Any = {
- column match {
- case "score" => score
- case "timestamp" | "_timestamp" => edge.ts
- case _ =>
- keyWithJs.get(column) match {
- case None => keyWithJs.get("props").map { js => (js \ column).as[JsValue] }.get
- case Some(x) => x
- }
- }
- }
-
- def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsValue = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
-
- val degrees = ListBuffer[JsValue]()
- val rawEdges = ListBuffer[(Map[String, JsValue], Double, (Any, Any, Any, Any))]()
-
- if (queryRequestWithResultLs.isEmpty) {
- Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr())
- } else {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs.head).get
- val query = queryRequest.query
- val queryParam = queryRequest.queryParam
-
- val orderByColumns = query.orderByColumns.filter { case (column, _) =>
- column match {
- case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true
- case _ =>
- queryParam.label.metaPropNames.contains(column)
- }
- }
-
- /** build result jsons */
- for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- queryParam = queryRequest.queryParam
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, query.filterOutFields))
- } {
- // edge to json
- val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
- val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
- if (edge.isDegree && fromOpt.isDefined) {
- degrees += Json.obj(
- "from" -> fromOpt.get,
- "label" -> queryRequest.queryParam.label.label,
- "direction" -> GraphUtil.fromDirection(edge.labelWithDir.dir),
- LabelMeta.degree.name -> innerValToJsValue(edge.propsWithTs(LabelMeta.degreeSeq).innerVal, InnerVal.LONG)
- )
- } else {
- val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam)
- val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match {
- case 0 =>
- (None, None, None, None)
- case 1 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, None, None, None)
- case 2 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, None, None)
- case 3 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, v3, None)
- case _ =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, v3, v4)
- }
-
- val currentEdge = (keyWithJs, score, orderByValues)
- rawEdges += currentEdge
- }
- }
-
- if (query.groupByColumns.isEmpty) {
- // ordering
- val edges = orderBy(query, orderByColumns, rawEdges).map(_._1)
-
- Json.obj(
- "size" -> edges.size,
- "degrees" -> degrees,
- "results" -> edges,
- "impressionId" -> query.impressionId()
- )
- } else {
- val grouped = rawEdges.groupBy { case (keyWithJs, _, _) =>
- val props = keyWithJs.get("props")
-
- for {
- column <- query.groupByColumns
- value <- keyWithJs.get(column) match {
- case None => props.flatMap { js => (js \ column).asOpt[JsValue] }
- case Some(x) => Some(x)
- }
- } yield column -> value
- }
-
- val groupedEdges = {
- for {
- (groupByKeyVals, groupedRawEdges) <- grouped
- } yield {
- val scoreSum = groupedRawEdges.map(x => x._2).sum
- // ordering
- val edges = orderBy(query, orderByColumns, groupedRawEdges).map(_._1)
- Json.obj(
- "groupBy" -> Json.toJson(groupByKeyVals.toMap),
- "scoreSum" -> scoreSum,
- "agg" -> edges
- )
- }
- }
-
- val groupedSortedJsons = groupedEdges.toList.sortBy { jsVal => -1 * (jsVal \ "scoreSum").as[Double] }
- Json.obj(
- "size" -> groupedEdges.size,
- "degrees" -> degrees,
- "results" -> groupedSortedJsons,
- "impressionId" -> query.impressionId()
- )
- }
- }
- }
-
- def verticesToJson(vertices: Iterable[Vertex]) = {
- Json.toJson(vertices.flatMap { v => vertexToJson(v) })
- }
-
- def propsToJson(edge: Edge, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- for {
- (seq, innerValWithTs) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
- labelMeta <- queryParam.label.metaPropsMap.get(seq)
- jsValue <- innerValToJsValue(innerValWithTs.innerVal, labelMeta.dataType)
- } yield labelMeta.name -> jsValue
- }
-
- private def edgeParent(parentEdges: Seq[EdgeWithScore], q: Query, queryParam: QueryParam): JsValue = {
- if (parentEdges.isEmpty) {
- JsNull
- } else {
- val parents = for {
- parent <- parentEdges
- (parentEdge, parentScore) = EdgeWithScore.unapply(parent).get
- parentQueryParam = QueryParam(parentEdge.labelWithDir)
- parents = edgeParent(parentEdge.parentEdges, q, parentQueryParam) if parents != JsNull
- } yield {
- val originalEdge = parentEdge.originalEdgeOpt.getOrElse(parentEdge)
- val edgeJson = edgeToJsonInner(originalEdge, parentScore, q, parentQueryParam) + ("parents" -> parents)
- Json.toJson(edgeJson)
- }
-
- Json.toJson(parents)
- }
- }
-
- /** TODO */
- def edgeToJsonInner(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- val (srcColumn, tgtColumn) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
-
- val kvMapOpt = for {
- from <- innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
- to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType)
- } yield {
- val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props"
-
- val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam)
- val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap
-
- val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) =>
- val jsValue = column match {
- case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp))
- case "from" => from
- case "to" => to
- case "label" => JsString(queryParam.label.label)
- case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
- case "_timestamp" | "timestamp" => JsNumber(edge.ts)
- case "score" => JsNumber(score)
- case "props" if propsMap.nonEmpty => Json.toJson(propsMap)
- case _ => JsNull
- }
-
- if (jsValue == JsNull) map else map + (column -> jsValue)
- }
- kvMap
- }
-
- kvMapOpt.getOrElse(Map.empty)
- }
-
- def edgeToJson(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- val kvs = edgeToJsonInner(edge, score, q, queryParam)
- if (kvs.nonEmpty && q.returnTree) kvs + ("parents" -> Json.toJson(edgeParent(edge.parentEdges, q, queryParam)))
- else kvs
- }
-
- def vertexToJson(vertex: Vertex): Option[JsObject] = {
- val serviceColumn = ServiceColumn.findById(vertex.id.colId)
-
- for {
- id <- innerValToJsValue(vertex.innerId, serviceColumn.columnType)
- } yield {
- Json.obj("serviceName" -> serviceColumn.service.serviceName,
- "columnName" -> serviceColumn.columnName,
- "id" -> id, "props" -> propsToJson(vertex),
- "timestamp" -> vertex.ts,
-// "belongsTo" -> vertex.belongLabelIds)
- "belongsTo" -> vertex.belongLabelIds.flatMap(Label.findByIdOpt(_).map(_.label)))
- }
- }
-
- private def keysToName(seqsToNames: Map[Int, String], props: Map[Int, InnerValLike]) = {
- for {
- (seq, value) <- props
- name <- seqsToNames.get(seq)
- } yield (name, value)
- }
-
- private def propsToJson(vertex: Vertex) = {
- val serviceColumn = vertex.serviceColumn
- val props = for {
- (propKey, innerVal) <- vertex.props
- columnMeta <- ColumnMeta.findByIdAndSeq(serviceColumn.id.get, propKey.toByte, useCache = true)
- jsValue <- innerValToJsValue(innerVal, columnMeta.dataType)
- } yield {
- (columnMeta.name -> jsValue)
- }
- props.toMap
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def propsToJson(edge: Edge) = {
- for {
- (seq, v) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
- metaProp <- edge.label.metaPropsMap.get(seq)
- jsValue <- innerValToJsValue(v.innerVal, metaProp.dataType)
- } yield {
- (metaProp.name, jsValue)
- }
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def summarizeWithListExclude(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsObject = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
-
-
- val groupedEdgesWithRank = (for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
- } yield {
- (edge, score)
- }).groupBy { case (edge, score) =>
- (edge.label.tgtColumn, edge.label.srcColumn, edge.tgtVertex.innerId)
- }
-
- val jsons = for {
- ((tgtColumn, srcColumn, target), edgesAndRanks) <- groupedEdgesWithRank
- (edges, ranks) = edgesAndRanks.groupBy(x => x._1.srcVertex).map(_._2.head).unzip
- tgtId <- innerValToJsValue(target, tgtColumn.columnType)
- } yield {
- Json.obj(tgtColumn.columnName -> tgtId,
- s"${srcColumn.columnName}s" ->
- edges.flatMap(edge => innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)), "scoreSum" -> ranks.sum)
- }
- val sortedJsons = jsons.toList.sortBy { jsObj => (jsObj \ "scoreSum").as[Double] }.reverse
- if (queryRequestWithResultLs.isEmpty) {
- Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
- } else {
- Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons,
- "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId())
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/PublishController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/PublishController.scala b/app/controllers/PublishController.scala
deleted file mode 100644
index b1495d5..0000000
--- a/app/controllers/PublishController.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.ExceptionHandler
-import config.Config
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.mvc._
-
-import scala.concurrent.Future
-
-object PublishController extends Controller {
-
- import ApplicationController._
- import ExceptionHandler._
- import play.api.libs.concurrent.Execution.Implicits._
-
- /**
- * never check validation on string. just redirect strings to kafka.
- */
- val serviceNotExistException = new RuntimeException(s"service is not created in s2graph. create service first.")
-
- // private def toService(topic: String): String = {
- // Service.findByName(topic).map(service => s"${service.serviceName}-${Config.PHASE}").getOrElse(throw serviceNotExistException)
- // }
- def publishOnly(topic: String) = withHeaderAsync(parse.text) { request =>
- if (!Config.IS_WRITE_SERVER) Future.successful(UNAUTHORIZED)
- // val kafkaTopic = toService(topic)
- val strs = request.body.split("\n")
- strs.foreach(str => {
- val keyedMessage = new ProducerRecord[Key, Val](Config.KAFKA_LOG_TOPIC, str)
- // val keyedMessage = new ProducerRecord[Key, Val](kafkaTopic, s"$str")
- // logger.debug(s"$kafkaTopic, $str")
- ExceptionHandler.enqueue(KafkaMessage(keyedMessage))
- })
- Future.successful(
- Ok("publish success.\n").withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
- )
- // try {
- //
- // } catch {
- // case e: Exception => Future.successful(BadRequest(e.getMessage))
- // }
- }
-
- def publish(topic: String) = publishOnly(topic)
-
- // def mutateBulk(topic: String) = Action.async(parse.text) { request =>
- // EdgeController.mutateAndPublish(Config.KAFKA_LOG_TOPIC, Config.KAFKA_FAIL_TOPIC, request.body).map { result =>
- // result.withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
- // }
- // }
- def mutateBulk(topic: String) = withHeaderAsync(parse.text) { request =>
- EdgeController.mutateAndPublish(request.body)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/QueryController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/QueryController.scala b/app/controllers/QueryController.scala
deleted file mode 100644
index f545b89..0000000
--- a/app/controllers/QueryController.scala
+++ /dev/null
@@ -1,310 +0,0 @@
-package controllers
-
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.types.{LabelWithDirection, VertexId}
-import com.kakao.s2graph.core.utils.logger
-import config.Config
-import play.api.libs.json.{JsArray, JsObject, JsValue, Json}
-import play.api.mvc.{Action, Controller, Result}
-
-import scala.concurrent._
-import scala.language.postfixOps
-import scala.util.Try
-
-object QueryController extends Controller with RequestParser {
-
- import ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits.defaultContext
-
- private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
-
- private def badQueryExceptionResults(ex: Exception) = Future.successful(BadRequest(Json.obj("message" -> ex.getMessage)).as(applicationJsonHeader))
-
- private def errorResults = Future.successful(Ok(PostProcess.timeoutResults).as(applicationJsonHeader))
-
- def getEdges() = withHeaderAsync(jsonParser) { request =>
- getEdgesInner(request.body)
- }
-
- def getEdgesExcluded = withHeaderAsync(jsonParser) { request =>
- getEdgesExcludedInner(request.body)
- }
-
- private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = {
- val filterOutQueryResultsLs = q.filterOutQuery match {
- case Some(filterOutQuery) => s2.getEdges(filterOutQuery)
- case None => Future.successful(Seq.empty)
- }
-
- for {
- queryResultsLs <- s2.getEdges(q)
- filterOutResultsLs <- filterOutQueryResultsLs
- } yield {
- val json = post(queryResultsLs, filterOutResultsLs)
- json
- }
- }
-
- private def calcSize(js: JsValue): Int = js match {
- case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0)
- case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum
- case _ => 0
- }
-
- private def getEdgesAsync(jsonQuery: JsValue)
- (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = {
- if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
- val fetch = eachQuery(post) _
-// logger.info(jsonQuery)
-
- Try {
- val future = jsonQuery match {
- case JsArray(arr) => Future.traverse(arr.map(toQuery(_)))(fetch).map(JsArray)
- case obj@JsObject(_) => fetch(toQuery(obj))
- case _ => throw BadQueryException("Cannot support")
- }
-
- future map { json => jsonResponse(json, "result_size" -> calcSize(json).toString) }
-
- } recover {
- case e: BadQueryException =>
- logger.error(s"$jsonQuery, $e", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"$jsonQuery, $e", e)
- errorResults
- } get
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- private def getEdgesExcludedAsync(jsonQuery: JsValue)
- (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = {
-
- if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
-
- Try {
- val q = toQuery(jsonQuery)
- val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
-
- val fetchFuture = s2.getEdges(q)
- val excludeFuture = s2.getEdges(filterOutQuery)
-
- for {
- queryResultLs <- fetchFuture
- exclude <- excludeFuture
- } yield {
- val json = post(queryResultLs, exclude)
- jsonResponse(json, "result_size" -> calcSize(json).toString)
- }
- } recover {
- case e: BadQueryException =>
- logger.error(s"$jsonQuery, $e", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"$jsonQuery, $e", e)
- errorResults
- } get
- }
-
- def getEdgesInner(jsonQuery: JsValue) = {
- getEdgesAsync(jsonQuery)(PostProcess.toSimpleVertexArrJson)
- }
-
- def getEdgesExcludedInner(jsValue: JsValue) = {
- getEdgesExcludedAsync(jsValue)(PostProcess.toSimpleVertexArrJson)
- }
-
- def getEdgesWithGrouping() = withHeaderAsync(jsonParser) { request =>
- getEdgesWithGroupingInner(request.body)
- }
-
- def getEdgesWithGroupingInner(jsonQuery: JsValue) = {
- getEdgesAsync(jsonQuery)(PostProcess.summarizeWithListFormatted)
- }
-
- def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonParser) { request =>
- getEdgesExcludedWithGroupingInner(request.body)
- }
-
- def getEdgesExcludedWithGroupingInner(jsonQuery: JsValue) = {
- getEdgesExcludedAsync(jsonQuery)(PostProcess.summarizeWithListExcludeFormatted)
- }
-
- def getEdgesGroupedInner(jsonQuery: JsValue) = {
- getEdgesAsync(jsonQuery)(PostProcess.summarizeWithList)
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def getEdgesGrouped() = withHeaderAsync(jsonParser) { request =>
- getEdgesGroupedInner(request.body)
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def getEdgesGroupedExcluded() = withHeaderAsync(jsonParser) { request =>
- getEdgesGroupedExcludedInner(request.body)
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def getEdgesGroupedExcludedInner(jsonQuery: JsValue): Future[Result] = {
- if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
-
- Try {
- val q = toQuery(jsonQuery)
- val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
-
- val fetchFuture = s2.getEdges(q)
- val excludeFuture = s2.getEdges(filterOutQuery)
-
- for {
- queryResultLs <- fetchFuture
- exclude <- excludeFuture
- } yield {
- val json = PostProcess.summarizeWithListExclude(queryResultLs, exclude)
- jsonResponse(json, "result_size" -> calcSize(json).toString)
- }
- } recover {
- case e: BadQueryException =>
- logger.error(s"$jsonQuery, $e", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"$jsonQuery, $e", e)
- errorResults
- } get
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def getEdgesGroupedExcludedFormatted = withHeaderAsync(jsonParser) { request =>
- getEdgesGroupedExcludedFormattedInner(request.body)
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def getEdgesGroupedExcludedFormattedInner(jsonQuery: JsValue): Future[Result] = {
- if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
-
- Try {
- val q = toQuery(jsonQuery)
- val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
-
- val fetchFuture = s2.getEdges(q)
- val excludeFuture = s2.getEdges(filterOutQuery)
-
- for {
- queryResultLs <- fetchFuture
- exclude <- excludeFuture
- } yield {
- val json = PostProcess.summarizeWithListExcludeFormatted(queryResultLs, exclude)
- jsonResponse(json, "result_size" -> calcSize(json).toString)
- }
- } recover {
- case e: BadQueryException =>
- logger.error(s"$jsonQuery, $e", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"$jsonQuery, $e", e)
- errorResults
- } get
- }
-
- def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) = Action.async { request =>
- if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized)
- val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
- checkEdgesInner(params)
- }
-
- /**
- * Vertex
- */
-
- def checkEdgesInner(jsValue: JsValue) = {
- try {
- val params = jsValue.as[List[JsValue]]
- var isReverted = false
- val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]()
- val quads = for {
- param <- params
- labelName <- (param \ "label").asOpt[String]
- direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out"))
- label <- Label.findByName(labelName)
- srcId <- jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion)
- tgtId <- jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion)
- } yield {
- val labelWithDir = LabelWithDirection(label.id.get, direction)
- labelWithDirs += labelWithDir
- val (src, tgt, dir) = if (direction == 1) {
- isReverted = true
- (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)),
- Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0)
- } else {
- (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)),
- Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0)
- }
-
- // logger.debug(s"SrcVertex: $src")
- // logger.debug(s"TgtVertex: $tgt")
- // logger.debug(s"direction: $dir")
- (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir)))
- }
-
- s2.checkEdges(quads).map { case queryRequestWithResultLs =>
- val edgeJsons = for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- convertedEdge = if (isReverted) edge.duplicateEdge else edge
- edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam)
- } yield Json.toJson(edgeJson)
-
- val json = Json.toJson(edgeJsons)
- jsonResponse(json, "result_size" -> edgeJsons.size.toString)
- }
- } catch {
- case e: Exception =>
- logger.error(s"$jsValue, $e", e)
- errorResults
- }
- }
-
- def checkEdges() = withHeaderAsync(jsonParser) { request =>
- if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized)
-
- checkEdgesInner(request.body)
- }
-
- def getVertices() = withHeaderAsync(jsonParser) { request =>
- getVerticesInner(request.body)
- }
-
- def getVerticesInner(jsValue: JsValue) = {
- if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader)
-
- val jsonQuery = jsValue
- val ts = System.currentTimeMillis()
- val props = "{}"
-
- Try {
- val vertices = jsonQuery.as[List[JsValue]].flatMap { js =>
- val serviceName = (js \ "serviceName").as[String]
- val columnName = (js \ "columnName").as[String]
- for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield {
- Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props)
- }
- }
-
- s2.getVertices(vertices) map { vertices =>
- val json = PostProcess.verticesToJson(vertices)
- jsonResponse(json, "result_size" -> calcSize(json).toString)
- }
- } recover {
- case e: play.api.libs.json.JsResultException =>
- logger.error(s"$jsonQuery, $e", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"$jsonQuery, $e", e)
- errorResults
- } get
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/RequestParser.scala
----------------------------------------------------------------------
diff --git a/app/controllers/RequestParser.scala b/app/controllers/RequestParser.scala
deleted file mode 100644
index e2ac8d6..0000000
--- a/app/controllers/RequestParser.scala
+++ /dev/null
@@ -1,460 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.GraphExceptions.{BadQueryException, ModelNotFoundException}
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.parsers.WhereParser
-import com.kakao.s2graph.core.types._
-import config.Config
-import play.Play
-import play.api.libs.json._
-
-import scala.util.{Failure, Success, Try}
-
-
-trait RequestParser extends JSONParser {
-
- import Management.JsonModel._
-
- val hardLimit = Config.QUERY_HARD_LIMIT
- val defaultLimit = 100
-
- lazy val defaultCluster = Play.application().configuration().getString("hbase.zookeeper.quorum")
- lazy val defaultCompressionAlgorithm = Play.application().configuration.getString("hbase.table.compression.algorithm")
-
- private def extractScoring(labelId: Int, value: JsValue) = {
- val ret = for {
- js <- parse[Option[JsObject]](value, "scoring")
- } yield {
- for {
- (k, v) <- js.fields
- labelOrderType <- LabelMeta.findByName(labelId, k)
- } yield {
- val value = v match {
- case n: JsNumber => n.as[Double]
- case _ => throw new Exception("scoring weight should be double.")
- }
- (labelOrderType.seq, value)
- }
- }
- ret
- }
-
- def extractInterval(label: Label, jsValue: JsValue) = {
- def extractKv(js: JsValue) = js match {
- case JsObject(obj) => obj
- case JsArray(arr) => arr.flatMap {
- case JsObject(obj) => obj
- case _ => throw new RuntimeException(s"cannot support json type $js")
- }
- case _ => throw new RuntimeException(s"cannot support json type: $js")
- }
-
- val ret = for {
- js <- parse[Option[JsObject]](jsValue, "interval")
- fromJs <- (js \ "from").asOpt[JsValue]
- toJs <- (js \ "to").asOpt[JsValue]
- } yield {
- val from = Management.toProps(label, extractKv(fromJs))
- val to = Management.toProps(label, extractKv(toJs))
- (from, to)
- }
-
- ret
- }
-
- def extractDuration(label: Label, jsValue: JsValue) = {
- for {
- js <- parse[Option[JsObject]](jsValue, "duration")
- } yield {
- val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue)
- val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue)
- (minTs, maxTs)
- }
- }
-
- def extractHas(label: Label, jsValue: JsValue) = {
- val ret = for {
- js <- parse[Option[JsObject]](jsValue, "has")
- } yield {
- for {
- (k, v) <- js.fields
- labelMeta <- LabelMeta.findByName(label.id.get, k)
- value <- jsValueToInnerVal(v, labelMeta.dataType, label.schemaVersion)
- } yield {
- labelMeta.seq -> value
- }
- }
- ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
- }
-
- def extractWhere(labelMap: Map[String, Label], jsValue: JsValue) = {
- (jsValue \ "where").asOpt[String] match {
- case None => Success(WhereParser.success)
- case Some(where) =>
- WhereParser(labelMap).parse(where) match {
- case s@Success(_) => s
- case Failure(ex) => throw BadQueryException(ex.getMessage, ex)
- }
- }
- }
-
- def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[Vertex] = {
- val vertices = for {
- label <- Label.findByName(labelName).toSeq
- serviceColumn = if (direction == "out") label.srcColumn else label.tgtColumn
- id <- ids
- innerId <- jsValueToInnerVal(id, serviceColumn.columnType, label.schemaVersion)
- } yield {
- Vertex(SourceVertexId(serviceColumn.id.get, innerId), System.currentTimeMillis())
- }
- vertices.toSeq
- }
-
- def toQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): Query = {
- try {
- val vertices =
- (for {
- value <- parse[List[JsValue]](jsValue, "srcVertices")
- serviceName = parse[String](value, "serviceName")
- column = parse[String](value, "columnName")
- } yield {
- val service = Service.findByName(serviceName).getOrElse(throw BadQueryException("service not found"))
- val col = ServiceColumn.find(service.id.get, column).getOrElse(throw BadQueryException("bad column name"))
- val (idOpt, idsOpt) = ((value \ "id").asOpt[JsValue], (value \ "ids").asOpt[List[JsValue]])
- for {
- idVal <- idOpt ++ idsOpt.toSeq.flatten
-
- /** bug, need to use labels schemaVersion */
- innerVal <- jsValueToInnerVal(idVal, col.columnType, col.schemaVersion)
- } yield {
- Vertex(SourceVertexId(col.id.get, innerVal), System.currentTimeMillis())
- }
- }).flatten
-
- if (vertices.isEmpty) throw BadQueryException("srcVertices`s id is empty")
-
- val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name))
- val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q => q.copy(filterOutFields = filterOutFields) }
- val steps = parse[Vector[JsValue]](jsValue, "steps")
- val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true)
- val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty)
- val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty)
- val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs =>
- for {
- js <- jsLs
- (column, orderJs) <- js.fields
- } yield {
- val ascending = orderJs.as[String].toUpperCase match {
- case "ASC" => true
- case "DESC" => false
- }
- column -> ascending
- }
- }.getOrElse(List("score" -> false, "timestamp" -> false))
- val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true)
- val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false)
-
- // TODO: throw exception, when label dosn't exist
- val labelMap = (for {
- js <- jsValue \\ "label"
- labelName <- js.asOpt[String]
- label <- Label.findByName(labelName)
- } yield (labelName, label)).toMap
-
- val querySteps =
- steps.zipWithIndex.map { case (step, stepIdx) =>
- val labelWeights = step match {
- case obj: JsObject =>
- val converted = for {
- (k, v) <- (obj \ "weights").asOpt[JsObject].getOrElse(Json.obj()).fields
- l <- Label.findByName(k)
- } yield {
- l.id.get -> v.toString().toDouble
- }
- converted.toMap
- case _ => Map.empty[Int, Double]
- }
- val queryParamJsVals = step match {
- case arr: JsArray => arr.as[List[JsValue]]
- case obj: JsObject => (obj \ "step").as[List[JsValue]]
- case _ => List.empty[JsValue]
- }
- val nextStepScoreThreshold = step match {
- case obj: JsObject => (obj \ "nextStepThreshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
- case _ => QueryParam.DefaultThreshold
- }
- val nextStepLimit = step match {
- case obj: JsObject => (obj \ "nextStepLimit").asOpt[Int].getOrElse(-1)
- case _ => -1
- }
- val cacheTTL = step match {
- case obj: JsObject => (obj \ "cacheTTL").asOpt[Int].getOrElse(-1)
- case _ => -1
- }
- val queryParams =
- for {
- labelGroup <- queryParamJsVals
- queryParam <- parseQueryParam(labelMap, labelGroup)
- } yield {
- val (_, columnName) =
- if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) {
- (queryParam.label.srcService.serviceName, queryParam.label.srcColumnName)
- } else {
- (queryParam.label.tgtService.serviceName, queryParam.label.tgtColumnName)
- }
- //FIXME:
- if (stepIdx == 0 && vertices.nonEmpty && !vertices.exists(v => v.serviceColumn.columnName == columnName)) {
- throw BadQueryException("srcVertices contains incompatiable serviceName or columnName with first step.")
- }
-
- queryParam
- }
- Step(queryParams.toList, labelWeights = labelWeights,
- // scoreThreshold = stepThreshold,
- nextStepScoreThreshold = nextStepScoreThreshold,
- nextStepLimit = nextStepLimit,
- cacheTTL = cacheTTL)
-
- }
-
- val ret = Query(
- vertices,
- querySteps,
- removeCycle = removeCycle,
- selectColumns = selectColumns,
- groupByColumns = groupByColumns,
- orderByColumns = orderByColumns,
- filterOutQuery = filterOutQuery,
- filterOutFields = filterOutFields,
- withScore = withScore,
- returnTree = returnTree
- )
- // logger.debug(ret.toString)
- ret
- } catch {
- case e: BadQueryException =>
- throw e
- case e: ModelNotFoundException =>
- throw BadQueryException(e.getMessage, e)
- case e: Exception =>
- throw BadQueryException(s"$jsValue, $e", e)
- }
- }
-
- private def parseQueryParam(labelMap: Map[String, Label], labelGroup: JsValue): Option[QueryParam] = {
- for {
- labelName <- parse[Option[String]](labelGroup, "label")
- } yield {
- val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found"))
- val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
- val limit = {
- parse[Option[Int]](labelGroup, "limit") match {
- case None => defaultLimit
- case Some(l) if l < 0 => Int.MaxValue
- case Some(l) if l >= 0 =>
- val default = hardLimit
- Math.min(l, default)
- }
- }
- val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0)
- val interval = extractInterval(label, labelGroup)
- val duration = extractDuration(label, labelGroup)
- val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList
- val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false)
- val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false)
- val hasFilter = extractHas(label, labelGroup)
- val labelWithDir = LabelWithDirection(label.id.get, direction)
- val indexNameOpt = (labelGroup \ "index").asOpt[String]
- val indexSeq = indexNameOpt match {
- case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq)
- case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
- }
- val where = extractWhere(labelMap, labelGroup)
- val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
- val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(Config.RPC_TIMEOUT)
- val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(Config.MAX_ATTEMPT)
- val tgtVertexInnerIdOpt = (labelGroup \ "_to").asOpt[JsValue].flatMap { jsVal =>
- jsValueToInnerVal(jsVal, label.tgtColumnWithDir(direction).columnType, label.schemaVersion)
- }
- val cacheTTL = (labelGroup \ "cacheTTL").asOpt[Long].getOrElse(-1L)
- val timeDecayFactor = (labelGroup \ "timeDecay").asOpt[JsObject].map { jsVal =>
- val initial = (jsVal \ "initial").asOpt[Double].getOrElse(1.0)
- val decayRate = (jsVal \ "decayRate").asOpt[Double].getOrElse(0.1)
- if (decayRate >= 1.0 || decayRate <= 0.0) throw new BadQueryException("decay rate should be 0.0 ~ 1.0")
- val timeUnit = (jsVal \ "timeUnit").asOpt[Double].getOrElse(60 * 60 * 24.0)
- TimeDecay(initial, decayRate, timeUnit)
- }
- val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
- // TODO: refactor this. dirty
- val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
-
- val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s)))
- val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue]
- val scorePropagateOp = (labelGroup \ "scorePropagateOp").asOpt[String].getOrElse("multiply")
- val sample = (labelGroup \ "sample").asOpt[Int].getOrElse(-1)
-
- // FIXME: Order of command matter
- QueryParam(labelWithDir)
- .sample(sample)
- .limit(offset, limit)
- .rank(RankParam(label.id.get, scoring))
- .exclude(exclude)
- .include(include)
- .interval(interval)
- .duration(duration)
- .has(hasFilter)
- .labelOrderSeq(indexSeq)
- .where(where)
- .duplicatePolicy(duplicate)
- .includeDegree(includeDegree)
- .rpcTimeout(rpcTimeout)
- .maxAttempt(maxAttempt)
- .tgtVertexInnerIdOpt(tgtVertexInnerIdOpt)
- .cacheTTLInMillis(cacheTTL)
- .timeDecay(timeDecayFactor)
- .threshold(threshold)
- .transformer(transformer)
- .scorePropagateOp(scorePropagateOp)
- }
- }
-
- private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = {
- (js \ key).validate[R]
- .fold(
- errors => {
- val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg")
- val e = Json.obj("args" -> key, "error" -> msg)
- throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
- },
- r => {
- r
- })
- }
-
- def toJsValues(jsValue: JsValue): List[JsValue] = {
- jsValue match {
- case obj: JsObject => List(obj)
- case arr: JsArray => arr.as[List[JsValue]]
- case _ => List.empty[JsValue]
- }
-
- }
-
- def toEdges(jsValue: JsValue, operation: String): List[Edge] = {
- toJsValues(jsValue).map(toEdge(_, operation))
- }
-
- def toEdge(jsValue: JsValue, operation: String) = {
-
- val srcId = parse[JsValue](jsValue, "from") match {
- case s: JsString => s.as[String]
- case o@_ => s"${o}"
- }
- val tgtId = parse[JsValue](jsValue, "to") match {
- case s: JsString => s.as[String]
- case o@_ => s"${o}"
- }
- val label = parse[String](jsValue, "label")
- val timestamp = parse[Long](jsValue, "timestamp")
- val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
- val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
- Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
-
- }
-
- def toVertices(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None) = {
- toJsValues(jsValue).map(toVertex(_, operation, serviceName, columnName))
- }
-
- def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = {
- val id = parse[JsValue](jsValue, "id")
- val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
- val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
- val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get
- val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
- Management.toVertex(ts, operation, id.toString, sName, cName, props.toString)
- }
-
- def toPropElements(jsObj: JsValue) = Try {
- val propName = (jsObj \ "name").as[String]
- val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
- val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
- case JsString(s) => s
- case _@js => js.toString
- }
- Prop(propName, defaultValue, dataType)
- }
-
- def toPropsElements(jsValue: JsValue): Seq[Prop] = for {
- jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil)
- } yield {
- val propName = (jsObj \ "name").as[String]
- val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
- val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
- case JsString(s) => s
- case _@js => js.toString
- }
- Prop(propName, defaultValue, dataType)
- }
-
- def toIndicesElements(jsValue: JsValue): Seq[Index] = for {
- jsObj <- jsValue.as[Seq[JsValue]]
- indexName = (jsObj \ "name").as[String]
- propNames = (jsObj \ "propNames").as[Seq[String]]
- } yield Index(indexName, propNames)
-
- def toLabelElements(jsValue: JsValue) = Try {
- val labelName = parse[String](jsValue, "label")
- val srcServiceName = parse[String](jsValue, "srcServiceName")
- val tgtServiceName = parse[String](jsValue, "tgtServiceName")
- val srcColumnName = parse[String](jsValue, "srcColumnName")
- val tgtColumnName = parse[String](jsValue, "tgtColumnName")
- val srcColumnType = parse[String](jsValue, "srcColumnType")
- val tgtColumnType = parse[String](jsValue, "tgtColumnType")
- val serviceName = (jsValue \ "serviceName").asOpt[String].getOrElse(tgtServiceName)
- val isDirected = (jsValue \ "isDirected").asOpt[Boolean].getOrElse(true)
-
- val allProps = toPropsElements(jsValue \ "props")
- val indices = toIndicesElements(jsValue \ "indices")
-
- val consistencyLevel = (jsValue \ "consistencyLevel").asOpt[String].getOrElse("weak")
-
- // expect new label don`t provide hTableName
- val hTableName = (jsValue \ "hTableName").asOpt[String]
- val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
- val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(HBaseType.DEFAULT_VERSION)
- val isAsync = (jsValue \ "isAsync").asOpt[Boolean].getOrElse(false)
- val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(defaultCompressionAlgorithm)
-
- (labelName, srcServiceName, srcColumnName, srcColumnType,
- tgtServiceName, tgtColumnName, tgtColumnType, isDirected, serviceName,
- indices, allProps, consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
- }
-
- def toIndexElements(jsValue: JsValue) = Try {
- val labelName = parse[String](jsValue, "label")
- val indices = toIndicesElements(jsValue \ "indices")
- (labelName, indices)
- }
-
- def toServiceElements(jsValue: JsValue) = {
- val serviceName = parse[String](jsValue, "serviceName")
- val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(defaultCluster)
- val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${Config.PHASE}")
- val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1)
- val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
- val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(defaultCompressionAlgorithm)
- (serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
- }
-
- def toServiceColumnElements(jsValue: JsValue) = Try {
- val serviceName = parse[String](jsValue, "serviceName")
- val columnName = parse[String](jsValue, "columnName")
- val columnType = parse[String](jsValue, "columnType")
- val props = toPropsElements(jsValue \ "props")
- (serviceName, columnName, columnType, props)
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/TestController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/TestController.scala b/app/controllers/TestController.scala
deleted file mode 100644
index 8558ae5..0000000
--- a/app/controllers/TestController.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package controllers
-
-import play.api.mvc.{Action, Controller}
-import util.TestDataLoader
-
-import scala.concurrent.Future
-
-
-object TestController extends Controller {
- import ApplicationController._
-
- def getRandomId() = withHeader(parse.anyContent) { request =>
- val id = TestDataLoader.randomId
- Ok(s"${id}")
- }
-
- def pingAsync() = Action.async(parse.json) { request =>
- Future.successful(Ok("Pong\n"))
- }
-
- def ping() = Action(parse.json) { request =>
- Ok("Pong\n")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/controllers/VertexController.scala
----------------------------------------------------------------------
diff --git a/app/controllers/VertexController.scala b/app/controllers/VertexController.scala
deleted file mode 100644
index 9c6de20..0000000
--- a/app/controllers/VertexController.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-package controllers
-
-
-import actors.QueueActor
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{ExceptionHandler, Graph, GraphExceptions}
-import config.Config
-import play.api.libs.json.{JsValue, Json}
-import play.api.mvc.{Controller, Result}
-
-import scala.concurrent.Future
-
-object VertexController extends Controller with RequestParser {
- private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
-
- import ExceptionHandler._
- import controllers.ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits._
-
- def tryMutates(jsValue: JsValue, operation: String, serviceNameOpt: Option[String] = None, columnNameOpt: Option[String] = None, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
- else {
- try {
- val vertices = toVertices(jsValue, operation, serviceNameOpt, columnNameOpt)
-
- for (vertex <- vertices) {
- if (vertex.isAsync)
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, vertex, None))
- else
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, vertex, None))
- }
-
- //FIXME:
- val verticesToStore = vertices.filterNot(v => v.isAsync)
-
- if (withWait) {
- val rets = s2.mutateVertices(verticesToStore, withWait = true)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = verticesToStore.map { vertex => QueueActor.router ! vertex; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"e"))
- case e: Exception =>
- logger.error(s"[Failed] tryMutates", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
- }
-
- def inserts() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert")
- }
-
- def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", withWait = true)
- }
-
- def insertsSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", Some(serviceName), Some(columnName))
- }
-
- def deletes() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete")
- }
-
- def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", withWait = true)
- }
-
- def deletesSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", Some(serviceName), Some(columnName))
- }
-
- def deletesAll() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "deleteAll")
- }
-
- def deletesAllSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "deleteAll", Some(serviceName), Some(columnName))
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/models/ExactCounterItem.scala
----------------------------------------------------------------------
diff --git a/app/models/ExactCounterItem.scala b/app/models/ExactCounterItem.scala
deleted file mode 100644
index 244c046..0000000
--- a/app/models/ExactCounterItem.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-package models
-
-import play.api.libs.json.{Json, Writes}
-
-/**
- * Created by alec on 15. 4. 15..
- */
-case class ExactCounterItem(ts: Long, count: Long, score: Double)
-
-case class ExactCounterIntervalItem(interval: String, dimension: Map[String, String], counter: Seq[ExactCounterItem])
-
-case class ExactCounterResultMeta(service: String, action: String, item: String)
-
-case class ExactCounterResult(meta: ExactCounterResultMeta, data: Seq[ExactCounterIntervalItem])
-
-object ExactCounterItem {
- implicit val writes = new Writes[ExactCounterItem] {
- def writes(item: ExactCounterItem) = Json.obj(
- "ts" -> item.ts,
- "time" -> tsFormat.format(item.ts),
- "count" -> item.count,
- "score" -> item.score
- )
- }
- implicit val reads = Json.reads[ExactCounterItem]
-}
-
-object ExactCounterIntervalItem {
- implicit val format = Json.format[ExactCounterIntervalItem]
-}
-
-object ExactCounterResultMeta {
- implicit val format = Json.format[ExactCounterResultMeta]
-}
-
-object ExactCounterResult {
- implicit val formats = Json.format[ExactCounterResult]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/models/RankCounterItem.scala
----------------------------------------------------------------------
diff --git a/app/models/RankCounterItem.scala b/app/models/RankCounterItem.scala
deleted file mode 100644
index aaa7df7..0000000
--- a/app/models/RankCounterItem.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-package models
-
-import play.api.libs.json.{Json, Writes}
-
-/**
- * Created by alec on 15. 4. 15..
- */
-case class RankCounterItem(rank: Int, id: String, score: Double)
-
-case class RankCounterDimensionItem(interval: String, ts: Long, dimension: String, total: Double, ranks: Seq[RankCounterItem])
-
-case class RankCounterResultMeta(service: String, action: String)
-
-case class RankCounterResult(meta: RankCounterResultMeta, data: Seq[RankCounterDimensionItem])
-
-object RankCounterItem {
- implicit val format = Json.format[RankCounterItem]
-}
-
-object RankCounterDimensionItem {
- implicit val writes = new Writes[RankCounterDimensionItem] {
- def writes(item: RankCounterDimensionItem) = Json.obj(
- "interval" -> item.interval,
- "ts" -> item.ts,
- "time" -> tsFormat.format(item.ts),
- "dimension" -> item.dimension,
- "total" -> item.total,
- "ranks" -> item.ranks
- )
- }
- implicit val reads = Json.reads[RankCounterDimensionItem]
-}
-
-object RankCounterResultMeta {
- implicit val format = Json.format[RankCounterResultMeta]
-}
-
-object RankCounterResult {
- implicit val format = Json.format[RankCounterResult]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/models/package.scala
----------------------------------------------------------------------
diff --git a/app/models/package.scala b/app/models/package.scala
deleted file mode 100644
index 17fa8e1..0000000
--- a/app/models/package.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-import java.text.SimpleDateFormat
-
-/**
- * Created by alec on 15. 4. 20..
- */
-package object models {
- def tsFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/app/util/TestDataLoader.scala
----------------------------------------------------------------------
diff --git a/app/util/TestDataLoader.scala b/app/util/TestDataLoader.scala
deleted file mode 100644
index 45a9b61..0000000
--- a/app/util/TestDataLoader.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-package util
-
-import java.io.File
-
-import scala.collection.mutable.{ArrayBuffer, HashMap, ListBuffer}
-import scala.io.Source
-import scala.util.Random
-
-
-object TestDataLoader {
- val step = 100
- val prob = 1.0
- val (testIds, testIdsHist, testIdsHistCnt) = loadSeeds("./talk_vertices.txt")
- val maxId = testIds.length
- // val randoms = (0 until 100).map{ i => new SecureRandom }
- // val idx = new AtomicInteger(0)
- // def randomId() = {
- // val r = randoms(idx.getAndIncrement() % randoms.size)
- // testAccountIds(r.nextInt(maxId))
- // }
- def randomId(histStep: Int) = {
- for {
- maxId <- testIdsHistCnt.get(histStep)
- rIdx = Random.nextInt(maxId.toInt)
- hist <- testIdsHist.get(histStep)
- id = hist(rIdx)
- } yield {
-// logger.debug(s"randomId: $histStep = $id[$rIdx / $maxId]")
- id
- }
- }
- def randomId() = {
- val id = testIds(Random.nextInt(maxId))
- // logger.debug(s"$id")
- id
- }
- private def loadSeeds(filePath: String) = {
- val histogram = new HashMap[Long, ListBuffer[Long]]
- val histogramCnt = new HashMap[Long, Long]
- val ids = new ArrayBuffer[Long]
-
- var idx = 0
-// logger.debug(s"$filePath start to load file.")
- for (line <- Source.fromFile(new File(filePath)).getLines) {
- // testAccountIds(idx) = line.toLong
-// if (idx % 10000 == 0) logger.debug(s"$idx")
- idx += 1
-
- val parts = line.split("\\t")
- val id = parts.head.toLong
- val count = parts.last.toLong / step
- if (count > 1 && Random.nextDouble < prob) {
- histogram.get(count) match {
- case None =>
- histogram.put(count, new ListBuffer[Long])
- histogram.get(count).get += id
- histogramCnt.put(count, 1)
- case Some(existed) =>
- existed += id
- histogramCnt.put(count, histogramCnt.getOrElse(count, 0L) + 1L)
- }
- ids += id
- }
-
- }
-// logger.debug(s"upload $filePath finished.")
-// logger.debug(s"${histogram.size}")
- (ids, histogram.map(t => (t._1 -> t._2.toArray[Long])), histogramCnt)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 03acb37..30e5b64 100755
--- a/build.sbt
+++ b/build.sbt
@@ -5,7 +5,7 @@ lazy val commonSettings = Seq(
scalaVersion := "2.11.7",
version := "0.12.1-SNAPSHOT",
scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation", "-feature", "-Xlint"),
- javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map{ case (key, value) => "-D" + key + "=" + value }.toSeq,
+ javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map { case (key, value) => "-D" + key + "=" + value }.toSeq,
testOptions in Test += Tests.Argument("-oDF"),
parallelExecution in Test := false,
resolvers ++= Seq(
@@ -14,13 +14,17 @@ lazy val commonSettings = Seq(
"Cloudera" at "https://repository.cloudera.com/artifactory/cloudera-repos",
"Twitter Maven" at "http://maven.twttr.com",
"sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
- "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases"
+ "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases",
+ "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/"
)
)
-lazy val root = project.in(file(".")).enablePlugins(PlayScala)
+Revolver.settings
+
+lazy val s2rest_play = project.enablePlugins(PlayScala)
.dependsOn(s2core, s2counter_core)
.settings(commonSettings: _*)
+ .settings(testOptions in Test += Tests.Argument("sequential"))
lazy val s2core = project.settings(commonSettings: _*)
@@ -35,17 +39,12 @@ lazy val s2counter_core = project.dependsOn(s2core)
lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
.settings(commonSettings: _*)
-libraryDependencies ++= Seq(
- ws,
- filters,
- "xalan" % "serializer" % "2.7.2", // Download in Intelli J(Download Source/Document)
- "com.github.danielwegener" % "logback-kafka-appender" % "0.0.3",
- "org.json4s" %% "json4s-native" % "3.2.11" % Test
-)
-
+lazy val root = (project in file("."))
+ .aggregate(s2core, s2rest_play)
+ .settings(commonSettings: _*)
lazy val runRatTask = taskKey[Unit]("Runs Apache rat on S2Graph")
runRatTask := {
"sh bin/run-rat.sh" !
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/conf/logger.xml
----------------------------------------------------------------------
diff --git a/conf/logger.xml b/conf/logger.xml
deleted file mode 100644
index 2d767c2..0000000
--- a/conf/logger.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<configuration>
-
- <conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel"/>
-
- <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>logs/application.log</file>
-
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>logs/application.%i.log</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
-
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>500MB</maxFileSize>
- </triggeringPolicy>
- <encoder>
- <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
- </encoder>
- </appender>
-
-
- <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>logs/error.log</file>
- <append>true</append>
- <encoder>
- <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
- </encoder>
- <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
- <fileNamePattern>logs/error.log.%d.%i</fileNamePattern>
- <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
- <maxFileSize>500MB</maxFileSize>
- </timeBasedFileNamingAndTriggeringPolicy>
- <maxHistory>3</maxHistory>
- </rollingPolicy>
- </appender>
-
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%coloredLevel %logger{15} - %message%n%xException%n</pattern>
- </encoder>
- </appender>
-
- <appender name="ACTOR" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>logs/actor.log</file>
- <append>true</append>
- <encoder>
- <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
- </encoder>
- <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
- <fileNamePattern>logs/actor.log.%d.%i</fileNamePattern>
- <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
- <maxFileSize>200MB</maxFileSize>
- </timeBasedFileNamingAndTriggeringPolicy>
- <maxHistory>7</maxHistory>
- </rollingPolicy>
- </appender>
-
- <logger name="play" level="INFO">
- <appender-ref ref="STDOUT"/>
- <appender-ref ref="FILE"/>
- </logger>
-
- <logger name="application" level="INFO">
- <appender-ref ref="STDOUT"/>
- <appender-ref ref="FILE"/>
- </logger>
-
- <logger name="error" level="INFO">
- <appender-ref ref="STDOUT"/>
- <appender-ref ref="ERROR"/>
- </logger>
-
- <logger name="actor" level="INFO">
- <appender-ref ref="ACTOR"/>
- </logger>
-
- <logger name="akka" level="INFO">
- <appender-ref ref="STDOUT"/>
- <appender-ref ref="FILE"/>
- </logger>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/conf/reference.conf
----------------------------------------------------------------------
diff --git a/conf/reference.conf b/conf/reference.conf
deleted file mode 100644
index d13be52..0000000
--- a/conf/reference.conf
+++ /dev/null
@@ -1,129 +0,0 @@
-# This is the main configuration file for the application.
-# ~~~~~
-
-# Secret key
-# ~~~~~
-# The secret key is used to secure cryptographics functions.
-#
-# This must be changed for production, but we recommend not changing it in this file.
-#
-# See http://www.playframework.com/documentation/latest/ApplicationSecret for more details.
-application.secret="/`==g^yr2DNnZGK_L^rguLZeR`60uLOVgY@OhyTv:maatl:Tl>9or/d1xME3b/Pi"
-
-# The application languages
-# ~~~~~
-application.langs="en"
-
-# Global object class
-# ~~~~~
-# Define the Global object class for this application.
-# Default to Global in the root package.
-# application.global=Global
-
-# Router
-# ~~~~~
-# Define the Router object to use for this application.
-# This router will be looked up first when the application is starting up,
-# so make sure this is the entry point.
-# Furthermore, it's assumed your route file is named properly.
-# So for an application router like `my.application.Router`,
-# you may need to define a router file `conf/my.application.routes`.
-# Default to Routes in the root package (and conf/routes)
-# application.router=my.application.Routes
-
-# Database configuration
-# ~~~~~
-# You can declare as many datasources as you want.
-# By convention, the default datasource is named `default`
-#
-# db.default.driver=org.h2.Driver
-# db.default.url="jdbc:h2:mem:play"
-# db.default.user=sa
-# db.default.password=""
-
-# Evolutions
-# ~~~~~
-# You can disable evolutions if needed
-# evolutionplugin=disabled
-
-# Logger
-# ~~~~~
-# You can also configure logback (http://logback.qos.ch/),
-# by providing an application-logger.xml file in the conf directory.
-
-# Root logger:
-logger.root=ERROR
-
-# Logger used by the framework:
-logger.play=INFO
-
-# Logger provided to your application:
-logger.application=DEBUG
-
-# APP PHASE
-phase=dev
-host=localhost
-
-# DB
-s2graph.models.table.name="models-dev"
-hbase.zookeeper.quorum=${host}
-db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
-# Query server
-is.query.server=true
-is.write.server=true
-query.hard.limit=100000
-
-# Local Cache
-cache.ttl.seconds=60
-cache.max.size=100000
-
-# HBASE
-#hbase.client.operation.timeout=1000
-#async.hbase.client.flush.interval=100
-hbase.table.compression.algorithm="gz"
-
-# Asynchbase
-hbase.client.retries.number=100
-hbase.rpcs.buffered_flush_interval=100
-hbase.rpc.timeout=0
-#hbase.nsre.high_watermark=1000000
-#hbase.timer.tick=5
-#hbase.timer.ticks_per_wheel=5
-
-# Kafka
-kafka.metadata.broker.list=${host}
-kafka.producer.pool.size=0
-
-# HTTP
-parsers.text.maxLength=512K
-parsers.json.maxLength=512K
-trustxforwarded=false
-
-# Local Queue Actor
-local.queue.actor.max.queue.size=100000
-local.queue.actor.rate.limit=1000000
-
-# local retry number
-max.retry.number=100
-max.back.off=50
-delete.all.fetch.size=10000
-hbase.fail.prob=-1.0
-
-# max allowd edges for deleteAll is multiply of above two configuration.
-
-# set global obejct package, TODO: remove global
-application.global=com.kakao.s2graph.rest.Global
-
-akka {
- loggers = ["akka.event.slf4j.Slf4jLogger"]
- loglevel = "DEBUG"
-}
-
-
-# Future cache.
-future.cache.max.size=1000000
-future.cache.max.idle.ttl=10000
-
-# Counter
-redis.instances = [${host}]
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/conf/routes
----------------------------------------------------------------------
diff --git a/conf/routes b/conf/routes
deleted file mode 100644
index df4a1ee..0000000
--- a/conf/routes
+++ /dev/null
@@ -1,124 +0,0 @@
-# Routes
-# This file defines all application routes (Higher priority routes first)
-# ~~~~
-
-
-# publish
-#POST /publish/:topic controllers.PublishController.publish(topic)
-POST /publish/:topic controllers.PublishController.mutateBulk(topic)
-POST /publishOnly/:topic controllers.PublishController.publishOnly(topic)
-
-#### Health Check
-#GET /health_check.html controllers.Assets.at(path="/public", file="health_check.html")
-GET /health_check.html controllers.ApplicationController.healthCheck()
-PUT /health_check/:isHealthy controllers.ApplicationController.updateHealthCheck(isHealthy: Boolean)
-
-## Edge
-POST /graphs/edges/insert controllers.EdgeController.inserts()
-POST /graphs/edges/insertWithWait controllers.EdgeController.insertsWithWait()
-POST /graphs/edges/insertBulk controllers.EdgeController.insertsBulk()
-POST /graphs/edges/delete controllers.EdgeController.deletes()
-POST /graphs/edges/deleteWithWait controllers.EdgeController.deletesWithWait()
-POST /graphs/edges/deleteAll controllers.EdgeController.deleteAll()
-POST /graphs/edges/update controllers.EdgeController.updates()
-POST /graphs/edges/updateWithWait controllers.EdgeController.updatesWithWait()
-POST /graphs/edges/increment controllers.EdgeController.increments()
-POST /graphs/edges/incrementCount controllers.EdgeController.incrementCounts()
-POST /graphs/edges/bulk controllers.EdgeController.mutateBulk()
-
-## Vertex
-POST /graphs/vertices/insert controllers.VertexController.inserts()
-POST /graphs/vertices/insertWithWait controllers.VertexController.insertsWithWait()
-POST /graphs/vertices/insert/:serviceName/:columnName controllers.VertexController.insertsSimple(serviceName, columnName)
-POST /graphs/vertices/delete controllers.VertexController.deletes()
-POST /graphs/vertices/deleteWithWait controllers.VertexController.deletesWithWait()
-POST /graphs/vertices/delete/:serviceName/:columnName controllers.VertexController.deletesSimple(serviceName, columnName)
-POST /graphs/vertices/deleteAll controllers.VertexController.deletesAll()
-POST /graphs/vertices/deleteAll/:serviceName/:columnName controllers.VertexController.deletesAllSimple(serviceName, columnName)
-
-
-### SELECT Edges
-POST /graphs/getEdges controllers.QueryController.getEdges()
-POST /graphs/getEdges/grouped controllers.QueryController.getEdgesWithGrouping()
-POST /graphs/getEdgesExcluded controllers.QueryController.getEdgesExcluded()
-POST /graphs/getEdgesExcluded/grouped controllers.QueryController.getEdgesExcludedWithGrouping()
-POST /graphs/checkEdges controllers.QueryController.checkEdges()
-
-### this will be deprecated
-POST /graphs/getEdgesGrouped controllers.QueryController.getEdgesGrouped()
-POST /graphs/getEdgesGroupedExcluded controllers.QueryController.getEdgesGroupedExcluded()
-POST /graphs/getEdgesGroupedExcludedFormatted controllers.QueryController.getEdgesGroupedExcludedFormatted()
-GET /graphs/getEdge/:srcId/:tgtId/:labelName/:direction controllers.QueryController.getEdge(srcId, tgtId, labelName, direction)
-
-
-### SELECT Vertices
-#POST /graphs/getVertex controllers.QueryController.getVertex()
-POST /graphs/getVertices controllers.QueryController.getVertices()
-
-
-#### ADMIN
-POST /graphs/createService controllers.AdminController.createService()
-GET /graphs/getService/:serviceName controllers.AdminController.getService(serviceName)
-GET /graphs/getLabels/:serviceName controllers.AdminController.getLabels(serviceName)
-POST /graphs/createLabel controllers.AdminController.createLabel()
-POST /graphs/addIndex controllers.AdminController.addIndex()
-GET /graphs/getLabel/:labelName controllers.AdminController.getLabel(labelName)
-PUT /graphs/deleteLabel/:labelName controllers.AdminController.deleteLabel(labelName)
-
-POST /graphs/addProp/:labelName controllers.AdminController.addProp(labelName)
-POST /graphs/createServiceColumn controllers.AdminController.createServiceColumn()
-PUT /graphs/deleteServiceColumn/:serviceName/:columnName controllers.AdminController.deleteServiceColumn(serviceName, columnName)
-POST /graphs/addServiceColumnProp/:serviceName/:columnName controllers.AdminController.addServiceColumnProp(serviceName, columnName)
-POST /graphs/addServiceColumnProps/:serviceName/:columnName controllers.AdminController.addServiceColumnProps(serviceName, columnName)
-GET /graphs/getServiceColumn/:serviceName/:columnName controllers.AdminController.getServiceColumn(serviceName, columnName)
-POST /graphs/createHTable controllers.AdminController.createHTable()
-
-
-
-
-#### TEST
-#GET /graphs/testGetEdges/:label/:limit/:friendCntStep controllers.QueryController.testGetEdges(label, limit: Int, friendCntStep: Int)
-#GET /graphs/testGetEdges2/:label1/:limit1/:label2/:limit2 controllers.QueryController.testGetEdges2(label1, limit1: Int, label2, limit2: Int)
-#GET /graphs/testGetEdges3/:label1/:limit1/:label2/:limit2/:label3/:limit3 controllers.QueryController.testGetEdges3(label1, limit1: Int, label2, limit2: Int, label3, limit3: Int)
-POST /ping controllers.TestController.ping()
-POST /pingAsync controllers.TestController.pingAsync()
-GET /graphs/testId controllers.TestController.getRandomId()
-
-# Map static resources from the /public folder to the /assets URL path
-GET /images/*file controllers.Assets.at(path="/public/images", file)
-GET /javascripts/*file controllers.Assets.at(path="/public/javascripts", file)
-GET /stylesheets/*file controllers.Assets.at(path="/public/stylesheets", file)
-GET /font-awesome-4.1.0/*file controllers.Assets.at(path="/public/font-awesome-4.1.0", file)
-GET /swagger/*file controllers.Assets.at(path="/public/swagger-ui", file)
-
-
-# AdminController API
-#GET /admin/services controllers.AdminController.allServices
-GET /admin/labels/:serviceName controllers.AdminController.getLabels(serviceName)
-#POST /admin/labels/delete/:zkAddr/:tableName/:labelIds/:minTs/:maxTs controllers.AdminController.deleteEdges(zkAddr, tableName, labelIds, minTs: Long, maxTs: Long)
-#POST /admin/labels/deleteAll/:zkAddr/:tableName/:minTs/:maxTs controllers.AdminController.deleteAllEdges(zkAddr, tableName, minTs: Long, maxTs: Long)
-#POST /admin/swapLabel/:oldLabelName/:newLabelName controllers.AdminController.swapLabel(oldLabelName, newLabelName)
-#GET /admin/reloadLabel/:labelName controllers.AdminController.reloadLabel(labelName)
-#POST /admin/getEdges controllers.AdminController.getEdges()
-POST /graphs/copyLabel/:oldLabelName/:newLabelName controllers.AdminController.copyLabel(oldLabelName, newLabelName)
-POST /graphs/renameLabel/:oldLabelName/:newLabelName controllers.AdminController.renameLabel(oldLabelName, newLabelName)
-POST /graphs/updateHTable/:labelName/:newHTableName controllers.AdminController.updateHTable(labelName, newHTableName)
-PUT /graphs/loadCache controllers.AdminController.loadCache()
-
-
-# Counter Admin API
-POST /counter/v1/:service/:action controllers.CounterController.createAction(service, action)
-GET /counter/v1/:service/:action controllers.CounterController.getAction(service, action)
-PUT /counter/v1/:service/:action controllers.CounterController.updateAction(service, action)
-PUT /counter/v1/:service/:action/prepare controllers.CounterController.prepareAction(service, action)
-DELETE /counter/v1/:service/:action controllers.CounterController.deleteAction(service, action)
-
-# Counter API
-GET /counter/v1/:service/:action/ranking controllers.CounterController.getRankingCountAsync(service, action)
-DELETE /counter/v1/:service/:action/ranking controllers.CounterController.deleteRankingCount(service, action)
-GET /counter/v1/:service/:action/:item controllers.CounterController.getExactCountAsync(service, action, item)
-PUT /counter/v1/:service/:action/:item controllers.CounterController.incrementCount(service, action, item)
-POST /counter/v1/mget controllers.CounterController.getExactCountAsyncMulti()
-
-# Experiment API
-POST /graphs/experiment/:accessToken/:experimentName/:uuid controllers.ExperimentController.experiment(accessToken, experimentName, uuid)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/conf/test.conf
----------------------------------------------------------------------
diff --git a/conf/test.conf b/conf/test.conf
deleted file mode 100644
index c51baef..0000000
--- a/conf/test.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-max.retry.number=10000
-hbase.fail.prob=0.1
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index f09608e..4c90a1f 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,10 +1,10 @@
-// The Typesafe repository
-resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
-
// Use the Play sbt plugin for Play projects
addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10")
// http://www.scalastyle.org/sbt.html
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
-resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/"
+// sbt revolver
+addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0")
+
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.3")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 94f4920..543365c 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -1,15 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
- <jmxConfigurator/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n
- </pattern>
- </encoder>
- </appender>
-
- <root level="info">
- <appender-ref ref="STDOUT"/>
- </root>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <withJansi>true</withJansi>
+ <encoder>
+ <pattern>
+ %d{ISO8601} [%highlight(%-5level)] [%gray(%logger{0})] [%thread] - %msg%n
+ </pattern>
+ </encoder>
+ </appender>
+
+ <root level="ERROR">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+ <logger name="application" level="DEBUG"></logger>
+
+ <logger name="error" level="DEBUG"></logger>
</configuration>
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f2311f25/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
index 727a3be..111f3df 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
@@ -38,7 +38,10 @@ object Graph {
"max.retry.number" -> java.lang.Integer.valueOf(100),
"max.back.off" -> java.lang.Integer.valueOf(100),
"hbase.fail.prob" -> java.lang.Double.valueOf(-0.1),
- "delete.all.fetch.size" -> java.lang.Integer.valueOf(1000)
+ "delete.all.fetch.size" -> java.lang.Integer.valueOf(1000),
+ "future.cache.max.size" -> java.lang.Integer.valueOf(100000),
+ "future.cache.expire.after.write" -> java.lang.Integer.valueOf(10000),
+ "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000)
)
var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs)
@@ -315,7 +318,8 @@ class Graph(_config: Config)(implicit ec: ExecutionContext) {
// val cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Seq[QueryResult]]()
val vertexCache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Option[Vertex]]()
- Model(config)
+ Model.apply(config)
+ Model.loadCache()
// TODO: Make storage client by config param
val storage: Storage = new AsynchbaseStorage(config, vertexCache)(ec)
@@ -345,5 +349,8 @@ class Graph(_config: Config)(implicit ec: ExecutionContext) {
def incrementCounts(edges: Seq[Edge]): Future[Seq[(Boolean, Long)]] = storage.incrementCounts(edges)
- def shutdown(): Unit = storage.flush()
+ def shutdown(): Unit = {
+ storage.flush()
+ Model.shutdown()
+ }
}