You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2016/03/15 10:28:17 UTC
[01/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Repository: incubator-s2graph
Updated Branches:
refs/heads/master e71264d0d -> b6fe32fc2
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
deleted file mode 100644
index a387ba5..0000000
--- a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-package benchmark
-
-import play.api.libs.json.JsNumber
-import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
-import play.libs.Json
-
-class JsonBenchmarkSpec extends BenchmarkCommon {
- "to json" >> {
- "json benchmark" >> {
-
- duration("map to json") {
- (0 to 10) foreach { n =>
- val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
- Json.toJson(numberMaps)
- }
- }
-
- duration("directMakeJson") {
- (0 to 10) foreach { n =>
- var jsObj = play.api.libs.json.Json.obj()
- (0 to 10).foreach { n =>
- jsObj += (n.toString -> JsNumber(n * n))
- }
- }
- }
-
- duration("map to json 2") {
- (0 to 50) foreach { n =>
- val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap
- Json.toJson(numberMaps)
- }
- }
-
- duration("directMakeJson 2") {
- (0 to 50) foreach { n =>
- var jsObj = play.api.libs.json.Json.obj()
- (0 to 10).foreach { n =>
- jsObj += (n.toString -> JsNumber(n * n))
- }
- }
- }
- true
- }
- true
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
deleted file mode 100644
index d2d3624..0000000
--- a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-package benchmark
-
-import com.kakao.s2graph.core.OrderingUtil._
-import com.kakao.s2graph.core.SeqMultiOrdering
-import play.api.libs.json.{JsNumber, JsValue}
-import play.api.test.PlaySpecification
-import play.api.{Application => PlayApplication}
-
-import scala.util.Random
-
-class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
- "OrderingUtilBenchmarkSpec" should {
-
- "performance MultiOrdering any" >> {
- val tupLs = (0 until 10) map { i =>
- Random.nextDouble() -> Random.nextLong()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(tup._1, tup._2)
- }
-
- val sorted1 = duration("TupleOrdering double,long") {
- (0 until 1000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }.map { x => x._1 }
-
- val sorted2 = duration("MultiOrdering double,long") {
- (0 until 1000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
- }
- seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
- }.map { x => x.head }
-
- sorted1.toString() must_== sorted2.toString()
- }
-
- "performance MultiOrdering double" >> {
- val tupLs = (0 until 50) map { i =>
- Random.nextDouble() -> Random.nextDouble()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(tup._1, tup._2)
- }
-
- duration("MultiOrdering double") {
- (0 until 1000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
- }
- }
-
- duration("TupleOrdering double") {
- (0 until 1000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- }
-
- 1 must_== 1
- }
-
- "performance MultiOrdering jsvalue" >> {
- val tupLs = (0 until 50) map { i =>
- Random.nextDouble() -> Random.nextLong()
- }
-
- val seqLs = tupLs.map { tup =>
- Seq(JsNumber(tup._1), JsNumber(tup._2))
- }
-
- val sorted1 = duration("TupleOrdering double,long") {
- (0 until 1000) foreach { _ =>
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
- tupLs.sortBy { case (x, y) =>
- -x -> -y
- }
- }
-
- val sorted2 = duration("MultiOrdering jsvalue") {
- (0 until 1000) foreach { _ =>
- seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
- }
- seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
- }
-
- 1 must_== 1
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
deleted file mode 100644
index 0c27a2a..0000000
--- a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-package benchmark
-import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
-import scala.annotation.tailrec
-import scala.util.Random
-
-class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
- "sample" should {
- implicit val app = FakeApplication()
-
- "sample benchmark" in new WithApplication(app) {
- @tailrec
- def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (set.size == n) set
- else randomInt(n, range, set + Random.nextInt(range))
- }
-
- // sample using random array
- def randomArraySample[T](num: Int, ls: List[T]): List[T] = {
- val randomNum = randomInt(num, ls.size)
- var sample = List.empty[T]
- var idx = 0
- ls.foreach { e =>
- if (randomNum.contains(idx)) sample = e :: sample
- idx += 1
- }
- sample
- }
-
- // sample using shuffle
- def shuffleSample[T](num: Int, ls: List[T]): List[T] = {
- Random.shuffle(ls).take(num)
- }
-
- // sample using random number generation
- def rngSample[T](num: Int, ls: List[T]): List[T] = {
- var sampled = List.empty[T]
- val N = ls.size // population
- var t = 0 // total input records dealt with
- var m = 0 // number of items selected so far
-
- while (m < num) {
- val u = Random.nextDouble()
- if ( (N - t)*u < num - m) {
- sampled = ls(t) :: sampled
- m += 1
- }
- t += 1
- }
- sampled
- }
-
- // test data
- val testLimit = 500000
- val testNum = 10
- val testData = (0 to 1000).toList
-
- // dummy for warm-up
- (0 to testLimit) foreach { n =>
- randomArraySample(testNum, testData)
- shuffleSample(testNum, testData)
- rngSample(testNum, testData)
- }
-
- duration("Random Array Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = randomArraySample(testNum, testData)
- }
- }
-
- duration("Shuffle Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = shuffleSample(testNum, testData)
- }
- }
-
- duration("RNG Sampling") {
- (0 to testLimit) foreach { _ =>
- val sampled = rngSample(testNum, testData)
- }
- }
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/controllers/PostProcessSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/PostProcessSpec.scala b/s2rest_play/test/controllers/PostProcessSpec.scala
deleted file mode 100644
index cea132a..0000000
--- a/s2rest_play/test/controllers/PostProcessSpec.scala
+++ /dev/null
@@ -1,112 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.{OrderingUtil, SeqMultiOrdering}
-import play.api.libs.json.{JsNumber, JsString, JsValue}
-import play.api.test.PlaySpecification
-
-class PostProcessSpec extends PlaySpecification {
- import OrderingUtil._
-
- "test order by json" >> {
- val jsLs: Seq[Seq[JsValue]] = Seq(
- Seq(JsNumber(0), JsString("a")),
- Seq(JsNumber(0), JsString("b")),
- Seq(JsNumber(1), JsString("a")),
- Seq(JsNumber(1), JsString("b")),
- Seq(JsNumber(2), JsString("c"))
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[Seq[JsValue]] = Seq(
- Seq(JsNumber(2), JsString("c")),
- Seq(JsNumber(1), JsString("a")),
- Seq(JsNumber(1), JsString("b")),
- Seq(JsNumber(0), JsString("a")),
- Seq(JsNumber(0), JsString("b"))
- )
-
- val orderParam: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] {
- override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = {
- val xe = x.iterator
- val ye = y.iterator
- val oe = orderParam.iterator
-
- while (xe.hasNext && ye.hasNext && oe.hasNext) {
- val (xev, yev) = oe.next() match {
- case true => xe.next() -> ye.next()
- case false => ye.next() -> xe.next()
- }
- val res = (xev, yev) match {
- case (JsNumber(xv), JsNumber(yv)) =>
- Ordering[BigDecimal].compare(xv, yv)
- case (JsString(xv), JsString(yv)) =>
- Ordering[String].compare(xv, yv)
- case _ => throw new Exception("type mismatch")
- }
- if (res != 0) return res
- }
-
- Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
- }
- })
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-
- "test order by primitive type" >> {
- val jsLs: Seq[Seq[Any]] = Seq(
- Seq(0, "a"),
- Seq(0, "b"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(2, "c")
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[Seq[Any]] = Seq(
- Seq(2, "c"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(0, "a"),
- Seq(0, "b")
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-
- "test order by primitive type with short ascending list" >> {
- val jsLs: Seq[Seq[Any]] = Seq(
- Seq(0, "a"),
- Seq(1, "b"),
- Seq(0, "b"),
- Seq(1, "a"),
- Seq(2, "c"),
- Seq(1, "c"),
- Seq(1, "d"),
- Seq(1, "f"),
- Seq(1, "e")
- )
-
- // number descending, string ascending(default)
- val sortedJsLs: Seq[Seq[Any]] = Seq(
- Seq(2, "c"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(1, "c"),
- Seq(1, "d"),
- Seq(1, "e"),
- Seq(1, "f"),
- Seq(0, "a"),
- Seq(0, "b")
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false)
- val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() must_== sortedJsLs.toString
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala
new file mode 100644
index 0000000..3d77e16
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala
@@ -0,0 +1,24 @@
+package org.apache.s2graph.rest.play.benchmark
+
+import org.specs2.mutable.Specification
+
+trait BenchmarkCommon extends Specification {
+ val wrapStr = s"\n=================================================="
+
+ def duration[T](prefix: String = "")(block: => T) = {
+ val startTs = System.currentTimeMillis()
+ val ret = block
+ val endTs = System.currentTimeMillis()
+ println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr")
+ ret
+ }
+
+ def durationWithReturn[T](prefix: String = "")(block: => T): (T, Long) = {
+ val startTs = System.currentTimeMillis()
+ val ret = block
+ val endTs = System.currentTimeMillis()
+ val duration = endTs - startTs
+// println(s"$wrapStr\n$prefix: took $duration ms$wrapStr")
+ (ret, duration)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala
new file mode 100644
index 0000000..52ef4c5
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala
@@ -0,0 +1,124 @@
+package org.apache.s2graph.rest.play.benchmark
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.core.types.{HBaseType, InnerVal, SourceVertexId}
+import play.api.test.{FakeApplication, PlaySpecification}
+
+import scala.collection.mutable
+import scala.collection.mutable.ListBuffer
+
+class GraphUtilSpec extends BenchmarkCommon with PlaySpecification {
+
+ def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean =
+ Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0
+
+ def betweenShort(value: Short, start: Short, end: Short): Boolean =
+ start <= value && value <= end
+
+
+ "GraphUtil" should {
+ "test murmur3 hash function distribution" in {
+ val testNum = 1000000
+ val bucketSize = Short.MaxValue / 40
+ val countsNew = new mutable.HashMap[Int, Int]()
+ val counts = new mutable.HashMap[Int, Int]()
+ for {
+ i <- (0 until testNum)
+ } {
+ val h = GraphUtil.murmur3(i.toString) / bucketSize
+ val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize
+ counts += (h -> (counts.getOrElse(h, 0) + 1))
+ countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1))
+ }
+ val all = counts.toList.sortBy { case (bucket, count) => count }.reverse
+ val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse
+ val top = all.take(10)
+ val bottom = all.takeRight(10)
+ val topNew = allNew.take(10)
+ val bottomNew = allNew.takeRight(10)
+ println(s"Top: $top")
+ println(s"Bottom: $bottom")
+ println("-" * 50)
+ println(s"TopNew: $topNew")
+ println(s"Bottom: $bottomNew")
+ true
+ }
+
+ "test murmur hash skew2" in {
+ running(FakeApplication()) {
+ import HBaseType._
+ val testNum = 1000000L
+ val regionCount = 40
+ val window = Int.MaxValue / regionCount
+ val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]()
+ for {
+ i <- (0 until regionCount)
+ } yield {
+ val startKey = Bytes.toBytes(i * window)
+ val endKey = Bytes.toBytes((i + 1) * window)
+ rangeBytes += (startKey.toList -> endKey.toList)
+ }
+
+
+
+ val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]()
+ val counts = new collection.mutable.HashMap[Short, Long]()
+ stats += (0 -> (rangeBytes.head -> 0L))
+
+ for (i <- (0L until testNum)) {
+ val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION))
+ val bytes = vertexId.bytes
+ val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString())
+ val shortVal = counts.getOrElse(shortKey, 0L) + 1L
+ counts += (shortKey -> shortVal)
+ var j = 0
+ var found = false
+ while (j < rangeBytes.size && !found) {
+ val (start, end) = rangeBytes(j)
+ if (between(bytes, start.toArray, end.toArray)) {
+ found = true
+ }
+ j += 1
+ }
+ val head = rangeBytes(j - 1)
+ val key = j - 1
+ val value = stats.get(key) match {
+ case None => 0L
+ case Some(v) => v._2 + 1
+ }
+ stats += (key -> (head, value))
+ }
+ val sorted = stats.toList.sortBy(kv => kv._2._2).reverse
+ println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
+ sorted.foreach { case (idx, ((start, end), cnt)) =>
+ val startShort = Bytes.toShort(start.take(2).toArray)
+ val endShort = Bytes.toShort(end.take(2).toArray)
+ val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
+ println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
+
+ }
+ println("\n" * 10)
+ println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
+ stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) =>
+ val startShort = Bytes.toShort(start.take(2).toArray)
+ val endShort = Bytes.toShort(end.take(2).toArray)
+ val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
+ println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
+
+ }
+ }
+ true
+ }
+
+ "Bytes compareTo" in {
+ val x = Array[Byte](11, -12, -26, -14, -23)
+ val startKey = Array[Byte](0, 0, 0, 0)
+ val endKey = Array[Byte](12, -52, -52, -52)
+ println(Bytes.compareTo(startKey, x))
+ println(Bytes.compareTo(endKey, x))
+ true
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala
new file mode 100644
index 0000000..ee12b8d
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala
@@ -0,0 +1,45 @@
+package org.apache.s2graph.rest.play.benchmark
+
+import play.api.libs.json.JsNumber
+import play.libs.Json
+
+class JsonBenchmarkSpec extends BenchmarkCommon {
+ "to json" >> {
+ "json benchmark" >> {
+
+ duration("map to json") {
+ (0 to 10) foreach { n =>
+ val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ Json.toJson(numberMaps)
+ }
+ }
+
+ duration("directMakeJson") {
+ (0 to 10) foreach { n =>
+ var jsObj = play.api.libs.json.Json.obj()
+ (0 to 10).foreach { n =>
+ jsObj += (n.toString -> JsNumber(n * n))
+ }
+ }
+ }
+
+ duration("map to json 2") {
+ (0 to 50) foreach { n =>
+ val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ Json.toJson(numberMaps)
+ }
+ }
+
+ duration("directMakeJson 2") {
+ (0 to 50) foreach { n =>
+ var jsObj = play.api.libs.json.Json.obj()
+ (0 to 10).foreach { n =>
+ jsObj += (n.toString -> JsNumber(n * n))
+ }
+ }
+ }
+ true
+ }
+ true
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala
new file mode 100644
index 0000000..cc194d7
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala
@@ -0,0 +1,98 @@
+package org.apache.s2graph.rest.play.benchmark
+
+import org.apache.s2graph.core.OrderingUtil._
+import org.apache.s2graph.core.{OrderingUtil, SeqMultiOrdering}
+import play.api.libs.json.{JsNumber, JsValue}
+
+import scala.util.Random
+
+class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
+ "OrderingUtilBenchmarkSpec" should {
+
+ "performance MultiOrdering any" >> {
+ val tupLs = (0 until 10) map { i =>
+ Random.nextDouble() -> Random.nextLong()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(tup._1, tup._2)
+ }
+
+ val sorted1 = duration("TupleOrdering double,long") {
+ (0 until 1000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }.map { x => x._1 }
+
+ val sorted2 = duration("MultiOrdering double,long") {
+ (0 until 1000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
+ }
+ seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
+ }.map { x => x.head }
+
+ sorted1.toString() must_== sorted2.toString()
+ }
+
+ "performance MultiOrdering double" >> {
+ val tupLs = (0 until 50) map { i =>
+ Random.nextDouble() -> Random.nextDouble()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(tup._1, tup._2)
+ }
+
+ duration("MultiOrdering double") {
+ (0 until 1000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
+ }
+ }
+
+ duration("TupleOrdering double") {
+ (0 until 1000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ }
+
+ 1 must_== 1
+ }
+
+ "performance MultiOrdering jsvalue" >> {
+ val tupLs = (0 until 50) map { i =>
+ Random.nextDouble() -> Random.nextLong()
+ }
+
+ val seqLs = tupLs.map { tup =>
+ Seq(JsNumber(tup._1), JsNumber(tup._2))
+ }
+
+ val sorted1 = duration("TupleOrdering double,long") {
+ (0 until 1000) foreach { _ =>
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+ tupLs.sortBy { case (x, y) =>
+ -x -> -y
+ }
+ }
+
+ val sorted2 = duration("MultiOrdering jsvalue") {
+ (0 until 1000) foreach { _ =>
+ seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
+ }
+ seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
+ }
+
+ 1 must_== 1
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala
new file mode 100644
index 0000000..2db95e4
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala
@@ -0,0 +1,87 @@
+package org.apache.s2graph.rest.play.benchmark
+
+import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
+
+import scala.annotation.tailrec
+import scala.util.Random
+
+class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
+ "sample" should {
+ implicit val app = FakeApplication()
+
+ "sample benchmark" in new WithApplication(app) {
+ @tailrec
+ def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (set.size == n) set
+ else randomInt(n, range, set + Random.nextInt(range))
+ }
+
+ // sample using random array
+ def randomArraySample[T](num: Int, ls: List[T]): List[T] = {
+ val randomNum = randomInt(num, ls.size)
+ var sample = List.empty[T]
+ var idx = 0
+ ls.foreach { e =>
+ if (randomNum.contains(idx)) sample = e :: sample
+ idx += 1
+ }
+ sample
+ }
+
+ // sample using shuffle
+ def shuffleSample[T](num: Int, ls: List[T]): List[T] = {
+ Random.shuffle(ls).take(num)
+ }
+
+ // sample using random number generation
+ def rngSample[T](num: Int, ls: List[T]): List[T] = {
+ var sampled = List.empty[T]
+ val N = ls.size // population
+ var t = 0 // total input records dealt with
+ var m = 0 // number of items selected so far
+
+ while (m < num) {
+ val u = Random.nextDouble()
+ if ( (N - t)*u < num - m) {
+ sampled = ls(t) :: sampled
+ m += 1
+ }
+ t += 1
+ }
+ sampled
+ }
+
+ // test data
+ val testLimit = 500000
+ val testNum = 10
+ val testData = (0 to 1000).toList
+
+ // dummy for warm-up
+ (0 to testLimit) foreach { n =>
+ randomArraySample(testNum, testData)
+ shuffleSample(testNum, testData)
+ rngSample(testNum, testData)
+ }
+
+ duration("Random Array Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = randomArraySample(testNum, testData)
+ }
+ }
+
+ duration("Shuffle Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = shuffleSample(testNum, testData)
+ }
+ }
+
+ duration("RNG Sampling") {
+ (0 to testLimit) foreach { _ =>
+ val sampled = rngSample(testNum, testData)
+ }
+ }
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala
new file mode 100644
index 0000000..998aef8
--- /dev/null
+++ b/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala
@@ -0,0 +1,112 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.{OrderingUtil, SeqMultiOrdering}
+import play.api.libs.json.{JsNumber, JsString, JsValue}
+import play.api.test.PlaySpecification
+
+class PostProcessSpec extends PlaySpecification {
+ import OrderingUtil._
+
+ "test order by json" >> {
+ val jsLs: Seq[Seq[JsValue]] = Seq(
+ Seq(JsNumber(0), JsString("a")),
+ Seq(JsNumber(0), JsString("b")),
+ Seq(JsNumber(1), JsString("a")),
+ Seq(JsNumber(1), JsString("b")),
+ Seq(JsNumber(2), JsString("c"))
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[Seq[JsValue]] = Seq(
+ Seq(JsNumber(2), JsString("c")),
+ Seq(JsNumber(1), JsString("a")),
+ Seq(JsNumber(1), JsString("b")),
+ Seq(JsNumber(0), JsString("a")),
+ Seq(JsNumber(0), JsString("b"))
+ )
+
+ val orderParam: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] {
+ override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = {
+ val xe = x.iterator
+ val ye = y.iterator
+ val oe = orderParam.iterator
+
+ while (xe.hasNext && ye.hasNext && oe.hasNext) {
+ val (xev, yev) = oe.next() match {
+ case true => xe.next() -> ye.next()
+ case false => ye.next() -> xe.next()
+ }
+ val res = (xev, yev) match {
+ case (JsNumber(xv), JsNumber(yv)) =>
+ Ordering[BigDecimal].compare(xv, yv)
+ case (JsString(xv), JsString(yv)) =>
+ Ordering[String].compare(xv, yv)
+ case _ => throw new Exception("type mismatch")
+ }
+ if (res != 0) return res
+ }
+
+ Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
+ }
+ })
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+
+ "test order by primitive type" >> {
+ val jsLs: Seq[Seq[Any]] = Seq(
+ Seq(0, "a"),
+ Seq(0, "b"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(2, "c")
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[Seq[Any]] = Seq(
+ Seq(2, "c"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(0, "a"),
+ Seq(0, "b")
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+
+ "test order by primitive type with short ascending list" >> {
+ val jsLs: Seq[Seq[Any]] = Seq(
+ Seq(0, "a"),
+ Seq(1, "b"),
+ Seq(0, "b"),
+ Seq(1, "a"),
+ Seq(2, "c"),
+ Seq(1, "c"),
+ Seq(1, "d"),
+ Seq(1, "f"),
+ Seq(1, "e")
+ )
+
+ // number descending, string ascending(default)
+ val sortedJsLs: Seq[Seq[Any]] = Seq(
+ Seq(2, "c"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(1, "c"),
+ Seq(1, "d"),
+ Seq(1, "e"),
+ Seq(1, "f"),
+ Seq(0, "a"),
+ Seq(0, "b")
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false)
+ val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() must_== sortedJsLs.toString
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/config/S2ConfigFactory.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/config/S2ConfigFactory.scala b/spark/src/main/scala/org/apache/s2graph/spark/config/S2ConfigFactory.scala
new file mode 100644
index 0000000..99d3ba4
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/config/S2ConfigFactory.scala
@@ -0,0 +1,20 @@
+package org.apache.s2graph.spark.config
+
+import com.typesafe.config.{Config, ConfigFactory}
+
+object S2ConfigFactory {
+ lazy val config: Config = _load
+
+ @deprecated("do not call explicitly. use config", "0.0.6")
+ def load(): Config = {
+ _load
+ }
+
+ def _load: Config = {
+ // default configuration file name : application.conf
+ val sysConfig = ConfigFactory.parseProperties(System.getProperties)
+
+ lazy val phase = if (!sysConfig.hasPath("phase")) "alpha" else sysConfig.getString("phase")
+ sysConfig.withFallback(ConfigFactory.parseResourcesAnySyntax(s"$phase.conf")).withFallback(ConfigFactory.load())
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/spark/HashMapParam.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/spark/HashMapParam.scala b/spark/src/main/scala/org/apache/s2graph/spark/spark/HashMapParam.scala
new file mode 100644
index 0000000..b8e73ca
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/spark/HashMapParam.scala
@@ -0,0 +1,55 @@
+package org.apache.s2graph.spark.spark
+
+import org.apache.spark.serializer.JavaSerializer
+import org.apache.spark.{AccumulableParam, SparkConf}
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+
+/*
+ * Allows a mutable HashMap[String, Int] to be used as an accumulator in Spark.
+ * Whenever we try to put (k, v2) into an accumulator that already contains (k, v1), the result
+ * will be a HashMap containing (k, v1 + v2).
+ *
+ * Would have been nice to extend GrowableAccumulableParam instead of redefining everything, but it's
+ * private to the spark package.
+ */
+
+class HashMapParam[K, V](op: (V, V) => V) extends AccumulableParam[MutableHashMap[K, V], (K, V)] {
+ type MapType = MutableHashMap[K, V]
+ type ElemType = (K, V)
+
+ def addAccumulator(acc: MapType, elem: ElemType): MapType = {
+ val (k1, v1) = elem
+ acc += acc.find(_._1 == k1).map {
+ case (k2, v2) => k2 -> op(v1, v2)
+ }.getOrElse(elem)
+
+ acc
+ }
+
+ /*
+ * This method is allowed to modify and return the first value for efficiency.
+ *
+ * @see org.apache.spark.GrowableAccumulableParam.addInPlace(r1: R, r2: R): R
+ */
+ def addInPlace(acc1: MapType, acc2: MapType): MapType = {
+ acc2.foreach(elem => addAccumulator(acc1, elem))
+ acc1
+ }
+
+ /*
+ * @see org.apache.spark.GrowableAccumulableParam.zero(initialValue: R): R
+ */
+ def zero(initialValue: MapType): MapType = {
+ val ser = new JavaSerializer(new SparkConf(false)).newInstance()
+ val copy = ser.deserialize[MapType](ser.serialize(initialValue))
+ copy.clear()
+ copy
+ }
+}
+
+object HashMapParam {
+ def apply[K, V](op: (V, V) => V): HashMapParam[K, V] = {
+ new HashMapParam[K, V](op)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/spark/RDDUtil.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/spark/RDDUtil.scala b/spark/src/main/scala/org/apache/s2graph/spark/spark/RDDUtil.scala
new file mode 100644
index 0000000..651d78b
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/spark/RDDUtil.scala
@@ -0,0 +1,13 @@
+package org.apache.s2graph.spark.spark
+
+import org.apache.spark.rdd.RDD
+
+object RDDUtil {
+ def rddIsNonEmpty[T](rdd: RDD[T]): Boolean = {
+ !rddIsEmpty(rdd)
+ }
+
+ def rddIsEmpty[T](rdd: RDD[T]): Boolean = {
+ rdd.mapPartitions(it => Iterator(!it.hasNext)).reduce(_ && _)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/spark/SparkApp.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/spark/SparkApp.scala b/spark/src/main/scala/org/apache/s2graph/spark/spark/SparkApp.scala
new file mode 100644
index 0000000..2e11904
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/spark/SparkApp.scala
@@ -0,0 +1,120 @@
+package org.apache.s2graph.spark.spark
+
+import kafka.serializer.StringDecoder
+import org.apache.spark.storage.StorageLevel
+import org.apache.spark.streaming.dstream.DStream
+import org.apache.spark.streaming.kafka.{KafkaUtils, StreamHelper}
+import org.apache.spark.streaming.{Duration, StreamingContext}
+import org.apache.spark.{Accumulable, Logging, SparkConf}
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+
+trait SparkApp extends Logging {
+ type HashMapAccumulable = Accumulable[MutableHashMap[String, Long], (String, Long)]
+
+ protected def args: Array[String] = _args
+
+ private var _args: Array[String] = _
+
+ private var streamHelper: StreamHelper = _
+
+ // should implement in derived class
+ def run()
+
+ def getArgs(index: Int) = args(index)
+
+ def main(args: Array[String]) {
+ _args = args
+ run()
+ }
+
+ def validateArgument(argNames: String*): Unit = {
+ if (args == null || args.length < argNames.length) {
+ System.err.println(s"Usage: ${getClass.getName} " + argNames.map(s => s"<$s>").mkString(" "))
+ System.exit(1)
+ }
+ }
+
+ def buildKafkaGroupId(topic: String, ext: String): String = {
+ val phase = System.getProperty("phase")
+
+ // use first topic for group id
+ val groupId = s"${topic.split(',')(0)}_$ext"
+
+ groupId + {
+ phase match {
+ case "real" | "production" => ""
+ case x => s"_$x"
+ }
+ }
+ }
+
+ def getStreamHelper(kafkaParam: Map[String, String]): StreamHelper = {
+ if (streamHelper == null) {
+ this.synchronized {
+ if (streamHelper == null) {
+ streamHelper = StreamHelper(kafkaParam)
+ }
+ }
+ }
+ streamHelper
+ }
+
+ def sparkConf(jobName: String): SparkConf = {
+ val conf = new SparkConf()
+ conf.setAppName(jobName)
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ conf.set("spark.streaming.unpersist", "true")
+ conf
+ }
+
+ def streamingContext(sparkConf: SparkConf, interval: Duration, checkPoint: Option[String] = None) = {
+ val ssc = new StreamingContext(sparkConf, interval)
+ checkPoint.foreach { dir =>
+ ssc.checkpoint(dir)
+ }
+
+ // for watch tower
+ ssc.addStreamingListener(new SubscriberListener(ssc))
+
+ ssc
+ }
+
+ def createKafkaPairStream(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, numPartition: Option[Int] = None): DStream[(String, String)] = {
+ val topicMap = topics.split(",").map((_, 1)).toMap
+ val stream = KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParam, topicMap, StorageLevel.MEMORY_AND_DISK_SER_2)
+ numPartition.map(n =>
+ stream.repartition(n)
+ ).getOrElse(stream)
+ }
+
+ def createKafkaValueStream(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, numPartition: Option[Int] = None): DStream[String] = {
+ createKafkaPairStream(ssc, kafkaParam, topics, numPartition).map(_._2)
+ }
+
+ def createKafkaPairStreamMulti(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, receiverCount: Int, numPartition: Option[Int] = None): DStream[(String, String)] = {
+ // wait until all executor is running
+ Stream.continually(ssc.sparkContext.getExecutorStorageStatus).takeWhile(_.length < receiverCount).foreach { arr =>
+ Thread.sleep(100)
+ }
+ Thread.sleep(1000)
+
+ val topicMap = topics.split(",").map((_, 1)).toMap
+
+ val stream = {
+ val streams = {
+ (1 to receiverCount) map { _ =>
+ KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParam, topicMap, StorageLevel.MEMORY_AND_DISK_SER_2)
+ }
+ }
+ ssc.union(streams)
+ }
+ numPartition.map(n =>
+ stream.repartition(n)
+ ).getOrElse(stream)
+ }
+
+ def createKafkaValueStreamMulti(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, receiverCount: Int, numPartition: Option[Int] = None): DStream[String] = {
+ createKafkaPairStreamMulti(ssc, kafkaParam, topics, receiverCount, numPartition).map(_._2)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/spark/SubscriberListener.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/spark/SubscriberListener.scala b/spark/src/main/scala/org/apache/s2graph/spark/spark/SubscriberListener.scala
new file mode 100644
index 0000000..f23debb
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/spark/SubscriberListener.scala
@@ -0,0 +1,20 @@
+package org.apache.s2graph.spark.spark
+
+import org.apache.spark.Logging
+import org.apache.spark.streaming.StreamingContext
+import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverError, StreamingListenerReceiverStarted, StreamingListenerReceiverStopped}
+
+class SubscriberListener(ssc: StreamingContext) extends StreamingListener with Logging {
+ override def onReceiverError(receiverError: StreamingListenerReceiverError): Unit = {
+ logInfo("onReceiverError")
+ }
+
+ override def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted): Unit = {
+ logInfo("onReceiverStarted")
+ }
+
+ override def onReceiverStopped(receiverStopped: StreamingListenerReceiverStopped): Unit = {
+ logInfo("onReceiverStopped")
+ ssc.stop()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/s2graph/spark/spark/WithKafka.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/s2graph/spark/spark/WithKafka.scala b/spark/src/main/scala/org/apache/s2graph/spark/spark/WithKafka.scala
new file mode 100644
index 0000000..f28b9cf
--- /dev/null
+++ b/spark/src/main/scala/org/apache/s2graph/spark/spark/WithKafka.scala
@@ -0,0 +1,69 @@
+package org.apache.s2graph.spark.spark
+
+import java.util.Properties
+
+import kafka.producer.{Producer, ProducerConfig}
+
+trait WithKafka {
+ def kafkaConf(brokerList: String) = {
+ val props = new Properties()
+ props.put("metadata.broker.list", brokerList)
+ props.put("request.required.acks", "0")
+ props.put("producer.type", "async")
+ props.put("serializer.class", "kafka.serializer.StringEncoder")
+ props.put("compression.codec", "1")
+ props.put("message.send.max.retries", "3")
+ props.put("batch.num.messages", "1000")
+ new ProducerConfig(props)
+ }
+
+ def producerConfig(brokerList: String, requireAcks: String = "1", producerType: String = "sync") = {
+ val props = new Properties()
+ props.setProperty("metadata.broker.list", brokerList)
+ props.setProperty("request.required.acks", requireAcks)
+ props.setProperty("producer.type", producerType)
+ props.setProperty("serializer.class", "kafka.serializer.StringEncoder")
+ props.setProperty("compression.codec", "snappy")
+ props.setProperty("message.send.max.retries", "1")
+ new ProducerConfig(props)
+ }
+
+ def getProducer[K, V](config: ProducerConfig): Producer[K, V] = {
+ new Producer[K, V](config)
+ }
+
+ def getProducer[K, V](brokers: String): Producer[K, V] = {
+ getProducer(producerConfig(brokers))
+ }
+
+ /**
+ * Kafka DefaultPartitioner
+ * @param k
+ * @param n
+ * @return
+ */
+ def getPartKey(k: Any, n: Int): Int = {
+ kafka.utils.Utils.abs(k.hashCode()) % n
+ }
+
+ def makeKafkaGroupId(topic: String, ext: String): String = {
+ val phase = System.getProperty("phase")
+
+ var groupId = s"${topic}_$ext"
+
+ groupId += {
+ System.getProperty("spark.master") match {
+ case x if x.startsWith("local") => "_local"
+ case _ => ""
+ }
+ }
+
+ groupId += {
+ phase match {
+ case "alpha" => "_alpha"
+ case _ => ""
+ }}
+
+ groupId
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDFunctions.scala b/spark/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDFunctions.scala
index 56be543..768bedb 100644
--- a/spark/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDFunctions.scala
+++ b/spark/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDDFunctions.scala
@@ -6,9 +6,6 @@ import org.apache.spark.rdd.RDD
import scala.language.implicitConversions
import scala.reflect.ClassTag
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 6..
- */
class KafkaRDDFunctions[T: ClassTag](self: RDD[T])
extends Logging
with Serializable
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/org/apache/spark/streaming/kafka/StreamHelper.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/spark/streaming/kafka/StreamHelper.scala b/spark/src/main/scala/org/apache/spark/streaming/kafka/StreamHelper.scala
index 782f87f..0baf397 100644
--- a/spark/src/main/scala/org/apache/spark/streaming/kafka/StreamHelper.scala
+++ b/spark/src/main/scala/org/apache/spark/streaming/kafka/StreamHelper.scala
@@ -8,12 +8,8 @@ import kafka.serializer.Decoder
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.{Logging, SparkException}
-
import scala.reflect.ClassTag
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 4. 22..
- */
case class StreamHelper(kafkaParams: Map[String, String]) extends Logging {
// helper for kafka zookeeper
lazy val kafkaHelper = KafkaHelper(kafkaParams)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/config/S2ConfigFactory.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/config/S2ConfigFactory.scala b/spark/src/main/scala/s2/config/S2ConfigFactory.scala
deleted file mode 100644
index 7666cdc..0000000
--- a/spark/src/main/scala/s2/config/S2ConfigFactory.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package s2.config
-
-import com.typesafe.config.{Config, ConfigFactory}
-
-/**
- * Created by alec on 15. 3. 4..
- */
-
-/**
- * phase에 따라 phase.conf 파일을 load 해주는 config factory
- */
-object S2ConfigFactory {
- lazy val config: Config = _load
-
- @deprecated("do not call explicitly. use config", "0.0.6")
- def load(): Config = {
- _load
- }
-
- def _load: Config = {
- // default configuration file name : application.conf
- val sysConfig = ConfigFactory.parseProperties(System.getProperties)
-
- lazy val phase = if (!sysConfig.hasPath("phase")) "alpha" else sysConfig.getString("phase")
- sysConfig.withFallback(ConfigFactory.parseResourcesAnySyntax(s"$phase.conf")).withFallback(ConfigFactory.load())
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/spark/HashMapParam.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/spark/HashMapParam.scala b/spark/src/main/scala/s2/spark/HashMapParam.scala
deleted file mode 100644
index a84c687..0000000
--- a/spark/src/main/scala/s2/spark/HashMapParam.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-package s2.spark
-
-import org.apache.spark.serializer.JavaSerializer
-import org.apache.spark.{AccumulableParam, SparkConf}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-/*
- * Allows a mutable HashMap[String, Int] to be used as an accumulator in Spark.
- * Whenever we try to put (k, v2) into an accumulator that already contains (k, v1), the result
- * will be a HashMap containing (k, v1 + v2).
- *
- * Would have been nice to extend GrowableAccumulableParam instead of redefining everything, but it's
- * private to the spark package.
- */
-
-class HashMapParam[K, V](op: (V, V) => V) extends AccumulableParam[MutableHashMap[K, V], (K, V)] {
- type MapType = MutableHashMap[K, V]
- type ElemType = (K, V)
-
- def addAccumulator(acc: MapType, elem: ElemType): MapType = {
- val (k1, v1) = elem
- acc += acc.find(_._1 == k1).map {
- case (k2, v2) => k2 -> op(v1, v2)
- }.getOrElse(elem)
-
- acc
- }
-
- /*
- * This method is allowed to modify and return the first value for efficiency.
- *
- * @see org.apache.spark.GrowableAccumulableParam.addInPlace(r1: R, r2: R): R
- */
- def addInPlace(acc1: MapType, acc2: MapType): MapType = {
- acc2.foreach(elem => addAccumulator(acc1, elem))
- acc1
- }
-
- /*
- * @see org.apache.spark.GrowableAccumulableParam.zero(initialValue: R): R
- */
- def zero(initialValue: MapType): MapType = {
- val ser = new JavaSerializer(new SparkConf(false)).newInstance()
- val copy = ser.deserialize[MapType](ser.serialize(initialValue))
- copy.clear()
- copy
- }
-}
-
-object HashMapParam {
- def apply[K, V](op: (V, V) => V): HashMapParam[K, V] = {
- new HashMapParam[K, V](op)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/spark/RDDUtil.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/spark/RDDUtil.scala b/spark/src/main/scala/s2/spark/RDDUtil.scala
deleted file mode 100644
index c5a371a..0000000
--- a/spark/src/main/scala/s2/spark/RDDUtil.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package s2.spark
-
-import org.apache.spark.rdd.RDD
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 14. 12. 23..
- */
-object RDDUtil {
- def rddIsNonEmpty[T](rdd: RDD[T]): Boolean = {
- !rddIsEmpty(rdd)
- }
-
- def rddIsEmpty[T](rdd: RDD[T]): Boolean = {
- rdd.mapPartitions(it => Iterator(!it.hasNext)).reduce(_ && _)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/spark/SparkApp.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/spark/SparkApp.scala b/spark/src/main/scala/s2/spark/SparkApp.scala
deleted file mode 100644
index e27b7ec..0000000
--- a/spark/src/main/scala/s2/spark/SparkApp.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-package s2.spark
-
-import kafka.serializer.StringDecoder
-import org.apache.spark.storage.StorageLevel
-import org.apache.spark.streaming.dstream.DStream
-import org.apache.spark.streaming.kafka.{KafkaUtils, StreamHelper}
-import org.apache.spark.streaming.{Duration, StreamingContext}
-import org.apache.spark.{Accumulable, Logging, SparkConf}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 14. 12. 26..
- */
-trait SparkApp extends Logging {
- type HashMapAccumulable = Accumulable[MutableHashMap[String, Long], (String, Long)]
-
- protected def args: Array[String] = _args
-
- private var _args: Array[String] = _
-
- private var streamHelper: StreamHelper = _
-
- // should implement in derived class
- def run()
-
- def getArgs(index: Int) = args(index)
-
- def main(args: Array[String]) {
- _args = args
- run()
- }
-
- def validateArgument(argNames: String*): Unit = {
- if (args == null || args.length < argNames.length) {
- System.err.println(s"Usage: ${getClass.getName} " + argNames.map(s => s"<$s>").mkString(" "))
- System.exit(1)
- }
- }
-
- def buildKafkaGroupId(topic: String, ext: String): String = {
- val phase = System.getProperty("phase")
-
- // use first topic for group id
- val groupId = s"${topic.split(',')(0)}_$ext"
-
- groupId + {
- phase match {
- case "real" | "production" => ""
- case x => s"_$x"
- }
- }
- }
-
- def getStreamHelper(kafkaParam: Map[String, String]): StreamHelper = {
- if (streamHelper == null) {
- this.synchronized {
- if (streamHelper == null) {
- streamHelper = StreamHelper(kafkaParam)
- }
- }
- }
- streamHelper
- }
-
- def sparkConf(jobName: String): SparkConf = {
- val conf = new SparkConf()
- conf.setAppName(jobName)
- conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- conf.set("spark.streaming.unpersist", "true")
- conf
- }
-
- def streamingContext(sparkConf: SparkConf, interval: Duration, checkPoint: Option[String] = None) = {
- val ssc = new StreamingContext(sparkConf, interval)
- checkPoint.foreach { dir =>
- ssc.checkpoint(dir)
- }
-
- // for watch tower
- ssc.addStreamingListener(new SubscriberListener(ssc))
-
- ssc
- }
-
- def createKafkaPairStream(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, numPartition: Option[Int] = None): DStream[(String, String)] = {
- val topicMap = topics.split(",").map((_, 1)).toMap
- val stream = KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParam, topicMap, StorageLevel.MEMORY_AND_DISK_SER_2)
- numPartition.map(n =>
- stream.repartition(n)
- ).getOrElse(stream)
- }
-
- def createKafkaValueStream(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, numPartition: Option[Int] = None): DStream[String] = {
- createKafkaPairStream(ssc, kafkaParam, topics, numPartition).map(_._2)
- }
-
- def createKafkaPairStreamMulti(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, receiverCount: Int, numPartition: Option[Int] = None): DStream[(String, String)] = {
- // wait until all executor is running
- Stream.continually(ssc.sparkContext.getExecutorStorageStatus).takeWhile(_.length < receiverCount).foreach { arr =>
- Thread.sleep(100)
- }
- Thread.sleep(1000)
-
- val topicMap = topics.split(",").map((_, 1)).toMap
-
- val stream = {
- val streams = {
- (1 to receiverCount) map { _ =>
- KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParam, topicMap, StorageLevel.MEMORY_AND_DISK_SER_2)
- }
- }
- ssc.union(streams)
- }
- numPartition.map(n =>
- stream.repartition(n)
- ).getOrElse(stream)
- }
-
- def createKafkaValueStreamMulti(ssc: StreamingContext, kafkaParam: Map[String, String], topics: String, receiverCount: Int, numPartition: Option[Int] = None): DStream[String] = {
- createKafkaPairStreamMulti(ssc, kafkaParam, topics, receiverCount, numPartition).map(_._2)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/spark/SubscriberListener.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/spark/SubscriberListener.scala b/spark/src/main/scala/s2/spark/SubscriberListener.scala
deleted file mode 100644
index 9e156a7..0000000
--- a/spark/src/main/scala/s2/spark/SubscriberListener.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package s2.spark
-
-import org.apache.spark.Logging
-import org.apache.spark.streaming.StreamingContext
-import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverError, StreamingListenerReceiverStarted, StreamingListenerReceiverStopped}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 1. 8..
- */
-
-class SubscriberListener(ssc: StreamingContext) extends StreamingListener with Logging {
- override def onReceiverError(receiverError: StreamingListenerReceiverError): Unit = {
- logInfo("onReceiverError")
- }
-
- override def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted): Unit = {
- logInfo("onReceiverStarted")
- }
-
- override def onReceiverStopped(receiverStopped: StreamingListenerReceiverStopped): Unit = {
- logInfo("onReceiverStopped")
- ssc.stop()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/main/scala/s2/spark/WithKafka.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/s2/spark/WithKafka.scala b/spark/src/main/scala/s2/spark/WithKafka.scala
deleted file mode 100644
index 9bd5944..0000000
--- a/spark/src/main/scala/s2/spark/WithKafka.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package s2.spark
-
-import java.util.Properties
-
-import kafka.producer.{Producer, ProducerConfig}
-
-trait WithKafka {
- def kafkaConf(brokerList: String) = {
- val props = new Properties()
- props.put("metadata.broker.list", brokerList)
- props.put("request.required.acks", "0")
- props.put("producer.type", "async")
- props.put("serializer.class", "kafka.serializer.StringEncoder")
- props.put("compression.codec", "1")
- props.put("message.send.max.retries", "3")
- props.put("batch.num.messages", "1000")
- new ProducerConfig(props)
- }
-
- def producerConfig(brokerList: String, requireAcks: String = "1", producerType: String = "sync") = {
- val props = new Properties()
- props.setProperty("metadata.broker.list", brokerList)
- props.setProperty("request.required.acks", requireAcks)
- props.setProperty("producer.type", producerType)
- props.setProperty("serializer.class", "kafka.serializer.StringEncoder")
- props.setProperty("compression.codec", "snappy")
- props.setProperty("message.send.max.retries", "1")
- new ProducerConfig(props)
- }
-
- def getProducer[K, V](config: ProducerConfig): Producer[K, V] = {
- new Producer[K, V](config)
- }
-
- def getProducer[K, V](brokers: String): Producer[K, V] = {
- getProducer(producerConfig(brokers))
- }
-
- /**
- * Kafka DefaultPartitioner
- * @param k
- * @param n
- * @return
- */
- def getPartKey(k: Any, n: Int): Int = {
- kafka.utils.Utils.abs(k.hashCode()) % n
- }
-
- def makeKafkaGroupId(topic: String, ext: String): String = {
- val phase = System.getProperty("phase")
-
- var groupId = s"${topic}_$ext"
-
- groupId += {
- System.getProperty("spark.master") match {
- case x if x.startsWith("local") => "_local"
- case _ => ""
- }
- }
-
- groupId += {
- phase match {
- case "alpha" => "_alpha"
- case _ => ""
- }}
-
- groupId
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/test/scala/org/apache/s2graph/spark/SparkAppTest.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/s2graph/spark/SparkAppTest.scala b/spark/src/test/scala/org/apache/s2graph/spark/SparkAppTest.scala
new file mode 100644
index 0000000..2d48a89
--- /dev/null
+++ b/spark/src/test/scala/org/apache/s2graph/spark/SparkAppTest.scala
@@ -0,0 +1,17 @@
+package org.apache.s2graph.spark
+
+import org.apache.s2graph.spark.spark.SparkApp
+import org.scalatest.{FunSuite, Matchers}
+
+object TestApp extends SparkApp {
+ override def run(): Unit = {
+ validateArgument("topic", "phase")
+ }
+}
+
+class SparkAppTest extends FunSuite with Matchers {
+ test("parse argument") {
+ TestApp.main(Array("s2graphInreal", "real"))
+ TestApp.getArgs(0) shouldEqual "s2graphInreal"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/test/scala/org/apache/s2graph/spark/TestStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/s2graph/spark/TestStreamingSpec.scala b/spark/src/test/scala/org/apache/s2graph/spark/TestStreamingSpec.scala
new file mode 100644
index 0000000..df6c7e5
--- /dev/null
+++ b/spark/src/test/scala/org/apache/s2graph/spark/TestStreamingSpec.scala
@@ -0,0 +1,31 @@
+package org.apache.s2graph.spark
+
+import org.apache.spark.streaming.{Seconds, StreamingContext}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.specs2.mutable.Specification
+import org.specs2.specification.BeforeAfterAll
+
+class TestStreamingSpec extends Specification with BeforeAfterAll {
+ private val master = "local[2]"
+ private val appName = "test_streaming"
+ private val batchDuration = Seconds(1)
+
+ private var sc: SparkContext = _
+ private var ssc: StreamingContext = _
+
+ override def beforeAll(): Unit = {
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ ssc = new StreamingContext(conf, batchDuration)
+
+ sc = ssc.sparkContext
+ }
+
+ override def afterAll(): Unit = {
+ if (ssc != null) {
+ ssc.stop()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/test/scala/s2/spark/SparkAppTest.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/s2/spark/SparkAppTest.scala b/spark/src/test/scala/s2/spark/SparkAppTest.scala
deleted file mode 100644
index 6bf2482..0000000
--- a/spark/src/test/scala/s2/spark/SparkAppTest.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-package s2.spark
-
-import org.scalatest.{FunSuite, Matchers}
-
-/**
- * Created by alec.k on 14. 12. 26..
- */
-
-object TestApp extends SparkApp {
- // 상속받은 클래스에서 구현해줘야 하는 함수
- override def run(): Unit = {
- validateArgument("topic", "phase")
- }
-}
-
-class SparkAppTest extends FunSuite with Matchers {
- test("parse argument") {
- TestApp.main(Array("s2graphInreal", "real"))
- TestApp.getArgs(0) shouldEqual "s2graphInreal"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/spark/src/test/scala/s2/spark/TestStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/s2/spark/TestStreamingSpec.scala b/spark/src/test/scala/s2/spark/TestStreamingSpec.scala
deleted file mode 100644
index bf15618..0000000
--- a/spark/src/test/scala/s2/spark/TestStreamingSpec.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-package s2.spark
-
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{SparkConf, SparkContext}
-import org.specs2.mutable.Specification
-import org.specs2.specification.BeforeAfterAll
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 17..
- */
-class TestStreamingSpec extends Specification with BeforeAfterAll {
- private val master = "local[2]"
- private val appName = "test_streaming"
- private val batchDuration = Seconds(1)
-
- private var sc: SparkContext = _
- private var ssc: StreamingContext = _
-
- override def beforeAll(): Unit = {
- val conf = new SparkConf()
- .setMaster(master)
- .setAppName(appName)
-
- ssc = new StreamingContext(conf, batchDuration)
-
- sc = ssc.sparkContext
- }
-
- override def afterAll(): Unit = {
- if (ssc != null) {
- ssc.stop()
- }
- }
-}
[02/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/CounterController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/CounterController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/CounterController.scala
new file mode 100644
index 0000000..4587d59
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/CounterController.scala
@@ -0,0 +1,744 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.s2graph.core.ExceptionHandler
+import org.apache.s2graph.core.ExceptionHandler.KafkaMessage
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter.core.v1.{RankingStorageRedis, ExactStorageAsyncHBase}
+import org.apache.s2graph.counter.core.v2.{RankingStorageGraph, ExactStorageGraph}
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.apache.s2graph.counter.models.{Counter, CounterModel}
+import org.apache.s2graph.counter.util.{ReduceMapValue, CartesianProduct, UnitConverter}
+import org.apache.s2graph.rest.play.config.CounterConfig
+import org.apache.s2graph.rest.play.models._
+import play.api.Play
+import play.api.libs.json.Reads._
+import play.api.libs.json._
+import play.api.mvc.{Action, Controller, Request}
+import scala.concurrent.Future
+import scala.util.{Failure, Success, Try}
+
+object CounterController extends Controller {
+ import play.api.libs.concurrent.Execution.Implicits.defaultContext
+
+ val config = Play.current.configuration.underlying
+ val s2config = new S2CounterConfig(config)
+
+ private val exactCounterMap = Map(
+ counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
+ counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
+ )
+ private val rankingCounterMap = Map(
+ counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
+ counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
+ )
+
+ private val tablePrefixMap = Map (
+ counter.VERSION_1 -> "s2counter",
+ counter.VERSION_2 -> "s2counter_v2"
+ )
+
+ private def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
+ private def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
+
+ lazy val counterModel = new CounterModel(config)
+
+ def getQueryString[T](key: String, default: String)(implicit request: Request[T]): String = {
+ request.getQueryString(key).getOrElse(default)
+ }
+
+ implicit val counterWrites = new Writes[Counter] {
+ override def writes(o: Counter): JsValue = Json.obj(
+ "version" -> o.version.toInt,
+ "autoComb" -> o.autoComb,
+ "dimension" -> o.dimension,
+ "useProfile" -> o.useProfile,
+ "bucketImpId" -> o.bucketImpId,
+ "useRank" -> o.useRank,
+ "intervalUnit" -> o.intervalUnit,
+ "ttl" -> o.ttl,
+ "dailyTtl" -> o.dailyTtl,
+ "rateAction" -> o.rateActionId.flatMap { actionId =>
+ counterModel.findById(actionId, useCache = false).map { actionPolicy =>
+ Json.obj("service" -> actionPolicy.service, "action" -> actionPolicy.action)
+ }
+ },
+ "rateBase" -> o.rateBaseId.flatMap { baseId =>
+ counterModel.findById(baseId, useCache = false).map { basePolicy =>
+ Json.obj("service" -> basePolicy.service, "action" -> basePolicy.action)
+ }
+ },
+ "rateThreshold" -> o.rateThreshold
+ )
+ }
+
+ def createAction(service: String, action: String) = Action(s2parse.json) { implicit request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case None =>
+ val body = request.body
+ val version = (body \ "version").asOpt[Int].map(_.toByte).getOrElse(counter.VERSION_2)
+ val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(true)
+ val dimension = (body \ "dimension").asOpt[String].getOrElse("")
+ val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(false)
+ val bucketImpId = (body \ "bucketImpId").asOpt[String]
+
+ val useExact = (body \ "useExact").asOpt[Boolean].getOrElse(true)
+ val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(true)
+
+ val intervalUnit = (body \ "intervalUnit").asOpt[String]
+ // 2 day
+ val ttl = (body \ "ttl").asOpt[Int].getOrElse(2 * 24 * 60 * 60)
+ val dailyTtl = (body \ "dailyTtl").asOpt[Int]
+ val regionMultiplier = (body \ "regionMultiplier").asOpt[Int].getOrElse(1)
+
+ val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
+ val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
+ val rateThreshold = (body \ "rateThreshold").asOpt[Int]
+
+ val rateActionId = {
+ for {
+ actionMap <- rateAction
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action)
+ } yield {
+ policy.id
+ }
+ }
+ val rateBaseId = {
+ for {
+ actionMap <- rateBase
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action)
+ } yield {
+ policy.id
+ }
+ }
+
+ val hbaseTable = {
+ Seq(tablePrefixMap(version), service, ttl) ++ dailyTtl mkString "_"
+ }
+
+ // find label
+ val itemType = Label.findByName(action, useCache = false) match {
+ case Some(label) =>
+ ItemType.withName(label.tgtColumnType.toUpperCase)
+ case None =>
+ val strItemType = (body \ "itemType").asOpt[String].getOrElse("STRING")
+ ItemType.withName(strItemType.toUpperCase)
+ }
+ val policy = Counter(useFlag = true, version, service, action, itemType, autoComb = autoComb, dimension,
+ useProfile = useProfile, bucketImpId, useRank = useRank, ttl, dailyTtl, Some(hbaseTable), intervalUnit,
+ rateActionId, rateBaseId, rateThreshold)
+
+ // prepare exact storage
+ exactCounter(version).prepare(policy)
+ if (useRank) {
+ // prepare ranking storage
+ rankingCounter(version).prepare(policy)
+ }
+ counterModel.createServiceAction(policy)
+ Ok(Json.toJson(Map("msg" -> s"created $service/$action")))
+ case Some(policy) =>
+ Ok(Json.toJson(Map("msg" -> s"already exist $service/$action")))
+ }
+ }
+
+ def getAction(service: String, action: String) = Action { request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(policy) =>
+ Ok(Json.toJson(policy))
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def updateAction(service: String, action: String) = Action(s2parse.json) { request =>
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(oldPolicy) =>
+ val body = request.body
+ val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(oldPolicy.autoComb)
+ val dimension = (body \ "dimension").asOpt[String].getOrElse(oldPolicy.dimension)
+ val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(oldPolicy.useProfile)
+ val bucketImpId = (body \ "bucketImpId").asOpt[String] match {
+ case Some(s) => Some(s)
+ case None => oldPolicy.bucketImpId
+ }
+
+ val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(oldPolicy.useRank)
+
+ val intervalUnit = (body \ "intervalUnit").asOpt[String] match {
+ case Some(s) => Some(s)
+ case None => oldPolicy.intervalUnit
+ }
+
+ val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
+ val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
+ val rateThreshold = (body \ "rateThreshold").asOpt[Int] match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateThreshold
+ }
+
+ val rateActionId = {
+ for {
+ actionMap <- rateAction
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ policy.id
+ }
+ } match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateActionId
+ }
+ val rateBaseId = {
+ for {
+ actionMap <- rateBase
+ service <- actionMap.get("service")
+ action <- actionMap.get("action")
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ policy.id
+ }
+ } match {
+ case Some(i) => Some(i)
+ case None => oldPolicy.rateBaseId
+ }
+
+ // new counter
+ val policy = Counter(id = oldPolicy.id, useFlag = oldPolicy.useFlag, oldPolicy.version, service, action, oldPolicy.itemType, autoComb = autoComb, dimension,
+ useProfile = useProfile, bucketImpId, useRank = useRank, oldPolicy.ttl, oldPolicy.dailyTtl, oldPolicy.hbaseTable, intervalUnit,
+ rateActionId, rateBaseId, rateThreshold)
+
+ counterModel.updateServiceAction(policy)
+ Ok(Json.toJson(Map("msg" -> s"updated $service/$action")))
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def prepareAction(service: String, action: String) = Action(s2parse.json) { request =>
+ // for data migration
+ counterModel.findByServiceAction(service, action, useCache = false) match {
+ case Some(policy) =>
+ val body = request.body
+ val version = (body \ "version").as[Int].toByte
+ if (version != policy.version) {
+ // change table name
+ val newTableName = Seq(tablePrefixMap(version), service, policy.ttl) ++ policy.dailyTtl mkString "_"
+ val newPolicy = policy.copy(version = version, hbaseTable = Some(newTableName))
+ exactCounter(version).prepare(newPolicy)
+ if (newPolicy.useRank) {
+ rankingCounter(version).prepare(newPolicy)
+ }
+ Ok(Json.toJson(Map("msg" -> s"prepare storage v$version $service/$action")))
+ } else {
+ Ok(Json.toJson(Map("msg" -> s"already prepared storage v$version $service/$action")))
+ }
+ case None =>
+ NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
+ }
+ }
+
+ def deleteAction(service: String, action: String) = Action.apply {
+ {
+ for {
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ Try {
+ exactCounter(policy.version).destroy(policy)
+ if (policy.useRank) {
+ rankingCounter(policy.version).destroy(policy)
+ }
+ counterModel.deleteServiceAction(policy)
+ } match {
+ case Success(v) =>
+ Ok(Json.toJson(Map("msg" -> s"deleted $service/$action")))
+ case Failure(ex) =>
+ throw ex
+ }
+ }
+ }.getOrElse(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+
+ def getExactCountAsync(service: String, action: String, item: String) = Action.async { implicit request =>
+ val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ val limit = getQueryString("limit", "1").toInt
+
+ val qsSum = request.getQueryString("sum")
+
+ val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
+ val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
+
+ val limitOpt = (optFrom, optTo) match {
+ case (Some(_), Some(_)) =>
+ None
+ case _ =>
+ Some(limit)
+ }
+
+ // find dimension
+ lazy val dimQueryValues = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').filter(_.nonEmpty).toSet)
+ }
+// Logger.warn(s"$dimQueryValues")
+
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
+ val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ try {
+// Logger.warn(s"$tqs $qsSum")
+ if (tqs.head.length > 1 && qsSum.isDefined) {
+ getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum).map { jsVal =>
+ Ok(jsVal)
+ }
+ } else {
+ getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues).map { jsVal =>
+ Ok(jsVal)
+ }
+ }
+ } catch {
+ case e: Exception =>
+ throw e
+// Future.successful(BadRequest(s"$service, $action, $item"))
+ }
+ case None =>
+ Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+ }
+
+ /**
+ * [{
+ * "service": , "action", "itemIds": [], "interval": string, "limit": int, "from": ts, "to": ts,
+ * "dimensions": [{"key": list[String]}]
+ * }]
+ * @return
+ */
+ private def parseExactCountParam(jsValue: JsValue) = {
+ val service = (jsValue \ "service").as[String]
+ val action = (jsValue \ "action").as[String]
+ val itemIds = (jsValue \ "itemIds").as[Seq[String]]
+ val intervals = (jsValue \ "intervals").asOpt[Seq[String]].getOrElse(Seq("t")).distinct.map(IntervalUnit.withName)
+ val limit = (jsValue \ "limit").asOpt[Int].getOrElse(1)
+ val from = (jsValue \ "from").asOpt[Long]
+ val to = (jsValue \ "to").asOpt[Long]
+ val sum = (jsValue \ "sum").asOpt[String]
+ val dimensions = {
+ for {
+ dimension <- (jsValue \ "dimensions").asOpt[Seq[JsObject]].getOrElse(Nil)
+ (k, vs) <- dimension.fields
+ } yield {
+ k -> vs.as[Seq[String]].toSet
+ }
+ }.toMap
+ (service, action, itemIds, intervals, limit, from, to, dimensions, sum)
+ }
+
+ def getExactCountAsyncMulti = Action.async(s2parse.json) { implicit request =>
+ val jsValue = request.body
+ try {
+ val futures = {
+ for {
+ jsObject <- jsValue.asOpt[List[JsObject]].getOrElse(Nil)
+ (service, action, itemIds, intervalUnits, limit, from, to, dimQueryValues, qsSum) = parseExactCountParam(jsObject)
+ optFrom = from.map(UnitConverter.toMillis)
+ optTo = to.map(UnitConverter.toMillis)
+ timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ policy <- counterModel.findByServiceAction(service, action).toSeq
+ item <- itemIds
+ } yield {
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
+ val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
+ val limitOpt = (optFrom, optTo) match {
+ case (Some(_), Some(_)) =>
+ None
+ case _ =>
+ Some(limit)
+ }
+
+// Logger.warn(s"$item, $limit, $optFrom, $optTo, $qsSum, $tqs")
+
+ if (tqs.head.length > 1 && qsSum.isDefined) {
+ getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum)
+ } else {
+ getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues)
+ }
+ }
+ }
+ Future.sequence(futures).map { rets =>
+ Ok(Json.toJson(rets))
+ }
+ } catch {
+ case e: Exception =>
+ throw e
+// Future.successful(BadRequest(s"$jsValue"))
+ }
+ }
+
+ private [controllers] def fetchedToResult(fetchedCounts: FetchedCountsGrouped, limitOpt: Option[Int]): Seq[ExactCounterIntervalItem] = {
+ for {
+ ((interval, dimKeyValues), values) <- fetchedCounts.intervalWithCountMap
+ } yield {
+ val counterItems = {
+ val sortedItems = values.toSeq.sortBy { case (eq, v) => -eq.tq.ts }
+ val limited = limitOpt match {
+ case Some(limit) => sortedItems.take(limit)
+ case None => sortedItems
+ }
+ for {
+ (eq, value) <- limited
+ } yield {
+ ExactCounterItem(eq.tq.ts, value, value.toDouble)
+ }
+ }
+ ExactCounterIntervalItem(interval.toString, dimKeyValues, counterItems)
+ }
+ }.toSeq
+
+ private def decayedToResult(decayedCounts: DecayedCounts): Seq[ExactCounterIntervalItem] = {
+ for {
+ (eq, score) <- decayedCounts.qualifierWithCountMap
+ } yield {
+ ExactCounterIntervalItem(eq.tq.q.toString, eq.dimKeyValues, Seq(ExactCounterItem(eq.tq.ts, score.toLong, score)))
+ }
+ }.toSeq
+
+ private def getExactCountToJs(policy: Counter,
+ item: String,
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ limitOpt: Option[Int],
+ dimQueryValues: Map[String, Set[String]]): Future[JsValue] = {
+ exactCounter(policy.version).getCountsAsync(policy, Seq(item), timeRange, dimQueryValues).map { seq =>
+ val items = {
+ for {
+ fetched <- seq
+ } yield {
+ fetchedToResult(fetched, limitOpt)
+ }
+ }.flatten
+ Json.toJson(ExactCounterResult(ExactCounterResultMeta(policy.service, policy.action, item), items))
+ }
+ }
+
+ private def getDecayedCountToJs(policy: Counter,
+ item: String,
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQueryValues: Map[String, Set[String]],
+ qsSum: Option[String]): Future[JsValue] = {
+ exactCounter(policy.version).getDecayedCountsAsync(policy, Seq(item), timeRange, dimQueryValues, qsSum).map { seq =>
+ val decayedCounts = seq.head
+ val meta = ExactCounterResultMeta(policy.service, policy.action, decayedCounts.exactKey.itemKey)
+ val intervalItems = decayedToResult(decayedCounts)
+ Json.toJson(ExactCounterResult(meta, intervalItems))
+ }
+ }
+
+ def getRankingCountAsync(service: String, action: String) = Action.async { implicit request =>
+ lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ lazy val limit = getQueryString("limit", "1").toInt
+ lazy val kValue = getQueryString("k", "10").toInt
+
+ lazy val qsSum = request.getQueryString("sum")
+
+ lazy val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
+ lazy val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
+
+ // find dimension
+ lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').toList)
+ }
+
+ val dimensions = {
+ for {
+ values <- CartesianProduct(dimensionMap.values.toList).toSeq
+ } yield {
+ dimensionMap.keys.zip(values).toMap
+ }
+ }
+
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optTo)
+ val dimKeys = {
+ for {
+ dimension <- dimensions
+ } yield {
+ dimension -> tqs.map(tq => RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension)))
+ }
+ }
+
+ // if tqs has only 1 tq, do not apply sum function
+ try {
+ val rankResult = {
+ if (tqs.length > 1 && qsSum.isDefined) {
+ getSumRankCounterResultAsync(policy, dimKeys, kValue, qsSum)
+ } else {
+ // no summary
+ Future.successful(getRankCounterResult(policy, dimKeys, kValue))
+ }
+ }
+
+ rankResult.map { result =>
+ Ok(Json.toJson(result))
+ }
+ } catch {
+ case e: UnsupportedOperationException =>
+ Future.successful(NotImplemented(Json.toJson(
+ Map("msg" -> e.getMessage)
+ )))
+ case e: Throwable =>
+ throw e
+ }
+ case None =>
+ Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
+ }
+ }
+
+ def deleteRankingCount(service: String, action: String) = Action.async { implicit request =>
+ lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
+ .map(IntervalUnit.withName)
+ lazy val limit = getQueryString("limit", "1").toInt
+
+ // find dimension
+ lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
+ (k.substring(1), v.mkString(",").split(',').toList)
+ }
+
+ val dimensions = {
+ for {
+ values <- CartesianProduct(dimensionMap.values.toList).toSeq
+ } yield {
+ dimensionMap.keys.zip(values).toMap
+ }
+ }
+
+ Future {
+ counterModel.findByServiceAction(service, action) match {
+ case Some(policy) =>
+ val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit)
+ val keys = {
+ for {
+ dimension <- dimensions
+ tq <- tqs
+ } yield {
+ RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension))
+ }
+ }
+
+ for {
+ key <- keys
+ } {
+ rankingCounter(policy.version).delete(key)
+ }
+
+ Ok(JsObject(
+ Seq(
+ ("msg", Json.toJson(s"delete ranking in $service.$action")),
+ ("items", Json.toJson({
+ for {
+ key <- keys
+ } yield {
+ s"${key.eq.tq.q}.${key.eq.tq.ts}.${key.eq.dimension}"
+ }
+ }))
+ )
+ ))
+ case None =>
+ NotFound(Json.toJson(
+ Map("msg" -> s"$service.$action not found")
+ ))
+ }
+ }
+ }
+
+ val reduceRateRankingValue = new ReduceMapValue[ExactKeyTrait, RateRankingValue](RateRankingValue.reduce, RateRankingValue(-1, -1))
+
+ // change format
+ private def getDecayedCountsAsync(policy: Counter,
+ items: Seq[String],
+ timeRange: (TimedQualifier, TimedQualifier),
+ dimension: Map[String, String],
+ qsSum: Option[String]): Future[Seq[(ExactKeyTrait, Double)]] = {
+ exactCounter(policy.version).getDecayedCountsAsync(policy, items, Seq(timeRange), dimension.mapValues(s => Set(s)), qsSum).map { seq =>
+ for {
+ DecayedCounts(exactKey, qcMap) <- seq
+ value <- qcMap.values
+ } yield {
+ exactKey -> value
+ }
+ }
+ }
+
+ def getSumRankCounterResultAsync(policy: Counter,
+ dimKeys: Seq[(Map[String, String], Seq[RankingKey])],
+ kValue: Int,
+ qsSum: Option[String]): Future[RankCounterResult] = {
+ val futures = {
+ for {
+ (dimension, keys) <- dimKeys
+ } yield {
+ val tqs = keys.map(rk => rk.eq.tq)
+ val (tqFrom, tqTo) = (tqs.last, tqs.head)
+ val items = rankingCounter(policy.version).getAllItems(keys, kValue)
+// Logger.warn(s"item count: ${items.length}")
+ val future = {
+ if (policy.isRateCounter) {
+ val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
+ val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
+
+ val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
+ }.toMap
+ }
+ val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
+ }.toMap
+ }
+ futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
+ reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
+// Logger.warn(s"$k -> $rrv")
+ k -> rrv.rankingValue.score
+ }.toSeq
+ }
+ }
+ else if (policy.isTrendCounter) {
+ val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
+ val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
+
+ val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
+ }.toMap
+ }
+ val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom.add(-1), tqTo.add(-1)), dimension, qsSum).map { seq =>
+ seq.map { case (k, score) =>
+ ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
+ }.toMap
+ }
+ futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
+ reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
+// Logger.warn(s"$k -> $rrv")
+ k -> rrv.rankingValue.score
+ }.toSeq
+ }
+ }
+ else {
+ getDecayedCountsAsync(policy, items, (tqFrom, tqTo), dimension, qsSum)
+ }
+ }
+ future.map { keyWithScore =>
+ val ranking = keyWithScore.sortBy(-_._2).take(kValue)
+ val rankCounterItems = {
+ for {
+ idx <- ranking.indices
+ (exactKey, score) = ranking(idx)
+ } yield {
+ val realId = policy.itemType match {
+ case ItemType.BLOB => exactCounter(policy.version).getBlobValue(policy, exactKey.itemKey)
+ .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} ${exactKey.itemKey}"))
+ case _ => exactKey.itemKey
+ }
+ RankCounterItem(idx + 1, realId, score)
+ }
+ }
+
+ val eq = ExactQualifier(tqFrom, dimension)
+ RankCounterDimensionItem(eq.tq.q.toString, eq.tq.ts, eq.dimension, -1, rankCounterItems)
+ }
+ }
+ }
+
+ Future.sequence(futures).map { dimensionResultList =>
+ RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
+ }
+ }
+
+ def getRankCounterResult(policy: Counter, dimKeys: Seq[(Map[String, String], Seq[RankingKey])], kValue: Int): RankCounterResult = {
+ val dimensionResultList = {
+ for {
+ (dimension, keys) <- dimKeys
+ key <- keys
+ } yield {
+ val rankingValue = rankingCounter(policy.version).getTopK(key, kValue)
+ val ranks = {
+ for {
+ rValue <- rankingValue.toSeq
+ idx <- rValue.values.indices
+ rank = idx + 1
+ } yield {
+ val (id, score) = rValue.values(idx)
+ val realId = policy.itemType match {
+ case ItemType.BLOB =>
+ exactCounter(policy.version)
+ .getBlobValue(policy, id)
+ .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} $id"))
+ case _ => id
+ }
+ RankCounterItem(rank, realId, score)
+ }
+ }
+ val eq = key.eq
+ val tq = eq.tq
+ RankCounterDimensionItem(tq.q.toString, tq.ts, eq.dimension, rankingValue.map(v => v.totalScore).getOrElse(0d), ranks)
+ }
+ }
+
+ RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
+ }
+
+ type Record = ProducerRecord[String, String]
+
+ def incrementCount(service: String, action: String, item: String) = Action.async(s2parse.json) { request =>
+ Future {
+ /**
+ * {
+ * timestamp: Long
+ * property: {}
+ * value: Int
+ * }
+ */
+ lazy val metaMap = Map("service" -> service, "action" -> action, "item" -> item)
+ counterModel.findByServiceAction(service, action).map { policy =>
+ val body = request.body
+ try {
+ val ts = (body \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis()).toString
+ val dimension = (body \ "dimension").asOpt[JsValue].getOrElse(Json.obj())
+ val property = (body \ "property").asOpt[JsValue].getOrElse(Json.obj())
+
+ val msg = List(ts, service, action, item, dimension, property).mkString("\t")
+
+ // produce to kafka
+ // hash partitioner by key
+ ExceptionHandler.enqueue(KafkaMessage(new Record(CounterConfig.KAFKA_TOPIC_COUNTER, s"$ts.$item", msg)))
+
+ Ok(Json.toJson(
+ Map(
+ "meta" -> metaMap
+ )
+ ))
+ }
+ catch {
+ case e: JsResultException =>
+ BadRequest(Json.toJson(
+ Map("msg" -> s"need timestamp.")
+ ))
+ }
+ }.getOrElse {
+ NotFound(Json.toJson(
+ Map("msg" -> s"$service.$action not found")
+ ))
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
new file mode 100644
index 0000000..6c83743
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
@@ -0,0 +1,219 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.core.rest.RequestParser
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.rest.play.actors.QueueActor
+import org.apache.s2graph.rest.play.config.Config
+import play.api.libs.json._
+import play.api.mvc.{Controller, Result}
+
+import scala.collection.Seq
+import scala.concurrent.Future
+
+object EdgeController extends Controller {
+
+ import ApplicationController._
+ import ExceptionHandler._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ private val s2: Graph = org.apache.s2graph.rest.play.Global.s2graph
+ private val requestParser: RequestParser = org.apache.s2graph.rest.play.Global.s2parser
+ private def jsToStr(js: JsValue): String = js match {
+ case JsString(s) => s
+ case _ => js.toString()
+ }
+
+ def toTsv(jsValue: JsValue, op: String): String = {
+ val ts = jsToStr(jsValue \ "timestamp")
+ val from = jsToStr(jsValue \ "from")
+ val to = jsToStr(jsValue \ "to")
+ val label = jsToStr(jsValue \ "label")
+ val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
+
+ (jsValue \ "direction").asOpt[String] match {
+ case None => Seq(ts, op, "e", from, to, label, props).mkString("\t")
+ case Some(dir) => Seq(ts, op, "e", from, to, label, props, dir).mkString("\t")
+ }
+ }
+
+ def tryMutates(jsValue: JsValue, operation: String, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+
+ else {
+ try {
+ logger.debug(s"$jsValue")
+ val (edges, jsOrgs) = requestParser.toEdgesWithOrg(jsValue, operation)
+
+ for ((edge, orgJs) <- edges.zip(jsOrgs)) {
+ if (edge.isAsync)
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, edge, Option(toTsv(orgJs, operation))))
+ else
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, edge, Option(toTsv(orgJs, operation))))
+ }
+
+ val edgesToStore = edges.filterNot(e => e.isAsync)
+
+ if (withWait) {
+ val rets = s2.mutateEdges(edgesToStore, withWait = true)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = edgesToStore.map { edge => QueueActor.router ! edge; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
+ case e: Exception =>
+ logger.error(s"mutateAndPublish: $e", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+ }
+
+ def mutateAndPublish(str: String, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+
+ logger.debug(s"$str")
+ val edgeStrs = str.split("\\n")
+
+ var vertexCnt = 0L
+ var edgeCnt = 0L
+ try {
+ val elements =
+ for (edgeStr <- edgeStrs; str <- GraphUtil.parseString(edgeStr); element <- Graph.toGraphElement(str)) yield {
+ element match {
+ case v: Vertex => vertexCnt += 1
+ case e: Edge => edgeCnt += 1
+ }
+ if (element.isAsync) {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, element, Some(str)))
+ } else {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, element, Some(str)))
+ }
+ element
+ }
+
+ //FIXME:
+ val elementsToStore = elements.filterNot(e => e.isAsync)
+ if (withWait) {
+ val rets = s2.mutateElements(elementsToStore, withWait)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = elementsToStore.map { element => QueueActor.router ! element; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
+ case e: Throwable =>
+ logger.error(s"mutateAndPublish: $e", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+
+ def mutateBulk() = withHeaderAsync(parse.text) { request =>
+ mutateAndPublish(request.body, withWait = false)
+ }
+
+ def mutateBulkWithWait() = withHeaderAsync(parse.text) { request =>
+ mutateAndPublish(request.body, withWait = true)
+ }
+
+ def inserts() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert")
+ }
+
+ def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", withWait = true)
+ }
+
+ def insertsBulk() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insertBulk")
+ }
+
+ def deletes() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete")
+ }
+
+ def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", withWait = true)
+ }
+
+ def updates() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "update")
+ }
+
+ def updatesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "update", withWait = true)
+ }
+
+ def increments() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "increment")
+ }
+
+ def incrementsWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "increment", withWait = true)
+ }
+
+ def incrementCounts() = withHeaderAsync(jsonParser) { request =>
+ val jsValue = request.body
+ val edges = requestParser.toEdges(jsValue, "incrementCount")
+
+ s2.incrementCounts(edges, withWait = true).map { results =>
+ val json = results.map { case (isSuccess, resultCount) =>
+ Json.obj("success" -> isSuccess, "result" -> resultCount)
+ }
+
+ jsonResponse(Json.toJson(json))
+ }
+ }
+
+ def deleteAll() = withHeaderAsync(jsonParser) { request =>
+// deleteAllInner(request.body, withWait = false)
+ deleteAllInner(request.body, withWait = true)
+ }
+
+ def deleteAllInner(jsValue: JsValue, withWait: Boolean) = {
+
+ /** logging for delete all request */
+ def enqueueLogMessage(ids: Seq[JsValue], labels: Seq[Label], ts: Long, direction: String, topicOpt: Option[String]) = {
+ val kafkaMessages = for {
+ id <- ids
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", jsToStr(id), jsToStr(id), label.label, "{}", direction).mkString("\t")
+ val topic = topicOpt.getOrElse {
+ if (label.isAsync) Config.KAFKA_LOG_TOPIC_ASYNC else Config.KAFKA_LOG_TOPIC
+ }
+
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+
+ ExceptionHandler.enqueues(kafkaMessages)
+ }
+
+ def deleteEach(labels: Seq[Label], direction: String, ids: Seq[JsValue], ts: Long, vertices: Seq[Vertex]) = {
+ enqueueLogMessage(ids, labels, ts, direction, None)
+ val future = s2.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
+ if (withWait) {
+ future
+ } else {
+ Future.successful(true)
+ }
+ }
+
+ val deleteFutures = jsValue.as[Seq[JsValue]].map { json =>
+ val (labels, direction, ids, ts, vertices) = requestParser.toDeleteParam(json)
+ if (labels.isEmpty || ids.isEmpty) Future.successful(true)
+ else deleteEach(labels, direction, ids, ts, vertices)
+ }
+
+ val deleteResults = Future.sequence(deleteFutures)
+ deleteResults.map { rst =>
+ logger.debug(s"deleteAllInner: $rst")
+ Ok(s"deleted... ${rst.toString()}")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ExperimentController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ExperimentController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ExperimentController.scala
new file mode 100644
index 0000000..b489a81
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ExperimentController.scala
@@ -0,0 +1,23 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.mysqls.Experiment
+import org.apache.s2graph.core.rest.RestHandler
+import play.api.mvc._
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+object ExperimentController extends Controller {
+ private val rest: RestHandler = org.apache.s2graph.rest.play.Global.s2rest
+
+ import ApplicationController._
+
+ def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(jsonText) { request =>
+ val body = request.body
+
+ val res = rest.doPost(request.uri, body, request.headers.get(Experiment.impressionKey))
+ res.body.map { case js =>
+ val headers = res.headers :+ ("result_size" -> rest.calcSize(js).toString)
+ jsonResponse(js, headers: _*)
+ } recoverWith ApplicationController.requestFallback(body)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
new file mode 100644
index 0000000..42710e1
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
@@ -0,0 +1,86 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.utils.logger
+import play.api.Play
+import play.api.libs.iteratee.Iteratee
+import play.api.libs.json.{JsValue, Json}
+import play.api.mvc._
+
+import scala.concurrent.Future
+import scala.util.control.NonFatal
+
+object s2parse extends BodyParsers {
+
+ import parse._
+
+ val defaultMaxTextLength = 1024 * 512
+ val defaultMaxJsonLength = 1024 * 512
+
+ def json: BodyParser[JsValue] = json(defaultMaxTextLength)
+
+ /**
+ * parseText with application/json header for Pre-Process text
+ */
+ def jsonText: BodyParser[String] = when(
+ _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
+ jsonText(defaultMaxTextLength),
+ createBadResult("Expecting text/json or application/json body")
+ )
+
+ private def jsonText(maxLength: Int): BodyParser[String] = BodyParser("json, maxLength=" + maxLength) { request =>
+ import play.api.libs.iteratee.Execution.Implicits.trampoline
+ import play.api.libs.iteratee.Traversable
+
+ Traversable.takeUpTo[Array[Byte]](maxLength)
+ .transform(Iteratee.consume[Array[Byte]]().map(c => new String(c, "UTF-8")))
+ .flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
+ }
+
+ def json(maxLength: Int): BodyParser[JsValue] = when(
+ _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
+ tolerantJson(maxLength),
+ createBadResult("Expecting text/json or application/json body")
+ )
+
+ def tolerantJson(maxLength: Int): BodyParser[JsValue] =
+ tolerantBodyParser[JsValue]("json", maxLength, "Invalid Json") { (request, bytes) =>
+ // Encoding notes: RFC 4627 requires that JSON be encoded in Unicode, and states that whether that's
+ // UTF-8, UTF-16 or UTF-32 can be auto detected by reading the first two bytes. So we ignore the declared
+ // charset and don't decode, we passing the byte array as is because Jackson supports auto detection.
+ Json.parse(bytes)
+ }
+
+ private def tolerantBodyParser[A](name: String, maxLength: Int, errorMessage: String)(parser: (RequestHeader, Array[Byte]) => A): BodyParser[A] =
+ BodyParser(name + ", maxLength=" + maxLength) { request =>
+ import play.api.libs.iteratee.Execution.Implicits.trampoline
+ import play.api.libs.iteratee.Traversable
+
+ import scala.util.control.Exception._
+
+ val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] =
+ Traversable.takeUpTo[Array[Byte]](maxLength).transform(
+ Iteratee.consume[Array[Byte]]().map { bytes =>
+ allCatch[A].either {
+ parser(request, bytes)
+ }.left.map {
+ case NonFatal(e) =>
+ val txt = new String(bytes)
+ logger.error(s"$errorMessage: $txt", e)
+ createBadResult(s"$errorMessage: $e")(request)
+ case t => throw t
+ }
+ }
+ ).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
+
+ bodyParser.mapM {
+ case Left(tooLarge) => Future.successful(Left(tooLarge))
+ case Right(Left(badResult)) => badResult.map(Left.apply)
+ case Right(Right(body)) => Future.successful(Right(body))
+ }
+ }
+
+ private def createBadResult(msg: String): RequestHeader => Future[Result] = { request =>
+ Play.maybeApplication.map(_.global.onBadRequest(request, msg))
+ .getOrElse(Future.successful(Results.BadRequest))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/PublishController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/PublishController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/PublishController.scala
new file mode 100644
index 0000000..fe478e6
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/PublishController.scala
@@ -0,0 +1,54 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.s2graph.core.ExceptionHandler
+import org.apache.s2graph.rest.play.config.Config
+import play.api.mvc._
+
+import scala.concurrent.Future
+
+object PublishController extends Controller {
+
+ import ApplicationController._
+ import ExceptionHandler._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ /**
+ * never check validation on string. just redirect strings to kafka.
+ */
+ val serviceNotExistException = new RuntimeException(s"service is not created in s2graph. create service first.")
+
+ // private def toService(topic: String): String = {
+ // Service.findByName(topic).map(service => s"${service.serviceName}-${Config.PHASE}").getOrElse(throw serviceNotExistException)
+ // }
+ def publishOnly(topic: String) = withHeaderAsync(parse.text) { request =>
+ if (!Config.IS_WRITE_SERVER) Future.successful(UNAUTHORIZED)
+ // val kafkaTopic = toService(topic)
+ val strs = request.body.split("\n")
+ strs.foreach(str => {
+ val keyedMessage = new ProducerRecord[Key, Val](Config.KAFKA_LOG_TOPIC, str)
+ // val keyedMessage = new ProducerRecord[Key, Val](kafkaTopic, s"$str")
+ // logger.debug(s"$kafkaTopic, $str")
+ ExceptionHandler.enqueue(KafkaMessage(keyedMessage))
+ })
+ Future.successful(
+ Ok("publish success.\n").withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
+ )
+ // try {
+ //
+ // } catch {
+ // case e: Exception => Future.successful(BadRequest(e.getMessage))
+ // }
+ }
+
+ def publish(topic: String) = publishOnly(topic)
+
+ // def mutateBulk(topic: String) = Action.async(parse.text) { request =>
+ // EdgeController.mutateAndPublish(Config.KAFKA_LOG_TOPIC, Config.KAFKA_FAIL_TOPIC, request.body).map { result =>
+ // result.withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
+ // }
+ // }
+ def mutateBulk(topic: String) = withHeaderAsync(parse.text) { request =>
+ EdgeController.mutateAndPublish(request.body)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/QueryController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/QueryController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/QueryController.scala
new file mode 100644
index 0000000..e509554
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/QueryController.scala
@@ -0,0 +1,52 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.JSONParser
+import org.apache.s2graph.core.mysqls.Experiment
+import org.apache.s2graph.core.rest.RestHandler
+import play.api.libs.json.Json
+import play.api.mvc._
+
+import scala.language.postfixOps
+
+object QueryController extends Controller with JSONParser {
+
+ import ApplicationController._
+ import play.api.libs.concurrent.Execution.Implicits.defaultContext
+
+ private val rest: RestHandler = org.apache.s2graph.rest.play.Global.s2rest
+
+ def delegate(request: Request[String]) = {
+ rest.doPost(request.uri, request.body, request.headers.get(Experiment.impressionKey)).body.map {
+ js =>
+ jsonResponse(js, "result_size" -> rest.calcSize(js).toString)
+ } recoverWith ApplicationController.requestFallback(request.body)
+ }
+
+ def getEdges() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesWithGrouping() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesExcluded() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonText)(delegate)
+
+ def checkEdges() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesGrouped() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesGroupedExcluded() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdgesGroupedExcludedFormatted() = withHeaderAsync(jsonText)(delegate)
+
+ def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) =
+ withHeaderAsync(jsonText) {
+ request =>
+ val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
+ rest.checkEdges(params).body.map {
+ js =>
+ jsonResponse(js, "result_size" -> rest.calcSize(js).toString)
+ } recoverWith ApplicationController.requestFallback(request.body)
+ }
+
+ def getVertices() = withHeaderAsync(jsonText)(delegate)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/VertexController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/VertexController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/VertexController.scala
new file mode 100644
index 0000000..b20fa37
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/VertexController.scala
@@ -0,0 +1,85 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.rest.RequestParser
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{ExceptionHandler, Graph, GraphExceptions}
+import org.apache.s2graph.rest.play.actors.QueueActor
+import org.apache.s2graph.rest.play.config.Config
+import play.api.libs.json.{JsValue, Json}
+import play.api.mvc.{Controller, Result}
+
+import scala.concurrent.Future
+
+object VertexController extends Controller {
+ private val s2: Graph = org.apache.s2graph.rest.play.Global.s2graph
+ private val requestParser: RequestParser = org.apache.s2graph.rest.play.Global.s2parser
+
+ import ApplicationController._
+ import ExceptionHandler._
+ import play.api.libs.concurrent.Execution.Implicits._
+
+ def tryMutates(jsValue: JsValue, operation: String, serviceNameOpt: Option[String] = None, columnNameOpt: Option[String] = None, withWait: Boolean = false): Future[Result] = {
+ if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
+ else {
+ try {
+ val vertices = requestParser.toVertices(jsValue, operation, serviceNameOpt, columnNameOpt)
+
+ for (vertex <- vertices) {
+ if (vertex.isAsync)
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, vertex, None))
+ else
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, vertex, None))
+ }
+
+ //FIXME:
+ val verticesToStore = vertices.filterNot(v => v.isAsync)
+
+ if (withWait) {
+ val rets = s2.mutateVertices(verticesToStore, withWait = true)
+ rets.map(Json.toJson(_)).map(jsonResponse(_))
+ } else {
+ val rets = verticesToStore.map { vertex => QueueActor.router ! vertex; true }
+ Future.successful(jsonResponse(Json.toJson(rets)))
+ }
+ } catch {
+ case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"e"))
+ case e: Exception =>
+ logger.error(s"[Failed] tryMutates", e)
+ Future.successful(InternalServerError(s"${e.getStackTrace}"))
+ }
+ }
+ }
+
+ def inserts() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert")
+ }
+
+ def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", withWait = true)
+ }
+
+ def insertsSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "insert", Some(serviceName), Some(columnName))
+ }
+
+ def deletes() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete")
+ }
+
+ def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", withWait = true)
+ }
+
+ def deletesSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "delete", Some(serviceName), Some(columnName))
+ }
+
+ def deletesAll() = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "deleteAll")
+ }
+
+ def deletesAllSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
+ tryMutates(request.body, "deleteAll", Some(serviceName), Some(columnName))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/models/ExactCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/models/ExactCounterItem.scala b/s2rest_play/app/org/apache/s2graph/rest/play/models/ExactCounterItem.scala
new file mode 100644
index 0000000..df10aab
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/models/ExactCounterItem.scala
@@ -0,0 +1,38 @@
+package org.apache.s2graph.rest.play.models
+
+import play.api.libs.json.{Json, Writes}
+
+/**
+ * Created by alec on 15. 4. 15..
+ */
+case class ExactCounterItem(ts: Long, count: Long, score: Double)
+
+case class ExactCounterIntervalItem(interval: String, dimension: Map[String, String], counter: Seq[ExactCounterItem])
+
+case class ExactCounterResultMeta(service: String, action: String, item: String)
+
+case class ExactCounterResult(meta: ExactCounterResultMeta, data: Seq[ExactCounterIntervalItem])
+
+object ExactCounterItem {
+ implicit val writes = new Writes[ExactCounterItem] {
+ def writes(item: ExactCounterItem) = Json.obj(
+ "ts" -> item.ts,
+ "time" -> tsFormat.format(item.ts),
+ "count" -> item.count,
+ "score" -> item.score
+ )
+ }
+ implicit val reads = Json.reads[ExactCounterItem]
+}
+
+object ExactCounterIntervalItem {
+ implicit val format = Json.format[ExactCounterIntervalItem]
+}
+
+object ExactCounterResultMeta {
+ implicit val format = Json.format[ExactCounterResultMeta]
+}
+
+object ExactCounterResult {
+ implicit val formats = Json.format[ExactCounterResult]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/models/RankCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/models/RankCounterItem.scala b/s2rest_play/app/org/apache/s2graph/rest/play/models/RankCounterItem.scala
new file mode 100644
index 0000000..3d9ef41
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/models/RankCounterItem.scala
@@ -0,0 +1,40 @@
+package org.apache.s2graph.rest.play.models
+
+import play.api.libs.json.{Json, Writes}
+
+/**
+ * Created by alec on 15. 4. 15..
+ */
+case class RankCounterItem(rank: Int, id: String, score: Double)
+
+case class RankCounterDimensionItem(interval: String, ts: Long, dimension: String, total: Double, ranks: Seq[RankCounterItem])
+
+case class RankCounterResultMeta(service: String, action: String)
+
+case class RankCounterResult(meta: RankCounterResultMeta, data: Seq[RankCounterDimensionItem])
+
+object RankCounterItem {
+ implicit val format = Json.format[RankCounterItem]
+}
+
+object RankCounterDimensionItem {
+ implicit val writes = new Writes[RankCounterDimensionItem] {
+ def writes(item: RankCounterDimensionItem) = Json.obj(
+ "interval" -> item.interval,
+ "ts" -> item.ts,
+ "time" -> tsFormat.format(item.ts),
+ "dimension" -> item.dimension,
+ "total" -> item.total,
+ "ranks" -> item.ranks
+ )
+ }
+ implicit val reads = Json.reads[RankCounterDimensionItem]
+}
+
+object RankCounterResultMeta {
+ implicit val format = Json.format[RankCounterResultMeta]
+}
+
+object RankCounterResult {
+ implicit val format = Json.format[RankCounterResult]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/models/package.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/models/package.scala b/s2rest_play/app/org/apache/s2graph/rest/play/models/package.scala
new file mode 100644
index 0000000..08b0d45
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/models/package.scala
@@ -0,0 +1,10 @@
+package org.apache.s2graph.rest.play
+
+import java.text.SimpleDateFormat
+
+/**
+ * Created by alec on 15. 4. 20..
+ */
+package object models {
+ def tsFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/util/TestDataLoader.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/util/TestDataLoader.scala b/s2rest_play/app/util/TestDataLoader.scala
deleted file mode 100644
index 45a9b61..0000000
--- a/s2rest_play/app/util/TestDataLoader.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-package util
-
-import java.io.File
-
-import scala.collection.mutable.{ArrayBuffer, HashMap, ListBuffer}
-import scala.io.Source
-import scala.util.Random
-
-
-object TestDataLoader {
- val step = 100
- val prob = 1.0
- val (testIds, testIdsHist, testIdsHistCnt) = loadSeeds("./talk_vertices.txt")
- val maxId = testIds.length
- // val randoms = (0 until 100).map{ i => new SecureRandom }
- // val idx = new AtomicInteger(0)
- // def randomId() = {
- // val r = randoms(idx.getAndIncrement() % randoms.size)
- // testAccountIds(r.nextInt(maxId))
- // }
- def randomId(histStep: Int) = {
- for {
- maxId <- testIdsHistCnt.get(histStep)
- rIdx = Random.nextInt(maxId.toInt)
- hist <- testIdsHist.get(histStep)
- id = hist(rIdx)
- } yield {
-// logger.debug(s"randomId: $histStep = $id[$rIdx / $maxId]")
- id
- }
- }
- def randomId() = {
- val id = testIds(Random.nextInt(maxId))
- // logger.debug(s"$id")
- id
- }
- private def loadSeeds(filePath: String) = {
- val histogram = new HashMap[Long, ListBuffer[Long]]
- val histogramCnt = new HashMap[Long, Long]
- val ids = new ArrayBuffer[Long]
-
- var idx = 0
-// logger.debug(s"$filePath start to load file.")
- for (line <- Source.fromFile(new File(filePath)).getLines) {
- // testAccountIds(idx) = line.toLong
-// if (idx % 10000 == 0) logger.debug(s"$idx")
- idx += 1
-
- val parts = line.split("\\t")
- val id = parts.head.toLong
- val count = parts.last.toLong / step
- if (count > 1 && Random.nextDouble < prob) {
- histogram.get(count) match {
- case None =>
- histogram.put(count, new ListBuffer[Long])
- histogram.get(count).get += id
- histogramCnt.put(count, 1)
- case Some(existed) =>
- existed += id
- histogramCnt.put(count, histogramCnt.getOrElse(count, 0L) + 1L)
- }
- ids += id
- }
-
- }
-// logger.debug(s"upload $filePath finished.")
-// logger.debug(s"${histogram.size}")
- (ids, histogram.map(t => (t._1 -> t._2.toArray[Long])), histogramCnt)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/conf/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/reference.conf b/s2rest_play/conf/reference.conf
index 6e76847..452c013 100644
--- a/s2rest_play/conf/reference.conf
+++ b/s2rest_play/conf/reference.conf
@@ -112,7 +112,7 @@ lock.expire.time=600000
# max allowd edges for deleteAll is multiply of above two configuration.
# set global obejct package, TODO: remove global
-application.global=com.kakao.s2graph.rest.Global
+application.global=org.apache.s2graph.rest.play.Global
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/conf/routes
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/routes b/s2rest_play/conf/routes
index 90838c8..f0115c9 100644
--- a/s2rest_play/conf/routes
+++ b/s2rest_play/conf/routes
@@ -4,123 +4,104 @@
# publish
-#POST /publish/:topic controllers.PublishController.publish(topic)
-POST /publish/:topic controllers.PublishController.mutateBulk(topic)
-POST /publishOnly/:topic controllers.PublishController.publishOnly(topic)
+POST /publish/:topic org.apache.s2graph.rest.play.controllers.PublishController.mutateBulk(topic)
+POST /publishOnly/:topic org.apache.s2graph.rest.play.controllers.PublishController.publishOnly(topic)
#### Health Check
-#GET /health_check.html controllers.Assets.at(path="/public", file="health_check.html")
-GET /health_check.html controllers.ApplicationController.healthCheck()
-PUT /health_check/:isHealthy controllers.ApplicationController.updateHealthCheck(isHealthy: Boolean)
+GET /health_check.html org.apache.s2graph.rest.play.controllers.ApplicationController.healthCheck()
+PUT /health_check/:isHealthy org.apache.s2graph.rest.play.controllers.ApplicationController.updateHealthCheck(isHealthy: Boolean)
## Edge
-POST /graphs/edges/insert controllers.EdgeController.inserts()
-POST /graphs/edges/insertWithWait controllers.EdgeController.insertsWithWait()
-POST /graphs/edges/insertBulk controllers.EdgeController.insertsBulk()
-POST /graphs/edges/delete controllers.EdgeController.deletes()
-POST /graphs/edges/deleteWithWait controllers.EdgeController.deletesWithWait()
-POST /graphs/edges/deleteAll controllers.EdgeController.deleteAll()
-POST /graphs/edges/update controllers.EdgeController.updates()
-POST /graphs/edges/updateWithWait controllers.EdgeController.updatesWithWait()
-POST /graphs/edges/increment controllers.EdgeController.increments()
-POST /graphs/edges/incrementWithWait controllers.EdgeController.incrementsWithWait()
-POST /graphs/edges/incrementCount controllers.EdgeController.incrementCounts()
-POST /graphs/edges/bulk controllers.EdgeController.mutateBulk()
-POST /graphs/edges/bulkWithWait controllers.EdgeController.mutateBulkWithWait()
+POST /graphs/edges/insert org.apache.s2graph.rest.play.controllers.EdgeController.inserts()
+POST /graphs/edges/insertWithWait org.apache.s2graph.rest.play.controllers.EdgeController.insertsWithWait()
+POST /graphs/edges/insertBulk org.apache.s2graph.rest.play.controllers.EdgeController.insertsBulk()
+POST /graphs/edges/delete org.apache.s2graph.rest.play.controllers.EdgeController.deletes()
+POST /graphs/edges/deleteWithWait org.apache.s2graph.rest.play.controllers.EdgeController.deletesWithWait()
+POST /graphs/edges/deleteAll org.apache.s2graph.rest.play.controllers.EdgeController.deleteAll()
+POST /graphs/edges/update org.apache.s2graph.rest.play.controllers.EdgeController.updates()
+POST /graphs/edges/updateWithWait org.apache.s2graph.rest.play.controllers.EdgeController.updatesWithWait()
+POST /graphs/edges/increment org.apache.s2graph.rest.play.controllers.EdgeController.increments()
+POST /graphs/edges/incrementWithWait org.apache.s2graph.rest.play.controllers.EdgeController.incrementsWithWait()
+POST /graphs/edges/incrementCount org.apache.s2graph.rest.play.controllers.EdgeController.incrementCounts()
+POST /graphs/edges/bulk org.apache.s2graph.rest.play.controllers.EdgeController.mutateBulk()
+POST /graphs/edges/bulkWithWait org.apache.s2graph.rest.play.controllers.EdgeController.mutateBulkWithWait()
## Vertex
-POST /graphs/vertices/insert controllers.VertexController.inserts()
-POST /graphs/vertices/insertWithWait controllers.VertexController.insertsWithWait()
-POST /graphs/vertices/insert/:serviceName/:columnName controllers.VertexController.insertsSimple(serviceName, columnName)
-POST /graphs/vertices/delete controllers.VertexController.deletes()
-POST /graphs/vertices/deleteWithWait controllers.VertexController.deletesWithWait()
-POST /graphs/vertices/delete/:serviceName/:columnName controllers.VertexController.deletesSimple(serviceName, columnName)
-POST /graphs/vertices/deleteAll controllers.VertexController.deletesAll()
-POST /graphs/vertices/deleteAll/:serviceName/:columnName controllers.VertexController.deletesAllSimple(serviceName, columnName)
+POST /graphs/vertices/insert org.apache.s2graph.rest.play.controllers.VertexController.inserts()
+POST /graphs/vertices/insertWithWait org.apache.s2graph.rest.play.controllers.VertexController.insertsWithWait()
+POST /graphs/vertices/insert/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.VertexController.insertsSimple(serviceName, columnName)
+POST /graphs/vertices/delete org.apache.s2graph.rest.play.controllers.VertexController.deletes()
+POST /graphs/vertices/deleteWithWait org.apache.s2graph.rest.play.controllers.VertexController.deletesWithWait()
+POST /graphs/vertices/delete/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.VertexController.deletesSimple(serviceName, columnName)
+POST /graphs/vertices/deleteAll org.apache.s2graph.rest.play.controllers.VertexController.deletesAll()
+POST /graphs/vertices/deleteAll/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.VertexController.deletesAllSimple(serviceName, columnName)
### SELECT Edges
-POST /graphs/getEdges controllers.QueryController.getEdges()
-POST /graphs/getEdges/grouped controllers.QueryController.getEdgesWithGrouping()
-POST /graphs/getEdgesExcluded controllers.QueryController.getEdgesExcluded()
-POST /graphs/getEdgesExcluded/grouped controllers.QueryController.getEdgesExcludedWithGrouping()
-POST /graphs/checkEdges controllers.QueryController.checkEdges()
+POST /graphs/getEdges org.apache.s2graph.rest.play.controllers.QueryController.getEdges()
+POST /graphs/getEdges/grouped org.apache.s2graph.rest.play.controllers.QueryController.getEdgesWithGrouping()
+POST /graphs/getEdgesExcluded org.apache.s2graph.rest.play.controllers.QueryController.getEdgesExcluded()
+POST /graphs/getEdgesExcluded/grouped org.apache.s2graph.rest.play.controllers.QueryController.getEdgesExcludedWithGrouping()
+POST /graphs/checkEdges org.apache.s2graph.rest.play.controllers.QueryController.checkEdges()
### this will be deprecated
-POST /graphs/getEdgesGrouped controllers.QueryController.getEdgesGrouped()
-POST /graphs/getEdgesGroupedExcluded controllers.QueryController.getEdgesGroupedExcluded()
-POST /graphs/getEdgesGroupedExcludedFormatted controllers.QueryController.getEdgesGroupedExcludedFormatted()
-GET /graphs/getEdge/:srcId/:tgtId/:labelName/:direction controllers.QueryController.getEdge(srcId, tgtId, labelName, direction)
+POST /graphs/getEdgesGrouped org.apache.s2graph.rest.play.controllers.QueryController.getEdgesGrouped()
+POST /graphs/getEdgesGroupedExcluded org.apache.s2graph.rest.play.controllers.QueryController.getEdgesGroupedExcluded()
+POST /graphs/getEdgesGroupedExcludedFormatted org.apache.s2graph.rest.play.controllers.QueryController.getEdgesGroupedExcludedFormatted()
+GET /graphs/getEdge/:srcId/:tgtId/:labelName/:direction org.apache.s2graph.rest.play.controllers.QueryController.getEdge(srcId, tgtId, labelName, direction)
### SELECT Vertices
-#POST /graphs/getVertex controllers.QueryController.getVertex()
-POST /graphs/getVertices controllers.QueryController.getVertices()
+POST /graphs/getVertices org.apache.s2graph.rest.play.controllers.QueryController.getVertices()
#### ADMIN
-POST /graphs/createService controllers.AdminController.createService()
-GET /graphs/getService/:serviceName controllers.AdminController.getService(serviceName)
-GET /graphs/getLabels/:serviceName controllers.AdminController.getLabels(serviceName)
-POST /graphs/createLabel controllers.AdminController.createLabel()
-POST /graphs/addIndex controllers.AdminController.addIndex()
-GET /graphs/getLabel/:labelName controllers.AdminController.getLabel(labelName)
-PUT /graphs/deleteLabel/:labelName controllers.AdminController.deleteLabel(labelName)
+POST /graphs/createService org.apache.s2graph.rest.play.controllers.AdminController.createService()
+GET /graphs/getService/:serviceName org.apache.s2graph.rest.play.controllers.AdminController.getService(serviceName)
+GET /graphs/getLabels/:serviceName org.apache.s2graph.rest.play.controllers.AdminController.getLabels(serviceName)
+POST /graphs/createLabel org.apache.s2graph.rest.play.controllers.AdminController.createLabel()
+POST /graphs/addIndex org.apache.s2graph.rest.play.controllers.AdminController.addIndex()
+GET /graphs/getLabel/:labelName org.apache.s2graph.rest.play.controllers.AdminController.getLabel(labelName)
+PUT /graphs/deleteLabel/:labelName org.apache.s2graph.rest.play.controllers.AdminController.deleteLabel(labelName)
+
+POST /graphs/addProp/:labelName org.apache.s2graph.rest.play.controllers.AdminController.addProp(labelName)
+POST /graphs/createServiceColumn org.apache.s2graph.rest.play.controllers.AdminController.createServiceColumn()
+PUT /graphs/deleteServiceColumn/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.AdminController.deleteServiceColumn(serviceName, columnName)
+POST /graphs/addServiceColumnProp/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.AdminController.addServiceColumnProp(serviceName, columnName)
+POST /graphs/addServiceColumnProps/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.AdminController.addServiceColumnProps(serviceName, columnName)
+GET /graphs/getServiceColumn/:serviceName/:columnName org.apache.s2graph.rest.play.controllers.AdminController.getServiceColumn(serviceName, columnName)
+POST /graphs/createHTable org.apache.s2graph.rest.play.controllers.AdminController.createHTable()
-POST /graphs/addProp/:labelName controllers.AdminController.addProp(labelName)
-POST /graphs/createServiceColumn controllers.AdminController.createServiceColumn()
-PUT /graphs/deleteServiceColumn/:serviceName/:columnName controllers.AdminController.deleteServiceColumn(serviceName, columnName)
-POST /graphs/addServiceColumnProp/:serviceName/:columnName controllers.AdminController.addServiceColumnProp(serviceName, columnName)
-POST /graphs/addServiceColumnProps/:serviceName/:columnName controllers.AdminController.addServiceColumnProps(serviceName, columnName)
-GET /graphs/getServiceColumn/:serviceName/:columnName controllers.AdminController.getServiceColumn(serviceName, columnName)
-POST /graphs/createHTable controllers.AdminController.createHTable()
+# AdminController API
+GET /admin/labels/:serviceName org.apache.s2graph.rest.play.controllers.AdminController.getLabels(serviceName)
+POST /graphs/copyLabel/:oldLabelName/:newLabelName org.apache.s2graph.rest.play.controllers.AdminController.copyLabel(oldLabelName, newLabelName)
+POST /graphs/renameLabel/:oldLabelName/:newLabelName org.apache.s2graph.rest.play.controllers.AdminController.renameLabel(oldLabelName, newLabelName)
+POST /graphs/updateHTable/:labelName/:newHTableName org.apache.s2graph.rest.play.controllers.AdminController.updateHTable(labelName, newHTableName)
+PUT /graphs/loadCache org.apache.s2graph.rest.play.controllers.AdminController.loadCache()
+
+
+# Counter Admin API
+POST /counter/v1/:service/:action org.apache.s2graph.rest.play.controllers.CounterController.createAction(service, action)
+GET /counter/v1/:service/:action org.apache.s2graph.rest.play.controllers.CounterController.getAction(service, action)
+PUT /counter/v1/:service/:action org.apache.s2graph.rest.play.controllers.CounterController.updateAction(service, action)
+PUT /counter/v1/:service/:action/prepare org.apache.s2graph.rest.play.controllers.CounterController.prepareAction(service, action)
+DELETE /counter/v1/:service/:action org.apache.s2graph.rest.play.controllers.CounterController.deleteAction(service, action)
+# Counter API
+GET /counter/v1/:service/:action/ranking org.apache.s2graph.rest.play.controllers.CounterController.getRankingCountAsync(service, action)
+DELETE /counter/v1/:service/:action/ranking org.apache.s2graph.rest.play.controllers.CounterController.deleteRankingCount(service, action)
+GET /counter/v1/:service/:action/:item org.apache.s2graph.rest.play.controllers.CounterController.getExactCountAsync(service, action, item)
+PUT /counter/v1/:service/:action/:item org.apache.s2graph.rest.play.controllers.CounterController.incrementCount(service, action, item)
+POST /counter/v1/mget org.apache.s2graph.rest.play.controllers.CounterController.getExactCountAsyncMulti()
+# Experiment API
+POST /graphs/experiment/:accessToken/:experimentName/:uuid org.apache.s2graph.rest.play.controllers.ExperimentController.experiment(accessToken, experimentName, uuid)
-#### TEST
-#GET /graphs/testGetEdges/:label/:limit/:friendCntStep controllers.QueryController.testGetEdges(label, limit: Int, friendCntStep: Int)
-#GET /graphs/testGetEdges2/:label1/:limit1/:label2/:limit2 controllers.QueryController.testGetEdges2(label1, limit1: Int, label2, limit2: Int)
-#GET /graphs/testGetEdges3/:label1/:limit1/:label2/:limit2/:label3/:limit3 controllers.QueryController.testGetEdges3(label1, limit1: Int, label2, limit2: Int, label3, limit3: Int)
-POST /ping controllers.TestController.ping()
-POST /pingAsync controllers.TestController.pingAsync()
-GET /graphs/testId controllers.TestController.getRandomId()
# Map static resources from the /public folder to the /assets URL path
GET /images/*file controllers.Assets.at(path="/public/images", file)
GET /javascripts/*file controllers.Assets.at(path="/public/javascripts", file)
GET /stylesheets/*file controllers.Assets.at(path="/public/stylesheets", file)
GET /font-awesome-4.1.0/*file controllers.Assets.at(path="/public/font-awesome-4.1.0", file)
-GET /swagger/*file controllers.Assets.at(path="/public/swagger-ui", file)
-
-
-# AdminController API
-#GET /admin/services controllers.AdminController.allServices
-GET /admin/labels/:serviceName controllers.AdminController.getLabels(serviceName)
-#POST /admin/labels/delete/:zkAddr/:tableName/:labelIds/:minTs/:maxTs controllers.AdminController.deleteEdges(zkAddr, tableName, labelIds, minTs: Long, maxTs: Long)
-#POST /admin/labels/deleteAll/:zkAddr/:tableName/:minTs/:maxTs controllers.AdminController.deleteAllEdges(zkAddr, tableName, minTs: Long, maxTs: Long)
-#POST /admin/swapLabel/:oldLabelName/:newLabelName controllers.AdminController.swapLabel(oldLabelName, newLabelName)
-#GET /admin/reloadLabel/:labelName controllers.AdminController.reloadLabel(labelName)
-#POST /admin/getEdges controllers.AdminController.getEdges()
-POST /graphs/copyLabel/:oldLabelName/:newLabelName controllers.AdminController.copyLabel(oldLabelName, newLabelName)
-POST /graphs/renameLabel/:oldLabelName/:newLabelName controllers.AdminController.renameLabel(oldLabelName, newLabelName)
-POST /graphs/updateHTable/:labelName/:newHTableName controllers.AdminController.updateHTable(labelName, newHTableName)
-PUT /graphs/loadCache controllers.AdminController.loadCache()
-
-
-# Counter Admin API
-POST /counter/v1/:service/:action controllers.CounterController.createAction(service, action)
-GET /counter/v1/:service/:action controllers.CounterController.getAction(service, action)
-PUT /counter/v1/:service/:action controllers.CounterController.updateAction(service, action)
-PUT /counter/v1/:service/:action/prepare controllers.CounterController.prepareAction(service, action)
-DELETE /counter/v1/:service/:action controllers.CounterController.deleteAction(service, action)
-
-# Counter API
-GET /counter/v1/:service/:action/ranking controllers.CounterController.getRankingCountAsync(service, action)
-DELETE /counter/v1/:service/:action/ranking controllers.CounterController.deleteRankingCount(service, action)
-GET /counter/v1/:service/:action/:item controllers.CounterController.getExactCountAsync(service, action, item)
-PUT /counter/v1/:service/:action/:item controllers.CounterController.incrementCount(service, action, item)
-POST /counter/v1/mget controllers.CounterController.getExactCountAsyncMulti()
-
-# Experiment API
-POST /graphs/experiment/:accessToken/:experimentName/:uuid controllers.ExperimentController.experiment(accessToken, experimentName, uuid)
+GET /swagger/*file controllers.Assets.at(path="/public/swagger-ui", file)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/benchmark/BenchmarkCommon.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/BenchmarkCommon.scala b/s2rest_play/test/benchmark/BenchmarkCommon.scala
deleted file mode 100644
index 48f84c8..0000000
--- a/s2rest_play/test/benchmark/BenchmarkCommon.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package benchmark
-
-import org.specs2.mutable.Specification
-
-trait BenchmarkCommon extends Specification {
- val wrapStr = s"\n=================================================="
-
- def duration[T](prefix: String = "")(block: => T) = {
- val startTs = System.currentTimeMillis()
- val ret = block
- val endTs = System.currentTimeMillis()
- println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr")
- ret
- }
-
- def durationWithReturn[T](prefix: String = "")(block: => T): (T, Long) = {
- val startTs = System.currentTimeMillis()
- val ret = block
- val endTs = System.currentTimeMillis()
- val duration = endTs - startTs
-// println(s"$wrapStr\n$prefix: took $duration ms$wrapStr")
- (ret, duration)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/test/benchmark/GraphUtilSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/GraphUtilSpec.scala b/s2rest_play/test/benchmark/GraphUtilSpec.scala
deleted file mode 100644
index b5ce93a..0000000
--- a/s2rest_play/test/benchmark/GraphUtilSpec.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-package benchmark
-
-import com.kakao.s2graph.core.{Management, GraphUtil}
-import com.kakao.s2graph.core.types.{SourceVertexId, HBaseType, InnerVal, VertexId}
-import org.apache.hadoop.hbase.util.Bytes
-import play.api.test.{FakeApplication, PlaySpecification}
-
-import scala.collection.mutable
-import scala.collection.mutable.ListBuffer
-import scala.util.Random
-
-class GraphUtilSpec extends BenchmarkCommon with PlaySpecification {
-
- def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean =
- Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0
-
- def betweenShort(value: Short, start: Short, end: Short): Boolean =
- start <= value && value <= end
-
-
- "GraphUtil" should {
- "test murmur3 hash function distribution" in {
- val testNum = 1000000
- val bucketSize = Short.MaxValue / 40
- val countsNew = new mutable.HashMap[Int, Int]()
- val counts = new mutable.HashMap[Int, Int]()
- for {
- i <- (0 until testNum)
- } {
- val h = GraphUtil.murmur3(i.toString) / bucketSize
- val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize
- counts += (h -> (counts.getOrElse(h, 0) + 1))
- countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1))
- }
- val all = counts.toList.sortBy { case (bucket, count) => count }.reverse
- val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse
- val top = all.take(10)
- val bottom = all.takeRight(10)
- val topNew = allNew.take(10)
- val bottomNew = allNew.takeRight(10)
- println(s"Top: $top")
- println(s"Bottom: $bottom")
- println("-" * 50)
- println(s"TopNew: $topNew")
- println(s"Bottom: $bottomNew")
- true
- }
-
- "test murmur hash skew2" in {
- running(FakeApplication()) {
- import HBaseType._
- val testNum = 1000000L
- val regionCount = 40
- val window = Int.MaxValue / regionCount
- val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]()
- for {
- i <- (0 until regionCount)
- } yield {
- val startKey = Bytes.toBytes(i * window)
- val endKey = Bytes.toBytes((i + 1) * window)
- rangeBytes += (startKey.toList -> endKey.toList)
- }
-
-
-
- val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]()
- val counts = new collection.mutable.HashMap[Short, Long]()
- stats += (0 -> (rangeBytes.head -> 0L))
-
- for (i <- (0L until testNum)) {
- val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION))
- val bytes = vertexId.bytes
- val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString())
- val shortVal = counts.getOrElse(shortKey, 0L) + 1L
- counts += (shortKey -> shortVal)
- var j = 0
- var found = false
- while (j < rangeBytes.size && !found) {
- val (start, end) = rangeBytes(j)
- if (between(bytes, start.toArray, end.toArray)) {
- found = true
- }
- j += 1
- }
- val head = rangeBytes(j - 1)
- val key = j - 1
- val value = stats.get(key) match {
- case None => 0L
- case Some(v) => v._2 + 1
- }
- stats += (key -> (head, value))
- }
- val sorted = stats.toList.sortBy(kv => kv._2._2).reverse
- println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
- sorted.foreach { case (idx, ((start, end), cnt)) =>
- val startShort = Bytes.toShort(start.take(2).toArray)
- val endShort = Bytes.toShort(end.take(2).toArray)
- val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
- println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
-
- }
- println("\n" * 10)
- println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount")
- stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) =>
- val startShort = Bytes.toShort(start.take(2).toArray)
- val endShort = Bytes.toShort(end.take(2).toArray)
- val count = counts.count(t => startShort <= t._1 && t._1 < endShort)
- println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count")
-
- }
- }
- true
- }
-
- "Bytes compareTo" in {
- val x = Array[Byte](11, -12, -26, -14, -23)
- val startKey = Array[Byte](0, 0, 0, 0)
- val endKey = Array[Byte](12, -52, -52, -52)
- println(Bytes.compareTo(startKey, x))
- println(Bytes.compareTo(endKey, x))
- true
- }
- }
-
-}
[03/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/CounterController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/CounterController.scala b/s2rest_play/app/controllers/CounterController.scala
deleted file mode 100644
index c2b4dc2..0000000
--- a/s2rest_play/app/controllers/CounterController.scala
+++ /dev/null
@@ -1,747 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.ExceptionHandler
-import com.kakao.s2graph.core.ExceptionHandler.KafkaMessage
-import com.kakao.s2graph.core.mysqls.Label
-import config.CounterConfig
-import models._
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.Play
-import play.api.libs.json.Reads._
-import play.api.libs.json._
-import play.api.mvc.{Action, Controller, Request}
-import s2.config.S2CounterConfig
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.counter.core.v1.{ExactStorageAsyncHBase, RankingStorageRedis}
-import s2.counter.core.v2.{ExactStorageGraph, RankingStorageGraph}
-import s2.models.Counter.ItemType
-import s2.models.{Counter, CounterModel}
-import s2.util.{CartesianProduct, ReduceMapValue, UnitConverter}
-
-import scala.concurrent.Future
-import scala.util.{Failure, Success, Try}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 22..
- */
-object CounterController extends Controller {
- import play.api.libs.concurrent.Execution.Implicits.defaultContext
-
- val config = Play.current.configuration.underlying
- val s2config = new S2CounterConfig(config)
-
- private val exactCounterMap = Map(
- s2.counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
- s2.counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
- )
- private val rankingCounterMap = Map(
- s2.counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
- s2.counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
- )
-
- private val tablePrefixMap = Map (
- s2.counter.VERSION_1 -> "s2counter",
- s2.counter.VERSION_2 -> "s2counter_v2"
- )
-
- private def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
- private def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
-
- lazy val counterModel = new CounterModel(config)
-
- def getQueryString[T](key: String, default: String)(implicit request: Request[T]): String = {
- request.getQueryString(key).getOrElse(default)
- }
-
- implicit val counterWrites = new Writes[Counter] {
- override def writes(o: Counter): JsValue = Json.obj(
- "version" -> o.version.toInt,
- "autoComb" -> o.autoComb,
- "dimension" -> o.dimension,
- "useProfile" -> o.useProfile,
- "bucketImpId" -> o.bucketImpId,
- "useRank" -> o.useRank,
- "intervalUnit" -> o.intervalUnit,
- "ttl" -> o.ttl,
- "dailyTtl" -> o.dailyTtl,
- "rateAction" -> o.rateActionId.flatMap { actionId =>
- counterModel.findById(actionId, useCache = false).map { actionPolicy =>
- Json.obj("service" -> actionPolicy.service, "action" -> actionPolicy.action)
- }
- },
- "rateBase" -> o.rateBaseId.flatMap { baseId =>
- counterModel.findById(baseId, useCache = false).map { basePolicy =>
- Json.obj("service" -> basePolicy.service, "action" -> basePolicy.action)
- }
- },
- "rateThreshold" -> o.rateThreshold
- )
- }
-
- def createAction(service: String, action: String) = Action(s2parse.json) { implicit request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case None =>
- val body = request.body
- val version = (body \ "version").asOpt[Int].map(_.toByte).getOrElse(s2.counter.VERSION_2)
- val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(true)
- val dimension = (body \ "dimension").asOpt[String].getOrElse("")
- val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(false)
- val bucketImpId = (body \ "bucketImpId").asOpt[String]
-
- val useExact = (body \ "useExact").asOpt[Boolean].getOrElse(true)
- val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(true)
-
- val intervalUnit = (body \ "intervalUnit").asOpt[String]
- // 2 day
- val ttl = (body \ "ttl").asOpt[Int].getOrElse(2 * 24 * 60 * 60)
- val dailyTtl = (body \ "dailyTtl").asOpt[Int]
- val regionMultiplier = (body \ "regionMultiplier").asOpt[Int].getOrElse(1)
-
- val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
- val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
- val rateThreshold = (body \ "rateThreshold").asOpt[Int]
-
- val rateActionId = {
- for {
- actionMap <- rateAction
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action)
- } yield {
- policy.id
- }
- }
- val rateBaseId = {
- for {
- actionMap <- rateBase
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action)
- } yield {
- policy.id
- }
- }
-
- val hbaseTable = {
- Seq(tablePrefixMap(version), service, ttl) ++ dailyTtl mkString "_"
- }
-
- // find label
- val itemType = Label.findByName(action, useCache = false) match {
- case Some(label) =>
- ItemType.withName(label.tgtColumnType.toUpperCase)
- case None =>
- val strItemType = (body \ "itemType").asOpt[String].getOrElse("STRING")
- ItemType.withName(strItemType.toUpperCase)
- }
- val policy = Counter(useFlag = true, version, service, action, itemType, autoComb = autoComb, dimension,
- useProfile = useProfile, bucketImpId, useRank = useRank, ttl, dailyTtl, Some(hbaseTable), intervalUnit,
- rateActionId, rateBaseId, rateThreshold)
-
- // prepare exact storage
- exactCounter(version).prepare(policy)
- if (useRank) {
- // prepare ranking storage
- rankingCounter(version).prepare(policy)
- }
- counterModel.createServiceAction(policy)
- Ok(Json.toJson(Map("msg" -> s"created $service/$action")))
- case Some(policy) =>
- Ok(Json.toJson(Map("msg" -> s"already exist $service/$action")))
- }
- }
-
- def getAction(service: String, action: String) = Action { request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(policy) =>
- Ok(Json.toJson(policy))
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def updateAction(service: String, action: String) = Action(s2parse.json) { request =>
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(oldPolicy) =>
- val body = request.body
- val autoComb = (body \ "autoComb").asOpt[Boolean].getOrElse(oldPolicy.autoComb)
- val dimension = (body \ "dimension").asOpt[String].getOrElse(oldPolicy.dimension)
- val useProfile = (body \ "useProfile").asOpt[Boolean].getOrElse(oldPolicy.useProfile)
- val bucketImpId = (body \ "bucketImpId").asOpt[String] match {
- case Some(s) => Some(s)
- case None => oldPolicy.bucketImpId
- }
-
- val useRank = (body \ "useRank").asOpt[Boolean].getOrElse(oldPolicy.useRank)
-
- val intervalUnit = (body \ "intervalUnit").asOpt[String] match {
- case Some(s) => Some(s)
- case None => oldPolicy.intervalUnit
- }
-
- val rateAction = (body \ "rateAction").asOpt[Map[String, String]]
- val rateBase = (body \ "rateBase").asOpt[Map[String, String]]
- val rateThreshold = (body \ "rateThreshold").asOpt[Int] match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateThreshold
- }
-
- val rateActionId = {
- for {
- actionMap <- rateAction
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- policy.id
- }
- } match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateActionId
- }
- val rateBaseId = {
- for {
- actionMap <- rateBase
- service <- actionMap.get("service")
- action <- actionMap.get("action")
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- policy.id
- }
- } match {
- case Some(i) => Some(i)
- case None => oldPolicy.rateBaseId
- }
-
- // new counter
- val policy = Counter(id = oldPolicy.id, useFlag = oldPolicy.useFlag, oldPolicy.version, service, action, oldPolicy.itemType, autoComb = autoComb, dimension,
- useProfile = useProfile, bucketImpId, useRank = useRank, oldPolicy.ttl, oldPolicy.dailyTtl, oldPolicy.hbaseTable, intervalUnit,
- rateActionId, rateBaseId, rateThreshold)
-
- counterModel.updateServiceAction(policy)
- Ok(Json.toJson(Map("msg" -> s"updated $service/$action")))
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def prepareAction(service: String, action: String) = Action(s2parse.json) { request =>
- // for data migration
- counterModel.findByServiceAction(service, action, useCache = false) match {
- case Some(policy) =>
- val body = request.body
- val version = (body \ "version").as[Int].toByte
- if (version != policy.version) {
- // change table name
- val newTableName = Seq(tablePrefixMap(version), service, policy.ttl) ++ policy.dailyTtl mkString "_"
- val newPolicy = policy.copy(version = version, hbaseTable = Some(newTableName))
- exactCounter(version).prepare(newPolicy)
- if (newPolicy.useRank) {
- rankingCounter(version).prepare(newPolicy)
- }
- Ok(Json.toJson(Map("msg" -> s"prepare storage v$version $service/$action")))
- } else {
- Ok(Json.toJson(Map("msg" -> s"already prepared storage v$version $service/$action")))
- }
- case None =>
- NotFound(Json.toJson(Map("msg" -> s"$service.$action not found")))
- }
- }
-
- def deleteAction(service: String, action: String) = Action.apply {
- {
- for {
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- Try {
- exactCounter(policy.version).destroy(policy)
- if (policy.useRank) {
- rankingCounter(policy.version).destroy(policy)
- }
- counterModel.deleteServiceAction(policy)
- } match {
- case Success(v) =>
- Ok(Json.toJson(Map("msg" -> s"deleted $service/$action")))
- case Failure(ex) =>
- throw ex
- }
- }
- }.getOrElse(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
-
- def getExactCountAsync(service: String, action: String, item: String) = Action.async { implicit request =>
- val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- val limit = getQueryString("limit", "1").toInt
-
- val qsSum = request.getQueryString("sum")
-
- val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
- val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
-
- val limitOpt = (optFrom, optTo) match {
- case (Some(_), Some(_)) =>
- None
- case _ =>
- Some(limit)
- }
-
- // find dimension
- lazy val dimQueryValues = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').filter(_.nonEmpty).toSet)
- }
-// Logger.warn(s"$dimQueryValues")
-
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
- val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- try {
-// Logger.warn(s"$tqs $qsSum")
- if (tqs.head.length > 1 && qsSum.isDefined) {
- getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum).map { jsVal =>
- Ok(jsVal)
- }
- } else {
- getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues).map { jsVal =>
- Ok(jsVal)
- }
- }
- } catch {
- case e: Exception =>
- throw e
-// Future.successful(BadRequest(s"$service, $action, $item"))
- }
- case None =>
- Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
- }
-
- /**
- * [{
- * "service": , "action", "itemIds": [], "interval": string, "limit": int, "from": ts, "to": ts,
- * "dimensions": [{"key": list[String]}]
- * }]
- * @return
- */
- private def parseExactCountParam(jsValue: JsValue) = {
- val service = (jsValue \ "service").as[String]
- val action = (jsValue \ "action").as[String]
- val itemIds = (jsValue \ "itemIds").as[Seq[String]]
- val intervals = (jsValue \ "intervals").asOpt[Seq[String]].getOrElse(Seq("t")).distinct.map(IntervalUnit.withName)
- val limit = (jsValue \ "limit").asOpt[Int].getOrElse(1)
- val from = (jsValue \ "from").asOpt[Long]
- val to = (jsValue \ "to").asOpt[Long]
- val sum = (jsValue \ "sum").asOpt[String]
- val dimensions = {
- for {
- dimension <- (jsValue \ "dimensions").asOpt[Seq[JsObject]].getOrElse(Nil)
- (k, vs) <- dimension.fields
- } yield {
- k -> vs.as[Seq[String]].toSet
- }
- }.toMap
- (service, action, itemIds, intervals, limit, from, to, dimensions, sum)
- }
-
- def getExactCountAsyncMulti = Action.async(s2parse.json) { implicit request =>
- val jsValue = request.body
- try {
- val futures = {
- for {
- jsObject <- jsValue.asOpt[List[JsObject]].getOrElse(Nil)
- (service, action, itemIds, intervalUnits, limit, from, to, dimQueryValues, qsSum) = parseExactCountParam(jsObject)
- optFrom = from.map(UnitConverter.toMillis)
- optTo = to.map(UnitConverter.toMillis)
- timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- policy <- counterModel.findByServiceAction(service, action).toSeq
- item <- itemIds
- } yield {
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optFrom, optTo)
- val timeRange = TimedQualifier.getTimeRange(intervalUnits, limit, optFrom, optTo)
- val limitOpt = (optFrom, optTo) match {
- case (Some(_), Some(_)) =>
- None
- case _ =>
- Some(limit)
- }
-
-// Logger.warn(s"$item, $limit, $optFrom, $optTo, $qsSum, $tqs")
-
- if (tqs.head.length > 1 && qsSum.isDefined) {
- getDecayedCountToJs(policy, item, timeRange, dimQueryValues, qsSum)
- } else {
- getExactCountToJs(policy, item, timeRange, limitOpt, dimQueryValues)
- }
- }
- }
- Future.sequence(futures).map { rets =>
- Ok(Json.toJson(rets))
- }
- } catch {
- case e: Exception =>
- throw e
-// Future.successful(BadRequest(s"$jsValue"))
- }
- }
-
- private [controllers] def fetchedToResult(fetchedCounts: FetchedCountsGrouped, limitOpt: Option[Int]): Seq[ExactCounterIntervalItem] = {
- for {
- ((interval, dimKeyValues), values) <- fetchedCounts.intervalWithCountMap
- } yield {
- val counterItems = {
- val sortedItems = values.toSeq.sortBy { case (eq, v) => -eq.tq.ts }
- val limited = limitOpt match {
- case Some(limit) => sortedItems.take(limit)
- case None => sortedItems
- }
- for {
- (eq, value) <- limited
- } yield {
- ExactCounterItem(eq.tq.ts, value, value.toDouble)
- }
- }
- ExactCounterIntervalItem(interval.toString, dimKeyValues, counterItems)
- }
- }.toSeq
-
- private def decayedToResult(decayedCounts: DecayedCounts): Seq[ExactCounterIntervalItem] = {
- for {
- (eq, score) <- decayedCounts.qualifierWithCountMap
- } yield {
- ExactCounterIntervalItem(eq.tq.q.toString, eq.dimKeyValues, Seq(ExactCounterItem(eq.tq.ts, score.toLong, score)))
- }
- }.toSeq
-
- private def getExactCountToJs(policy: Counter,
- item: String,
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- limitOpt: Option[Int],
- dimQueryValues: Map[String, Set[String]]): Future[JsValue] = {
- exactCounter(policy.version).getCountsAsync(policy, Seq(item), timeRange, dimQueryValues).map { seq =>
- val items = {
- for {
- fetched <- seq
- } yield {
- fetchedToResult(fetched, limitOpt)
- }
- }.flatten
- Json.toJson(ExactCounterResult(ExactCounterResultMeta(policy.service, policy.action, item), items))
- }
- }
-
- private def getDecayedCountToJs(policy: Counter,
- item: String,
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQueryValues: Map[String, Set[String]],
- qsSum: Option[String]): Future[JsValue] = {
- exactCounter(policy.version).getDecayedCountsAsync(policy, Seq(item), timeRange, dimQueryValues, qsSum).map { seq =>
- val decayedCounts = seq.head
- val meta = ExactCounterResultMeta(policy.service, policy.action, decayedCounts.exactKey.itemKey)
- val intervalItems = decayedToResult(decayedCounts)
- Json.toJson(ExactCounterResult(meta, intervalItems))
- }
- }
-
- def getRankingCountAsync(service: String, action: String) = Action.async { implicit request =>
- lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- lazy val limit = getQueryString("limit", "1").toInt
- lazy val kValue = getQueryString("k", "10").toInt
-
- lazy val qsSum = request.getQueryString("sum")
-
- lazy val optFrom = request.getQueryString("from").filter(!_.isEmpty).map(UnitConverter.toMillis)
- lazy val optTo = request.getQueryString("to").filter(!_.isEmpty).map(UnitConverter.toMillis)
-
- // find dimension
- lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').toList)
- }
-
- val dimensions = {
- for {
- values <- CartesianProduct(dimensionMap.values.toList).toSeq
- } yield {
- dimensionMap.keys.zip(values).toMap
- }
- }
-
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit, optTo)
- val dimKeys = {
- for {
- dimension <- dimensions
- } yield {
- dimension -> tqs.map(tq => RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension)))
- }
- }
-
- // if tqs has only 1 tq, do not apply sum function
- try {
- val rankResult = {
- if (tqs.length > 1 && qsSum.isDefined) {
- getSumRankCounterResultAsync(policy, dimKeys, kValue, qsSum)
- } else {
- // no summary
- Future.successful(getRankCounterResult(policy, dimKeys, kValue))
- }
- }
-
- rankResult.map { result =>
- Ok(Json.toJson(result))
- }
- } catch {
- case e: UnsupportedOperationException =>
- Future.successful(NotImplemented(Json.toJson(
- Map("msg" -> e.getMessage)
- )))
- case e: Throwable =>
- throw e
- }
- case None =>
- Future.successful(NotFound(Json.toJson(Map("msg" -> s"$service.$action not found"))))
- }
- }
-
- def deleteRankingCount(service: String, action: String) = Action.async { implicit request =>
- lazy val intervalUnits = getQueryString("interval", getQueryString("step", "t")).split(',').toSeq
- .map(IntervalUnit.withName)
- lazy val limit = getQueryString("limit", "1").toInt
-
- // find dimension
- lazy val dimensionMap = request.queryString.filterKeys { k => k.charAt(0) == ':' }.map { case (k, v) =>
- (k.substring(1), v.mkString(",").split(',').toList)
- }
-
- val dimensions = {
- for {
- values <- CartesianProduct(dimensionMap.values.toList).toSeq
- } yield {
- dimensionMap.keys.zip(values).toMap
- }
- }
-
- Future {
- counterModel.findByServiceAction(service, action) match {
- case Some(policy) =>
- val tqs = TimedQualifier.getQualifiersToLimit(intervalUnits, limit)
- val keys = {
- for {
- dimension <- dimensions
- tq <- tqs
- } yield {
- RankingKey(policy.id, policy.version, ExactQualifier(tq, dimension))
- }
- }
-
- for {
- key <- keys
- } {
- rankingCounter(policy.version).delete(key)
- }
-
- Ok(JsObject(
- Seq(
- ("msg", Json.toJson(s"delete ranking in $service.$action")),
- ("items", Json.toJson({
- for {
- key <- keys
- } yield {
- s"${key.eq.tq.q}.${key.eq.tq.ts}.${key.eq.dimension}"
- }
- }))
- )
- ))
- case None =>
- NotFound(Json.toJson(
- Map("msg" -> s"$service.$action not found")
- ))
- }
- }
- }
-
- val reduceRateRankingValue = new ReduceMapValue[ExactKeyTrait, RateRankingValue](RateRankingValue.reduce, RateRankingValue(-1, -1))
-
- // change format
- private def getDecayedCountsAsync(policy: Counter,
- items: Seq[String],
- timeRange: (TimedQualifier, TimedQualifier),
- dimension: Map[String, String],
- qsSum: Option[String]): Future[Seq[(ExactKeyTrait, Double)]] = {
- exactCounter(policy.version).getDecayedCountsAsync(policy, items, Seq(timeRange), dimension.mapValues(s => Set(s)), qsSum).map { seq =>
- for {
- DecayedCounts(exactKey, qcMap) <- seq
- value <- qcMap.values
- } yield {
- exactKey -> value
- }
- }
- }
-
- def getSumRankCounterResultAsync(policy: Counter,
- dimKeys: Seq[(Map[String, String], Seq[RankingKey])],
- kValue: Int,
- qsSum: Option[String]): Future[RankCounterResult] = {
- val futures = {
- for {
- (dimension, keys) <- dimKeys
- } yield {
- val tqs = keys.map(rk => rk.eq.tq)
- val (tqFrom, tqTo) = (tqs.last, tqs.head)
- val items = rankingCounter(policy.version).getAllItems(keys, kValue)
-// Logger.warn(s"item count: ${items.length}")
- val future = {
- if (policy.isRateCounter) {
- val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
- val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
-
- val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
- }.toMap
- }
- val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
- }.toMap
- }
- futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
- reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
-// Logger.warn(s"$k -> $rrv")
- k -> rrv.rankingValue.score
- }.toSeq
- }
- }
- else if (policy.isTrendCounter) {
- val actionPolicy = policy.rateActionId.flatMap(counterModel.findById(_)).get
- val basePolicy = policy.rateBaseId.flatMap(counterModel.findById(_)).get
-
- val futureAction = getDecayedCountsAsync(actionPolicy, items, (tqFrom, tqTo), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(score, -1)
- }.toMap
- }
- val futureBase = getDecayedCountsAsync(basePolicy, items, (tqFrom.add(-1), tqTo.add(-1)), dimension, qsSum).map { seq =>
- seq.map { case (k, score) =>
- ExactKey(policy, k.itemKey, checkItemType = false) -> RateRankingValue(-1, score)
- }.toMap
- }
- futureAction.zip(futureBase).map { case (actionScores, baseScores) =>
- reduceRateRankingValue(actionScores, baseScores).map { case (k, rrv) =>
-// Logger.warn(s"$k -> $rrv")
- k -> rrv.rankingValue.score
- }.toSeq
- }
- }
- else {
- getDecayedCountsAsync(policy, items, (tqFrom, tqTo), dimension, qsSum)
- }
- }
- future.map { keyWithScore =>
- val ranking = keyWithScore.sortBy(-_._2).take(kValue)
- val rankCounterItems = {
- for {
- idx <- ranking.indices
- (exactKey, score) = ranking(idx)
- } yield {
- val realId = policy.itemType match {
- case ItemType.BLOB => exactCounter(policy.version).getBlobValue(policy, exactKey.itemKey)
- .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} ${exactKey.itemKey}"))
- case _ => exactKey.itemKey
- }
- RankCounterItem(idx + 1, realId, score)
- }
- }
-
- val eq = ExactQualifier(tqFrom, dimension)
- RankCounterDimensionItem(eq.tq.q.toString, eq.tq.ts, eq.dimension, -1, rankCounterItems)
- }
- }
- }
-
- Future.sequence(futures).map { dimensionResultList =>
- RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
- }
- }
-
- def getRankCounterResult(policy: Counter, dimKeys: Seq[(Map[String, String], Seq[RankingKey])], kValue: Int): RankCounterResult = {
- val dimensionResultList = {
- for {
- (dimension, keys) <- dimKeys
- key <- keys
- } yield {
- val rankingValue = rankingCounter(policy.version).getTopK(key, kValue)
- val ranks = {
- for {
- rValue <- rankingValue.toSeq
- idx <- rValue.values.indices
- rank = idx + 1
- } yield {
- val (id, score) = rValue.values(idx)
- val realId = policy.itemType match {
- case ItemType.BLOB =>
- exactCounter(policy.version)
- .getBlobValue(policy, id)
- .getOrElse(throw new Exception(s"not found blob id. ${policy.service}.${policy.action} $id"))
- case _ => id
- }
- RankCounterItem(rank, realId, score)
- }
- }
- val eq = key.eq
- val tq = eq.tq
- RankCounterDimensionItem(tq.q.toString, tq.ts, eq.dimension, rankingValue.map(v => v.totalScore).getOrElse(0d), ranks)
- }
- }
-
- RankCounterResult(RankCounterResultMeta(policy.service, policy.action), dimensionResultList)
- }
-
- type Record = ProducerRecord[String, String]
-
- def incrementCount(service: String, action: String, item: String) = Action.async(s2parse.json) { request =>
- Future {
- /**
- * {
- * timestamp: Long
- * property: {}
- * value: Int
- * }
- */
- lazy val metaMap = Map("service" -> service, "action" -> action, "item" -> item)
- counterModel.findByServiceAction(service, action).map { policy =>
- val body = request.body
- try {
- val ts = (body \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis()).toString
- val dimension = (body \ "dimension").asOpt[JsValue].getOrElse(Json.obj())
- val property = (body \ "property").asOpt[JsValue].getOrElse(Json.obj())
-
- val msg = List(ts, service, action, item, dimension, property).mkString("\t")
-
- // produce to kafka
- // hash partitioner by key
- ExceptionHandler.enqueue(KafkaMessage(new Record(CounterConfig.KAFKA_TOPIC_COUNTER, s"$ts.$item", msg)))
-
- Ok(Json.toJson(
- Map(
- "meta" -> metaMap
- )
- ))
- }
- catch {
- case e: JsResultException =>
- BadRequest(Json.toJson(
- Map("msg" -> s"need timestamp.")
- ))
- }
- }.getOrElse {
- NotFound(Json.toJson(
- Map("msg" -> s"$service.$action not found")
- ))
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/EdgeController.scala b/s2rest_play/app/controllers/EdgeController.scala
deleted file mode 100644
index a8e1a41..0000000
--- a/s2rest_play/app/controllers/EdgeController.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-package controllers
-
-import actors.QueueActor
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{Label}
-import com.kakao.s2graph.core.rest.RequestParser
-import com.kakao.s2graph.core.utils.logger
-import config.Config
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.libs.json._
-import play.api.mvc.{Controller, Result}
-
-import scala.collection.Seq
-import scala.concurrent.Future
-
-object EdgeController extends Controller {
-
- import ExceptionHandler._
- import controllers.ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits._
-
- private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
- private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
- private def jsToStr(js: JsValue): String = js match {
- case JsString(s) => s
- case _ => js.toString()
- }
-
- def toTsv(jsValue: JsValue, op: String): String = {
- val ts = jsToStr(jsValue \ "timestamp")
- val from = jsToStr(jsValue \ "from")
- val to = jsToStr(jsValue \ "to")
- val label = jsToStr(jsValue \ "label")
- val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
-
- (jsValue \ "direction").asOpt[String] match {
- case None => Seq(ts, op, "e", from, to, label, props).mkString("\t")
- case Some(dir) => Seq(ts, op, "e", from, to, label, props, dir).mkString("\t")
- }
- }
-
- def tryMutates(jsValue: JsValue, operation: String, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
-
- else {
- try {
- logger.debug(s"$jsValue")
- val (edges, jsOrgs) = requestParser.toEdgesWithOrg(jsValue, operation)
-
- for ((edge, orgJs) <- edges.zip(jsOrgs)) {
- if (edge.isAsync)
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, edge, Option(toTsv(orgJs, operation))))
- else
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, edge, Option(toTsv(orgJs, operation))))
- }
-
- val edgesToStore = edges.filterNot(e => e.isAsync)
-
- if (withWait) {
- val rets = s2.mutateEdges(edgesToStore, withWait = true)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = edgesToStore.map { edge => QueueActor.router ! edge; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
- case e: Exception =>
- logger.error(s"mutateAndPublish: $e", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
- }
-
- def mutateAndPublish(str: String, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
-
- logger.debug(s"$str")
- val edgeStrs = str.split("\\n")
-
- var vertexCnt = 0L
- var edgeCnt = 0L
- try {
- val elements =
- for (edgeStr <- edgeStrs; str <- GraphUtil.parseString(edgeStr); element <- Graph.toGraphElement(str)) yield {
- element match {
- case v: Vertex => vertexCnt += 1
- case e: Edge => edgeCnt += 1
- }
- if (element.isAsync) {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, element, Some(str)))
- } else {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, element, Some(str)))
- }
- element
- }
-
- //FIXME:
- val elementsToStore = elements.filterNot(e => e.isAsync)
- if (withWait) {
- val rets = s2.mutateElements(elementsToStore, withWait)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = elementsToStore.map { element => QueueActor.router ! element; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"$e"))
- case e: Throwable =>
- logger.error(s"mutateAndPublish: $e", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
-
- def mutateBulk() = withHeaderAsync(parse.text) { request =>
- mutateAndPublish(request.body, withWait = false)
- }
-
- def mutateBulkWithWait() = withHeaderAsync(parse.text) { request =>
- mutateAndPublish(request.body, withWait = true)
- }
-
- def inserts() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert")
- }
-
- def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", withWait = true)
- }
-
- def insertsBulk() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insertBulk")
- }
-
- def deletes() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete")
- }
-
- def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", withWait = true)
- }
-
- def updates() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "update")
- }
-
- def updatesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "update", withWait = true)
- }
-
- def increments() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "increment")
- }
-
- def incrementsWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "increment", withWait = true)
- }
-
- def incrementCounts() = withHeaderAsync(jsonParser) { request =>
- val jsValue = request.body
- val edges = requestParser.toEdges(jsValue, "incrementCount")
-
- s2.incrementCounts(edges, withWait = true).map { results =>
- val json = results.map { case (isSuccess, resultCount) =>
- Json.obj("success" -> isSuccess, "result" -> resultCount)
- }
-
- jsonResponse(Json.toJson(json))
- }
- }
-
- def deleteAll() = withHeaderAsync(jsonParser) { request =>
-// deleteAllInner(request.body, withWait = false)
- deleteAllInner(request.body, withWait = true)
- }
-
- def deleteAllInner(jsValue: JsValue, withWait: Boolean) = {
-
- /** logging for delete all request */
- def enqueueLogMessage(ids: Seq[JsValue], labels: Seq[Label], ts: Long, direction: String, topicOpt: Option[String]) = {
- val kafkaMessages = for {
- id <- ids
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", jsToStr(id), jsToStr(id), label.label, "{}", direction).mkString("\t")
- val topic = topicOpt.getOrElse {
- if (label.isAsync) Config.KAFKA_LOG_TOPIC_ASYNC else Config.KAFKA_LOG_TOPIC
- }
-
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
-
- ExceptionHandler.enqueues(kafkaMessages)
- }
-
- def deleteEach(labels: Seq[Label], direction: String, ids: Seq[JsValue], ts: Long, vertices: Seq[Vertex]) = {
- enqueueLogMessage(ids, labels, ts, direction, None)
- val future = s2.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
- if (withWait) {
- future
- } else {
- Future.successful(true)
- }
- }
-
- val deleteFutures = jsValue.as[Seq[JsValue]].map { json =>
- val (labels, direction, ids, ts, vertices) = requestParser.toDeleteParam(json)
- if (labels.isEmpty || ids.isEmpty) Future.successful(true)
- else deleteEach(labels, direction, ids, ts, vertices)
- }
-
- val deleteResults = Future.sequence(deleteFutures)
- deleteResults.map { rst =>
- logger.debug(s"deleteAllInner: $rst")
- Ok(s"deleted... ${rst.toString()}")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/ExperimentController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ExperimentController.scala b/s2rest_play/app/controllers/ExperimentController.scala
deleted file mode 100644
index e48b0f1..0000000
--- a/s2rest_play/app/controllers/ExperimentController.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package controllers
-
-
-import com.kakao.s2graph.core.mysqls.Experiment
-import com.kakao.s2graph.core.rest.RestHandler
-import com.kakao.s2graph.core.utils.logger
-import play.api.mvc._
-import scala.concurrent.ExecutionContext.Implicits.global
-
-object ExperimentController extends Controller {
- private val rest: RestHandler = com.kakao.s2graph.rest.Global.s2rest
-
- import ApplicationController._
-
- def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(jsonText) { request =>
- val body = request.body
-
- val res = rest.doPost(request.uri, body, request.headers.get(Experiment.impressionKey))
- res.body.map { case js =>
- val headers = res.headers :+ ("result_size" -> rest.calcSize(js).toString)
- jsonResponse(js, headers: _*)
- } recoverWith ApplicationController.requestFallback(body)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/JsonBodyParser.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/JsonBodyParser.scala b/s2rest_play/app/controllers/JsonBodyParser.scala
deleted file mode 100644
index 4339eb4..0000000
--- a/s2rest_play/app/controllers/JsonBodyParser.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.Play
-import play.api.libs.iteratee.{Execution, Iteratee}
-import play.api.libs.json.{JsValue, Json}
-import play.api.mvc._
-
-import scala.concurrent.Future
-import scala.util.control.NonFatal
-
-object s2parse extends BodyParsers {
-
- import parse._
-
- val defaultMaxTextLength = 1024 * 512
- val defaultMaxJsonLength = 1024 * 512
-
- def json: BodyParser[JsValue] = json(defaultMaxTextLength)
-
- /**
- * parseText with application/json header for Pre-Process text
- */
- def jsonText: BodyParser[String] = when(
- _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
- jsonText(defaultMaxTextLength),
- createBadResult("Expecting text/json or application/json body")
- )
-
- private def jsonText(maxLength: Int): BodyParser[String] = BodyParser("json, maxLength=" + maxLength) { request =>
- import play.api.libs.iteratee.Execution.Implicits.trampoline
- import play.api.libs.iteratee.Traversable
-
- Traversable.takeUpTo[Array[Byte]](maxLength)
- .transform(Iteratee.consume[Array[Byte]]().map(c => new String(c, "UTF-8")))
- .flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
- }
-
- def json(maxLength: Int): BodyParser[JsValue] = when(
- _.contentType.exists(m => m.equalsIgnoreCase("text/json") || m.equalsIgnoreCase("application/json")),
- tolerantJson(maxLength),
- createBadResult("Expecting text/json or application/json body")
- )
-
- def tolerantJson(maxLength: Int): BodyParser[JsValue] =
- tolerantBodyParser[JsValue]("json", maxLength, "Invalid Json") { (request, bytes) =>
- // Encoding notes: RFC 4627 requires that JSON be encoded in Unicode, and states that whether that's
- // UTF-8, UTF-16 or UTF-32 can be auto detected by reading the first two bytes. So we ignore the declared
- // charset and don't decode, we passing the byte array as is because Jackson supports auto detection.
- Json.parse(bytes)
- }
-
- private def tolerantBodyParser[A](name: String, maxLength: Int, errorMessage: String)(parser: (RequestHeader, Array[Byte]) => A): BodyParser[A] =
- BodyParser(name + ", maxLength=" + maxLength) { request =>
- import play.api.libs.iteratee.Execution.Implicits.trampoline
- import play.api.libs.iteratee.Traversable
-
- import scala.util.control.Exception._
-
- val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] =
- Traversable.takeUpTo[Array[Byte]](maxLength).transform(
- Iteratee.consume[Array[Byte]]().map { bytes =>
- allCatch[A].either {
- parser(request, bytes)
- }.left.map {
- case NonFatal(e) =>
- val txt = new String(bytes)
- logger.error(s"$errorMessage: $txt", e)
- createBadResult(s"$errorMessage: $e")(request)
- case t => throw t
- }
- }
- ).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
-
- bodyParser.mapM {
- case Left(tooLarge) => Future.successful(Left(tooLarge))
- case Right(Left(badResult)) => badResult.map(Left.apply)
- case Right(Right(body)) => Future.successful(Right(body))
- }
- }
-
- private def createBadResult(msg: String): RequestHeader => Future[Result] = { request =>
- Play.maybeApplication.map(_.global.onBadRequest(request, msg))
- .getOrElse(Future.successful(Results.BadRequest))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/PublishController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/PublishController.scala b/s2rest_play/app/controllers/PublishController.scala
deleted file mode 100644
index b1495d5..0000000
--- a/s2rest_play/app/controllers/PublishController.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.ExceptionHandler
-import config.Config
-import org.apache.kafka.clients.producer.ProducerRecord
-import play.api.mvc._
-
-import scala.concurrent.Future
-
-object PublishController extends Controller {
-
- import ApplicationController._
- import ExceptionHandler._
- import play.api.libs.concurrent.Execution.Implicits._
-
- /**
- * never check validation on string. just redirect strings to kafka.
- */
- val serviceNotExistException = new RuntimeException(s"service is not created in s2graph. create service first.")
-
- // private def toService(topic: String): String = {
- // Service.findByName(topic).map(service => s"${service.serviceName}-${Config.PHASE}").getOrElse(throw serviceNotExistException)
- // }
- def publishOnly(topic: String) = withHeaderAsync(parse.text) { request =>
- if (!Config.IS_WRITE_SERVER) Future.successful(UNAUTHORIZED)
- // val kafkaTopic = toService(topic)
- val strs = request.body.split("\n")
- strs.foreach(str => {
- val keyedMessage = new ProducerRecord[Key, Val](Config.KAFKA_LOG_TOPIC, str)
- // val keyedMessage = new ProducerRecord[Key, Val](kafkaTopic, s"$str")
- // logger.debug(s"$kafkaTopic, $str")
- ExceptionHandler.enqueue(KafkaMessage(keyedMessage))
- })
- Future.successful(
- Ok("publish success.\n").withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
- )
- // try {
- //
- // } catch {
- // case e: Exception => Future.successful(BadRequest(e.getMessage))
- // }
- }
-
- def publish(topic: String) = publishOnly(topic)
-
- // def mutateBulk(topic: String) = Action.async(parse.text) { request =>
- // EdgeController.mutateAndPublish(Config.KAFKA_LOG_TOPIC, Config.KAFKA_FAIL_TOPIC, request.body).map { result =>
- // result.withHeaders(CONNECTION -> "Keep-Alive", "Keep-Alive" -> "timeout=10, max=10")
- // }
- // }
- def mutateBulk(topic: String) = withHeaderAsync(parse.text) { request =>
- EdgeController.mutateAndPublish(request.body)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/QueryController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/QueryController.scala b/s2rest_play/app/controllers/QueryController.scala
deleted file mode 100644
index d133370..0000000
--- a/s2rest_play/app/controllers/QueryController.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.Experiment
-import com.kakao.s2graph.core.rest.RestHandler
-import play.api.libs.json.{Json}
-import play.api.mvc._
-
-import scala.language.postfixOps
-
-object QueryController extends Controller with JSONParser {
-
- import ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits.defaultContext
-
- private val rest: RestHandler = com.kakao.s2graph.rest.Global.s2rest
-
- def delegate(request: Request[String]) = {
- rest.doPost(request.uri, request.body, request.headers.get(Experiment.impressionKey)).body.map {
- js =>
- jsonResponse(js, "result_size" -> rest.calcSize(js).toString)
- } recoverWith ApplicationController.requestFallback(request.body)
- }
-
- def getEdges() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesWithGrouping() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesExcluded() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonText)(delegate)
-
- def checkEdges() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesGrouped() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesGroupedExcluded() = withHeaderAsync(jsonText)(delegate)
-
- def getEdgesGroupedExcludedFormatted() = withHeaderAsync(jsonText)(delegate)
-
- def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) =
- withHeaderAsync(jsonText) {
- request =>
- val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
- rest.checkEdges(params).body.map {
- js =>
- jsonResponse(js, "result_size" -> rest.calcSize(js).toString)
- } recoverWith ApplicationController.requestFallback(request.body)
- }
-
- def getVertices() = withHeaderAsync(jsonText)(delegate)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/TestController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/TestController.scala b/s2rest_play/app/controllers/TestController.scala
deleted file mode 100644
index 8558ae5..0000000
--- a/s2rest_play/app/controllers/TestController.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package controllers
-
-import play.api.mvc.{Action, Controller}
-import util.TestDataLoader
-
-import scala.concurrent.Future
-
-
-object TestController extends Controller {
- import ApplicationController._
-
- def getRandomId() = withHeader(parse.anyContent) { request =>
- val id = TestDataLoader.randomId
- Ok(s"${id}")
- }
-
- def pingAsync() = Action.async(parse.json) { request =>
- Future.successful(Ok("Pong\n"))
- }
-
- def ping() = Action(parse.json) { request =>
- Ok("Pong\n")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/VertexController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/VertexController.scala b/s2rest_play/app/controllers/VertexController.scala
deleted file mode 100644
index 977c2fc..0000000
--- a/s2rest_play/app/controllers/VertexController.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-package controllers
-
-
-import actors.QueueActor
-import com.kakao.s2graph.core.rest.RequestParser
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{ExceptionHandler, Graph, GraphExceptions}
-import config.Config
-import play.api.libs.json.{JsValue, Json}
-import play.api.mvc.{Controller, Result}
-
-import scala.concurrent.Future
-
-object VertexController extends Controller {
- private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
- private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
-
- import ExceptionHandler._
- import controllers.ApplicationController._
- import play.api.libs.concurrent.Execution.Implicits._
-
- def tryMutates(jsValue: JsValue, operation: String, serviceNameOpt: Option[String] = None, columnNameOpt: Option[String] = None, withWait: Boolean = false): Future[Result] = {
- if (!Config.IS_WRITE_SERVER) Future.successful(Unauthorized)
- else {
- try {
- val vertices = requestParser.toVertices(jsValue, operation, serviceNameOpt, columnNameOpt)
-
- for (vertex <- vertices) {
- if (vertex.isAsync)
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC_ASYNC, vertex, None))
- else
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_LOG_TOPIC, vertex, None))
- }
-
- //FIXME:
- val verticesToStore = vertices.filterNot(v => v.isAsync)
-
- if (withWait) {
- val rets = s2.mutateVertices(verticesToStore, withWait = true)
- rets.map(Json.toJson(_)).map(jsonResponse(_))
- } else {
- val rets = verticesToStore.map { vertex => QueueActor.router ! vertex; true }
- Future.successful(jsonResponse(Json.toJson(rets)))
- }
- } catch {
- case e: GraphExceptions.JsonParseException => Future.successful(BadRequest(s"e"))
- case e: Exception =>
- logger.error(s"[Failed] tryMutates", e)
- Future.successful(InternalServerError(s"${e.getStackTrace}"))
- }
- }
- }
-
- def inserts() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert")
- }
-
- def insertsWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", withWait = true)
- }
-
- def insertsSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "insert", Some(serviceName), Some(columnName))
- }
-
- def deletes() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete")
- }
-
- def deletesWithWait() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", withWait = true)
- }
-
- def deletesSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "delete", Some(serviceName), Some(columnName))
- }
-
- def deletesAll() = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "deleteAll")
- }
-
- def deletesAllSimple(serviceName: String, columnName: String) = withHeaderAsync(jsonParser) { request =>
- tryMutates(request.body, "deleteAll", Some(serviceName), Some(columnName))
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/models/ExactCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/ExactCounterItem.scala b/s2rest_play/app/models/ExactCounterItem.scala
deleted file mode 100644
index 244c046..0000000
--- a/s2rest_play/app/models/ExactCounterItem.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-package models
-
-import play.api.libs.json.{Json, Writes}
-
-/**
- * Created by alec on 15. 4. 15..
- */
-case class ExactCounterItem(ts: Long, count: Long, score: Double)
-
-case class ExactCounterIntervalItem(interval: String, dimension: Map[String, String], counter: Seq[ExactCounterItem])
-
-case class ExactCounterResultMeta(service: String, action: String, item: String)
-
-case class ExactCounterResult(meta: ExactCounterResultMeta, data: Seq[ExactCounterIntervalItem])
-
-object ExactCounterItem {
- implicit val writes = new Writes[ExactCounterItem] {
- def writes(item: ExactCounterItem) = Json.obj(
- "ts" -> item.ts,
- "time" -> tsFormat.format(item.ts),
- "count" -> item.count,
- "score" -> item.score
- )
- }
- implicit val reads = Json.reads[ExactCounterItem]
-}
-
-object ExactCounterIntervalItem {
- implicit val format = Json.format[ExactCounterIntervalItem]
-}
-
-object ExactCounterResultMeta {
- implicit val format = Json.format[ExactCounterResultMeta]
-}
-
-object ExactCounterResult {
- implicit val formats = Json.format[ExactCounterResult]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/models/RankCounterItem.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/RankCounterItem.scala b/s2rest_play/app/models/RankCounterItem.scala
deleted file mode 100644
index aaa7df7..0000000
--- a/s2rest_play/app/models/RankCounterItem.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-package models
-
-import play.api.libs.json.{Json, Writes}
-
-/**
- * Created by alec on 15. 4. 15..
- */
-case class RankCounterItem(rank: Int, id: String, score: Double)
-
-case class RankCounterDimensionItem(interval: String, ts: Long, dimension: String, total: Double, ranks: Seq[RankCounterItem])
-
-case class RankCounterResultMeta(service: String, action: String)
-
-case class RankCounterResult(meta: RankCounterResultMeta, data: Seq[RankCounterDimensionItem])
-
-object RankCounterItem {
- implicit val format = Json.format[RankCounterItem]
-}
-
-object RankCounterDimensionItem {
- implicit val writes = new Writes[RankCounterDimensionItem] {
- def writes(item: RankCounterDimensionItem) = Json.obj(
- "interval" -> item.interval,
- "ts" -> item.ts,
- "time" -> tsFormat.format(item.ts),
- "dimension" -> item.dimension,
- "total" -> item.total,
- "ranks" -> item.ranks
- )
- }
- implicit val reads = Json.reads[RankCounterDimensionItem]
-}
-
-object RankCounterResultMeta {
- implicit val format = Json.format[RankCounterResultMeta]
-}
-
-object RankCounterResult {
- implicit val format = Json.format[RankCounterResult]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/models/package.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/models/package.scala b/s2rest_play/app/models/package.scala
deleted file mode 100644
index 17fa8e1..0000000
--- a/s2rest_play/app/models/package.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-import java.text.SimpleDateFormat
-
-/**
- * Created by alec on 15. 4. 20..
- */
-package object models {
- def tsFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
new file mode 100644
index 0000000..8474b88
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
@@ -0,0 +1,81 @@
+package org.apache.s2graph.rest.play
+
+import java.util.concurrent.Executors
+
+import org.apache.s2graph.core.rest.{RequestParser, RestHandler}
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{ExceptionHandler, Graph, Management}
+import org.apache.s2graph.rest.play.actors.QueueActor
+import org.apache.s2graph.rest.play.config.Config
+import org.apache.s2graph.rest.play.controllers.ApplicationController
+import play.api.Application
+import play.api.mvc.{WithFilters, _}
+import play.filters.gzip.GzipFilter
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.io.Source
+import scala.util.Try
+
+object Global extends WithFilters(new GzipFilter()) {
+ var s2graph: Graph = _
+ var storageManagement: Management = _
+ var s2parser: RequestParser = _
+ var s2rest: RestHandler = _
+
+ // Application entry point
+ override def onStart(app: Application) {
+ ApplicationController.isHealthy = false
+
+ val numOfThread = Runtime.getRuntime.availableProcessors()
+ val threadPool = Executors.newFixedThreadPool(numOfThread)
+ val ec = ExecutionContext.fromExecutor(threadPool)
+
+ val config = Config.conf.underlying
+
+ // init s2graph with config
+ s2graph = new Graph(config)(ec)
+ storageManagement = new Management(s2graph)
+ s2parser = new RequestParser(s2graph.config) // merged config
+ s2rest = new RestHandler(s2graph)(ec)
+
+ QueueActor.init(s2graph)
+
+ if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ ExceptionHandler.apply(config)
+ }
+
+ val defaultHealthOn = Config.conf.getBoolean("app.health.on").getOrElse(true)
+ ApplicationController.deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
+
+ ApplicationController.isHealthy = defaultHealthOn
+ logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
+ }
+
+ override def onStop(app: Application) {
+ QueueActor.shutdown()
+
+ if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ ExceptionHandler.shutdown()
+ }
+
+ /**
+ * shutdown hbase client for flush buffers.
+ */
+ s2graph.shutdown()
+ }
+
+ override def onError(request: RequestHeader, ex: Throwable): Future[Result] = {
+ logger.error(s"onError => ip:${request.remoteAddress}, request:${request}", ex)
+ Future.successful(Results.InternalServerError)
+ }
+
+ override def onHandlerNotFound(request: RequestHeader): Future[Result] = {
+ logger.error(s"onHandlerNotFound => ip:${request.remoteAddress}, request:${request}")
+ Future.successful(Results.NotFound)
+ }
+
+ override def onBadRequest(request: RequestHeader, error: String): Future[Result] = {
+ logger.error(s"onBadRequest => ip:${request.remoteAddress}, request:$request, error:$error")
+ Future.successful(Results.BadRequest(error))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/actors/QueueActor.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/actors/QueueActor.scala b/s2rest_play/app/org/apache/s2graph/rest/play/actors/QueueActor.scala
new file mode 100644
index 0000000..2559fd1
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/actors/QueueActor.scala
@@ -0,0 +1,89 @@
+package org.apache.s2graph.rest.play.actors
+
+import java.util.concurrent.TimeUnit
+
+import akka.actor._
+import org.apache.s2graph.core.ExceptionHandler._
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{ExceptionHandler, Graph, GraphElement}
+import org.apache.s2graph.rest.play.actors.Protocol.FlushAll
+import org.apache.s2graph.rest.play.config.Config
+import play.api.Play.current
+import play.api.libs.concurrent.Akka
+
+import scala.collection.mutable
+import scala.concurrent.duration.Duration
+
+object Protocol {
+
+ case object Flush
+
+ case object FlushAll
+
+}
+
+object QueueActor {
+ /** we are throttling down here so fixed number of actor to constant */
+ var router: ActorRef = _
+
+ // Akka.system.actorOf(props(), name = "queueActor")
+ def init(s2: Graph) = {
+ router = Akka.system.actorOf(props(s2))
+ }
+
+ def shutdown() = {
+ router ! FlushAll
+ Akka.system.shutdown()
+ Thread.sleep(Config.ASYNC_HBASE_CLIENT_FLUSH_INTERVAL * 2)
+ }
+
+ def props(s2: Graph): Props = Props(classOf[QueueActor], s2)
+}
+
+class QueueActor(s2: Graph) extends Actor with ActorLogging {
+
+ import Protocol._
+
+ implicit val ec = context.system.dispatcher
+ // logger.error(s"QueueActor: $self")
+ val queue = mutable.Queue.empty[GraphElement]
+ var queueSize = 0L
+ val maxQueueSize = Config.LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE
+ val timeUnitInMillis = 10
+ val rateLimitTimeStep = 1000 / timeUnitInMillis
+ val rateLimit = Config.LOCAL_QUEUE_ACTOR_RATE_LIMIT / rateLimitTimeStep
+
+
+ context.system.scheduler.schedule(Duration.Zero, Duration(timeUnitInMillis, TimeUnit.MILLISECONDS), self, Flush)
+
+ override def receive: Receive = {
+ case element: GraphElement =>
+
+ if (queueSize > maxQueueSize) {
+ ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_FAIL_TOPIC, element, None))
+ } else {
+ queueSize += 1L
+ queue.enqueue(element)
+ }
+
+ case Flush =>
+ val elementsToFlush =
+ if (queue.size < rateLimit) queue.dequeueAll(_ => true)
+ else (0 until rateLimit).map(_ => queue.dequeue())
+
+ val flushSize = elementsToFlush.size
+
+ queueSize -= elementsToFlush.length
+ s2.mutateElements(elementsToFlush)
+
+ if (flushSize > 0) {
+ logger.info(s"flush: $flushSize, $queueSize")
+ }
+
+ case FlushAll =>
+ s2.mutateElements(queue)
+ context.stop(self)
+
+ case _ => logger.error("unknown protocol")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
new file mode 100644
index 0000000..0254506
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
@@ -0,0 +1,41 @@
+package org.apache.s2graph.rest.play.config
+
+import play.api.Play
+
+object Config {
+ // HBASE
+ lazy val HBASE_ZOOKEEPER_QUORUM = conf.getString("hbase.zookeeper.quorum").getOrElse("localhost")
+
+
+ // HBASE CLIENT
+ lazy val ASYNC_HBASE_CLIENT_FLUSH_INTERVAL = conf.getInt("async.hbase.client.flush.interval").getOrElse(1000).toShort
+ lazy val RPC_TIMEOUT = conf.getInt("hbase.client.operation.timeout").getOrElse(1000)
+ lazy val MAX_ATTEMPT = conf.getInt("hbase.client.operation.maxAttempt").getOrElse(3)
+
+ // PHASE
+ lazy val PHASE = conf.getString("phase").getOrElse("dev")
+ lazy val conf = Play.current.configuration
+
+ // CACHE
+ lazy val CACHE_TTL_SECONDS = conf.getInt("cache.ttl.seconds").getOrElse(600)
+ lazy val CACHE_MAX_SIZE = conf.getInt("cache.max.size").getOrElse(10000)
+
+ //KAFKA
+ lazy val KAFKA_METADATA_BROKER_LIST = conf.getString("kafka.metadata.broker.list").getOrElse("localhost")
+ lazy val KAFKA_PRODUCER_POOL_SIZE = conf.getInt("kafka.producer.pool.size").getOrElse(0)
+ lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}"
+ lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async"
+ lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"
+
+ // is query or write
+ lazy val IS_QUERY_SERVER = conf.getBoolean("is.query.server").getOrElse(true)
+ lazy val IS_WRITE_SERVER = conf.getBoolean("is.write.server").getOrElse(true)
+
+
+ // query limit per step
+ lazy val QUERY_HARD_LIMIT = conf.getInt("query.hard.limit").getOrElse(300)
+
+ // local queue actor
+ lazy val LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE = conf.getInt("local.queue.actor.max.queue.size").getOrElse(10000)
+ lazy val LOCAL_QUEUE_ACTOR_RATE_LIMIT = conf.getInt("local.queue.actor.rate.limit").getOrElse(1000)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/config/CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/config/CounterConfig.scala b/s2rest_play/app/org/apache/s2graph/rest/play/config/CounterConfig.scala
new file mode 100644
index 0000000..0bcdd81
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/config/CounterConfig.scala
@@ -0,0 +1,10 @@
+package org.apache.s2graph.rest.play.config
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 9. 3..
+ */
+object CounterConfig {
+ // kafka
+ lazy val KAFKA_TOPIC_COUNTER = s"s2counter-${Config.PHASE}"
+ lazy val KAFKA_TOPIC_COUNTER_TRX = s"s2counter-trx-${Config.PHASE}"
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
new file mode 100644
index 0000000..00e0741
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
@@ -0,0 +1,424 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.Management
+import org.apache.s2graph.core.mysqls._
+import org.apache.s2graph.core.rest.RequestParser
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.functional.syntax._
+import play.api.libs.json._
+import play.api.mvc
+import play.api.mvc.{Action, Controller}
+
+import scala.util.{Failure, Success, Try}
+
+object AdminController extends Controller {
+
+ import ApplicationController._
+ private val management: Management = org.apache.s2graph.rest.play.Global.storageManagement
+ private val requestParser: RequestParser = org.apache.s2graph.rest.play.Global.s2parser
+
+ /**
+ * admin message formatter
+ * @tparam T
+ */
+ trait AdminMessageFormatter[T] {
+ def toJson(msg: T): JsValue
+ }
+
+ object AdminMessageFormatter {
+ implicit def jsValueToJson[T <: JsValue] = new AdminMessageFormatter[T] {
+ def toJson(js: T) = js
+ }
+
+ implicit val stringToJson = new AdminMessageFormatter[String] {
+ def toJson(js: String) = Json.obj("message" -> js)
+ }
+ }
+
+ def format[T: AdminMessageFormatter](f: JsValue => play.mvc.Result)(message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ f(formatter.toJson(message))
+ }
+
+ /**
+ * ok response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def ok[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ Ok(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ /**
+ * bad request response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def bad[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ BadRequest(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ /**
+ * not found response
+ * @param message
+ * @tparam T
+ * @return
+ */
+ def notFound[T: AdminMessageFormatter](message: T) = {
+ val formatter = implicitly[AdminMessageFormatter[T]]
+ NotFound(formatter.toJson(message)).as(applicationJsonHeader)
+ }
+
+ private[AdminController] def tryResponse[T, R: AdminMessageFormatter](res: Try[T])(callback: T => R): mvc.Result = res match {
+ case Success(m) =>
+ val ret = callback(m)
+ logger.info(ret.toString)
+ ok(ret)
+ case Failure(error) =>
+ logger.error(error.getMessage, error)
+ error match {
+ case JsResultException(e) => bad(JsError.toFlatJson(e))
+ case _ => bad(error.getMessage)
+ }
+ }
+
+ def optionResponse[T, R: AdminMessageFormatter](res: Option[T])(callback: T => R): mvc.Result = res match {
+ case Some(m) => ok(callback(m))
+ case None => notFound("not found")
+ }
+
+ /**
+ * load all model cache
+ * @return
+ */
+ def loadCache() = Action { request =>
+ val startTs = System.currentTimeMillis()
+
+ if (!ApplicationController.isHealthy) {
+ loadCacheInner()
+ }
+
+ ok(s"${System.currentTimeMillis() - startTs}")
+ }
+
+ def loadCacheInner() = {
+ Service.findAll()
+ ServiceColumn.findAll()
+ Label.findAll()
+ LabelMeta.findAll()
+ LabelIndex.findAll()
+ ColumnMeta.findAll()
+ }
+
+ /**
+ * read
+ */
+
+ /**
+ * get service info
+ * @param serviceName
+ * @return
+ */
+ def getService(serviceName: String) = Action { request =>
+ val serviceOpt = Management.findService(serviceName)
+ optionResponse(serviceOpt)(_.toJson)
+ }
+
+ /**
+ * get label info
+ * @param labelName
+ * @return
+ */
+ def getLabel(labelName: String) = Action { request =>
+ val labelOpt = Management.findLabel(labelName)
+ optionResponse(labelOpt)(_.toJson)
+ }
+
+ /**
+ * get all labels of service
+ * @param serviceName
+ * @return
+ */
+ def getLabels(serviceName: String) = Action { request =>
+ Service.findByName(serviceName) match {
+ case None => notFound(s"Service $serviceName not found")
+ case Some(service) =>
+ val src = Label.findBySrcServiceId(service.id.get)
+ val tgt = Label.findByTgtServiceId(service.id.get)
+
+ ok(Json.obj("from" -> src.map(_.toJson), "to" -> tgt.map(_.toJson)))
+ }
+ }
+
+ /**
+ * get service columns
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def getServiceColumn(serviceName: String, columnName: String) = Action { request =>
+ val serviceColumnOpt = for {
+ service <- Service.findByName(serviceName)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName, useCache = false)
+ } yield serviceColumn
+
+ optionResponse(serviceColumnOpt)(_.toJson)
+ }
+
+ /**
+ * create
+ */
+
+ /**
+ * create service
+ * @return
+ */
+ def createService() = Action(parse.json) { request =>
+ val serviceTry = createServiceInner(request.body)
+ tryResponse(serviceTry)(_.toJson)
+ }
+
+
+ def createServiceInner(jsValue: JsValue) = {
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = requestParser.toServiceElements(jsValue)
+ management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ }
+
+ /**
+ * create label
+ * @return
+ */
+ def createLabel() = Action(parse.json) { request =>
+ val ret = createLabelInner(request.body)
+ tryResponse(ret)(_.toJson)
+ }
+
+
+ def createLabelInner(json: JsValue) = for {
+ labelArgs <- requestParser.toLabelElements(json)
+ label <- (management.createLabel _).tupled(labelArgs)
+ } yield label
+
+ /**
+ * add index
+ * @return
+ */
+ def addIndex() = Action(parse.json) { request =>
+ val ret = addIndexInner(request.body)
+ tryResponse(ret)(_.label + " is updated")
+ }
+
+ def addIndexInner(json: JsValue) = for {
+ (labelName, indices) <- requestParser.toIndexElements(json)
+ label <- Management.addIndex(labelName, indices)
+ } yield label
+
+ /**
+ * create service column
+ * @return
+ */
+ def createServiceColumn() = Action(parse.json) { request =>
+ val serviceColumnTry = createServiceColumnInner(request.body)
+ tryResponse(serviceColumnTry) { (columns: Seq[ColumnMeta]) => Json.obj("metas" -> columns.map(_.toJson)) }
+ }
+
+ def createServiceColumnInner(jsValue: JsValue) = for {
+ (serviceName, columnName, columnType, props) <- requestParser.toServiceColumnElements(jsValue)
+ serviceColumn <- Management.createServiceColumn(serviceName, columnName, columnType, props)
+ } yield serviceColumn
+
+ /**
+ * delete
+ */
+
+ /**
+ * delete label
+ * @param labelName
+ * @return
+ */
+ def deleteLabel(labelName: String) = Action { request =>
+ val deleteLabelTry = deleteLabelInner(labelName)
+ tryResponse(deleteLabelTry)(labelName => labelName + " is deleted")
+ }
+
+ def deleteLabelInner(labelName: String) = Management.deleteLabel(labelName)
+
+ /**
+ * delete servieColumn
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def deleteServiceColumn(serviceName: String, columnName: String) = Action { request =>
+ val serviceColumnTry = deleteServiceColumnInner(serviceName, columnName)
+ tryResponse(serviceColumnTry)(columnName => columnName + " is deleted")
+ }
+
+ def deleteServiceColumnInner(serviceName: String, columnName: String) =
+ Management.deleteColumn(serviceName, columnName)
+
+ /**
+ * update
+ */
+
+ /**
+ * add Prop to label
+ * @param labelName
+ * @return
+ */
+ def addProp(labelName: String) = Action(parse.json) { request =>
+ val labelMetaTry = addPropInner(labelName, request.body)
+ tryResponse(labelMetaTry)(_.toJson)
+ }
+
+ def addPropInner(labelName: String, js: JsValue) = for {
+ prop <- requestParser.toPropElements(js)
+ labelMeta <- Management.addProp(labelName, prop)
+ } yield labelMeta
+
+ /**
+ * add prop to serviceColumn
+ * @param serviceName
+ * @param columnName
+ * @return
+ */
+ def addServiceColumnProp(serviceName: String, columnName: String) = Action(parse.json) { request =>
+ addServiceColumnPropInner(serviceName, columnName)(request.body) match {
+ case None => bad(s"can`t find service with $serviceName or can`t find serviceColumn with $columnName")
+ case Some(m) => Ok(m.toJson).as(applicationJsonHeader)
+ }
+ }
+
+ def addServiceColumnPropInner(serviceName: String, columnName: String)(js: JsValue) = {
+ for {
+ service <- Service.findByName(serviceName)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName)
+ prop <- requestParser.toPropElements(js).toOption
+ } yield {
+ ColumnMeta.findOrInsert(serviceColumn.id.get, prop.name, prop.defaultValue)
+ }
+ }
+
+ /**
+ * add props to serviceColumn
+ * @param serviecName
+ * @param columnName
+ * @return
+ */
+ def addServiceColumnProps(serviecName: String, columnName: String) = Action(parse.json) { request =>
+ val jsObjs = request.body.asOpt[List[JsObject]].getOrElse(List.empty[JsObject])
+ val newProps = for {
+ js <- jsObjs
+ newProp <- addServiceColumnPropInner(serviecName, columnName)(js)
+ } yield newProp
+ ok(s"${newProps.size} is added.")
+ }
+
+ /**
+ * copy label
+ * @param oldLabelName
+ * @param newLabelName
+ * @return
+ */
+ def copyLabel(oldLabelName: String, newLabelName: String) = Action { request =>
+ val copyTry = management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
+ tryResponse(copyTry)(_.label + "created")
+ }
+
+ /**
+ * rename label
+ * @param oldLabelName
+ * @param newLabelName
+ * @return
+ */
+ def renameLabel(oldLabelName: String, newLabelName: String) = Action { request =>
+ Label.findByName(oldLabelName) match {
+ case None => NotFound.as(applicationJsonHeader)
+ case Some(label) =>
+ Management.updateLabelName(oldLabelName, newLabelName)
+ ok(s"Label was updated")
+ }
+ }
+
+ /**
+ * update HTable for a label
+ * @param labelName
+ * @param newHTableName
+ * @return
+ */
+ def updateHTable(labelName: String, newHTableName: String) = Action { request =>
+ val updateTry = Management.updateHTable(labelName, newHTableName)
+ tryResponse(updateTry)(_.toString + " label(s) updated.")
+ }
+
+
+ case class HTableParams(cluster: String, hTableName: String,
+ preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
+
+ override def toString(): String = {
+ s"""HtableParams
+ |-- cluster : $cluster
+ |-- hTableName : $hTableName
+ |-- preSplitSize : $preSplitSize
+ |-- hTableTTL : $hTableTTL
+ |-- compressionAlgorithm : $compressionAlgorithm
+ |""".stripMargin
+ }
+ }
+
+ implicit object HTableParamsJsonConverter extends Format[HTableParams] {
+ def reads(json: JsValue): JsResult[HTableParams] = (
+ (__ \ "cluster").read[String] and
+ (__ \ "hTableName").read[String] and
+ (__ \ "preSplitSize").read[Int] and
+ (__ \ "hTableTTL").readNullable[Int] and
+ (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
+
+ def writes(o: HTableParams): JsValue = Json.obj(
+ "cluster" -> o.cluster,
+ "hTableName" -> o.hTableName,
+ "preSplitSize" -> o.preSplitSize,
+ "hTableTTL" -> o.hTableTTL,
+ "compressionAlgorithm" -> o.compressionAlgorithm
+ )
+ }
+
+ implicit object JsErrorJsonWriter extends Writes[JsError] {
+ def writes(o: JsError): JsValue = Json.obj(
+ "errors" -> JsArray(
+ o.errors.map {
+ case (path, validationErrors) => Json.obj(
+ "path" -> Json.toJson(path.toString()),
+ "validationErrors" -> JsArray(validationErrors.map(validationError => Json.obj(
+ "message" -> JsString(validationError.message),
+ "args" -> JsArray(validationError.args.map(_ match {
+ case x: Int => JsNumber(x)
+ case x => JsString(x.toString)
+ }))
+ )))
+ )
+ }
+ )
+ )
+ }
+
+ def createHTable() = Action { request =>
+
+ // Management.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
+ request.body.asJson.map(_.validate[HTableParams] match {
+ case JsSuccess(hTableParams, _) => {
+ management.createTable(hTableParams.cluster, hTableParams.hTableName, List("e", "v"),
+ hTableParams.preSplitSize, hTableParams.hTableTTL,
+ hTableParams.compressionAlgorithm.getOrElse(Management.DefaultCompressionAlgorithm))
+ logger.info(hTableParams.toString())
+ ok(s"HTable was created.")
+ }
+ case err@JsError(_) => bad(Json.toJson(err))
+ }).getOrElse(bad("Invalid Json."))
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
new file mode 100644
index 0000000..3482f6d
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
@@ -0,0 +1,101 @@
+package org.apache.s2graph.rest.play.controllers
+
+import org.apache.s2graph.core.GraphExceptions.BadQueryException
+import org.apache.s2graph.core.PostProcess
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.iteratee.Enumerator
+import play.api.libs.json.{JsString, JsValue}
+import play.api.mvc._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object ApplicationController extends Controller {
+
+ var isHealthy = true
+ var deployInfo = ""
+ val applicationJsonHeader = "application/json"
+
+ val jsonParser: BodyParser[JsValue] = s2parse.json
+
+ val jsonText: BodyParser[String] = s2parse.jsonText
+
+ private def badQueryExceptionResults(ex: Exception) =
+ Future.successful(BadRequest(PostProcess.badRequestResults(ex)).as(applicationJsonHeader))
+
+ private def errorResults =
+ Future.successful(Ok(PostProcess.emptyResults).as(applicationJsonHeader))
+
+ def requestFallback(body: String): PartialFunction[Throwable, Future[Result]] = {
+ case e: BadQueryException =>
+ logger.error(s"{$body}, ${e.getMessage}", e)
+ badQueryExceptionResults(e)
+ case e: Exception =>
+ logger.error(s"${body}, ${e.getMessage}", e)
+ errorResults
+ }
+
+ def updateHealthCheck(isHealthy: Boolean) = Action { request =>
+ this.isHealthy = isHealthy
+ Ok(this.isHealthy + "\n")
+ }
+
+ def healthCheck() = withHeader(parse.anyContent) { request =>
+ if (isHealthy) Ok(deployInfo)
+ else NotFound
+ }
+
+ def jsonResponse(json: JsValue, headers: (String, String)*) =
+ if (ApplicationController.isHealthy) {
+ Ok(json).as(applicationJsonHeader).withHeaders(headers: _*)
+ } else {
+ Result(
+ header = ResponseHeader(OK),
+ body = Enumerator(json.toString.getBytes()),
+ connection = HttpConnection.Close
+ ).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*)
+ }
+
+ def toLogMessage[A](request: Request[A], result: Result)(startedAt: Long): String = {
+ val duration = System.currentTimeMillis() - startedAt
+ val isQueryRequest = result.header.headers.contains("result_size")
+ val resultSize = result.header.headers.getOrElse("result_size", "-1")
+
+ try {
+ val body = request.body match {
+ case AnyContentAsJson(jsValue) => jsValue match {
+ case JsString(str) => str
+ case _ => jsValue.toString
+ }
+ case AnyContentAsEmpty => ""
+ case _ => request.body.toString
+ }
+
+ val str =
+ if (isQueryRequest)
+ s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
+ else
+ s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
+
+ str
+ } finally {
+ /* pass */
+ }
+ }
+
+ def withHeaderAsync[A](bodyParser: BodyParser[A])(block: Request[A] => Future[Result])(implicit ex: ExecutionContext) =
+ Action.async(bodyParser) { request =>
+ val startedAt = System.currentTimeMillis()
+ block(request).map { r =>
+ logger.info(toLogMessage(request, r)(startedAt))
+ r
+ }
+ }
+
+ def withHeader[A](bodyParser: BodyParser[A])(block: Request[A] => Result) =
+ Action(bodyParser) { request: Request[A] =>
+ val startedAt = System.currentTimeMillis()
+ val r = block(request)
+ logger.info(toLogMessage(request, r)(startedAt))
+ r
+ }
+}
[24/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/HBaseRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/HBaseRDDFunctions.scala b/loader/src/main/scala/spark/HBaseRDDFunctions.scala
deleted file mode 100644
index 8ff8d58..0000000
--- a/loader/src/main/scala/spark/HBaseRDDFunctions.scala
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import java.util
-
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hbase.{HConstants, TableName}
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.spark.rdd.RDD
-
-import scala.collection.immutable.HashMap
-import scala.reflect.ClassTag
-
-/**
- * HBaseRDDFunctions contains a set of implicit functions that can be
- * applied to a Spark RDD so that we can easily interact with HBase
- */
-object HBaseRDDFunctions
-{
-
- /**
- * These are implicit methods for a RDD that contains any type of
- * data.
- *
- * @param rdd This is for rdd of any type
- * @tparam T This is any type
- */
- implicit class GenericHBaseRDDFunctions[T](val rdd: RDD[T]) {
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * put. This will not return a new RDD. Think of it like a foreach
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param f The function that will turn the RDD values
- * into HBase Put objects.
- */
- def hbaseBulkPut(hc: HBaseContext,
- tableName: TableName,
- f: (T) => Put): Unit = {
- hc.bulkPut(rdd, tableName, f)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * get. This will return a new RDD. Think about it as a RDD map
- * function. In that every RDD value will get a new value out of
- * HBase. That new value will populate the newly generated RDD.
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param batchSize How many gets to execute in a single batch
- * @param f The function that will turn the RDD values
- * in HBase Get objects
- * @param convertResult The function that will convert a HBase
- * Result object into a value that will go
- * into the resulting RDD
- * @tparam R The type of Object that will be coming
- * out of the resulting RDD
- * @return A resulting RDD with type R objects
- */
- def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
- tableName: TableName, batchSize:Int,
- f: (T) => Get, convertResult: (Result) => R): RDD[R] = {
- hc.bulkGet[T, R](tableName, batchSize, rdd, f, convertResult)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * get. This will return a new RDD. Think about it as a RDD map
- * function. In that every RDD value will get a new value out of
- * HBase. That new value will populate the newly generated RDD.
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param batchSize How many gets to execute in a single batch
- * @param f The function that will turn the RDD values
- * in HBase Get objects
- * @return A resulting RDD with type R objects
- */
- def hbaseBulkGet(hc: HBaseContext,
- tableName: TableName, batchSize:Int,
- f: (T) => Get): RDD[(ImmutableBytesWritable, Result)] = {
- hc.bulkGet[T, (ImmutableBytesWritable, Result)](tableName,
- batchSize, rdd, f,
- result => if (result != null && result.getRow != null) {
- (new ImmutableBytesWritable(result.getRow), result)
- } else {
- null
- })
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * Delete. This will not return a new RDD.
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param tableName The tableName that the deletes will be sent to
- * @param f The function that will convert the RDD value into
- * a HBase Delete Object
- * @param batchSize The number of Deletes to be sent in a single batch
- */
- def hbaseBulkDelete(hc: HBaseContext,
- tableName: TableName, f:(T) => Delete, batchSize:Int): Unit = {
- hc.bulkDelete(rdd, tableName, f, batchSize)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's
- * foreachPartition method. This will ack very much like a normal RDD
- * foreach method but for the fact that you will now have a HBase connection
- * while iterating through the values.
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param f This function will get an iterator for a Partition of an
- * RDD along with a connection object to HBase
- */
- def hbaseForeachPartition(hc: HBaseContext,
- f: (Iterator[T], Connection) => Unit): Unit = {
- hc.foreachPartition(rdd, f)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's
- * mapPartitions method. This will ask very much like a normal RDD
- * map partitions method but for the fact that you will now have a
- * HBase connection while iterating through the values
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param f This function will get an iterator for a Partition of an
- * RDD along with a connection object to HBase
- * @tparam R This is the type of objects that will go into the resulting
- * RDD
- * @return A resulting RDD of type R
- */
- def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
- f: (Iterator[T], Connection) => Iterator[R]):
- RDD[R] = {
- hc.mapPartitions[T,R](rdd, f)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's
- * bulkLoad method.
- *
- * A Spark Implementation of HBase Bulk load
- *
- * This will take the content from an existing RDD then sort and shuffle
- * it with respect to region splits. The result of that sort and shuffle
- * will be written to HFiles.
- *
- * After this function is executed the user will have to call
- * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
- *
- * Also note this version of bulk load is different from past versions in
- * that it includes the qualifier as part of the sort process. The
- * reason for this is to be able to support rows will very large number
- * of columns.
- *
- * @param tableName The HBase table we are loading into
- * @param flatMap A flapMap function that will make every row in the RDD
- * into N cells for the bulk load
- * @param stagingDir The location on the FileSystem to bulk load into
- * @param familyHFileWriteOptionsMap Options that will define how the HFile for a
- * column family is written
- * @param compactionExclude Compaction excluded for the HFiles
- * @param maxSize Max size for the HFiles before they roll
- */
- def hbaseBulkLoad(hc: HBaseContext,
- tableName: TableName,
- flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
- stagingDir:String,
- familyHFileWriteOptionsMap:
- util.Map[Array[Byte], FamilyHFileWriteOptions] =
- new util.HashMap[Array[Byte], FamilyHFileWriteOptions](),
- compactionExclude: Boolean = false,
- maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):Unit = {
- hc.bulkLoad(rdd, tableName,
- flatMap, stagingDir, familyHFileWriteOptionsMap,
- compactionExclude, maxSize)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/JavaHBaseContext.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/JavaHBaseContext.scala b/loader/src/main/scala/spark/JavaHBaseContext.scala
deleted file mode 100644
index 39ddf2a..0000000
--- a/loader/src/main/scala/spark/JavaHBaseContext.scala
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.TableName
-import org.apache.hadoop.hbase.client.{Connection, Delete, Get, Put, Result, Scan}
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
-import org.apache.spark.api.java.function.{FlatMapFunction, Function, VoidFunction}
-import org.apache.spark.streaming.api.java.JavaDStream
-
-import scala.collection.JavaConversions._
-import scala.reflect.ClassTag
-
-/**
- * This is the Java Wrapper over HBaseContext which is written in
- * Scala. This class will be used by developers that want to
- * work with Spark or Spark Streaming in Java
- *
- * @param jsc This is the JavaSparkContext that we will wrap
- * @param config This is the config information to out HBase cluster
- */
-class JavaHBaseContext(@transient jsc: JavaSparkContext,
- @transient config: Configuration) extends Serializable {
- val hbaseContext = new HBaseContext(jsc.sc, config)
-
- /**
- * A simple enrichment of the traditional Spark javaRdd foreachPartition.
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * @param javaRdd Original javaRdd with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the RDD values and a HConnection object to interact
- * with HBase
- */
- def foreachPartition[T](javaRdd: JavaRDD[T],
- f: VoidFunction[(java.util.Iterator[T], Connection)]) = {
-
- hbaseContext.foreachPartition(javaRdd.rdd,
- (it: Iterator[T], conn: Connection) => {
- f.call((it, conn))
- })
- }
-
- /**
- * A simple enrichment of the traditional Spark Streaming dStream foreach
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * @param javaDstream Original DStream with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the JavaDStream values and a HConnection object to
- * interact with HBase
- */
- def foreachPartition[T](javaDstream: JavaDStream[T],
- f: VoidFunction[(Iterator[T], Connection)]) = {
- hbaseContext.foreachPartition(javaDstream.dstream,
- (it: Iterator[T], conn: Connection) => f.call(it, conn))
- }
-
- /**
- * A simple enrichment of the traditional Spark JavaRDD mapPartition.
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * Note: Make sure to partition correctly to avoid memory issue when
- * getting data from HBase
- *
- * @param javaRdd Original JavaRdd with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the RDD values and a HConnection object to interact
- * with HBase
- * @return Returns a new RDD generated by the user definition
- * function just like normal mapPartition
- */
- def mapPartitions[T, R](javaRdd: JavaRDD[T],
- f: FlatMapFunction[(java.util.Iterator[T],
- Connection), R]): JavaRDD[R] = {
-
- def fn = (it: Iterator[T], conn: Connection) =>
- asScalaIterator(
- f.call((asJavaIterator(it), conn)).iterator()
- )
-
- JavaRDD.fromRDD(hbaseContext.mapPartitions(javaRdd.rdd,
- (iterator: Iterator[T], connection: Connection) =>
- fn(iterator, connection))(fakeClassTag[R]))(fakeClassTag[R])
- }
-
- /**
- * A simple enrichment of the traditional Spark Streaming JavaDStream
- * mapPartition.
- *
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * Note: Make sure to partition correctly to avoid memory issue when
- * getting data from HBase
- *
- * @param javaDstream Original JavaDStream with data to iterate over
- * @param mp Function to be given a iterator to iterate through
- * the JavaDStream values and a HConnection object to
- * interact with HBase
- * @return Returns a new JavaDStream generated by the user
- * definition function just like normal mapPartition
- */
- def streamMap[T, U](javaDstream: JavaDStream[T],
- mp: Function[(Iterator[T], Connection), Iterator[U]]):
- JavaDStream[U] = {
- JavaDStream.fromDStream(hbaseContext.streamMapPartitions(javaDstream.dstream,
- (it: Iterator[T], conn: Connection) =>
- mp.call(it, conn))(fakeClassTag[U]))(fakeClassTag[U])
- }
-
- /**
- * A simple abstraction over the HBaseContext.foreachPartition method.
- *
- * It allow addition support for a user to take JavaRDD
- * and generate puts and send them to HBase.
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param javaRdd Original JavaRDD with data to iterate over
- * @param tableName The name of the table to put into
- * @param f Function to convert a value in the JavaRDD
- * to a HBase Put
- */
- def bulkPut[T](javaRdd: JavaRDD[T],
- tableName: TableName,
- f: Function[(T), Put]) {
-
- hbaseContext.bulkPut(javaRdd.rdd, tableName, (t: T) => f.call(t))
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamMapPartition method.
- *
- * It allow addition support for a user to take a JavaDStream and
- * generate puts and send them to HBase.
- *
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param javaDstream Original DStream with data to iterate over
- * @param tableName The name of the table to put into
- * @param f Function to convert a value in
- * the JavaDStream to a HBase Put
- */
- def streamBulkPut[T](javaDstream: JavaDStream[T],
- tableName: TableName,
- f: Function[T, Put]) = {
- hbaseContext.streamBulkPut(javaDstream.dstream,
- tableName,
- (t: T) => f.call(t))
- }
-
- /**
- * A simple abstraction over the HBaseContext.foreachPartition method.
- *
- * It allow addition support for a user to take a JavaRDD and
- * generate delete and send them to HBase.
- *
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param javaRdd Original JavaRDD with data to iterate over
- * @param tableName The name of the table to delete from
- * @param f Function to convert a value in the JavaRDD to a
- * HBase Deletes
- * @param batchSize The number of deletes to batch before sending to HBase
- */
- def bulkDelete[T](javaRdd: JavaRDD[T], tableName: TableName,
- f: Function[T, Delete], batchSize: Integer) {
- hbaseContext.bulkDelete(javaRdd.rdd, tableName, (t: T) => f.call(t), batchSize)
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamBulkMutation method.
- *
- * It allow addition support for a user to take a JavaDStream and
- * generate Delete and send them to HBase.
- *
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param javaDStream Original DStream with data to iterate over
- * @param tableName The name of the table to delete from
- * @param f Function to convert a value in the JavaDStream to a
- * HBase Delete
- * @param batchSize The number of deletes to be sent at once
- */
- def streamBulkDelete[T](javaDStream: JavaDStream[T],
- tableName: TableName,
- f: Function[T, Delete],
- batchSize: Integer) = {
- hbaseContext.streamBulkDelete(javaDStream.dstream, tableName,
- (t: T) => f.call(t),
- batchSize)
- }
-
- /**
- * A simple abstraction over the HBaseContext.mapPartition method.
- *
- * It allow addition support for a user to take a JavaRDD and generates a
- * new RDD based on Gets and the results they bring back from HBase
- *
- * @param tableName The name of the table to get from
- * @param batchSize batch size of how many gets to retrieve in a single fetch
- * @param javaRdd Original JavaRDD with data to iterate over
- * @param makeGet Function to convert a value in the JavaRDD to a
- * HBase Get
- * @param convertResult This will convert the HBase Result object to
- * what ever the user wants to put in the resulting
- * JavaRDD
- * @return New JavaRDD that is created by the Get to HBase
- */
- def bulkGet[T, U](tableName: TableName,
- batchSize: Integer,
- javaRdd: JavaRDD[T],
- makeGet: Function[T, Get],
- convertResult: Function[Result, U]): JavaRDD[U] = {
-
- JavaRDD.fromRDD(hbaseContext.bulkGet[T, U](tableName,
- batchSize,
- javaRdd.rdd,
- (t: T) => makeGet.call(t),
- (r: Result) => {
- convertResult.call(r)
- })(fakeClassTag[U]))(fakeClassTag[U])
-
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamMap method.
- *
- * It allow addition support for a user to take a DStream and
- * generates a new DStream based on Gets and the results
- * they bring back from HBase
- *
-
- * @param tableName The name of the table to get from
- * @param batchSize The number of gets to be batched together
- * @param javaDStream Original DStream with data to iterate over
- * @param makeGet Function to convert a value in the JavaDStream to a
- * HBase Get
- * @param convertResult This will convert the HBase Result object to
- * what ever the user wants to put in the resulting
- * JavaDStream
- * @return New JavaDStream that is created by the Get to HBase
- */
- def streamBulkGet[T, U](tableName: TableName,
- batchSize: Integer,
- javaDStream: JavaDStream[T],
- makeGet: Function[T, Get],
- convertResult: Function[Result, U]) {
- JavaDStream.fromDStream(hbaseContext.streamBulkGet(tableName,
- batchSize,
- javaDStream.dstream,
- (t: T) => makeGet.call(t),
- (r: Result) => convertResult.call(r))(fakeClassTag[U]))(fakeClassTag[U])
- }
-
- /**
- * This function will use the native HBase TableInputFormat with the
- * given scan object to generate a new JavaRDD
- *
- * @param tableName The name of the table to scan
- * @param scans The HBase scan object to use to read data from HBase
- * @param f Function to convert a Result object from HBase into
- * What the user wants in the final generated JavaRDD
- * @return New JavaRDD with results from scan
- */
- def hbaseRDD[U](tableName: TableName,
- scans: Scan,
- f: Function[(ImmutableBytesWritable, Result), U]):
- JavaRDD[U] = {
- JavaRDD.fromRDD(
- hbaseContext.hbaseRDD[U](tableName,
- scans,
- (v: (ImmutableBytesWritable, Result)) =>
- f.call(v._1, v._2))(fakeClassTag[U]))(fakeClassTag[U])
- }
-
- /**
- * A overloaded version of HBaseContext hbaseRDD that define the
- * type of the resulting JavaRDD
- *
- * @param tableName The name of the table to scan
- * @param scans The HBase scan object to use to read data from HBase
- * @return New JavaRDD with results from scan
- */
- def hbaseRDD(tableName: TableName,
- scans: Scan):
- JavaRDD[(ImmutableBytesWritable, Result)] = {
- JavaRDD.fromRDD(hbaseContext.hbaseRDD(tableName, scans))
- }
-
- /**
- * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
- *
- * This method is used to keep ClassTags out of the external Java API, as the Java compiler
- * cannot produce them automatically. While this ClassTag-faking does please the compiler,
- * it can cause problems at runtime if the Scala API relies on ClassTags for correctness.
- *
- * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
- * just worse performance or security issues.
- * For instance, an Array[AnyRef] can hold any type T,
- * but may lose primitive
- * specialization.
- */
- private[spark]
- def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/KeyFamilyQualifier.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/KeyFamilyQualifier.scala b/loader/src/main/scala/spark/KeyFamilyQualifier.scala
deleted file mode 100644
index 4a00b63..0000000
--- a/loader/src/main/scala/spark/KeyFamilyQualifier.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import java.io.Serializable
-
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * This is the key to be used for sorting and shuffling.
- *
- * We will only partition on the rowKey but we will sort on all three
- *
- * @param rowKey Record RowKey
- * @param family Record ColumnFamily
- * @param qualifier Cell Qualifier
- */
-class KeyFamilyQualifier(val rowKey:Array[Byte], val family:Array[Byte], val qualifier:Array[Byte])
- extends Comparable[KeyFamilyQualifier] with Serializable {
- override def compareTo(o: KeyFamilyQualifier): Int = {
- var result = Bytes.compareTo(rowKey, o.rowKey)
- if (result == 0) {
- result = Bytes.compareTo(family, o.family)
- if (result == 0) result = Bytes.compareTo(qualifier, o.qualifier)
- }
- result
- }
- override def toString: String = {
- Bytes.toString(rowKey) + ":" + Bytes.toString(family) + ":" + Bytes.toString(qualifier)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/GraphSubscriber.scala b/loader/src/main/scala/subscriber/GraphSubscriber.scala
deleted file mode 100644
index f4f7865..0000000
--- a/loader/src/main/scala/subscriber/GraphSubscriber.scala
+++ /dev/null
@@ -1,339 +0,0 @@
-package subscriber
-
-
-import java.util
-
-import com.kakao.s2graph.core.{Graph, _}
-import com.typesafe.config.{Config, ConfigFactory}
-import kafka.javaapi.producer.Producer
-import kafka.producer.KeyedMessage
-import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
-import org.apache.hadoop.hbase.client._
-import org.apache.spark.{Accumulable, SparkContext}
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.HashMap
-import scala.concurrent.ExecutionContext
-
-object GraphConfig {
- var database = ""
- var zkQuorum = ""
- var kafkaBrokers = ""
- var cacheTTL = s"${60 * 60 * 1}"
- def apply(phase: String, dbUrl: Option[String], zkAddr: Option[String], kafkaBrokerList: Option[String]): Config = {
- database = dbUrl.getOrElse("jdbc:mysql://localhost:3306/graph_dev")
- zkQuorum = zkAddr.getOrElse("localhost")
-
-// val newConf = new util.HashMap[String, Object]()
-// newConf.put("hbase.zookeeper.quorum", zkQuorum)
-// newConf.put("db.default.url", database)
-// newConf.put("kafka.metadata.broker.list", kafkaBrokers)
- val newConf =
- if (kafkaBrokerList.isEmpty) Map("hbase.zookeeper.quorum" -> zkQuorum, "db.default.url" -> database, "cache.ttl.seconds" -> cacheTTL)
- else Map("hbase.zookeeper.quorum" -> zkQuorum, "db.default.url" -> database, "kafka.metadata.broker.list" -> kafkaBrokers, "cache.ttl.seconds" -> cacheTTL)
-
- ConfigFactory.parseMap(newConf).withFallback(Graph.DefaultConfig)
- }
-}
-
-object GraphSubscriberHelper extends WithKafka {
-
-
- type HashMapAccumulable = Accumulable[HashMap[String, Long], (String, Long)]
-
-
- lazy val producer = new Producer[String, String](kafkaConf(GraphConfig.kafkaBrokers))
- var config: Config = _
- private val writeBufferSize = 1024 * 1024 * 8
- private val sleepPeriod = 10000
- private val maxTryNum = 10
-
- var g: Graph = null
- var management: Management = null
- val conns = new scala.collection.mutable.HashMap[String, Connection]()
-
- def toOption(s: String) = {
- s match {
- case "" | "none" => None
- case _ => Some(s)
- }
- }
-
- def apply(phase: String, dbUrl: String, zkQuorum: String, kafkaBrokerList: String): Unit = {
- config = GraphConfig(phase, toOption(dbUrl), toOption(zkQuorum), toOption(kafkaBrokerList))
-
- if (g == null) {
- val ec = ExecutionContext.Implicits.global
- g = new Graph(config)(ec)
- management = new Management(g)(ec)
- }
- }
-
- def getConn(zkQuorum: String): Connection = {
- conns.getOrElseUpdate(zkQuorum, {
- val hbaseConf = HBaseConfiguration.create()
- hbaseConf.set("hbase.zookeeper.quorum", zkQuorum)
- ConnectionFactory.createConnection(hbaseConf)
- })
- conns(zkQuorum)
- }
- // def apply(phase: String, dbUrl: Option[String], zkQuorum: Option[String], kafkaBrokerList: Option[String]): Unit = {
- // Graph.apply(GraphConfig(phase, dbUrl, zkQuorum, kafkaBrokerList))(ExecutionContext.Implicits.global)
- // }
- def report(key: String, value: Option[String], topic: String = "report") = {
- val msg = Seq(Some(key), value).flatten.mkString("\t")
- val kafkaMsg = new KeyedMessage[String, String](topic, msg)
- producer.send(kafkaMsg)
- }
-
- def toGraphElements(msgs: Seq[String], labelMapping: Map[String, String] = Map.empty)
- (statFunc: (String, Int) => Unit): Iterable[GraphElement] = {
- (for (msg <- msgs) yield {
- statFunc("total", 1)
- Graph.toGraphElement(msg, labelMapping) match {
- case Some(e) if e.isInstanceOf[Edge] =>
- statFunc("EdgeParseOk", 1)
- e.asInstanceOf[Edge]
- case Some(v) if v.isInstanceOf[Vertex] =>
- statFunc("VertexParseOk", 1)
- v.asInstanceOf[Vertex]
- case Some(x) =>
- throw new RuntimeException(s">>>>> GraphSubscriber.toGraphElements: parsing failed. ${x.serviceName}")
- case None =>
- throw new RuntimeException(s"GraphSubscriber.toGraphElements: parsing failed. $msg")
- }
-
- }).toList
- }
-
-// private def storeRec(zkQuorum: String, tableName: String, puts: List[Put], elementsSize: Int, tryNum: Int)
-// (statFunc: (String, Int) => Unit, statPrefix: String = "edge"): Unit = {
-// if (tryNum <= 0) {
-// statFunc("errorStore", elementsSize)
-// throw new RuntimeException(s"retry failed after $maxTryNum")
-// }
-// val conn = getConn(zkQuorum)
-// val mutator = conn.getBufferedMutator(TableName.valueOf(tableName))
-// // val table = conn.getTable(TableName.valueOf(tableName))
-// // table.setAutoFlush(false, false)
-//
-// try {
-// puts.foreach { put => put.setDurability(Durability.ASYNC_WAL) }
-// mutator.mutate(puts)
-// // table.put(puts)
-// statFunc(s"$statPrefix:storeOk", elementsSize)
-// } catch {
-// case e: Throwable =>
-// e.printStackTrace()
-// Thread.sleep(sleepPeriod)
-// storeRec(zkQuorum, tableName, puts, elementsSize, tryNum - 1)(statFunc)
-// } finally {
-// mutator.close()
-// // table.close()
-// }
-// }
-//
-// def storeDegreeBulk(zkQuorum: String, tableName: String)
-// (degrees: Iterable[(String, String, String, Int)], labelMapping: Map[String, String] = Map.empty)
-// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
-// val counts = HashMap[String, Long]()
-// val statFunc = storeStat(counts)(mapAccOpt) _
-//
-// for {
-// (vertexId, labelName, direction, degreeVal) <- degrees
-// incrementRequests <- TransferToHFile.buildDegreePutRequests(vertexId, labelName, direction, degreeVal)
-// } {
-// storeRec(zkQuorum, tableName, incrementRequests, degrees.size, maxTryNum)(statFunc, "degree")
-// }
-// counts
-// }
-// def storeBulk(zkQuorum: String, tableName: String)
-// (msgs: Seq[String], labelMapping: Map[String, String] = Map.empty, autoCreateEdge: Boolean = false)
-// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
-//
-// val counts = HashMap[String, Long]()
-// val statFunc = storeStat(counts)(mapAccOpt) _
-// val elements = toGraphElements(msgs, labelMapping)(statFunc)
-//
-// val puts = elements.flatMap { element =>
-// element match {
-// case v: Vertex if v.op == GraphUtil.operations("insert") || v.op == GraphUtil.operations("insertBulk") =>
-// v.buildPuts()
-// case e: Edge if e.op == GraphUtil.operations("insert") || e.op == GraphUtil.operations("insertBulk") =>
-// EdgeWriter(e).insertBulkForLoader(autoCreateEdge)
-// case _ => Nil
-// }
-// } toList
-//
-// storeRec(zkQuorum, tableName, puts, msgs.size, maxTryNum)(statFunc)
-// counts
-// }
-
- def storeStat(counts: HashMap[String, Long])(mapAccOpt: Option[HashMapAccumulable])(key: String, value: Int) = {
- counts.put(key, counts.getOrElse(key, 0L) + value)
- mapAccOpt match {
- case None =>
- case Some(mapAcc) => mapAcc += (key -> value)
- }
- }
-
- def toLabelMapping(lableMapping: String): Map[String, String] = {
- (for {
- token <- lableMapping.split(",")
- inner = token.split(":") if inner.length == 2
- } yield {
- (inner.head, inner.last)
- }).toMap
- }
-
- def isValidQuorum(quorum: String) = {
- quorum.split(",").size > 1
- }
-}
-
-//object GraphSubscriber extends SparkApp with WithKafka {
-// val sleepPeriod = 5000
-// val usages =
-// s"""
-// |/**
-// |* this job read edge format(TSV) from HDFS file system then bulk load edges into s2graph. assumes that newLabelName is already created by API.
-// |* params:
-// |* 0. hdfsPath: where is your data in hdfs. require full path with hdfs:// predix
-// |* 1. dbUrl: jdbc database connection string to specify database for meta.
-// |* 2. labelMapping: oldLabel:newLabel delimited by ,
-// |* 3. zkQuorum: target hbase zkQuorum where this job will publish data to.
-// |* 4. hTableName: target hbase physical table name where this job will publish data to.
-// |* 5. batchSize: how many edges will be batched for Put request to target hbase.
-// |* 6. kafkaBrokerList: using kafka as fallback queue. when something goes wrong during batch, data needs to be replay will be stored in kafka.
-// |* 7. kafkaTopic: fallback queue topic.
-// |* 8. edgeAutoCreate: true if need to create reversed edge automatically.
-// |*
-// |* after this job finished, s2graph will have data with sequence corresponding newLabelName.
-// |* change this newLabelName to ogirinalName if you want to online replace of label.
-// |*
-// |*/
-// """.stripMargin
-//
-// override def run() = {
-// /**
-// * Main function
-// */
-// println(args.toList)
-//// if (args.length != 10) {
-//// System.err.println(usages)
-//// System.exit(1)
-//// }
-// val hdfsPath = args(0)
-// val dbUrl = args(1)
-// val labelMapping = GraphSubscriberHelper.toLabelMapping(args(2))
-//
-// val zkQuorum = args(3)
-// val hTableName = args(4)
-// val batchSize = args(5).toInt
-// val kafkaBrokerList = args(6)
-// val kafkaTopic = args(7)
-// val edgeAutoCreate = args(8).toBoolean
-// val vertexDegreePathOpt = if (args.length >= 10) GraphSubscriberHelper.toOption(args(9)) else None
-//
-// val conf = sparkConf(s"$hdfsPath: GraphSubscriber")
-// val sc = new SparkContext(conf)
-// val mapAcc = sc.accumulable(HashMap.empty[String, Long], "counter")(HashMapParam[String, Long](_ + _))
-//
-//
-// if (!GraphSubscriberHelper.isValidQuorum(zkQuorum)) throw new RuntimeException(s"$zkQuorum is not valid.")
-//
-// /** this job expect only one hTableName. all labels in this job will be stored in same physical hbase table */
-// try {
-//
-// import GraphSubscriberHelper._
-// // set local driver setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
-//
-// /** copy when oldLabel exist and newLabel done exist. otherwise ignore. */
-//
-// if (labelMapping.isEmpty) {
-// // pass
-// } else {
-// for {
-// (oldLabelName, newLabelName) <- labelMapping
-// } {
-// Management.copyLabel(oldLabelName, newLabelName, toOption(hTableName))
-// }
-// }
-//
-// vertexDegreePathOpt.foreach { vertexDegreePath =>
-// val vertexDegrees = sc.textFile(vertexDegreePath).filter(line => line.split("\t").length == 4).map { line =>
-// val tokens = line.split("\t")
-// (tokens(0), tokens(1), tokens(2), tokens(3).toInt)
-// }
-// vertexDegrees.foreachPartition { partition =>
-//
-// // init Graph
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
-//
-// partition.grouped(batchSize).foreach { msgs =>
-// try {
-// val start = System.currentTimeMillis()
-// val counts = GraphSubscriberHelper.storeDegreeBulk(zkQuorum, hTableName)(msgs, labelMapping)(Some(mapAcc))
-// for ((k, v) <- counts) {
-// mapAcc +=(k, v)
-// }
-// val duration = System.currentTimeMillis() - start
-// println(s"[Success]: store, $mapAcc, $duration, $zkQuorum, $hTableName")
-// } catch {
-// case e: Throwable =>
-// println(s"[Failed]: store $e")
-//
-// msgs.foreach { msg =>
-// GraphSubscriberHelper.report(msg.toString(), Some(e.getMessage()), topic = kafkaTopic)
-// }
-// }
-// }
-// }
-// }
-//
-//
-// val msgs = sc.textFile(hdfsPath)
-// msgs.foreachPartition(partition => {
-// // set executor setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
-//
-// partition.grouped(batchSize).foreach { msgs =>
-// try {
-// val start = System.currentTimeMillis()
-// // val counts =
-// // GraphSubscriberHelper.store(msgs, GraphSubscriberHelper.toOption(newLabelName))(Some(mapAcc))
-// val counts =
-// GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping, edgeAutoCreate)(Some(mapAcc))
-//
-// for ((k, v) <- counts) {
-// mapAcc +=(k, v)
-// }
-// val duration = System.currentTimeMillis() - start
-// println(s"[Success]: store, $mapAcc, $duration, $zkQuorum, $hTableName")
-// } catch {
-// case e: Throwable =>
-// println(s"[Failed]: store $e")
-//
-// msgs.foreach { msg =>
-// GraphSubscriberHelper.report(msg, Some(e.getMessage()), topic = kafkaTopic)
-// }
-// }
-// }
-// })
-//
-// logInfo(s"counter: $mapAcc")
-// println(s"Stats: ${mapAcc}")
-//
-// } catch {
-// case e: Throwable =>
-// println(s"job failed with exception: $e")
-// throw e
-// }
-//
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/GraphSubscriberStreaming.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/GraphSubscriberStreaming.scala b/loader/src/main/scala/subscriber/GraphSubscriberStreaming.scala
deleted file mode 100644
index 1063560..0000000
--- a/loader/src/main/scala/subscriber/GraphSubscriberStreaming.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-package subscriber
-
-import org.apache.hadoop.hbase.HBaseConfiguration
-import org.apache.hadoop.hbase.client.{ConnectionFactory}
-import org.apache.spark.streaming.Durations._
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-
-//object GraphSubscriberStreaming extends SparkApp with WithKafka {
-// val usages =
-// s"""
-// |/**
-// |this job consume edges/vertices from kafka topic then load them into s2graph.
-// |params:
-// | 1. kafkaZkQuorum: kafka zk address to consume events
-// | 2. brokerList: kafka cluster`s broker list.
-// | 3. topics: , delimited list of topics to consume
-// | 4. intervalInSec: batch interval for this job.
-// | 5. batchSize: how many edges/vertices will be grouped for bulk mutations.
-// | 6. hbaseZkQuorum: s2graph zookeeper address.
-// | 7. hTableName: physical hbase table name.
-// | 8. labelMapping: oldLabel:newLabel delimited by ,
-// |*/
-// """.stripMargin
-// override def run() = {
-// if (args.length != 9) {
-// System.err.println(usages)
-// System.exit(1)
-// }
-// val kafkaZkQuorum = args(0)
-// val brokerList = args(1)
-// val topics = args(2)
-// val intervalInSec = seconds(args(3).toLong)
-// val dbUrl = args(4)
-// val batchSize = args(5).toInt
-// val hbaseZkQuorum = args(6)
-// val hTableName = args(7)
-// val labelMapping = GraphSubscriberHelper.toLabelMapping(args(8))
-//
-//
-// if (!GraphSubscriberHelper.isValidQuorum(hbaseZkQuorum))
-// throw new RuntimeException(s"$hbaseZkQuorum is not valid.")
-//
-// val conf = sparkConf(s"$topics: GraphSubscriberStreaming")
-// val ssc = streamingContext(conf, intervalInSec)
-// val sc = ssc.sparkContext
-//
-// val groupId = topics.replaceAll(",", "_") + "_stream"
-// val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
-//
-// val kafkaParams = Map(
-// "zookeeper.connect" -> kafkaZkQuorum,
-// "group.id" -> groupId,
-// "zookeeper.connection.timeout.ms" -> "10000",
-// "metadata.broker.list" -> brokerList,
-// "auto.offset.reset" -> "largest")
-//
-// val stream = createKafkaValueStreamMulti(ssc, kafkaParams, topics, 8, None).flatMap(s => s.split("\n"))
-//
-// val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
-//
-//
-// stream.foreachRDD(rdd => {
-//
-// rdd.foreachPartition(partition => {
-// // set executor setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, hbaseZkQuorum, brokerList)
-//
-// partition.grouped(batchSize).foreach { msgs =>
-// try {
-// val start = System.currentTimeMillis()
-// // val counts =
-// // GraphSubscriberHelper.store(msgs, GraphSubscriberHelper.toOption(newLabelName))(Some(mapAcc))
-// val counts =
-// GraphSubscriberHelper.storeBulk(hbaseZkQuorum, hTableName)(msgs, labelMapping)(Some(mapAcc))
-//
-// for ((k, v) <- counts) {
-// mapAcc +=(k, v)
-// }
-// val duration = System.currentTimeMillis() - start
-// println(s"[Success]: store, $mapAcc, $duration, $hbaseZkQuorum, $hTableName")
-// } catch {
-// case e: Throwable =>
-// println(s"[Failed]: store $e")
-//
-// msgs.foreach { msg =>
-// GraphSubscriberHelper.report(msg, Some(e.getMessage()), topic = fallbackTopic)
-// }
-// }
-// }
-// })
-// })
-//
-//
-// logInfo(s"counter: ${mapAcc.value}")
-// println(s"counter: ${mapAcc.value}")
-// ssc.start()
-// ssc.awaitTermination()
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/KafkaToHdfs.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/KafkaToHdfs.scala b/loader/src/main/scala/subscriber/KafkaToHdfs.scala
deleted file mode 100644
index 785cb69..0000000
--- a/loader/src/main/scala/subscriber/KafkaToHdfs.scala
+++ /dev/null
@@ -1,111 +0,0 @@
-package subscriber
-
-import org.apache.spark.streaming.Durations._
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-
-/**
- * Created by shon on 9/2/15.
- */
-//object KafkaToHdfs extends SparkApp with WithKafka {
-// val usages =
-// s"""
-// |/**
-// |this job consume edges/vertices from kafka topic then load them into s2graph.
-// |params:
-// | 1. kafkaZkQuorum: kafka zk address to consume events
-// | 2. brokerList: kafka cluster`s broker list.
-// | 3. topics: , delimited list of topics to consume
-// | 4. intervalInSec: batch interval for this job.
-// | 5. batchSize: how many edges/vertices will be grouped for bulk mutations.
-// | 6. hbaseZkQuorum: s2graph zookeeper address.
-// | 7. hTableName: physical hbase table name.
-// | 8. labelMapping: oldLabel:newLabel delimited by ,
-// |*/
-// """.stripMargin
-// override def run() = {
-// if (args.length != 9) {
-// System.err.println(usages)
-// System.exit(1)
-// }
-// val kafkaZkQuorum = args(0)
-// val brokerList = args(1)
-// val topics = args(2)
-// val intervalInSec = seconds(args(3).toLong)
-// val dbUrl = args(4)
-// val batchSize = args(5).toInt
-// val hbaseZkQuorum = args(6)
-// val hTableName = args(7)
-// val labelMapping = GraphSubscriberHelper.toLabelMapping(args(8))
-//
-//
-// if (!GraphSubscriberHelper.isValidQuorum(hbaseZkQuorum))
-// throw new RuntimeException(s"$hbaseZkQuorum is not valid.")
-//
-// val conf = sparkConf(s"$topics: GraphSubscriberStreaming")
-// val ssc = streamingContext(conf, intervalInSec)
-// val sc = ssc.sparkContext
-//
-// val groupId = topics.replaceAll(",", "_") + "_stream"
-// val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
-//
-// val kafkaParams = Map(
-// "zookeeper.connect" -> kafkaZkQuorum,
-// "group.id" -> groupId,
-// "zookeeper.connection.timeout.ms" -> "10000",
-// "metadata.broker.list" -> brokerList,
-// "auto.offset.reset" -> "largest")
-//
-// val stream = createKafkaValueStreamMulti(ssc, kafkaParams, topics, 8, None).flatMap(s => s.split("\n"))
-//
-// val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
-//
-//
-// stream.foreachRDD(rdd => {
-//
-// rdd.foreachPartition(partition => {
-// // set executor setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, hbaseZkQuorum, brokerList)
-//
-// partition.grouped(batchSize).foreach { msgs =>
-// try {
-// val start = System.currentTimeMillis()
-// // val counts =
-// // GraphSubscriberHelper.store(msgs, GraphSubscriberHelper.toOption(newLabelName))(Some(mapAcc))
-// val dummyStatFunc = (s: String, i: Int) => {}
-// val elements = GraphSubscriberHelper.toGraphElements(msgs)(dummyStatFunc)
-// for {
-// element <- elements
-// } {
-// element.serviceName
-// }
-//// val counts =
-//// GraphSubscriberHelper.storeBulk(hbaseZkQuorum, hTableName)(msgs, labelMapping)(Some(mapAcc))
-////
-//// for ((k, v) <- counts) {
-//// mapAcc +=(k, v)
-//// }
-// val duration = System.currentTimeMillis() - start
-// println(s"[Success]: store, $mapAcc, $duration, $hbaseZkQuorum, $hTableName")
-// } catch {
-// case e: Throwable =>
-// println(s"[Failed]: store $e")
-//
-// msgs.foreach { msg =>
-// GraphSubscriberHelper.report(msg, Some(e.getMessage()), topic = fallbackTopic)
-// }
-// }
-// }
-// })
-// })
-//
-//
-// logInfo(s"counter: ${mapAcc.value}")
-// println(s"counter: ${mapAcc.value}")
-// ssc.start()
-// ssc.awaitTermination()
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/TestEdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/TestEdgeBuilder.scala b/loader/src/main/scala/subscriber/TestEdgeBuilder.scala
deleted file mode 100644
index f5f43f8..0000000
--- a/loader/src/main/scala/subscriber/TestEdgeBuilder.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-package subscriber
-
-import org.apache.spark.SparkContext
-import s2.spark.{SparkApp, WithKafka}
-
-import scala.util.Random
-
-/**
- * Created by shon on 7/27/15.
- */
-//object TestEdgeBuilder extends SparkApp with WithKafka {
-// val sleepPeriod = 5000
-// val usages =
-// s"""
-// |/**
-// |0: numOfRows =
-// |*/
-// """.stripMargin
-//
-// override def run() = {
-// /**
-// * label definition can be found on migrate/s2graph/bmt.schema
-// * Main function
-// * numOfRows: number of rows
-// * numOfCols: number of cols
-// * numOfMetas: number of metas
-// *
-// */
-// println(args.toList)
-// val conf = sparkConf(s"TestEdgeBuilder")
-// val sc = new SparkContext(conf)
-// val phase = args(0)
-// val dbUrl = args(1)
-// val zkQuorum = args(2)
-// val hTableName = args(3)
-// val labelName = args(4)
-// val metaName = args(5)
-//
-// val numOfRows = if (args.length >= 7) args(6).toInt else 100000
-// val numOfCols = if (args.length >= 8) args(7).toInt else 10000
-// val dimOfCols = if (args.length >= 9) args(8).toInt else 10000
-// val numOfSlice = if (args.length >= 10) args(9).toInt else 10
-// val batchSize = if (args.length >= 11) args(10).toInt else 100
-//
-// sc.parallelize((0 until numOfRows), numOfSlice).foreachPartition { partition =>
-//
-// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, "none")
-//
-// partition.grouped(batchSize).foreach { rows =>
-// for {
-// rowId <- rows
-// } {
-// val ts = System.currentTimeMillis()
-// val msgs = for {
-// colId <- (0 until numOfCols)
-// metaId = Random.nextInt(dimOfCols)
-// } yield {
-// List(ts, "insertBulk", "edge", rowId, colId, labelName, s"""{"$metaName": $metaId}""").mkString("\t")
-// }
-// GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs)(None)
-// }
-// }
-// }
-// }
-//}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/TransferToHFile.scala b/loader/src/main/scala/subscriber/TransferToHFile.scala
deleted file mode 100644
index 516bb39..0000000
--- a/loader/src/main/scala/subscriber/TransferToHFile.scala
+++ /dev/null
@@ -1,195 +0,0 @@
-package subscriber
-
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{LabelMeta, Label}
-import com.kakao.s2graph.core.types.{InnerValLikeWithTs, LabelWithDirection, SourceVertexId}
-import org.apache.hadoop.hbase.client.Put
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
-import org.apache.hadoop.hbase.mapreduce.{TableOutputFormat}
-import org.apache.hadoop.hbase._
-import org.apache.hadoop.hbase.regionserver.BloomType
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.spark.{SparkContext}
-import org.apache.spark.rdd.RDD
-import org.hbase.async.{PutRequest}
-import play.api.libs.json.Json
-import s2.spark.{SparkApp}
-import spark.{FamilyHFileWriteOptions, KeyFamilyQualifier, HBaseContext}
-import scala.collection.JavaConversions._
-
-
-object TransferToHFile extends SparkApp with JSONParser {
-
- val usages =
- s"""
- |create HFiles for hbase table on zkQuorum specified.
- |note that hbase table is created already and pre-splitted properly.
- |
- |params:
- |0. input: hdfs path for tsv file(bulk format).
- |1. output: hdfs path for storing HFiles.
- |2. zkQuorum: running hbase cluster zkQuorum.
- |3. tableName: table name for this bulk upload.
- |4. dbUrl: db url for parsing to graph element.
- """.stripMargin
-
- //TODO: Process AtomicIncrementRequest too.
- /** build key values */
- case class DegreeKey(vertexIdStr: String, labelName: String, direction: String)
-
- private def insertBulkForLoaderAsync(edge: Edge, createRelEdges: Boolean = true): List[PutRequest] = {
- val relEdges = if (createRelEdges) edge.relatedEdges else List(edge)
- buildPutRequests(edge.toSnapshotEdge) ++ relEdges.toList.flatMap { e =>
- e.edgesWithIndex.flatMap { indexEdge => buildPutRequests(indexEdge) }
- }
- }
-
- def buildDegrees(msgs: RDD[String], labelMapping: Map[String, String], edgeAutoCreate: Boolean) = {
- for {
- msg <- msgs
- tokens = GraphUtil.split(msg)
- if tokens(2) == "e" || tokens(2) == "edge"
- tempDirection = if (tokens.length == 7) "out" else tokens(7)
- direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
- reverseDirection = if (direction == "out") "in" else "out"
- convertedLabelName = labelMapping.get(tokens(5)).getOrElse(tokens(5))
- (vertexIdStr, vertexIdStrReversed) = (tokens(3), tokens(4))
- degreeKey = DegreeKey(vertexIdStr, convertedLabelName, direction)
- degreeKeyReversed = DegreeKey(vertexIdStrReversed, convertedLabelName, reverseDirection)
- extra = if (edgeAutoCreate) List(degreeKeyReversed -> 1L) else Nil
- output <- List(degreeKey -> 1L) ++ extra
- } yield output
- }
- def buildPutRequests(snapshotEdge: SnapshotEdge): List[PutRequest] = {
- val kvs = GraphSubscriberHelper.g.storage.snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
- kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
- }
- def buildPutRequests(indexEdge: IndexEdge): List[PutRequest] = {
- val kvs = GraphSubscriberHelper.g.storage.indexEdgeSerializer(indexEdge).toKeyValues.toList
- kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
- }
- def buildDegreePutRequests(vertexId: String, labelName: String, direction: String, degreeVal: Long): List[PutRequest] = {
- val label = Label.findByName(labelName).getOrElse(throw new RuntimeException(s"$labelName is not found in DB."))
- val dir = GraphUtil.directions(direction)
- val innerVal = jsValueToInnerVal(Json.toJson(vertexId), label.srcColumnWithDir(dir).columnType, label.schemaVersion).getOrElse {
- throw new RuntimeException(s"$vertexId can not be converted into innerval")
- }
- val vertex = Vertex(SourceVertexId(label.srcColumn.id.get, innerVal))
-
- val ts = System.currentTimeMillis()
- val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
- val labelWithDir = LabelWithDirection(label.id.get, dir)
- val edge = Edge(vertex, vertex, labelWithDir, propsWithTs=propsWithTs)
-
- edge.edgesWithIndex.flatMap { indexEdge =>
- GraphSubscriberHelper.g.storage.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
- new PutRequest(kv.table, kv.row, kv.cf, Array.empty[Byte], Bytes.toBytes(degreeVal), kv.timestamp)
- }
- }
- }
-
- def toKeyValues(degreeKeyVals: Seq[(DegreeKey, Long)]): Iterator[KeyValue] = {
- val kvs = for {
- (key, value) <- degreeKeyVals
- putRequest <- buildDegreePutRequests(key.vertexIdStr, key.labelName, key.direction, value)
- } yield {
- val p = putRequest
- val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
- kv
- }
- kvs.toIterator
- }
-
- def toKeyValues(strs: Seq[String], labelMapping: Map[String, String], autoEdgeCreate: Boolean): Iterator[KeyValue] = {
- val kvs = for {
- s <- strs
- element <- Graph.toGraphElement(s, labelMapping).toSeq if element.isInstanceOf[Edge]
- edge = element.asInstanceOf[Edge]
- putRequest <- insertBulkForLoaderAsync(edge, autoEdgeCreate)
- } yield {
- val p = putRequest
- val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
-
- // println(s"[Edge]: $edge\n[Put]: $p\n[KeyValue]: ${kv.getRow.toList}, ${kv.getQualifier.toList}, ${kv.getValue.toList}, ${kv.getTimestamp}")
-
- kv
- }
- kvs.toIterator
- }
-
-
- override def run() = {
- val input = args(0)
- val tmpPath = args(1)
- val zkQuorum = args(2)
- val tableName = args(3)
- val dbUrl = args(4)
- val maxHFilePerResionServer = args(5).toInt
- val labelMapping = if (args.length >= 7) GraphSubscriberHelper.toLabelMapping(args(6)) else Map.empty[String, String]
- val autoEdgeCreate = if (args.length >= 8) args(7).toBoolean else false
- val buildDegree = if (args.length >= 9) args(8).toBoolean else true
- val compressionAlgorithm = if (args.length >= 10) args(9) else "lz4"
- val conf = sparkConf(s"$input: TransferToHFile")
- conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
- conf.set("spark.kryoserializer.buffer.mb", "24")
-
- val sc = new SparkContext(conf)
-
- Management.createTable(zkQuorum, tableName, List("e", "v"), maxHFilePerResionServer, None, compressionAlgorithm)
-
- /** set up hbase init */
- val hbaseConf = HBaseConfiguration.create()
- hbaseConf.set("hbase.zookeeper.quorum", zkQuorum)
- hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, tableName)
- hbaseConf.set("hadoop.tmp.dir", s"/tmp/$tableName")
-
-
- val rdd = sc.textFile(input)
-
-
- val kvs = rdd.mapPartitions { iter =>
- val phase = System.getProperty("phase")
- GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
- toKeyValues(iter.toSeq, labelMapping, autoEdgeCreate)
- }
- //
- // val newRDD = if (!buildDegree) new HFileRDD(kvs)
- // else {
- // val degreeKVs = buildDegrees(rdd, labelMapping, autoEdgeCreate).reduceByKey { (agg, current) =>
- // agg + current
- // }.mapPartitions { iter =>
- // val phase = System.getProperty("phase")
- // GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
- // toKeyValues(iter.toSeq)
- // }
- // new HFileRDD(kvs ++ degreeKVs)
- // }
- //
- // newRDD.toHFile(hbaseConf, zkQuorum, tableName, maxHFilePerResionServer, tmpPath)
- val merged = if (!buildDegree) kvs
- else {
- kvs ++ buildDegrees(rdd, labelMapping, autoEdgeCreate).reduceByKey { (agg, current) =>
- agg + current
- }.mapPartitions { iter =>
- val phase = System.getProperty("phase")
- GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
- toKeyValues(iter.toSeq)
- }
- }
-
- val hbaseSc = new HBaseContext(sc, hbaseConf)
- def flatMap(kv: KeyValue): Iterator[(KeyFamilyQualifier, Array[Byte])] = {
- val k = new KeyFamilyQualifier(kv.getRow(), kv.getFamily(), kv.getQualifier())
- val v = kv.getValue()
- Seq((k -> v)).toIterator
- }
- val familyOptions = new FamilyHFileWriteOptions(Algorithm.LZ4.getName.toUpperCase,
- BloomType.ROW.name().toUpperCase, 32768, DataBlockEncoding.FAST_DIFF.name().toUpperCase)
- val familyOptionsMap = Map("e".getBytes() -> familyOptions, "v".getBytes() -> familyOptions)
-
- hbaseSc.bulkLoad(merged, TableName.valueOf(tableName), flatMap, tmpPath, familyOptionsMap)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/VertexDegreeBuilder.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/VertexDegreeBuilder.scala b/loader/src/main/scala/subscriber/VertexDegreeBuilder.scala
deleted file mode 100644
index c8255dd..0000000
--- a/loader/src/main/scala/subscriber/VertexDegreeBuilder.scala
+++ /dev/null
@@ -1,102 +0,0 @@
-package subscriber
-
-/**
- * Created by shon on 7/3/15.
- */
-
-import com.kakao.s2graph.core.{GraphUtil, Management}
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import s2.spark.{HashMapParam, WithKafka, SparkApp}
-
-import scala.collection.mutable
-
-/**
- * Created by shon on 7/3/15.
- */
-
-//object VertexDegreeBuilder extends SparkApp with WithKafka {
-// val sleepPeriod = 5000
-// val usages =
-// s"""
-// |/**
-// |* this job read edge format(TSV) from HDFS file system then bulk load edges into s2graph. assumes that newLabelName is already created by API.
-// |* params:
-// |* 0. hdfsPath: where is your data in hdfs. require full path with hdfs:// predix
-// |* 1. dbUrl: jdbc database connection string to specify database for meta.
-// |* 2. labelMapping: oldLabel:newLabel delimited by ,
-// |* 3. outputPath: degree output Path.
-// |* 4. edgeAutoCreate: true if need to create reversed edge automatically.
-// |*
-// |* after this job finished, s2graph will have data with sequence corresponding newLabelName.
-// |* change this newLabelName to ogirinalName if you want to online replace of label.
-// |*
-// |*/
-// """.stripMargin
-//
-// override def run() = {
-// /**
-// * Main function
-// */
-// println(args.toList)
-// if (args.length != 5) {
-// System.err.println(usages)
-// System.exit(1)
-// }
-// val hdfsPath = args(0)
-// val dbUrl = args(1)
-// val labelMapping = GraphSubscriberHelper.toLabelMapping(args(2))
-// val outputPath = args(3)
-// val edgeAutoCreate = args(4).toBoolean
-//
-// val conf = sparkConf(s"$hdfsPath: VertexDegreeBuilder")
-// val sc = new SparkContext(conf)
-// val mapAcc = sc.accumulable(mutable.HashMap.empty[String, Long], "counter")(HashMapParam[String, Long](_ + _))
-//
-// /** this job expect only one hTableName. all labels in this job will be stored in same physical hbase table */
-// try {
-//
-// // set local driver setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
-//
-// /** copy when oldLabel exist and newLabel done exist. otherwise ignore. */
-//
-//
-// val msgs = sc.textFile(hdfsPath)
-//
-// /** change assumption here. this job only take care of one label data */
-// val degreeStart: RDD[((String, String, String), Int)] = msgs.filter { msg =>
-// val tokens = GraphUtil.split(msg)
-// (tokens(2) == "e" || tokens(2) == "edge")
-// } flatMap { msg =>
-// val tokens = GraphUtil.split(msg)
-// val tempDirection = if (tokens.length == 7) "out" else tokens(7)
-// val direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
-// val reverseDirection = if (direction == "out") "in" else "out"
-// val convertedLabelName = labelMapping.get(tokens(5)).getOrElse(tokens(5))
-// val (vertexWithLabel, reverseVertexWithLabel) = if (direction == "out") {
-// (
-// (tokens(3), convertedLabelName, direction),
-// (tokens(4), convertedLabelName, reverseDirection)
-// )
-// } else {
-// (
-// (tokens(4), convertedLabelName, direction),
-// (tokens(3), convertedLabelName, reverseDirection)
-// )
-// }
-// if (edgeAutoCreate) {
-// List((vertexWithLabel -> 1), (reverseVertexWithLabel -> 1))
-// } else {
-// List((vertexWithLabel -> 1))
-// }
-// }
-// val vertexDegrees = degreeStart.reduceByKey(_ + _)
-// vertexDegrees.map { case ((vertexId, labelName, dir), degreeVal) =>
-// Seq(vertexId, labelName, dir, degreeVal).mkString("\t")
-// }.saveAsTextFile(outputPath)
-//
-// }
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/WalLogStat.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/WalLogStat.scala b/loader/src/main/scala/subscriber/WalLogStat.scala
deleted file mode 100644
index f5db2c1..0000000
--- a/loader/src/main/scala/subscriber/WalLogStat.scala
+++ /dev/null
@@ -1,94 +0,0 @@
-package subscriber
-
-import java.text.SimpleDateFormat
-import java.util.Date
-
-import com.kakao.s2graph.core.Graph
-import kafka.producer.KeyedMessage
-import kafka.serializer.StringDecoder
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.HasOffsetRanges
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-
-object WalLogStat extends SparkApp with WithKafka {
-
- override def run() = {
-
- validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "statTopic")
-
- val kafkaZkQuorum = args(0)
- val brokerList = args(1)
- val topics = args(2)
- val intervalInSec = seconds(args(3).toLong)
- val dbUrl = args(4)
- val statTopic = args(5)
-
-
- val conf = sparkConf(s"$topics: ${getClass.getSimpleName}")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- val groupId = topics.replaceAll(",", "_") + "_stat"
-
- val kafkaParams = Map(
- "zookeeper.connect" -> kafkaZkQuorum,
- "group.id" -> groupId,
- "metadata.broker.list" -> brokerList,
- "zookeeper.connection.timeout.ms" -> "10000",
- "auto.offset.reset" -> "largest")
-
- val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
- val statProducer = getProducer[String, String](brokerList)
-
- stream.foreachRDD { (rdd, time) =>
-
- val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
-
- val ts = time.milliseconds
-
- val elements = rdd.mapPartitions { partition =>
- // set executor setting.
- val phase = System.getProperty("phase")
- GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
- partition.map { case (key, msg) =>
- Graph.toGraphElement(msg) match {
- case Some(elem) =>
- val serviceName = elem.serviceName
- msg.split("\t", 7) match {
- case Array(_, operation, log_type, _, _, label, _*) =>
- Seq(serviceName, label, operation, log_type).mkString("\t")
- case _ =>
- Seq("no_service_name", "no_label", "no_operation", "parsing_error").mkString("\t")
- }
- case None =>
- Seq("no_service_name", "no_label", "no_operation", "no_element_error").mkString("\t")
- }
- }
- }
-
- val countByKey = elements.map(_ -> 1L).reduceByKey(_ + _).collect()
- val totalCount = countByKey.map(_._2).sum
- val keyedMessage = countByKey.map { case (key, value) =>
- new KeyedMessage[String, String](statTopic, s"$ts\t$key\t$value\t$totalCount")
- }
-
- statProducer.send(keyedMessage: _*)
-
- elements.mapPartitionsWithIndex { (i, part) =>
- // commit offset range
- val osr = offsets(i)
- getStreamHelper(kafkaParams).commitConsumerOffset(osr)
- Iterator.empty
- }.foreach {
- (_: Nothing) => ()
- }
-
- }
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/subscriber/WalLogToHDFS.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/WalLogToHDFS.scala b/loader/src/main/scala/subscriber/WalLogToHDFS.scala
deleted file mode 100644
index 44902c2..0000000
--- a/loader/src/main/scala/subscriber/WalLogToHDFS.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-package subscriber
-
-import java.text.SimpleDateFormat
-import java.util.Date
-
-import com.kakao.s2graph.core.Graph
-import kafka.serializer.StringDecoder
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.HasOffsetRanges
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-
-object WalLogToHDFS extends SparkApp with WithKafka {
-
- override def run() = {
-
- validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "outputPath", "hiveDatabase", "hiveTable", "splitListPath")
-
- val kafkaZkQuorum = args(0)
- val brokerList = args(1)
- val topics = args(2)
- val intervalInSec = seconds(args(3).toLong)
- val dbUrl = args(4)
- val outputPath = args(5)
- val hiveDatabase = args(6)
- val hiveTable = args(7)
- val splitListPath = args(8)
-
- val conf = sparkConf(s"$topics: WalLogToHDFS")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- val groupId = topics.replaceAll(",", "_") + "_stream"
- val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
-
- val kafkaParams = Map(
- "zookeeper.connect" -> kafkaZkQuorum,
- "group.id" -> groupId,
- "metadata.broker.list" -> brokerList,
- "zookeeper.connection.timeout.ms" -> "10000",
- "auto.offset.reset" -> "largest")
-
- val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
-
- val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
-
- val hdfsBlockSize = 134217728 // 128M
- val hiveContext = new HiveContext(sc)
- var splits = Array[String]()
- var excludeLabels = Set[String]()
- var excludeServices = Set[String]()
- stream.foreachRDD { (rdd, time) =>
- try {
- val read = sc.textFile(splitListPath).collect().map(_.split("=")).flatMap {
- case Array(value) => Some(("split", value))
- case Array(key, value) => Some((key, value))
- case _ => None
- }
- splits = read.filter(_._1 == "split").map(_._2)
- excludeLabels = read.filter(_._1 == "exclude_label").map(_._2).toSet
- excludeServices = read.filter(_._1 == "exclude_service").map(_._2).toSet
- } catch {
- case _: Throwable => // use previous information
- }
-
- val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
-
- val elements = rdd.mapPartitions { partition =>
- // set executor setting.
- val phase = System.getProperty("phase")
- GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
-
- partition.flatMap { case (key, msg) =>
- val optMsg = Graph.toGraphElement(msg).flatMap { element =>
- val arr = msg.split("\t", 7)
- val service = element.serviceName
- val label = arr(5)
- val n = arr.length
-
- if (excludeServices.contains(service) || excludeLabels.contains(label)) {
- None
- } else if(n == 6) {
- Some(Seq(msg, "{}", service).mkString("\t"))
- }
- else if(n == 7) {
- Some(Seq(msg, service).mkString("\t"))
- }
- else {
- None
- }
- }
- optMsg
- }
- }
-
- val ts = time.milliseconds
- val dateId = new SimpleDateFormat("yyyy-MM-dd").format(new Date(ts))
-
- /** make sure that `elements` are not running at the same time */
- val elementsWritten = {
- elements.cache()
- (Array("all") ++ splits).foreach {
- case split if split == "all" =>
- val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
- elements.saveAsTextFile(path)
- case split =>
- val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
- val strlen = split.length
- val splitData = elements.filter(_.takeRight(strlen) == split).cache()
- val totalSize = splitData
- .mapPartitions { iterator =>
- val s = iterator.map(_.length.toLong).sum
- Iterator.single(s)
- }
- .sum
- .toLong
- val numPartitions = math.max(1, (totalSize / hdfsBlockSize.toDouble).toInt)
- splitData.coalesce(math.min(splitData.partitions.length, numPartitions)).saveAsTextFile(path)
- splitData.unpersist()
- }
- elements.unpersist()
- elements
- }
-
- elementsWritten.mapPartitionsWithIndex { (i, part) =>
- // commit offset range
- val osr = offsets(i)
- getStreamHelper(kafkaParams).commitConsumerOffset(osr)
- Iterator.empty
- }.foreach {
- (_: Nothing) => ()
- }
-
- (Array("all") ++ splits).foreach { split =>
- val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
- hiveContext.sql(s"use $hiveDatabase")
- hiveContext.sql(s"alter table $hiveTable add partition (split='$split', date_id='$dateId', ts='$ts') location '$path'")
- }
- }
-
- logInfo(s"counter: ${mapAcc.value}")
- println(s"counter: ${mapAcc.value}")
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/test/scala/org/apache/s2graph/loader/subscriber/GraphSubscriberTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/GraphSubscriberTest.scala b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/GraphSubscriberTest.scala
new file mode 100644
index 0000000..1f93134
--- /dev/null
+++ b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/GraphSubscriberTest.scala
@@ -0,0 +1,45 @@
+package org.apache.s2graph.loader.subscriber
+
+import org.apache.s2graph.core.Management
+import org.apache.s2graph.spark.spark.WithKafka
+import org.scalatest.{ FunSuite, Matchers }
+import play.api.libs.json.{JsBoolean, JsNumber}
+
+class GraphSubscriberTest extends FunSuite with Matchers with WithKafka {
+ val phase = "dev"
+ val dbUrl = "jdbc:mysql://localhost:3306/graph_dev"
+ val zkQuorum = "localhost"
+ val kafkaBrokerList = "localhost:9099"
+ val currentTs = System.currentTimeMillis()
+ val op = "insertBulk"
+ val testLabelName = "s2graph_label_test"
+ val labelToReplace = "s2graph_label_test_new"
+ val serviceName = "s2graph"
+ val columnName = "user_id"
+ val columnType = "long"
+ val indexProps = Seq("time" -> JsNumber(0), "weight" -> JsNumber(0))
+ val props = Seq("is_hidden" -> JsBoolean(false), "is_blocked" -> JsBoolean(false))
+ val hTableName = "s2graph-dev_new"
+ val ttl = 86000
+ val testStrings = List("1431788400000\tinsertBulk\te\t147229417\t99240432\ts2graph_label_test\t{\"is_hidden\": true}")
+
+ GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
+
+ test("GraphSubscriberHelper.store") {
+ // actually we need to delete labelToReplace first for each test.
+ val labelMapping = Map(testLabelName -> labelToReplace)
+ GraphSubscriberHelper.management.copyLabel(testLabelName, labelToReplace, Some(hTableName))
+
+//
+// val msgs = (for {
+// i <- (1 until 10)
+// j <- (100 until 110)
+// } yield {
+// s"$currentTs\t$op\tedge\t$i\t$j\t$testLabelName"
+// }).toSeq
+ val msgs = testStrings
+
+// val stat = GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping = labelMapping, autoCreateEdge = false)(None)
+// println(stat)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
new file mode 100644
index 0000000..b0dd80d
--- /dev/null
+++ b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
@@ -0,0 +1,169 @@
+package org.apache.s2graph.loader.subscriber
+
+import org.apache.s2graph.core.Management
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest._
+import TransferToHFile._
+
+/**
+ * Created by Eric on 2015. 12. 2..
+ */
+class TransferToHFileTest extends FlatSpec with BeforeAndAfterAll with Matchers {
+
+ private val master = "local[2]"
+ private val appName = "example-spark"
+
+ private var sc: SparkContext = _
+
+ val dataWithoutDir =
+ """
+ |1447686000000 insertBulk e a b friends_rel {}
+ |1447686000000 insertBulk e a c friends_rel {}
+ |1447686000000 insertBulk e a d friends_rel {}
+ |1447686000000 insertBulk e b d friends_rel {}
+ |1447686000000 insertBulk e b e friends_rel {}
+ """.stripMargin.trim
+
+ val dataWithDir =
+ """
+ |1447686000000 insertBulk e a b friends_rel {} out
+ |1447686000000 insertBulk e b a friends_rel {} in
+ |1447686000000 insertBulk e a c friends_rel {} out
+ |1447686000000 insertBulk e c a friends_rel {} in
+ |1447686000000 insertBulk e a d friends_rel {} out
+ |1447686000000 insertBulk e d a friends_rel {} in
+ |1447686000000 insertBulk e b d friends_rel {} out
+ |1447686000000 insertBulk e d b friends_rel {} in
+ |1447686000000 insertBulk e b e friends_rel {} out
+ |1447686000000 insertBulk e e b friends_rel {} in
+ """.stripMargin.trim
+
+ override def beforeAll(): Unit = {
+ println("### beforeAll")
+
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ // 1. create service
+ if(Management.findService("loader-test").isEmpty) {
+ println(">>> create service...")
+ Management.createService("loader-test", "localhost", "loader-test-dev", 1, None, "gz")
+ }
+
+ // 2. create label
+ if(Management.findLabel("friends_rel").isEmpty) {
+ println(">>> create label...")
+ Management.createLabel(
+ "friends_rel",
+ "loader-test", "user_id", "string",
+ "loader-test", "user_id", "string",
+ true,
+ "loader-test",
+ Seq(),
+ Seq(),
+ "weak",
+ None, None,
+ HBaseType.DEFAULT_VERSION,
+ false,
+ Management.defaultCompressionAlgorithm
+ )
+ }
+
+ // create spark context
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ sc = new SparkContext(conf)
+ }
+
+ override def afterAll(): Unit = {
+ println("### afterALL")
+ if (sc != null) {
+ sc.stop()
+ }
+
+ Management.deleteLabel("friends_rel")
+ }
+
+ "buildDegreePutRequest" should "transform degree to PutRequest" in {
+ val putReqs = buildDegreePutRequests("a", "friends_rel", "out", 3L)
+ putReqs.size should equal(1)
+ }
+
+ "toKeyValues" should "transform edges to KeyValues on edge format data without direction" in {
+ val rdd = sc.parallelize(dataWithoutDir.split("\n"))
+
+ val kvs = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
+ }
+ kvs.foreach(println)
+ // edges * 2 (snapshot edges + indexed edges)
+ kvs.count() should equal(10)
+
+
+ val kvsAutoCreated = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], true)
+ }
+
+ // edges * 3 (snapshot edges + indexed edges + reverse edges)
+ kvsAutoCreated.count() should equal(15)
+ }
+
+ "toKeyValues" should "transform edges to KeyValues on edge format data with direction" in {
+ val rdd = sc.parallelize(dataWithDir.split("\n"))
+
+ val kvs = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
+ }
+
+ // edges * 2 (snapshot edges + indexed edges)
+ kvs.count() should equal(20)
+ }
+
+ "buildDegrees" should "build degrees on edge format data without direction" in {
+ val rdd = sc.parallelize(dataWithoutDir.split("\n"))
+
+ // autoCreate = false
+ val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+ degrees.size should equal(2)
+
+ degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+
+
+ // autoCreate = true
+ val degreesAutoCreated = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], true).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+ degreesAutoCreated.size should equal(6)
+
+ degreesAutoCreated should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+ degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
+ degreesAutoCreated should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
+ degreesAutoCreated should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
+ degreesAutoCreated should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
+ }
+
+ "buildDegrees" should "build degrees on edge format data with direction" in {
+ val rdd = sc.parallelize(dataWithDir.split("\n"))
+
+ val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+
+ degrees.size should equal(6)
+
+ degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+ degrees should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
+ degrees should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
+ degrees should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
+ degrees should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/subscriber/GraphSubscriberTest.scala b/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
deleted file mode 100644
index f927509..0000000
--- a/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-package subscriber
-
-import com.kakao.s2graph.core.Management
-import org.scalatest.{ FunSuite, Matchers }
-import play.api.libs.json.{JsBoolean, JsNumber}
-import s2.spark.WithKafka
-
-class GraphSubscriberTest extends FunSuite with Matchers with WithKafka {
- val phase = "dev"
- val dbUrl = "jdbc:mysql://localhost:3306/graph_dev"
- val zkQuorum = "localhost"
- val kafkaBrokerList = "localhost:9099"
- val currentTs = System.currentTimeMillis()
- val op = "insertBulk"
- val testLabelName = "s2graph_label_test"
- val labelToReplace = "s2graph_label_test_new"
- val serviceName = "s2graph"
- val columnName = "user_id"
- val columnType = "long"
- val indexProps = Seq("time" -> JsNumber(0), "weight" -> JsNumber(0))
- val props = Seq("is_hidden" -> JsBoolean(false), "is_blocked" -> JsBoolean(false))
- val hTableName = "s2graph-dev_new"
- val ttl = 86000
- val testStrings = List("1431788400000\tinsertBulk\te\t147229417\t99240432\ts2graph_label_test\t{\"is_hidden\": true}")
-
- GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
-
- test("GraphSubscriberHelper.store") {
- // actually we need to delete labelToReplace first for each test.
- val labelMapping = Map(testLabelName -> labelToReplace)
- GraphSubscriberHelper.management.copyLabel(testLabelName, labelToReplace, Some(hTableName))
-
-//
-// val msgs = (for {
-// i <- (1 until 10)
-// j <- (100 until 110)
-// } yield {
-// s"$currentTs\t$op\tedge\t$i\t$j\t$testLabelName"
-// }).toSeq
- val msgs = testStrings
-
-// val stat = GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping = labelMapping, autoCreateEdge = false)(None)
-// println(stat)
- }
-}
[10/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala
new file mode 100644
index 0000000..6b3babb
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/JsonParserTest.scala
@@ -0,0 +1,66 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.types.{InnerVal, InnerValLike}
+import org.scalatest.{FunSuite, Matchers}
+
+class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser {
+
+ import InnerVal._
+ import types.HBaseType._
+
+ val innerValsPerVersion = for {
+ version <- List(VERSION2, VERSION1)
+ } yield {
+ val innerVals = List(
+ (InnerVal.withStr("ABC123", version), STRING),
+ (InnerVal.withNumber(23, version), BYTE),
+ (InnerVal.withNumber(23, version), INT),
+ (InnerVal.withNumber(Int.MaxValue, version), INT),
+ (InnerVal.withNumber(Int.MinValue, version), INT),
+ (InnerVal.withNumber(Long.MaxValue, version), LONG),
+ (InnerVal.withNumber(Long.MinValue, version), LONG),
+ (InnerVal.withBoolean(true, version), BOOLEAN)
+ )
+ val doubleVals = if (version == VERSION2) {
+ List(
+ (InnerVal.withDouble(Double.MaxValue, version), DOUBLE),
+ (InnerVal.withDouble(Double.MinValue, version), DOUBLE),
+ (InnerVal.withDouble(0.1, version), DOUBLE),
+ (InnerVal.withFloat(Float.MinValue, version), FLOAT),
+ (InnerVal.withFloat(Float.MaxValue, version), FLOAT),
+ (InnerVal.withFloat(0.9f, version), FLOAT)
+ )
+ } else {
+ List.empty[(InnerValLike, String)]
+ }
+ (innerVals ++ doubleVals, version)
+ }
+
+ def testInnerValToJsValue(innerValWithDataTypes: Seq[(InnerValLike, String)],
+ version: String) = {
+ for {
+ (innerVal, dataType) <- innerValWithDataTypes
+ } {
+ val jsValueOpt = innerValToJsValue(innerVal, dataType)
+ val decodedOpt = jsValueOpt.flatMap { jsValue =>
+ jsValueToInnerVal(jsValue, dataType, version)
+ }
+ println(s"jsValue: $jsValueOpt")
+ println(s"innerVal: $decodedOpt")
+ (decodedOpt.isDefined && innerVal == decodedOpt.get) shouldBe true
+ }
+ }
+
+ test("aa") {
+ val innerVal = InnerVal.withStr("abc", VERSION2)
+ val tmp = innerValToJsValue(innerVal, "string")
+ println(tmp)
+ }
+ test("JsValue <-> InnerVal with dataType") {
+ for {
+ (innerValWithDataTypes, version) <- innerValsPerVersion
+ } {
+ testInnerValToJsValue(innerValWithDataTypes, version)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala
new file mode 100644
index 0000000..e3f1ed5
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/OrderingUtilTest.scala
@@ -0,0 +1,130 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.OrderingUtil._
+import org.scalatest.{FunSuite, Matchers}
+import play.api.libs.json.JsString
+
+class OrderingUtilTest extends FunSuite with Matchers {
+ test("test SeqMultiOrdering") {
+ val jsLs: Seq[Seq[Any]] = Seq(
+ Seq(0, "a"),
+ Seq(0, "b"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(2, "c")
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[Seq[Any]] = Seq(
+ Seq(2, "c"),
+ Seq(1, "a"),
+ Seq(1, "b"),
+ Seq(0, "a"),
+ Seq(0, "b")
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() should equal(sortedJsLs.toString())
+ }
+
+ test("test tuple 1 TupleMultiOrdering") {
+ val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, None, None, None),
+ (0, None, None, None),
+ (1, None, None, None),
+ (1, None, None, None),
+ (2, None, None, None)
+ )
+
+ val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (2, None, None, None),
+ (1, None, None, None),
+ (1, None, None, None),
+ (0, None, None, None),
+ (0, None, None, None)
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false)
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() should equal(sortedJsLs.toString())
+ }
+
+ test("test tuple 2 TupleMultiOrdering") {
+ val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, "a", None, None),
+ (0, "b", None, None),
+ (1, "a", None, None),
+ (1, "b", None, None),
+ (2, "c", None, None)
+ )
+
+ // number descending, string ascending
+ val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (2, "c", None, None),
+ (1, "a", None, None),
+ (1, "b", None, None),
+ (0, "a", None, None),
+ (0, "b", None, None)
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(false, true)
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() should equal(sortedJsLs.toString())
+ }
+
+ test("test tuple 3 TupleMultiOrdering") {
+ val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, "a", 0l, None),
+ (0, "a", 1l, None),
+ (0, "b", 0l, None),
+ (1, "a", 0l, None),
+ (1, "b", 0l, None),
+ (2, "c", 0l, None)
+ )
+
+ val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, "a", 1l, None),
+ (0, "a", 0l, None),
+ (0, "b", 0l, None),
+ (1, "a", 0l, None),
+ (1, "b", 0l, None),
+ (2, "c", 0l, None)
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(true, true, false)
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() should equal(sortedJsLs.toString())
+ }
+
+ test("test tuple 4 TupleMultiOrdering") {
+ val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, "a", 0l, JsString("a")),
+ (0, "a", 0l, JsString("b")),
+ (0, "a", 1l, JsString("a")),
+ (0, "b", 0l, JsString("b")),
+ (1, "a", 0l, JsString("b")),
+ (1, "b", 0l, JsString("b")),
+ (2, "c", 0l, JsString("b"))
+ )
+
+ val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
+ (0, "a", 0l, JsString("b")),
+ (0, "a", 0l, JsString("a")),
+ (0, "a", 1l, JsString("a")),
+ (0, "b", 0l, JsString("b")),
+ (1, "a", 0l, JsString("b")),
+ (1, "b", 0l, JsString("b")),
+ (2, "c", 0l, JsString("b"))
+ )
+
+ val ascendingLs: Seq[Boolean] = Seq(true, true, true, false)
+ val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
+
+ resultJsLs.toString() should equal(sortedJsLs.toString())
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala
new file mode 100644
index 0000000..5b324e1
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/QueryParamTest.scala
@@ -0,0 +1,86 @@
+package org.apache.s2graph.core
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.types.LabelWithDirection
+import org.scalatest.{FunSuite, Matchers}
+
+class QueryParamTest extends FunSuite with Matchers with TestCommon {
+// val version = HBaseType.VERSION2
+// val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString)
+// test("EdgeTransformer toInnerValOpt") {
+//
+// /** only labelNameV2 has string type output */
+// val jsVal = Json.arr(Json.arr("_to"), Json.arr("phone_number.$", "phone_number"), Json.arr("age.$", "age"))
+// val transformer = EdgeTransformer(queryParamV2, jsVal)
+// val convertedLs = transformer.transform(testEdge, None)
+//
+// convertedLs(0).tgtVertex.innerId.toString == "10" shouldBe true
+// convertedLs(1).tgtVertex.innerId.toString == "phone_number.xxxx" shouldBe true
+// convertedLs(2).tgtVertex.innerId.toString == "age.20" shouldBe true
+// true
+// }
+
+ val dummyRequests = {
+ for {
+ id <- 0 until 1000
+ } yield {
+ Bytes.toBytes(id)
+ }
+ }
+
+ test("QueryParam toCacheKey bytes") {
+ val startedAt = System.nanoTime()
+ val queryParam = QueryParam(LabelWithDirection(1, 0))
+
+ for {
+ i <- dummyRequests.indices
+ x = queryParam.toCacheKey(dummyRequests(i))
+ } {
+ for {
+ j <- dummyRequests.indices if i != j
+ y = queryParam.toCacheKey(dummyRequests(j))
+ } {
+ x should not equal y
+ }
+ }
+
+ dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
+ val xHash = queryParam.toCacheKey(x)
+ val yHash = queryParam.toCacheKey(y)
+// println(xHash, yHash)
+ xHash should be(yHash)
+ }
+ val duration = System.nanoTime() - startedAt
+
+ println(s">> bytes: $duration")
+ }
+
+ test("QueryParam toCacheKey with variable params") {
+ val startedAt = System.nanoTime()
+ val queryParam = QueryParam(LabelWithDirection(1, 0))
+
+ dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
+ x shouldBe y
+ queryParam.limit(0, 10)
+ var xHash = queryParam.toCacheKey(x)
+ xHash shouldBe queryParam.toCacheKey(y)
+ queryParam.limit(1, 10)
+ var yHash = queryParam.toCacheKey(y)
+ queryParam.toCacheKey(x) shouldBe yHash
+// println(xHash, yHash)
+ xHash should not be yHash
+
+ queryParam.limit(0, 10)
+ xHash = queryParam.toCacheKey(x)
+ queryParam.limit(0, 11)
+ yHash = queryParam.toCacheKey(y)
+
+ xHash should not be yHash
+ }
+
+ val duration = System.nanoTime() - startedAt
+
+ println(s">> diff: $duration")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala
new file mode 100644
index 0000000..2c16972
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommon.scala
@@ -0,0 +1,188 @@
+package org.apache.s2graph.core
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta}
+import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLikeWithTs, LabelWithDirection}
+
+
+trait TestCommon {
+ val ts = System.currentTimeMillis()
+ val testServiceId = 1
+ val testColumnId = 1
+ val testLabelId = 1
+ val testDir = GraphUtil.directions("out")
+ val testOp = GraphUtil.operations("insert")
+ val testLabelOrderSeq = LabelIndex.DefaultSeq
+ val testLabelWithDir = LabelWithDirection(testLabelId, testDir)
+ val labelMeta = LabelMeta
+
+ def equalsTo(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) == 0
+
+ def largerThan(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) > 0
+
+ def lessThan(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) < 0
+
+ def lessThanEqual(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) <= 0
+
+ /** */
+ import HBaseType.{VERSION1, VERSION2}
+ private val tsValSmall = InnerVal.withLong(ts, VERSION1)
+ private val tsValLarge = InnerVal.withLong(ts + 1, VERSION1)
+ private val boolValSmall = InnerVal.withBoolean(false, VERSION1)
+ private val boolValLarge = InnerVal.withBoolean(true, VERSION1)
+ private val doubleValSmall = InnerVal.withDouble(-0.1, VERSION1)
+ private val doubleValLarge = InnerVal.withDouble(0.1, VERSION1)
+ private val toSeq = LabelMeta.toSeq.toInt
+ private val toVal = InnerVal.withLong(Long.MinValue, VERSION1)
+
+
+ private val tsValSmallV2 = InnerVal.withLong(ts, VERSION2)
+ private val tsValLargeV2 = InnerVal.withLong(ts + 1, VERSION2)
+ private val boolValSmallV2 = InnerVal.withBoolean(false, VERSION2)
+ private val boolValLargeV2 = InnerVal.withBoolean(true, VERSION2)
+ private val doubleValSmallV2 = InnerVal.withDouble(-0.1, VERSION2)
+ private val doubleValLargeV2 = InnerVal.withDouble(0.1, VERSION2)
+ private val toValV2 = InnerVal.withLong(Long.MinValue, VERSION2)
+
+ val intVals = (Int.MinValue until Int.MinValue + 10) ++ (-129 to -126) ++ (-1 to 1) ++ (126 to 129) ++
+ (Int.MaxValue - 10 until Int.MaxValue)
+ val intInnerVals = intVals.map { v => InnerVal.withNumber(BigDecimal(v), VERSION1) }
+
+ val intInnerValsV2 = intVals.map { v => InnerVal.withNumber(BigDecimal(v), VERSION2) }
+
+ val stringVals = List("abc", "abd", "ac", "aca", "b")
+ val stringInnerVals = stringVals.map { s => InnerVal.withStr(s, VERSION1)}
+ val stringInnerValsV2 = stringVals.map { s => InnerVal.withStr(s, VERSION2)}
+
+ val numVals = (Long.MinValue until Long.MinValue + 10).map(BigDecimal(_)) ++
+ (Int.MinValue until Int.MinValue + 10).map(BigDecimal(_)) ++
+ (Int.MaxValue - 10 until Int.MaxValue).map(BigDecimal(_)) ++
+ (Long.MaxValue - 10 until Long.MaxValue).map(BigDecimal(_))
+ val numInnerVals = numVals.map { n => InnerVal.withLong(n.toLong, VERSION1)}
+ val numInnerValsV2 = numVals.map { n => InnerVal.withNumber(n, VERSION2)}
+
+ val doubleStep = Double.MaxValue / 5
+ val doubleVals = (Double.MinValue until 0 by doubleStep).map(BigDecimal(_)) ++
+ (-9999.9 until -9994.1 by 1.1).map(BigDecimal(_)) ++
+ (-128.0 until 128.0 by 1.2).map(BigDecimal(_)) ++
+ (129.0 until 142.0 by 1.1).map(BigDecimal(_)) ++
+ (doubleStep until Double.MaxValue by doubleStep).map(BigDecimal(_))
+ val doubleInnerVals = doubleVals.map { d => InnerVal.withDouble(d.toDouble, VERSION1)}
+ val doubleInnerValsV2 = doubleVals.map { d => InnerVal.withDouble(d.toDouble, VERSION2)}
+
+ /** version 1 string order is broken */
+ val idxPropsLs = Seq(
+ Seq((0 -> tsValSmall), (1 -> boolValSmall), (2 -> InnerVal.withStr("ac", VERSION1)),(toSeq -> toVal)),
+ Seq((0 -> tsValSmall), (1 -> boolValSmall), (2 -> InnerVal.withStr("ab", VERSION1)), (toSeq -> toVal)),
+ Seq((0 -> tsValSmall), (1 -> boolValSmall), (2-> InnerVal.withStr("b", VERSION1)), (toSeq -> toVal)),
+ Seq((0 -> tsValSmall), (1 -> boolValLarge), (2 -> InnerVal.withStr("b", VERSION1)), (toSeq -> toVal)),
+ Seq((0 -> tsValLarge), (1 -> boolValSmall), (2 -> InnerVal.withStr("a", VERSION1)), (toSeq -> toVal))
+ ).map(seq => seq.map(t => t._1.toByte -> t._2 ))
+
+ val idxPropsLsV2 = Seq(
+ Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 -> doubleValSmallV2), (toSeq -> toValV2)),
+ Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 -> doubleValLargeV2), (toSeq -> toValV2)),
+ Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("ab", VERSION2)), (3 -> doubleValLargeV2), (toSeq -> toValV2)),
+ Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2-> InnerVal.withStr("b", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2)),
+ Seq((0 -> tsValSmallV2), (1 -> boolValLargeV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2)),
+ Seq((0 -> tsValLargeV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2))
+ ).map(seq => seq.map(t => t._1.toByte -> t._2 ) )
+
+ val idxPropsWithTsLs = idxPropsLs.map { idxProps =>
+ idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }
+ }
+ val idxPropsWithTsLsV2 = idxPropsLsV2.map { idxProps =>
+ idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }
+ }
+ //
+ // def testOrder(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]],
+ // innerVals: Iterable[InnerValLike], skipHashBytes: Boolean = false)
+ // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
+ // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
+ // /** check if increasing target vertex id is ordered properly with same indexProps */
+ // val rets = for {
+ // idxProps <- idxPropsLs
+ // } yield {
+ // val head = createFunc(idxProps, innerVals.head)
+ // val start = head
+ // var prev = head
+ // val rets = for {
+ // innerVal <- innerVals.tail
+ // } yield {
+ // val current = createFunc(idxProps, innerVal)
+ // val bytes = current.bytes
+ // val decoded = fromBytesFunc(bytes)
+ // println(s"current: $current")
+ // println(s"decoded: $decoded")
+ //
+ // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
+ // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
+ // val (isSame, orderPreserved) = (current, decoded) match {
+ // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
+ // /** _to is used in indexProps */
+ // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
+ // case _ =>
+ // (current == decoded, lessThan(currentBytes, prevBytes))
+ // }
+ //
+ // println(s"$current ${bytes.toList}")
+ // println(s"$prev ${prev.bytes.toList}")
+ // println(s"SerDe[$isSame], Order[$orderPreserved]")
+ // prev = current
+ // isSame && orderPreserved
+ // }
+ // rets.forall(x => x)
+ // }
+ // rets.forall(x => x)
+ // }
+ // def testOrderReverse(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]], innerVals: Iterable[InnerValLike],
+ // skipHashBytes: Boolean = false)
+ // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
+ // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
+ // /** check if increasing target vertex id is ordered properly with same indexProps */
+ // val rets = for {
+ // innerVal <- innerVals
+ // } yield {
+ // val head = createFunc(idxPropsLs.head, innerVal)
+ // val start = head
+ // var prev = head
+ // val rets = for {
+ // idxProps <- idxPropsLs.tail
+ // } yield {
+ // val current = createFunc(idxProps, innerVal)
+ // val bytes = current.bytes
+ // val decoded = fromBytesFunc(bytes)
+ // println(s"current: $current")
+ // println(s"decoded: $decoded")
+ //
+ // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
+ // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
+ // val (isSame, orderPreserved) = (current, decoded) match {
+ // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
+ // /** _to is used in indexProps */
+ // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
+ // case _ =>
+ // (current == decoded, lessThan(currentBytes, prevBytes))
+ // }
+ //
+ // println(s"$current ${bytes.toList}")
+ // println(s"$prev ${prev.bytes.toList}")
+ // println(s"SerDe[$isSame], Order[$orderPreserved]")
+ // prev = current
+ // isSame && orderPreserved
+ // }
+ //
+ // rets.forall(x => x)
+ // }
+ //
+ // rets.forall(x => x)
+ // }
+ //
+ //
+ // def putToKeyValues(put: PutRequest) = {
+ // val ts = put.timestamp()
+ // for ((q, v) <- put.qualifiers().zip(put.values)) yield {
+ // new KeyValue(put.key(), put.family(), q, ts, v)
+ // }
+ // }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
new file mode 100644
index 0000000..74b4197
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
@@ -0,0 +1,194 @@
+package org.apache.s2graph.core
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, Service, ServiceColumn}
+import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection}
+import scalikejdbc.AutoSession
+
+import scala.concurrent.ExecutionContext
+
+trait TestCommonWithModels {
+
+ import InnerVal._
+ import types.HBaseType._
+
+ var graph: Graph = _
+ var config: Config = _
+ var management: Management = _
+
+ def initTests() = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+ management = new Management(graph)
+
+ implicit val session = AutoSession
+
+ deleteTestLabel()
+ deleteTestService()
+
+ createTestService()
+ createTestLabel()
+ }
+
+ def zkQuorum = config.getString("hbase.zookeeper.quorum")
+
+ def cluster = config.getString("hbase.zookeeper.quorum")
+
+ implicit val session = AutoSession
+
+ val serviceName = "_test_service"
+ val serviceNameV2 = "_test_service_v2"
+ val serviceNameV3 = "_test_service_v3"
+ val serviceNameV4 = "_test_service_v4"
+
+ val columnName = "user_id"
+ val columnNameV2 = "user_id_v2"
+ val columnNameV3 = "user_id_v3"
+ val columnNameV4 = "user_id_v4"
+
+ val columnType = "long"
+ val columnTypeV2 = "long"
+ val columnTypeV3 = "long"
+ val columnTypeV4 = "long"
+
+ val tgtColumnName = "itme_id"
+ val tgtColumnNameV2 = "item_id_v2"
+ val tgtColumnNameV3 = "item_id_v3"
+ val tgtColumnNameV4 = "item_id_v4"
+
+ val tgtColumnType = "string"
+ val tgtColumnTypeV2 = "string"
+ val tgtColumnTypeV3 = "string"
+ val tgtColumnTypeV4 = "string"
+
+ val hTableName = "_test_cases"
+ val preSplitSize = 0
+
+ val labelName = "_test_label"
+ val labelNameV2 = "_test_label_v2"
+ val labelNameV3 = "_test_label_v3"
+ val labelNameV4 = "_test_label_v4"
+
+ val undirectedLabelName = "_test_label_undirected"
+ val undirectedLabelNameV2 = "_test_label_undirected_v2"
+
+ val testProps = Seq(
+ Prop("affinity_score", "0.0", DOUBLE),
+ Prop("is_blocked", "false", BOOLEAN),
+ Prop("time", "0", INT),
+ Prop("weight", "0", INT),
+ Prop("is_hidden", "true", BOOLEAN),
+ Prop("phone_number", "xxx-xxx-xxxx", STRING),
+ Prop("score", "0.1", FLOAT),
+ Prop("age", "10", INT)
+ )
+ val testIdxProps = Seq(Index("_PK", Seq("_timestamp", "affinity_score")))
+ val consistencyLevel = "strong"
+ val hTableTTL = None
+
+
+ def createTestService() = {
+ implicit val session = AutoSession
+ management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
+ management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
+ management.createService(serviceNameV3, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
+ management.createService(serviceNameV4, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
+ }
+
+ def deleteTestService() = {
+ implicit val session = AutoSession
+ Management.deleteService(serviceName)
+ Management.deleteService(serviceNameV2)
+ Management.deleteService(serviceNameV3)
+ Management.deleteService(serviceNameV4)
+ }
+
+ def deleteTestLabel() = {
+ implicit val session = AutoSession
+ Management.deleteLabel(labelName)
+ Management.deleteLabel(labelNameV2)
+ Management.deleteLabel(undirectedLabelName)
+ Management.deleteLabel(undirectedLabelNameV2)
+ }
+
+ def createTestLabel() = {
+ implicit val session = AutoSession
+ management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
+ isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4")
+
+ management.createLabel(labelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
+ isDirected = true, serviceNameV2, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
+
+ management.createLabel(labelNameV3, serviceNameV3, columnNameV3, columnTypeV3, serviceNameV3, tgtColumnNameV3, tgtColumnTypeV3,
+ isDirected = true, serviceNameV3, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4")
+
+ management.createLabel(labelNameV4, serviceNameV4, columnNameV4, columnTypeV4, serviceNameV4, tgtColumnNameV4, tgtColumnTypeV4,
+ isDirected = true, serviceNameV4, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4")
+
+ management.createLabel(undirectedLabelName, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType,
+ isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4")
+
+ management.createLabel(undirectedLabelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
+ isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
+ }
+
+ def service = Service.findByName(serviceName, useCache = false).get
+
+ def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
+
+ def serviceV3 = Service.findByName(serviceNameV3, useCache = false).get
+
+ def serviceV4 = Service.findByName(serviceNameV4, useCache = false).get
+
+ def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
+
+ def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
+
+ def columnV3 = ServiceColumn.find(serviceV3.id.get, columnNameV3, useCache = false).get
+
+ def columnV4 = ServiceColumn.find(serviceV4.id.get, columnNameV4, useCache = false).get
+
+ def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
+
+ def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
+
+ def tgtColumnV3 = ServiceColumn.find(serviceV3.id.get, tgtColumnNameV3, useCache = false).get
+
+ def tgtColumnV4 = ServiceColumn.find(serviceV4.id.get, tgtColumnNameV4, useCache = false).get
+
+ def label = Label.findByName(labelName, useCache = false).get
+
+ def labelV2 = Label.findByName(labelNameV2, useCache = false).get
+
+ def labelV3 = Label.findByName(labelNameV3, useCache = false).get
+
+ def labelV4 = Label.findByName(labelNameV4, useCache = false).get
+
+ def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
+
+ def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
+
+ def dir = GraphUtil.directions("out")
+
+ def op = GraphUtil.operations("insert")
+
+ def labelOrderSeq = LabelIndex.DefaultSeq
+
+ def labelWithDir = LabelWithDirection(label.id.get, dir)
+
+ def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
+
+ def labelWithDirV3 = LabelWithDirection(labelV3.id.get, dir)
+
+ def labelWithDirV4 = LabelWithDirection(labelV4.id.get, dir)
+
+ def queryParam = QueryParam(labelWithDir)
+
+ def queryParamV2 = QueryParam(labelWithDirV2)
+
+ def queryParamV3 = QueryParam(labelWithDirV3)
+
+ def queryParamV4 = QueryParam(labelWithDirV4)
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala
new file mode 100644
index 0000000..5b32a0f
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/models/ModelTest.scala
@@ -0,0 +1,130 @@
+package org.apache.s2graph.core.models
+
+import org.apache.s2graph.core.TestCommonWithModels
+import org.apache.s2graph.core.mysqls.Label
+import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers}
+
+class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll {
+ override def beforeAll(): Unit = {
+ initTests()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
+
+ // val serviceName = "testService"
+ // val newServiceName = "newTestService"
+ // val cluster = "localhost"
+ // val hbaseTableName = "s2graph-dev"
+ // val columnName = "user_id"
+ // val columnType = "long"
+ // val labelName = "model_test_label"
+ // val newLabelName = "new_model_test_label"
+ // val columnMetaName = "is_valid_user"
+ // val labelMetaName = "is_hidden"
+ // val hbaseTableTTL = -1
+ // val id = 1
+ //
+ // val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster,
+ // "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1))
+ // val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get,
+ // "columnName" -> columnName, "columnType" -> columnType))
+ // val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte))
+ // val label = HLabel(Map("id" -> id, "label" -> labelName,
+ // "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType,
+ // "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType,
+ // "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get,
+ // "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1
+ // ))
+ // val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte,
+ // "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false))
+ // val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte,
+ // "metaSeqs" -> "0", "formular" -> "none"))
+ test("test Label.findByName") {
+ val labelOpt = Label.findByName(labelName, useCache = false)
+ println(labelOpt)
+ labelOpt.isDefined shouldBe true
+ val indices = labelOpt.get.indices
+ indices.size > 0 shouldBe true
+ println(indices)
+ val defaultIndexOpt = labelOpt.get.defaultIndex
+ println(defaultIndexOpt)
+ defaultIndexOpt.isDefined shouldBe true
+ val metas = labelOpt.get.metaProps
+ println(metas)
+ metas.size > 0 shouldBe true
+ val srcService = labelOpt.get.srcService
+ println(srcService)
+ val tgtService = labelOpt.get.tgtService
+ println(tgtService)
+ val service = labelOpt.get.service
+ println(service)
+ val srcColumn = labelOpt.get.srcService
+ println(srcColumn)
+ val tgtColumn = labelOpt.get.tgtService
+ println(tgtColumn)
+ }
+ // test("test create") {
+ // service.create()
+ // HService.findByName(serviceName, useCache = false) == Some(service)
+ //
+ // serviceColumn.create()
+ // HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn)
+ //
+ // columnMeta.create()
+ // HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta)
+ //
+ // label.create()
+ // HLabel.findByName(labelName, useCache = false) == Some(label)
+ //
+ // labelMeta.create()
+ // HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta)
+ //
+ // labelIndex.create()
+ // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex)
+ // }
+ //
+ // test("test update") {
+ // service.update("cluster", "...")
+ // HService.findById(service.id.get, useCache = false).cluster == "..."
+ //
+ // service.update("serviceName", newServiceName)
+ // assert(HService.findByName(serviceName, useCache = false) == None)
+ // HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get}
+ //
+ // label.update("label", newLabelName)
+ // HLabel.findById(label.id.get, useCache = false).label == "newLabelName"
+ //
+ // label.update("consistencyLevel", "strong")
+ // HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" &&
+ // HLabel.findByName(newLabelName).isDefined &&
+ // HLabel.findByName(labelName) == None
+ //
+ // }
+ // test("test read by index") {
+ // val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false)
+ // val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false)
+ // labels.length == 1 &&
+ // labels.head == label
+ // idxs.length == 1 &&
+ // idxs.head == labelIndex
+ // }
+ // test("test delete") {
+ //// HLabel.findByName(labelName).foreach { label =>
+ //// label.deleteAll()
+ //// }
+ // HLabel.findByName(newLabelName).foreach { label =>
+ // label.deleteAll()
+ // }
+ // HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty &&
+ // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty
+ //
+ // service.deleteAll()
+ // }
+
+ // test("test labelIndex") {
+ // println(HLabelIndex.findByLabelIdAll(1))
+ // }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala
new file mode 100644
index 0000000..ccae6b6
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala
@@ -0,0 +1,64 @@
+package org.apache.s2graph.core.mysqls
+
+import java.util.Properties
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import scalikejdbc._
+
+class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
+ val Ttl = 2
+ override def beforeAll(): Unit = {
+ /*
+ maxSize = config.getInt("cache.max.size")
+ ttl = config.getInt("cache.ttl.seconds")
+ */
+ val props = new Properties()
+ props.setProperty("cache.ttl.seconds", Ttl.toString)
+ Model.apply(ConfigFactory.load(ConfigFactory.parseProperties(props)))
+
+ implicit val session = AutoSession
+ sql"""DELETE FROM buckets""".update().apply()
+ sql"""DELETE FROM experiments""".update().apply()
+
+ val expId = sql"""INSERT INTO experiments(service_id, service_name, `name`, description) VALUES(1, "s1", "exp1", "")""".updateAndReturnGeneratedKey().apply()
+ sql"""INSERT INTO
+ buckets(experiment_id, modular, http_verb, api_path, request_body, impression_id)
+ VALUES($expId, "1~100", "POST", "/a/b/c", "None", "imp1")""".update().apply()
+
+ }
+
+ "Experiment" should "find bucket list" in {
+ Experiment.findBy(1, "exp1") should not be empty
+
+ Experiment.findBy(1, "exp1").foreach { exp =>
+ val bucket = exp.buckets.head
+ bucket.impressionId should equal("imp1")
+ }
+ }
+
+ it should "update bucket list after cache ttl time" in {
+ Experiment.findBy(1, "exp1").foreach { exp =>
+ val bucket = exp.buckets.head
+ bucket.impressionId should equal("imp1")
+
+ implicit val session = AutoSession
+
+ sql"""UPDATE buckets SET impression_id = "imp2" WHERE id = ${bucket.id}""".update().apply()
+ }
+
+ // sleep ttl time
+ Thread.sleep((Ttl + 1) * 1000)
+
+ // update experiment and bucket
+ Experiment.findBy(1, "exp1").foreach(exp => exp.buckets)
+
+ // wait for cache updating
+ Thread.sleep(1 * 1000)
+
+ // should be updated
+ Experiment.findBy(1, "exp1").foreach { exp =>
+ exp.buckets.head.impressionId should equal("imp2")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
new file mode 100644
index 0000000..f251654
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
@@ -0,0 +1,234 @@
+package org.apache.s2graph.core.parsers
+
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.rest.TemplateHelper
+import org.apache.s2graph.core.types._
+import org.scalatest.{FunSuite, Matchers}
+import play.api.libs.json.Json
+
+class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
+ initTests()
+
+ // dummy data for dummy edge
+ initTests()
+
+ import HBaseType.{VERSION1, VERSION2}
+
+ val ts = System.currentTimeMillis()
+ val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
+
+ def ids(version: String) = {
+ val colId = if (version == VERSION2) columnV2.id.get else column.id.get
+ val srcId = SourceVertexId(colId, InnerVal.withLong(1, version))
+ val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version))
+
+ val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version))
+ val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version))
+
+ val srcVertex = Vertex(srcId, ts)
+ val tgtVertex = Vertex(tgtId, ts)
+ val srcVertexStr = Vertex(srcIdStr, ts)
+ val tgtVertexStr = Vertex(tgtIdStr, ts)
+ (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version)
+ }
+
+
+ def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = {
+ val whereOpt = WhereParser(label).parse(sql)
+ whereOpt.isSuccess shouldBe true
+
+ println("=================================================================")
+ println(sql)
+ println(whereOpt.get)
+
+ val ret = whereOpt.get.filter(edge)
+ if (ret != expected) {
+ println("==================")
+ println(s"$whereOpt")
+ println(s"$edge")
+ println("==================")
+ }
+ ret shouldBe expected
+ }
+
+ test("check where clause not nested") {
+ for {
+ (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
+ } {
+ /** test for each version */
+ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
+ val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
+ val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
+ val f = validate(label)(edge) _
+
+ /** labelName label is long-long relation */
+ f(s"_to=${tgtVertex.innerId.toString}")(true)
+
+ // currently this throw exception since label`s _to is long type.
+ f(s"_to=19230495")(false)
+ f(s"_to!=19230495")(true)
+ }
+ }
+
+ test("check where clause nested") {
+ for {
+ (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
+ } {
+ /** test for each version */
+ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
+ val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
+ val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
+
+ val f = validate(label)(edge) _
+
+ // time == 3
+ f("time >= 3")(true)
+ f("time > 2")(true)
+ f("time <= 3")(true)
+ f("time < 2")(false)
+
+ f("(time in (1, 2, 3) and is_blocked = true) or is_hidden = false")(false)
+ f("(time in (1, 2, 3) or is_blocked = true) or is_hidden = false")(true)
+ f("(time in (1, 2, 3) and is_blocked = true) or is_hidden = true")(true)
+ f("(time in (1, 2, 3) or is_blocked = true) and is_hidden = true")(true)
+
+ f("((time in ( 1, 2, 3) and weight between 1 and 10) or is_hidden=false)")(true)
+ f("(time in (1, 2, 4 ) or weight between 1 and 9) or (is_hidden = false)")(false)
+ f("(time in ( 1,2,4 ) or weight between 1 and 9) or is_hidden= true")(true)
+ f("(time in (1,2,3) or weight between 1 and 10) and is_hidden =false")(false)
+ }
+ }
+
+ test("check where clause with from/to long") {
+ for {
+ (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
+ } {
+ /** test for each version */
+ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
+ val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
+ val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir
+ val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner)
+ val lname = if (schemaVer == VERSION2) labelNameV2 else labelName
+ val f = validate(label)(edge) _
+
+ f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true)
+ f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true)
+ f(s"_from = ${tgtVertex.innerId.value} and _to = 102934")(false)
+ f(s"_from = -1")(false)
+ f(s"_from in (-1, -0.1)")(false)
+ }
+ }
+
+
+ test("check where clause with parent") {
+ for {
+ (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
+ } {
+ /** test for each version */
+ val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 1, "name" -> "abc")
+ val parentJs = Json.obj("is_hidden" -> false, "is_blocked" -> false, "weight" -> 20, "time" -> 3, "name" -> "a")
+
+ val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
+ val parentPropsInner = Management.toProps(label, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
+
+ val grandParentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner)
+ val parentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner,
+ parentEdges = Seq(EdgeWithScore(grandParentEdge, 1.0)))
+ val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner,
+ parentEdges = Seq(EdgeWithScore(parentEdge, 1.0)))
+
+ println(edge.toString)
+ println(parentEdge.toString)
+ println(grandParentEdge.toString)
+
+ val f = validate(label)(edge) _
+
+ // Compare edge's prop(`_from`) with edge's prop(`name`)
+ f("_from = 1")(true)
+ f("_to = 2")(true)
+ f("_from = 123")(false)
+ f("_from = time")(true)
+
+ // Compare edge's prop(`weight`) with edge's prop(`time`)
+ f("weight = time")(false)
+ f("weight = is_blocked")(false)
+
+ // Compare edge's prop(`weight`) with parent edge's prop(`weight`)
+ f("_parent.is_blocked = is_blocked")(true)
+ f("is_hidden = _parent.is_hidden")(false)
+ f("_parent.weight = weight")(false)
+
+ // Compare edge's prop(`is_hidden`) with parent of parent edge's prop(`is_hidden`)
+ f("_parent._parent.is_hidden = is_hidden")(false)
+ f("_parent._parent.is_blocked = is_blocked")(true)
+ f("_parent._parent.weight = weight")(false)
+ f("_parent._parent.weight = _parent.weight")(true)
+ }
+ }
+
+ test("replace reserved") {
+ val ts = 0
+ import TemplateHelper._
+
+ calculate(ts, 1, "hour") should be(hour + ts)
+ calculate(ts, 1, "day") should be(day + ts)
+
+ calculate(ts + 10, 1, "HOUR") should be(hour + ts + 10)
+ calculate(ts + 10, 1, "DAY") should be(day + ts + 10)
+
+ val body = """{
+ "day": ${1day},
+ "hour": ${1hour},
+ "-day": "${-10 day}",
+ "-hour": ${-10 hour},
+ "now": "${now}"
+ }
+ """
+
+ val parsed = replaceVariable(ts, body)
+ val json = Json.parse(parsed)
+
+ (json \ "day").as[Long] should be (1 * day + ts)
+ (json \ "hour").as[Long] should be (1 * hour + ts)
+
+ (json \ "-day").as[Long] should be (-10 * day + ts)
+ (json \ "-hour").as[Long] should be (-10 * hour + ts)
+
+ (json \ "now").as[Long] should be (ts)
+
+ val otherBody = """{
+ "nextday": "${next_day}",
+ "3dayago": "${next_day - 3 day}",
+ "nexthour": "${next_hour}"
+ }"""
+
+ val currentTs = System.currentTimeMillis()
+ val expectedDayTs = currentTs / day * day + day
+ val expectedHourTs = currentTs / hour * hour + hour
+ val threeDayAgo = expectedDayTs - 3 * day
+ val currentTsLs = (1 until 1000).map(currentTs + _)
+
+ currentTsLs.foreach { ts =>
+ val parsed = replaceVariable(ts, otherBody)
+ val json = Json.parse(parsed)
+
+ (json \ "nextday").as[Long] should be(expectedDayTs)
+ (json \ "nexthour").as[Long] should be(expectedHourTs)
+ (json \ "3dayago").as[Long] should be(threeDayAgo)
+ }
+ }
+
+ // test("time decay") {
+ // val ts = System.currentTimeMillis()
+ //
+ // for {
+ // i <- (0 until 10)
+ // } {
+ // val timeUnit = 60 * 60
+ // val diff = i * timeUnit
+ // val x = TimeDecay(1.0, 0.05, timeUnit)
+ // println(x.decay(diff))
+ // }
+ // }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
new file mode 100644
index 0000000..7b601b6
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
@@ -0,0 +1,33 @@
+package org.apache.s2graph.core.storage.hbase
+
+import org.scalatest.{FunSuite, Matchers}
+
+class AsynchbaseStorageTest extends FunSuite with Matchers {
+
+ /** need secured cluster */
+// test("test secure cluster connection") {
+// val config = ConfigFactory.parseMap(
+// Map(
+// "hbase.zookeeper.quorum" -> "localhost",
+// "hbase.security.auth.enable" -> "true",
+// "hbase.security.authentication" -> "kerberos",
+// "hbase.kerberos.regionserver.principal" -> "hbase/_HOST@LOCAL.HBASE",
+// "hbase.sasl.clientconfig" -> "Client",
+// "java.security.krb5.conf" -> "krb5.conf",
+// "java.security.auth.login.config" -> "async-client.jaas.conf")
+// )
+//
+// val client = AsynchbaseStorage.makeClient(config)
+// val table = "test".getBytes()
+//
+// val putRequest = new PutRequest(table, "a".getBytes(), "e".getBytes, "a".getBytes, "a".getBytes)
+// val getRequest = new GetRequest(table, "a".getBytes(), "e".getBytes)
+// val ret = client.put(putRequest).join()
+// val kvs = client.get(getRequest).join()
+// for {
+// kv <- kvs
+// } {
+// println(kv.toString)
+// }
+// }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
new file mode 100644
index 0000000..84e7458
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
@@ -0,0 +1,81 @@
+package org.apache.s2graph.core.storage.hbase
+
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta}
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.{IndexEdge, TestCommonWithModels, Vertex}
+import org.scalatest.{FunSuite, Matchers}
+
+
+class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels {
+ initTests()
+
+ /**
+ * check if storage serializer/deserializer can translate from/to bytes array.
+ * @param l: label for edge.
+ * @param ts: timestamp for edge.
+ * @param to: to VertexId for edge.
+ * @param props: expected props of edge.
+ */
+ def check(l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = {
+ val from = InnerVal.withLong(1, l.schemaVersion)
+ val vertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, from)
+ val tgtVertexId = TargetVertexId(HBaseType.DEFAULT_COL_ID, to)
+ val vertex = Vertex(vertexId, ts)
+ val tgtVertex = Vertex(tgtVertexId, ts)
+ val labelWithDir = LabelWithDirection(l.id.get, 0)
+
+ val indexEdge = IndexEdge(vertex, tgtVertex, labelWithDir, 0, ts, LabelIndex.DefaultSeq, props)
+ val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam,
+ graph.storage.indexEdgeSerializer(indexEdge).toKeyValues, l.schemaVersion, None)
+
+ _indexEdgeOpt should not be empty
+ indexEdge should be(_indexEdgeOpt.get)
+ }
+
+
+ /** note that props have to be properly set up for equals */
+ test("test serializer/deserializer for index edge.") {
+ val ts = System.currentTimeMillis()
+ for {
+ l <- Seq(label, labelV2, labelV3, labelV4)
+ } {
+ val to = InnerVal.withLong(101, l.schemaVersion)
+ val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
+ val props = Map(LabelMeta.timeStampSeq -> tsInnerVal,
+ 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion))
+
+ check(l, ts, to, props)
+ }
+ }
+
+ test("test serializer/deserializer for degree edge.") {
+ val ts = System.currentTimeMillis()
+ for {
+ l <- Seq(label, labelV2, labelV3, labelV4)
+ } {
+ val to = InnerVal.withStr("0", l.schemaVersion)
+ val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
+ val props = Map(
+ LabelMeta.degreeSeq -> InnerVal.withLong(10, l.schemaVersion),
+ LabelMeta.timeStampSeq -> tsInnerVal)
+
+ check(l, ts, to, props)
+ }
+ }
+
+ test("test serializer/deserializer for incrementCount index edge.") {
+ val ts = System.currentTimeMillis()
+ for {
+ l <- Seq(label, labelV2, labelV3, labelV4)
+ } {
+ val to = InnerVal.withLong(101, l.schemaVersion)
+
+ val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
+ val props = Map(LabelMeta.timeStampSeq -> tsInnerVal,
+ 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion),
+ LabelMeta.countSeq -> InnerVal.withLong(10, l.schemaVersion))
+
+ check(l, ts, to, props)
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala
new file mode 100644
index 0000000..607f46d
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/types/InnerValTest.scala
@@ -0,0 +1,128 @@
+package org.apache.s2graph.core.types
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.TestCommonWithModels
+import org.scalatest.{FunSuite, Matchers}
+
+class InnerValTest extends FunSuite with Matchers with TestCommonWithModels {
+ initTests()
+
+ import HBaseType.VERSION2
+ val decimals = List(
+ BigDecimal(Long.MinValue),
+ BigDecimal(Int.MinValue),
+ BigDecimal(Double.MinValue),
+ BigDecimal(Float.MinValue),
+ BigDecimal(Short.MinValue),
+ BigDecimal(Byte.MinValue),
+
+ BigDecimal(-1),
+ BigDecimal(0),
+ BigDecimal(1),
+ BigDecimal(Long.MaxValue),
+ BigDecimal(Int.MaxValue),
+ BigDecimal(Double.MaxValue),
+ BigDecimal(Float.MaxValue),
+ BigDecimal(Short.MaxValue),
+ BigDecimal(Byte.MaxValue)
+ )
+ val booleans = List(
+ false, true
+ )
+ val strings = List(
+ "abc", "abd", "ac", "aca"
+ )
+ val texts = List(
+ (0 until 1000).map(x => "a").mkString
+ )
+ val blobs = List(
+ (0 until 1000).map(x => Byte.MaxValue).toArray
+ )
+ def testEncodeDecode(ranges: List[InnerValLike], version: String) = {
+ for {
+ innerVal <- ranges
+ } {
+ val bytes = innerVal.bytes
+ val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version)
+ innerVal == decoded shouldBe true
+ bytes.length == numOfBytesUsed shouldBe true
+ }
+ }
+ // test("big decimal") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)}
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("text") {
+ // for {
+ // version <- List(VERSION2)
+ // } {
+ // val innerVals = texts.map { t => InnerVal.withStr(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("string") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = strings.map { t => InnerVal.withStr(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("blob") {
+ // for {
+ // version <- List(VERSION2)
+ // } {
+ // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ // test("boolean") {
+ // for {
+ // version <- List(VERSION2, VERSION1)
+ // } {
+ // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) }
+ // testEncodeDecode(innerVals, version)
+ // }
+ // }
+ test("korean") {
+ val small = InnerVal.withStr("가", VERSION2)
+ val large = InnerVal.withStr("나", VERSION2)
+ val smallBytes = small.bytes
+ val largeBytes = large.bytes
+
+ println (Bytes.compareTo(smallBytes, largeBytes))
+ true
+ }
+ // test("innerVal") {
+ // val srcVal = InnerVal.withLong(44391298, VERSION2)
+ // val srcValV1 = InnerVal.withLong(44391298, VERSION1)
+ // val tgtVal = InnerVal.withLong(7295564, VERSION2)
+ //
+ // val a = VertexId(0, srcVal)
+ // val b = SourceVertexId(0, srcVal)
+ // val c = TargetVertexId(0, srcVal)
+ // val aa = VertexId(0, srcValV1)
+ // val bb = SourceVertexId(0, srcValV1)
+ // val cc = TargetVertexId(0, srcValV1)
+ // println(a.bytes.toList)
+ // println(b.bytes.toList)
+ // println(c.bytes.toList)
+ //
+ // println(aa.bytes.toList)
+ // println(bb.bytes.toList)
+ // println(cc.bytes.toList)
+ // }
+ // test("aa") {
+ // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes
+ // val pbr = new SimplePositionedByteRange(bytes)
+ // pbr.setOffset(1)
+ // println(pbr.getPosition)
+ // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr)
+ // println(pbr.getPosition)
+ // true
+ // }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/TrxLog.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/TrxLog.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/TrxLog.scala
new file mode 100644
index 0000000..530d775
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/TrxLog.scala
@@ -0,0 +1,25 @@
+package org.apache.s2graph.counter
+
+import play.api.libs.json.Json
+
+// item1 -> likedCount -> month:2015-10, 1
+// edge
+ // policyId = Label.findByName(likedCount).id.get
+ // item = edge.srcVertexId
+ // results =
+case class TrxLog(success: Boolean, policyId: Int, item: String, results: Iterable[TrxLogResult])
+
+// interval = m, ts = 2015-10, "age.gender.20.M", 1, 2
+case class TrxLogResult(interval: String, ts: Long, dimension: String, value: Long, result: Long = -1)
+
+object TrxLogResult {
+ implicit val writes = Json.writes[TrxLogResult]
+ implicit val reads = Json.reads[TrxLogResult]
+ implicit val formats = Json.format[TrxLogResult]
+}
+
+object TrxLog {
+ implicit val writes = Json.writes[TrxLog]
+ implicit val reads = Json.reads[TrxLog]
+ implicit val formats = Json.format[TrxLog]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/ConfigFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/ConfigFunctions.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/ConfigFunctions.scala
new file mode 100644
index 0000000..219568b
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/ConfigFunctions.scala
@@ -0,0 +1,30 @@
+package org.apache.s2graph.counter.config
+
+import com.typesafe.config.Config
+
+import scala.collection.JavaConversions._
+import scala.reflect.ClassTag
+
+abstract class ConfigFunctions(conf: Config) {
+ def getOrElse[T: ClassTag](key: String, default: T): T = {
+ val ret = if (conf.hasPath(key)) (default match {
+ case _: String => conf.getString(key)
+ case _: Int | _: Integer => conf.getInt(key)
+ case _: Float | _: Double => conf.getDouble(key)
+ case _: Boolean => conf.getBoolean(key)
+ case _ => default
+ }).asInstanceOf[T]
+ else default
+ println(s"${this.getClass.getName}: $key -> $ret")
+ ret
+ }
+
+ def getConfigMap(path: String): Map[String, String] = {
+ conf.getConfig(path).entrySet().map { entry =>
+ val key = s"$path.${entry.getKey}"
+ val value = conf.getString(key)
+ println(s"${this.getClass.getName}: $key -> $value")
+ key -> value
+ }.toMap
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/S2CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/S2CounterConfig.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/S2CounterConfig.scala
new file mode 100644
index 0000000..c9e1e38
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/config/S2CounterConfig.scala
@@ -0,0 +1,44 @@
+package org.apache.s2graph.counter.config
+
+import com.typesafe.config.Config
+
+import scala.collection.JavaConversions._
+
+class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
+ // HBase
+ lazy val HBASE_ZOOKEEPER_QUORUM = getOrElse("hbase.zookeeper.quorum", "")
+ lazy val HBASE_TABLE_NAME = getOrElse("hbase.table.name", "s2counter")
+ lazy val HBASE_TABLE_POOL_SIZE = getOrElse("hbase.table.pool.size", 100)
+ lazy val HBASE_CONNECTION_POOL_SIZE = getOrElse("hbase.connection.pool.size", 10)
+
+ lazy val HBASE_CLIENT_IPC_POOL_SIZE = getOrElse("hbase.client.ipc.pool.size", 5)
+ lazy val HBASE_CLIENT_MAX_TOTAL_TASKS = getOrElse("hbase.client.max.total.tasks", 100)
+ lazy val HBASE_CLIENT_MAX_PERSERVER_TASKS = getOrElse("hbase.client.max.perserver.tasks", 5)
+ lazy val HBASE_CLIENT_MAX_PERREGION_TASKS = getOrElse("hbase.client.max.perregion.tasks", 1)
+ lazy val HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD = getOrElse("hbase.client.scanner.timeout.period", 300)
+ lazy val HBASE_CLIENT_OPERATION_TIMEOUT = getOrElse("hbase.client.operation.timeout", 100)
+ lazy val HBASE_CLIENT_RETRIES_NUMBER = getOrElse("hbase.client.retries.number", 1)
+
+ // MySQL
+ lazy val DB_DEFAULT_DRIVER = getOrElse("db.default.driver", "com.mysql.jdbc.Driver")
+ lazy val DB_DEFAULT_URL = getOrElse("db.default.url", "")
+ lazy val DB_DEFAULT_USER = getOrElse("db.default.user", "graph")
+ lazy val DB_DEFAULT_PASSWORD = getOrElse("db.default.password", "graph")
+
+ // Redis
+ lazy val REDIS_INSTANCES = (for {
+ s <- config.getStringList("redis.instances")
+ } yield {
+ val sp = s.split(':')
+ (sp(0), if (sp.length > 1) sp(1).toInt else 6379)
+ }).toList
+
+ // Graph
+ lazy val GRAPH_URL = getOrElse("s2graph.url", "http://localhost:9000")
+ lazy val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
+
+ // Cache
+ lazy val CACHE_TTL_SECONDS = getOrElse("cache.ttl.seconds", 600)
+ lazy val CACHE_MAX_SIZE = getOrElse("cache.max.size", 10000)
+ lazy val CACHE_NEGATIVE_TTL_SECONDS = getOrElse("cache.negative.ttl.seconds", CACHE_TTL_SECONDS)
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/BytesUtil.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/BytesUtil.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/BytesUtil.scala
new file mode 100644
index 0000000..0f48b0b
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/BytesUtil.scala
@@ -0,0 +1,13 @@
+package org.apache.s2graph.counter.core
+
+
+trait BytesUtil {
+ def getRowKeyPrefix(id: Int): Array[Byte]
+
+ def toBytes(key: ExactKeyTrait): Array[Byte]
+ def toBytes(eq: ExactQualifier): Array[Byte]
+ def toBytes(tq: TimedQualifier): Array[Byte]
+
+ def toExactQualifier(bytes: Array[Byte]): ExactQualifier
+ def toTimedQualifier(bytes: Array[Byte]): TimedQualifier
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactCounter.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactCounter.scala
new file mode 100644
index 0000000..a7e40ae
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactCounter.scala
@@ -0,0 +1,243 @@
+package org.apache.s2graph.counter.core
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit.IntervalUnit
+import org.apache.s2graph.counter.{TrxLogResult, TrxLog}
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit.IntervalUnit
+import org.apache.s2graph.counter.decay.ExpDecayFormula
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.util.{FunctionParser, CollectionCacheConfig, CollectionCache}
+import org.slf4j.LoggerFactory
+import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext, Future}
+
+
+
+case class ExactCounterRow(key: ExactKeyTrait, value: Map[ExactQualifier, Long])
+case class FetchedCounts(exactKey: ExactKeyTrait, qualifierWithCountMap: Map[ExactQualifier, Long])
+case class DecayedCounts(exactKey: ExactKeyTrait, qualifierWithCountMap: Map[ExactQualifier, Double])
+case class FetchedCountsGrouped(exactKey: ExactKeyTrait, intervalWithCountMap: Map[(IntervalUnit, Map[String, String]), Map[ExactQualifier, Long]])
+
+class ExactCounter(config: Config, storage: ExactStorage) {
+ import ExactCounter._
+
+ val syncDuration = Duration(10, SECONDS)
+ private val log = LoggerFactory.getLogger(getClass)
+
+ val storageStatusCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(1000, 60, negativeCache = false, 60))
+
+ // dimension: age, value of ages
+ def getCountAsync(policy: Counter,
+ itemId: String,
+ intervals: Seq[IntervalUnit],
+ from: Long,
+ to: Long,
+ dimension: Map[String, Set[String]])
+ (implicit ex: ExecutionContext): Future[Option[FetchedCountsGrouped]] = {
+ for {
+ fetchedCounts <- getCountsAsync(policy, Seq(itemId),
+ intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))), dimension)
+ } yield {
+ fetchedCounts.headOption
+ }
+ }
+
+ // multi item, time range and multi dimension
+ def getCountsAsync(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
+ storage.get(policy, items, timeRange, dimQuery)
+ }
+
+ def getCount(policy: Counter,
+ itemId: String,
+ intervals: Seq[IntervalUnit],
+ from: Long,
+ to: Long,
+ dimension: Map[String, Set[String]])
+ (implicit ex: ExecutionContext): Option[FetchedCountsGrouped] = {
+ getCounts(policy, Seq(itemId),
+ intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))), dimension).headOption
+ }
+
+ def getCount(policy: Counter,
+ itemId: String,
+ intervals: Seq[IntervalUnit],
+ from: Long,
+ to: Long)
+ (implicit ex: ExecutionContext): Option[FetchedCounts] = {
+ val future = storage.get(policy,
+ Seq(itemId),
+ intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))))
+ Await.result(future, syncDuration).headOption
+ }
+
+ // multi item, time range and multi dimension
+ def getCounts(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ex: ExecutionContext): Seq[FetchedCountsGrouped] = {
+ Await.result(storage.get(policy, items, timeRange, dimQuery), syncDuration)
+ }
+
+ def getRelatedCounts(policy: Counter, keyWithQualifiers: Seq[(String, Seq[ExactQualifier])])
+ (implicit ex: ExecutionContext): Map[String, Map[ExactQualifier, Long]] = {
+ val queryKeyWithQualifiers = {
+ for {
+ (itemKey, qualifiers) <- keyWithQualifiers
+ } yield {
+ val relKey = ExactKey(policy.id, policy.version, policy.itemType, itemKey)
+ (relKey, qualifiers)
+ }
+ }
+ val future = storage.get(policy, queryKeyWithQualifiers)
+
+ for {
+ FetchedCounts(exactKey, exactQualifierToLong) <- Await.result(future, syncDuration)
+ } yield {
+ exactKey.itemKey -> exactQualifierToLong
+ }
+ }.toMap
+
+ def getPastCounts(policy: Counter, keyWithQualifiers: Seq[(String, Seq[ExactQualifier])])
+ (implicit ex: ExecutionContext): Map[String, Map[ExactQualifier, Long]] = {
+ // query paste count
+ val queryKeyWithQualifiers = {
+ for {
+ (itemKey, qualifiers) <- keyWithQualifiers
+ } yield {
+ val relKey = ExactKey(policy.id, policy.version, policy.itemType, itemKey)
+ (relKey, qualifiers.map(eq => eq.copy(tq = eq.tq.add(-1))))
+ }
+ }
+ val future = storage.get(policy, queryKeyWithQualifiers)
+
+ for {
+ FetchedCounts(exactKey, exactQualifierToLong) <- Await.result(future, syncDuration)
+ } yield {
+ // restore tq
+ exactKey.itemKey -> exactQualifierToLong.map { case (eq, v) =>
+ eq.copy(tq = eq.tq.add(1)) -> v
+ }
+ }
+ }.toMap
+
+ def getDecayedCountsAsync(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQuery: Map[String, Set[String]],
+ qsSum: Option[String])(implicit ex: ExecutionContext): Future[Seq[DecayedCounts]] = {
+ val groupedTimeRange = timeRange.groupBy(_._1.q)
+ getCountsAsync(policy, items, timeRange, dimQuery).map { seq =>
+ for {
+ FetchedCountsGrouped(k, intervalWithCountMap) <- seq
+ } yield {
+ DecayedCounts(k, {
+ for {
+ ((interval, dimKeyValues), grouped) <- intervalWithCountMap
+ } yield {
+ val (tqFrom, tqTo) = groupedTimeRange(interval).head
+ val formula = {
+ for {
+ strSum <- qsSum
+ (func, arg) <- FunctionParser(strSum)
+ } yield {
+ // apply function
+ func.toLowerCase match {
+ case "exp_decay" => ExpDecayFormula.byMeanLifeTime(arg.toLong * TimedQualifier.getTsUnit(interval))
+ case _ => throw new UnsupportedOperationException(s"unknown function: $strSum")
+ }
+ }
+ }
+ ExactQualifier(tqFrom, dimKeyValues) -> {
+ grouped.map { case (eq, count) =>
+ formula match {
+ case Some(decay) =>
+ decay(count, tqTo.ts - eq.tq.ts)
+ case None =>
+ count
+ }
+ }.sum
+ }
+ }
+ })
+ }
+ }
+ }
+
+ def updateCount(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Seq[TrxLog] = {
+ ready(policy) match {
+ case true =>
+ val updateResults = storage.update(policy, counts)
+ for {
+ (exactKey, values) <- counts
+ results = updateResults.getOrElse(exactKey, Nil.toMap)
+ } yield {
+ TrxLog(results.nonEmpty, exactKey.policyId, exactKey.itemKey, makeTrxLogResult(values, results))
+ }
+ case false =>
+ Nil
+ }
+ }
+
+ def deleteCount(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
+ storage.delete(policy, keys)
+ }
+
+ private def makeTrxLogResult(values: ExactValueMap, results: ExactValueMap): Seq[TrxLogResult] = {
+ for {
+ (eq, value) <- values
+ } yield {
+ val result = results.getOrElse(eq, -1l)
+ TrxLogResult(eq.tq.q.toString, eq.tq.ts, eq.dimension, value, result)
+ }
+ }.toSeq
+
+ def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
+ storage.insertBlobValue(policy, keys)
+ }
+
+ def getBlobValue(policy: Counter, blobId: String): Option[String] = {
+ storage.getBlobValue(policy, blobId)
+ }
+
+ def prepare(policy: Counter) = {
+ storage.prepare(policy)
+ }
+
+ def destroy(policy: Counter) = {
+ storage.destroy(policy)
+ }
+
+ def ready(policy: Counter): Boolean = {
+ storageStatusCache.withCache(s"${policy.id}") {
+ val ready = storage.ready(policy)
+ if (!ready) {
+ // if key is not in cache, log message
+ log.warn(s"${policy.service}.${policy.action} storage is not ready.")
+ }
+ Some(ready)
+ }.getOrElse(false)
+ }
+}
+
+object ExactCounter {
+ object ColumnFamily extends Enumeration {
+ type ColumnFamily = Value
+
+ val SHORT = Value("s")
+ val LONG = Value("l")
+ }
+ import IntervalUnit._
+ val intervalsMap = Map(MINUTELY -> ColumnFamily.SHORT, HOURLY -> ColumnFamily.SHORT,
+ DAILY -> ColumnFamily.LONG, MONTHLY -> ColumnFamily.LONG, TOTAL -> ColumnFamily.LONG)
+
+ val blobCF = ColumnFamily.LONG.toString.getBytes
+ val blobColumn = "b".getBytes
+
+ type ExactValueMap = Map[ExactQualifier, Long]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactKey.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactKey.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactKey.scala
new file mode 100644
index 0000000..b472649
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactKey.scala
@@ -0,0 +1,26 @@
+package org.apache.s2graph.counter.core
+
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.apache.s2graph.counter.models.Counter.ItemType.ItemType
+import org.apache.s2graph.counter.util.Hashes
+
+trait ExactKeyTrait {
+ def policyId: Int
+ def version: Byte
+ def itemType: ItemType
+ def itemKey: String
+}
+
+case class ExactKey(policyId: Int, version: Byte, itemType: ItemType, itemKey: String) extends ExactKeyTrait
+case class BlobExactKey(policyId: Int, version: Byte, itemType: ItemType, itemKey: String, itemId: String) extends ExactKeyTrait
+
+object ExactKey {
+ def apply(policy: Counter, itemId: String, checkItemType: Boolean): ExactKeyTrait = {
+ if (checkItemType && policy.itemType == ItemType.BLOB) {
+ BlobExactKey(policy.id, policy.version, ItemType.BLOB, Hashes.sha1(itemId), itemId)
+ } else {
+ ExactKey(policy.id, policy.version, policy.itemType, itemId)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactQualifier.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactQualifier.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactQualifier.scala
new file mode 100644
index 0000000..96104bf
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactQualifier.scala
@@ -0,0 +1,73 @@
+package org.apache.s2graph.counter.core
+
+import java.util
+import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit.IntervalUnit
+import scala.collection.JavaConversions._
+
+case class ExactQualifier(tq: TimedQualifier, dimKeyValues: Map[String, String], dimension: String) {
+ def checkDimensionEquality(dimQuery: Map[String, Set[String]]): Boolean = {
+// println(s"self: $dimKeyValues, query: $dimQuery")
+ dimQuery.size == dimKeyValues.size && {
+ for {
+ (k, v) <- dimKeyValues
+ } yield {
+ dimQuery.get(k).exists(qv => qv.isEmpty || qv.contains(v))
+ }
+ }.forall(x => x)
+ }
+}
+
+object ExactQualifier {
+ val cache: LoadingCache[String, Map[String, String]] = CacheBuilder.newBuilder()
+ .maximumSize(10000)
+ .build(
+ new CacheLoader[String, Map[String, String]]() {
+ def load(s: String): Map[String, String] = {
+ strToDimensionMap(s)
+ }
+ }
+ )
+
+ def apply(tq: TimedQualifier, dimension: String): ExactQualifier = {
+ ExactQualifier(tq, cache.get(dimension), dimension)
+ }
+
+ def apply(tq: TimedQualifier, dimKeyValues: Map[String, String]): ExactQualifier = {
+ ExactQualifier(tq, dimKeyValues, makeDimensionStr(dimKeyValues))
+ }
+
+ def makeSortedDimension(dimKeyValues: Map[String, String]): Iterator[String] = {
+ val sortedDimKeyValues = new util.TreeMap[String, String](dimKeyValues)
+ sortedDimKeyValues.keysIterator ++ sortedDimKeyValues.valuesIterator
+ }
+
+ def makeDimensionStr(dimKeyValues: Map[String, String]): String = {
+ makeSortedDimension(dimKeyValues).mkString(".")
+ }
+
+ def getQualifiers(intervals: Seq[IntervalUnit], ts: Long, dimKeyValues: Map[String, String]): Seq[ExactQualifier] = {
+ for {
+ tq <- TimedQualifier.getQualifiers(intervals, ts)
+ } yield {
+ ExactQualifier(tq, dimKeyValues, makeDimensionStr(dimKeyValues))
+ }
+ }
+
+ def strToDimensionMap(dimension: String): Map[String, String] = {
+ val dimSp = {
+ val sp = dimension.split('.')
+ if (dimension == ".") {
+ Array("", "")
+ }
+ else if (dimension.nonEmpty && dimension.last == '.') {
+ sp ++ Array("")
+ } else {
+ sp
+ }
+ }
+ val dimKey = dimSp.take(dimSp.length / 2)
+ val dimVal = dimSp.takeRight(dimSp.length / 2)
+ dimKey.zip(dimVal).toMap
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactStorage.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactStorage.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactStorage.scala
new file mode 100644
index 0000000..c34e631
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/ExactStorage.scala
@@ -0,0 +1,32 @@
+package org.apache.s2graph.counter.core
+
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.models.Counter
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait ExactStorage {
+ // for range query and check dimension
+ def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(TimedQualifier, TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]]
+ // for range query
+ def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(TimedQualifier, TimedQualifier)])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]]
+ // for query exact qualifier
+ def get(policy: Counter,
+ queries: Seq[(ExactKeyTrait, Seq[ExactQualifier])])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]]
+ def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap]
+ def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit
+ def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean]
+ def getBlobValue(policy: Counter, blobId: String): Option[String]
+
+ def prepare(policy: Counter): Unit
+ def destroy(policy: Counter): Unit
+ def ready(policy: Counter): Boolean
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingCounter.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingCounter.scala
new file mode 100644
index 0000000..e3ea1b8
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingCounter.scala
@@ -0,0 +1,101 @@
+package org.apache.s2graph.counter.core
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.util.{CollectionCacheConfig, CollectionCache}
+import org.slf4j.LoggerFactory
+import scala.collection.JavaConversions._
+
+case class RankingRow(key: RankingKey, value: Map[String, RankingValue])
+case class RateRankingRow(key: RankingKey, value: Map[String, RateRankingValue])
+
+class RankingCounter(config: Config, storage: RankingStorage) {
+ private val log = LoggerFactory.getLogger(getClass)
+
+ val storageStatusCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(1000, 60, negativeCache = false, 60))
+
+ val cache: LoadingCache[RankingKey, RankingResult] = CacheBuilder.newBuilder()
+ .maximumSize(1000000)
+ .expireAfterWrite(10l, TimeUnit.MINUTES)
+ .build(
+ new CacheLoader[RankingKey, RankingResult]() {
+ def load(rankingKey: RankingKey): RankingResult = {
+// log.warn(s"cache load: $rankingKey")
+ storage.getTopK(rankingKey, Int.MaxValue).getOrElse(RankingResult(-1, Nil))
+ }
+ }
+ )
+
+ def getTopK(rankingKey: RankingKey, k: Int = Int.MaxValue): Option[RankingResult] = {
+ val tq = rankingKey.eq.tq
+ if (TimedQualifier.getQualifiers(Seq(tq.q), System.currentTimeMillis()).head == tq) {
+ // do not use cache
+ storage.getTopK(rankingKey, k)
+ }
+ else {
+ val result = cache.get(rankingKey)
+ if (result.values.nonEmpty) {
+ Some(result.copy(values = result.values.take(k)))
+ }
+ else {
+ None
+ }
+ }
+ }
+
+ def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
+ storage.update(key, value, k)
+ }
+
+ def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
+ storage.update(values, k)
+ }
+
+ def delete(key: RankingKey): Unit = {
+ storage.delete(key)
+ }
+
+ def getAllItems(keys: Seq[RankingKey], k: Int = Int.MaxValue): Seq[String] = {
+ val oldKeys = keys.filter(key => TimedQualifier.getQualifiers(Seq(key.eq.tq.q), System.currentTimeMillis()).head != key.eq.tq)
+ val cached = cache.getAllPresent(oldKeys)
+ val missed = keys.diff(cached.keys.toSeq)
+ val found = storage.getTopK(missed, k)
+
+// log.warn(s"cached: ${cached.size()}, missed: ${missed.size}")
+
+ for {
+ (key, result) <- found
+ } {
+ cache.put(key, result)
+ }
+
+ for {
+ (key, RankingResult(totalScore, values)) <- cached ++ found
+ (item, score) <- values
+ } yield {
+ item
+ }
+ }.toSeq.distinct
+
+ def prepare(policy: Counter): Unit = {
+ storage.prepare(policy)
+ }
+
+ def destroy(policy: Counter): Unit = {
+ storage.destroy(policy)
+ }
+
+ def ready(policy: Counter): Boolean = {
+ storageStatusCache.withCache(s"${policy.id}") {
+ Some(storage.ready(policy))
+ }.getOrElse(false)
+ }
+}
+
+object RankingCounter {
+ type RankingValueMap = Map[String, RankingValue]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingKey.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingKey.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingKey.scala
new file mode 100644
index 0000000..9e3dc9a
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingKey.scala
@@ -0,0 +1,4 @@
+package org.apache.s2graph.counter.core
+
+
+case class RankingKey(policyId: Int, version: Byte, eq: ExactQualifier)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingResult.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingResult.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingResult.scala
new file mode 100644
index 0000000..e738614
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingResult.scala
@@ -0,0 +1,3 @@
+package org.apache.s2graph.counter.core
+
+case class RankingResult(totalScore: Double, values: Seq[(String, Double)])
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingStorage.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingStorage.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingStorage.scala
new file mode 100644
index 0000000..34af94f
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingStorage.scala
@@ -0,0 +1,17 @@
+package org.apache.s2graph.counter.core
+
+import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.models.Counter
+
+
+trait RankingStorage {
+ def getTopK(key: RankingKey, k: Int): Option[RankingResult]
+ def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)]
+ def update(key: RankingKey, value: RankingValueMap, k: Int): Unit
+ def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit
+ def delete(key: RankingKey)
+
+ def prepare(policy: Counter): Unit
+ def destroy(policy: Counter): Unit
+ def ready(policy: Counter): Boolean
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingValue.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingValue.scala
new file mode 100644
index 0000000..7691f35
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RankingValue.scala
@@ -0,0 +1,15 @@
+package org.apache.s2graph.counter.core
+
+/**
+ * ranking score and increment value
+ * @param score ranking score
+ * @param increment increment value for v1
+ */
+case class RankingValue(score: Double, increment: Double)
+
+object RankingValue {
+ def reduce(r1: RankingValue, r2: RankingValue): RankingValue = {
+ // maximum score and sum of increment
+ RankingValue(math.max(r1.score, r2.score), r1.increment + r2.increment)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RateRankingValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RateRankingValue.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RateRankingValue.scala
new file mode 100644
index 0000000..370926f
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/RateRankingValue.scala
@@ -0,0 +1,15 @@
+package org.apache.s2graph.counter.core
+
+case class RateRankingValue(actionScore: Double, baseScore: Double) {
+ // increment score do not use.
+ lazy val rankingValue: RankingValue = {
+ RankingValue(actionScore / math.max(1d, baseScore), 0)
+ }
+}
+
+object RateRankingValue {
+ def reduce(r1: RateRankingValue, r2: RateRankingValue): RateRankingValue = {
+ // maximum score and sum of increment
+ RateRankingValue(math.max(r1.actionScore, r2.actionScore), math.max(r1.baseScore, r2.baseScore))
+ }
+}
[11/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala
new file mode 100644
index 0000000..d67de7c
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/EdgeTest.scala
@@ -0,0 +1,535 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs, VertexId}
+import org.apache.s2graph.core.utils.logger
+import org.scalatest.FunSuite
+
+class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
+ initTests()
+
+ test("toLogString") {
+ val testLabelName = labelNameV2
+ val bulkQueries = List(
+ ("1445240543366", "update", "{\"is_blocked\":true}"),
+ ("1445240543362", "insert", "{\"is_hidden\":false}"),
+ ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
+ ("1445240543363", "delete", "{}"),
+ ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
+
+ val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
+
+ val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
+ Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
+ }).mkString("\n")
+
+ val expected = Seq(
+ Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"),
+ Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"),
+ Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"),
+ Seq("1445240543363", "delete", "e", "1", "2", testLabelName),
+ Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}")
+ ).map(_.mkString("\t")).mkString("\n")
+
+ assert(bulkEdge === expected)
+ }
+
+ test("buildOperation") {
+ val schemaVersion = "v2"
+ val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
+ val srcVertex = Vertex(vertexId)
+ val tgtVertex = srcVertex
+
+ val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
+
+ val snapshotEdge = None
+ val propsWithTs = Map(timestampProp)
+ val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
+ val newVersion = 0L
+
+ val newPropsWithTs = Map(
+ timestampProp,
+ 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1)
+ )
+
+ val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
+ logger.info(edgeMutate.toLogString)
+
+ assert(edgeMutate.newSnapshotEdge.isDefined)
+ assert(edgeMutate.edgesToInsert.nonEmpty)
+ assert(edgeMutate.edgesToDelete.isEmpty)
+ }
+
+ test("buildMutation: snapshotEdge: None with newProps") {
+ val schemaVersion = "v2"
+ val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
+ val srcVertex = Vertex(vertexId)
+ val tgtVertex = srcVertex
+
+ val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
+
+ val snapshotEdge = None
+ val propsWithTs = Map(timestampProp)
+ val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
+ val newVersion = 0L
+
+ val newPropsWithTs = Map(
+ timestampProp,
+ 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1)
+ )
+
+ val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
+ logger.info(edgeMutate.toLogString)
+
+ assert(edgeMutate.newSnapshotEdge.isDefined)
+ assert(edgeMutate.edgesToInsert.nonEmpty)
+ assert(edgeMutate.edgesToDelete.isEmpty)
+ }
+
+ test("buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests") {
+ val schemaVersion = "v2"
+ val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
+ val srcVertex = Vertex(vertexId)
+ val tgtVertex = srcVertex
+
+ val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
+
+ val snapshotEdge = None
+ val propsWithTs = Map(timestampProp)
+ val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
+ val newVersion = 0L
+
+ val newPropsWithTs = propsWithTs
+
+ val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
+ logger.info(edgeMutate.toLogString)
+
+ assert(edgeMutate.newSnapshotEdge.isEmpty)
+ assert(edgeMutate.edgesToInsert.isEmpty)
+ assert(edgeMutate.edgesToDelete.isEmpty)
+ }
+
+ test("buildMutation: All props older than snapshotEdge's LastDeletedAt") {
+ val schemaVersion = "v2"
+ val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
+ val srcVertex = Vertex(vertexId)
+ val tgtVertex = srcVertex
+
+ val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
+ val oldPropsWithTs = Map(
+ timestampProp,
+ LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
+ )
+
+ val propsWithTs = Map(
+ timestampProp,
+ 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 2),
+ LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
+ )
+
+ val snapshotEdge =
+ Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs))
+
+ val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
+ val newVersion = 0L
+ val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
+ logger.info(edgeMutate.toLogString)
+
+ assert(edgeMutate.newSnapshotEdge.nonEmpty)
+ assert(edgeMutate.edgesToInsert.isEmpty)
+ assert(edgeMutate.edgesToDelete.isEmpty)
+ }
+
+ test("buildMutation: All props newer than snapshotEdge's LastDeletedAt") {
+ val schemaVersion = "v2"
+ val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
+ val srcVertex = Vertex(vertexId)
+ val tgtVertex = srcVertex
+
+ val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
+ val oldPropsWithTs = Map(
+ timestampProp,
+ LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
+ )
+
+ val propsWithTs = Map(
+ timestampProp,
+ 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 4),
+ LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
+ )
+
+ val snapshotEdge =
+ Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs))
+
+ val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
+ val newVersion = 0L
+ val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
+ logger.info(edgeMutate.toLogString)
+
+ assert(edgeMutate.newSnapshotEdge.nonEmpty)
+ assert(edgeMutate.edgesToInsert.nonEmpty)
+ assert(edgeMutate.edgesToDelete.isEmpty)
+ }
+}
+
+//import com.kakao.s2graph.core.types._
+//import org.hbase.async.PutRequest
+//import org.scalatest.{FunSuite, Matchers}
+//
+///**
+// * Created by shon on 5/29/15.
+// */
+//class EdgeTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels {
+//
+//
+// import HBaseType.{VERSION1, VERSION2}
+//
+//
+// def srcVertex(innerVal: InnerValLike)(version: String) = {
+// val colId = if (version == VERSION1) column.id.get else columnV2.id.get
+// Vertex(SourceVertexId(colId, innerVal), ts)
+// }
+//
+// def tgtVertex(innerVal: InnerValLike)(version: String) = {
+// val colId = if (version == VERSION1) column.id.get else columnV2.id.get
+// Vertex(TargetVertexId(colId, innerVal), ts)
+// }
+//
+//
+// def testEdges(version: String) = {
+// val innerVals = if (version == VERSION1) intInnerVals else intInnerValsV2
+// val tgtV = tgtVertex(innerVals.head)(version)
+// innerVals.tail.map { intInnerVal =>
+// val labelWithDirection = if (version == VERSION1) labelWithDir else labelWithDirV2
+// val idxPropsList = if (version == VERSION1) idxPropsLs else idxPropsLsV2
+// idxPropsList.map { idxProps =>
+// val currentTs = idxProps.toMap.get(0.toByte).get.toString.toLong
+// val idxPropsWithTs = idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, currentTs) }
+//
+// Edge(srcVertex(intInnerVal)(version), tgtV, labelWithDirection, op, currentTs, currentTs, idxPropsWithTs.toMap)
+// }
+// }
+// }
+//
+// def testPropsUpdate(oldProps: Map[Byte, InnerValLikeWithTs],
+// newProps: Map[Byte, InnerValLikeWithTs],
+// expected: Map[Byte, Any],
+// expectedShouldUpdate: Boolean)
+// (f: PropsPairWithTs => (Map[Byte, InnerValLikeWithTs], Boolean))(version: String) = {
+//
+// val timestamp = newProps.toList.head._2.ts
+// val (updated, shouldUpdate) = f((oldProps, newProps, timestamp, version))
+// val rets = for {
+// (k, v) <- expected
+// } yield {
+// v match {
+// case v: String =>
+// v match {
+// case "left" => updated.get(k).isDefined && updated(k) == oldProps(k)
+// case "right" => updated.get(k).isDefined && updated(k) == newProps(k)
+// case "none" => updated.get(k).isEmpty
+// }
+// case value: InnerValLikeWithTs => updated.get(k).get == value
+// case _ => throw new RuntimeException(s"not supported keyword: $v")
+// }
+// }
+// println(rets)
+// rets.forall(x => x) && shouldUpdate == expectedShouldUpdate
+// }
+//
+// def testEdgeWithIndex(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = {
+// val rets = for {
+// edgeForSameTgtVertex <- edges
+// } yield {
+// val head = edgeForSameTgtVertex.head
+// val start = head
+// var prev = head
+// val rets = for {
+// edge <- edgeForSameTgtVertex.tail
+// } yield {
+// println(s"prevEdge: $prev")
+// println(s"currentEdge: $edge")
+// val prevEdgeWithIndex = edge.edgesWithIndex
+// val edgesWithIndex = edge.edgesWithIndex
+//
+// /** test encodeing decoding */
+// for {
+// edgeWithIndex <- edge.edgesWithIndex
+// put <- edgeWithIndex.buildPutsAsync()
+// kv <- putToKeyValues(put.asInstanceOf[PutRequest])
+// } {
+// val decoded = Edge.toEdge(kv, queryParam, None, Seq())
+// val comp = decoded.isDefined && decoded.get == edge
+// println(s"${decoded.get}")
+// println(s"$edge")
+// println(s"${decoded.get == edge}")
+// comp shouldBe true
+//
+// /** test order
+// * same source, target vertex. same indexProps keys.
+// * only difference is indexProps values so comparing qualifier is good enough
+// * */
+// for {
+// prevEdgeWithIndex <- prev.edgesWithIndex
+// } {
+// println(edgeWithIndex.qualifier)
+// println(prevEdgeWithIndex.qualifier)
+// println(edgeWithIndex.qualifier.bytes.toList)
+// println(prevEdgeWithIndex.qualifier.bytes.toList)
+// /** since index of this test label only use 0, 1 as indexProps
+// * if 0, 1 is not different then qualifier bytes should be same
+// * */
+// val comp = lessThanEqual(edgeWithIndex.qualifier.bytes, prevEdgeWithIndex.qualifier.bytes)
+// comp shouldBe true
+// }
+// }
+// prev = edge
+// }
+// }
+// }
+//
+// def testInvertedEdge(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = {
+// val rets = for {
+// edgeForSameTgtVertex <- edges
+// } yield {
+// val head = edgeForSameTgtVertex.head
+// val start = head
+// var prev = head
+// val rets = for {
+// edge <- edgeForSameTgtVertex.tail
+// } yield {
+// println(s"prevEdge: $prev")
+// println(s"currentEdge: $edge")
+// val prevEdgeWithIndexInverted = prev.toSnapshotEdge
+// val edgeWithInvertedIndex = edge.toSnapshotEdge
+// /** test encode decoding */
+//
+// val put = edgeWithInvertedIndex.buildPutAsync()
+// for {
+// kv <- putToKeyValues(put)
+// } yield {
+// val decoded = Edge.toSnapshotEdge(kv, queryParam, None, isInnerCall = false, Seq())
+// val comp = decoded.isDefined && decoded.get == edge
+// println(s"${decoded.get}")
+// println(s"$edge")
+// println(s"${decoded.get == edge}")
+// comp shouldBe true
+//
+// /** no need to test ordering because qualifier only use targetVertexId */
+// }
+// prev = edge
+// }
+// }
+// }
+//
+// test("insert for edgesWithIndex version 2") {
+// val version = VERSION2
+// testEdgeWithIndex(testEdges(version))(queryParamV2)
+// }
+// test("insert for edgesWithIndex version 1") {
+// val version = VERSION1
+// testEdgeWithIndex(testEdges(version))(queryParam)
+// }
+//
+// test("insert for edgeWithInvertedIndex version 1") {
+// val version = VERSION1
+// testInvertedEdge(testEdges(version))(queryParam)
+// }
+//
+// test("insert for edgeWithInvertedIndex version 2") {
+// val version = VERSION2
+// testInvertedEdge(testEdges(version))(queryParamV2)
+// }
+//
+//
+// // /** test cases for each operation */
+//
+// def oldProps(timestamp: Long, version: String) = {
+// Map(
+// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp - 2, timestamp - 2, version),
+// 1.toByte -> InnerValLikeWithTs.withLong(0L, timestamp, version),
+// 2.toByte -> InnerValLikeWithTs.withLong(1L, timestamp - 1, version),
+// 4.toByte -> InnerValLikeWithTs.withStr("old", timestamp - 1, version)
+// )
+// }
+//
+// def newProps(timestamp: Long, version: String) = {
+// Map(
+// 2.toByte -> InnerValLikeWithTs.withLong(-10L, timestamp, version),
+// 3.toByte -> InnerValLikeWithTs.withLong(20L, timestamp, version)
+// )
+// }
+//
+// def deleteProps(timestamp: Long, version: String) = Map(
+// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp, timestamp, version)
+// )
+//
+// /** upsert */
+// test("Edge.buildUpsert") {
+// val shouldUpdate = true
+// val oldState = oldProps(ts, VERSION2)
+// val newState = newProps(ts + 1, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "none",
+// 2.toByte -> "right",
+// 3.toByte -> "right",
+// 4.toByte -> "none")
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true
+// }
+// test("Edge.buildUpsert shouldUpdate false") {
+// val shouldUpdate = false
+// val oldState = oldProps(ts, VERSION2)
+// val newState = newProps(ts - 10, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> "left",
+// 3.toByte -> "none",
+// 4.toByte -> "left")
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true
+// }
+//
+// /** update */
+// test("Edge.buildUpdate") {
+// val shouldUpdate = true
+// val oldState = oldProps(ts, VERSION2)
+// val newState = newProps(ts + 1, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> "right",
+// 3.toByte -> "right",
+// 4.toByte -> "left"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true
+// }
+// test("Edge.buildUpdate shouldUpdate false") {
+// val shouldUpdate = false
+// val oldState = oldProps(ts, VERSION2)
+// val newState = newProps(ts - 10, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> "left",
+// 3.toByte -> "none",
+// 4.toByte -> "left"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true
+// }
+//
+// /** delete */
+// test("Edge.buildDelete") {
+// val shouldUpdate = true
+// val oldState = oldProps(ts, VERSION2)
+// val newState = deleteProps(ts + 1, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "right",
+// 1.toByte -> "none",
+// 2.toByte -> "none",
+// 4.toByte -> "none"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true
+// }
+// test("Edge.buildDelete shouldUpdate false") {
+// val shouldUpdate = false
+// val oldState = oldProps(ts, VERSION2)
+// val newState = deleteProps(ts - 10, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> "left",
+// 4.toByte -> "left"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true
+// }
+//
+// /** increment */
+// test("Edge.buildIncrement") {
+// val shouldUpdate = true
+// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte)
+// val newState = newProps(ts + 1, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> InnerValLikeWithTs.withLong(-9L, ts - 1, VERSION2),
+// 3.toByte -> "right"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true
+// }
+// test("Edge.buildIncrement shouldRepalce false") {
+// val shouldUpdate = false
+// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte)
+// val newState = newProps(ts - 10, VERSION2)
+// val expected = Map(
+// labelMeta.lastDeletedAt -> "left",
+// 1.toByte -> "left",
+// 2.toByte -> "left"
+// )
+// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true
+// }
+//
+// test("Edge`s srcVertex") {
+//
+// val version = VERSION2
+// val srcId = InnerVal.withLong(10, version)
+// val tgtId = InnerVal.withStr("abc", version)
+// val srcColumn = columnV2
+// val tgtColumn = tgtColumnV2
+// val srcVertexId = VertexId(srcColumn.id.get, srcId)
+// val tgtVertexId = VertexId(tgtColumn.id.get, tgtId)
+//
+// val srcVertex = Vertex(srcVertexId)
+// val tgtVertex = Vertex(tgtVertexId)
+//
+// val labelId = undirectedLabelV2.id.get
+//
+// val outDir = LabelWithDirection(labelId, GraphUtil.directions("out"))
+// val inDir = LabelWithDirection(labelId, GraphUtil.directions("in"))
+// val bothDir = LabelWithDirection(labelId, GraphUtil.directions("undirected"))
+//
+// val op = GraphUtil.operations("insert")
+//
+//
+// val bothEdge = Edge(srcVertex, tgtVertex, bothDir)
+// println(s"edge: $bothEdge")
+// bothEdge.relatedEdges.foreach { edge =>
+// println(edge)
+// }
+//
+// }
+// test("edge buildIncrementBulk") {
+//
+// /**
+// * 172567371 List(97, 74, 2, 117, -74, -44, -76, 0, 0, 4, 8, 2)
+//169116518 List(68, -110, 2, 117, -21, 124, -103, 0, 0, 4, 9, 2)
+//11646834 List(17, 33, 2, 127, 78, 72, -115, 0, 0, 4, 9, 2)
+//148171217 List(62, 54, 2, 119, 43, 22, 46, 0, 0, 4, 9, 2)
+//116315188 List(41, 86, 2, 121, 17, 43, -53, 0, 0, 4, 9, 2)
+//180667876 List(48, -82, 2, 117, 59, 58, 27, 0, 0, 4, 8, 2)
+//4594410 List(82, 29, 2, 127, -71, -27, 21, 0, 0, 4, 8, 2)
+//151435444 List(1, 105, 2, 118, -7, 71, 75, 0, 0, 4, 8, 2)
+//168460895 List(67, -35, 2, 117, -11, 125, -96, 0, 0, 4, 9, 2)
+//7941614 List(115, 67, 2, 127, -122, -46, 17, 0, 0, 4, 8, 2)
+//171169732 List(61, -42, 2, 117, -52, 40, 59, 0, 0, 4, 9, 2)
+//174381375 List(91, 2, 2, 117, -101, 38, -64, 0, 0, 4, 9, 2)
+//12754019 List(9, -80, 2, 127, 61, 99, -100, 0, 0, 4, 9, 2)
+//175518092 List(111, 32, 2, 117, -119, -50, 115, 0, 0, 4, 8, 2)
+//174748531 List(28, -81, 2, 117, -107, -116, -116, 0, 0, 4, 8, 2)
+//
+// */
+// // val incrementsOpt = Edge.buildIncrementDegreeBulk("169116518", "talk_friend_long_term_agg_test", "out", 10)
+// //
+// // for {
+// // increments <- incrementsOpt
+// // increment <- increments
+// // (cf, qs) <- increment.getFamilyMapOfLongs
+// // (q, v) <- qs
+// // } {
+// // println(increment.getRow.toList)
+// // println(q.toList)
+// // println(v)
+// // }
+// //List(68, -110, -29, -4, 116, -24, 124, -37, 0, 0, 52, -44, 2)
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala
new file mode 100644
index 0000000..13f0b0a
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala
@@ -0,0 +1,226 @@
+package org.apache.s2graph.core.Integrate
+
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import play.api.libs.json.{JsObject, Json}
+
+class CrudTest extends IntegrateCommon {
+ import CrudHelper._
+ import TestUtil._
+
+ test("test CRUD") {
+ var tcNum = 0
+ var tcString = ""
+ var bulkQueries = List.empty[(Long, String, String)]
+ var expected = Map.empty[String, String]
+
+ val curTime = System.currentTimeMillis
+ val t1 = curTime + 0
+ val t2 = curTime + 1
+ val t3 = curTime + 2
+ val t4 = curTime + 3
+ val t5 = curTime + 4
+
+ val tcRunner = new CrudTestRunner()
+ tcNum = 1
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 2
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 3
+ tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 4
+ tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 5
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 6
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 7
+ tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 8
+ tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 9
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 10
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 11
+ tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 12
+ tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 13
+ tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
+ bulkQueries = List(
+ (t5, "update", "{\"is_blocked\": true}"),
+ (t1, "insert", "{\"is_hidden\": false}"),
+ (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
+ (t2, "delete", ""),
+ (t4, "update", "{\"time\": 1, \"weight\": -10}"))
+ expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ }
+
+
+ object CrudHelper {
+
+ class CrudTestRunner {
+ var seed = 0
+
+ def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
+ for {
+ labelName <- List(testLabelName, testLabelName2)
+ i <- 0 until NumOfEachTest
+ } {
+ seed += 1
+ val srcId = seed.toString
+ val tgtId = srcId
+
+ val maxTs = opWithProps.map(t => t._1).max
+
+ /** insert edges */
+ println(s"---- TC${tcNum}_init ----")
+ val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
+ TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
+ })
+
+ TestUtil.insertEdgesSync(bulkEdges: _*)
+
+ for {
+ label <- Label.findByName(labelName)
+ direction <- List("out", "in")
+ cacheTTL <- List(-1L)
+ } {
+ val (serviceName, columnName, id, otherId) = direction match {
+ case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
+ case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
+ }
+
+ val qId = if (labelName == testLabelName) id else "\"" + id + "\""
+ val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
+
+ val jsResult = TestUtil.getEdgesSync(query)
+
+ val results = jsResult \ "results"
+ val deegrees = (jsResult \ "degrees").as[List[JsObject]]
+ val propsLs = (results \\ "props").seq
+ (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
+
+ val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
+ val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
+
+ from should be(id.toString)
+ to should be(otherId.toString)
+ (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
+
+ for ((key, expectedVal) <- expected) {
+ propsLs.last.as[JsObject].keys.contains(key) should be(true)
+ (propsLs.last \ key).toString should be(expectedVal)
+ }
+ }
+ }
+ }
+
+ def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = Json.parse(
+ s""" { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id } ],
+ "steps": [ [ {
+ "label": "$labelName",
+ "direction": "$dir",
+ "offset": 0,
+ "limit": 10,
+ "cacheTTL": $cacheTTL }]]}""")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
new file mode 100644
index 0000000..86bc85c
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
@@ -0,0 +1,309 @@
+package org.apache.s2graph.core.Integrate
+
+import com.typesafe.config._
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.core.rest.{RequestParser, RestHandler}
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{Graph, GraphUtil, Management, PostProcess}
+import org.scalatest._
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext, Future}
+
+trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
+
+ import TestUtil._
+
+ var graph: Graph = _
+ var parser: RequestParser = _
+ var management: Management = _
+ var config: Config = _
+
+ override def beforeAll = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+ management = new Management(graph)
+ parser = new RequestParser(graph.config)
+ initTestData()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
+
+ /**
+ * Make Service, Label, Vertex for integrate test
+ */
+ def initTestData() = {
+ println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ Management.deleteService(testServiceName)
+
+ // 1. createService
+ val jsValue = Json.parse(createService)
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) =
+ parser.toServiceElements(jsValue)
+
+ val tryRes =
+ management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ println(s">> Service created : $createService, $tryRes")
+
+ val labelNames = Map(testLabelName -> testLabelNameCreate,
+ testLabelName2 -> testLabelName2Create,
+ testLabelNameV1 -> testLabelNameV1Create,
+ testLabelNameWeak -> testLabelNameWeakCreate)
+
+ for {
+ (labelName, create) <- labelNames
+ } {
+ Management.deleteLabel(labelName)
+ Label.findByName(labelName, useCache = false) match {
+ case None =>
+ val json = Json.parse(create)
+ val tryRes = for {
+ labelArgs <- parser.toLabelElements(json)
+ label <- (management.createLabel _).tupled(labelArgs)
+ } yield label
+
+ tryRes.get
+ case Some(label) =>
+ println(s">> Label already exist: $create, $label")
+ }
+ }
+
+ val vertexPropsKeys = List("age" -> "int")
+
+ vertexPropsKeys.map { case (key, keyType) =>
+ Management.addVertexProp(testServiceName, testColumnName, key, keyType)
+ }
+
+ println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ }
+
+
+ /**
+ * Test Helpers
+ */
+ object TestUtil {
+ implicit def ec = scala.concurrent.ExecutionContext.global
+
+ // def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
+ // val arr = for {
+ // (label, dir, from, to) <- params
+ // } yield {
+ // Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
+ // }
+ //
+ // val s = Json.toJson(arr)
+ // s
+ // }
+
+ def deleteAllSync(jsValue: JsValue) = {
+ val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
+ val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json)
+ val future = graph.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
+
+ future
+ })
+
+ Await.result(future, HttpRequestWaitingTime)
+ }
+
+ def getEdgesSync(queryJson: JsValue): JsValue = {
+ logger.info(Json.prettyPrint(queryJson))
+ val restHandler = new RestHandler(graph)
+ Await.result(restHandler.getEdgesAsync(queryJson)(PostProcess.toSimpleVertexArrJson), HttpRequestWaitingTime)
+ }
+
+ def insertEdgesSync(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ val jsResult = Await.result(req, HttpRequestWaitingTime)
+
+ jsResult
+ }
+
+ def insertEdgesAsync(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ req
+ }
+
+ def toEdge(elems: Any*): String = elems.mkString("\t")
+
+ // common tables
+ val testServiceName = "s2graph"
+ val testLabelName = "s2graph_label_test"
+ val testLabelName2 = "s2graph_label_test_2"
+ val testLabelNameV1 = "s2graph_label_test_v1"
+ val testLabelNameWeak = "s2graph_label_test_weak"
+ val testColumnName = "user_id_test"
+ val testColumnType = "long"
+ val testTgtColumnName = "item_id_test"
+ val testHTableName = "test-htable"
+ val newHTableName = "new-htable"
+ val index1 = "idx_1"
+ val index2 = "idx_2"
+
+ val NumOfEachTest = 30
+ val HttpRequestWaitingTime = Duration("60 seconds")
+
+ val createService = s"""{"serviceName" : "$testServiceName"}"""
+
+ val testLabelNameCreate =
+ s"""
+ {
+ "label": "$testLabelName",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testColumnName",
+ "tgtColumnType": "long",
+ "indices": [
+ {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
+ {"name": "$index2", "propNames": ["_timestamp"]}
+ ],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "schemaVersion": "v4",
+ "compressionAlgorithm": "gz",
+ "hTableName": "$testHTableName"
+ }"""
+
+ val testLabelName2Create =
+ s"""
+ {
+ "label": "$testLabelName2",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": false,
+ "schemaVersion": "v3",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameV1Create =
+ s"""
+ {
+ "label": "$testLabelNameV1",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "${testTgtColumnName}_v1",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": true,
+ "schemaVersion": "v1",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameWeakCreate =
+ s"""
+ {
+ "label": "$testLabelNameWeak",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "weak",
+ "isDirected": true,
+ "compressionAlgorithm": "gz"
+ }"""
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala
new file mode 100644
index 0000000..232e685
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala
@@ -0,0 +1,906 @@
+package org.apache.s2graph.core.Integrate
+
+import org.apache.s2graph.core.utils.logger
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json.{JsNumber, JsValue, Json}
+
+class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import TestUtil._
+
+ val insert = "insert"
+ val e = "e"
+ val weight = "weight"
+ val is_hidden = "is_hidden"
+
+ test("interval") {
+ def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "index": "$index",
+ "interval": {
+ "from": [ { "$prop": $fromVal } ],
+ "to": [ { "$prop": $toVal } ]
+ }
+ }
+ ]]
+ }
+ """)
+
+ var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
+ (edges \ "size").toString should be("2")
+
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
+ (edges \ "size").toString should be("2")
+ }
+
+ test("get edge with where condition") {
+ def queryWhere(id: Int, where: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "where": "${where}"
+ }
+ ]]
+ }""")
+
+ var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdgesSync(queryWhere(0, "_from=0"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("get edge exclude") {
+ def queryExclude(id: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ },
+ {
+ "label": "${testLabelName}",
+ "direction": "in",
+ "offset": 0,
+ "limit": 2,
+ "exclude": true
+ }
+ ]]
+ }""")
+
+ val result = getEdgesSync(queryExclude(0))
+ (result \ "results").as[List[JsValue]].size should be(1)
+ }
+
+ test("get edge groupBy property") {
+ def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
+ Json.obj(
+ "groupBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+ }
+
+ val result = getEdgesSync(queryGroupBy(0, Seq("weight")))
+ (result \ "size").as[Int] should be(2)
+ val weights = (result \ "results" \\ "groupBy").map { js =>
+ (js \ "weight").as[Int]
+ }
+ weights should contain(30)
+ weights should contain(40)
+
+ weights should not contain (10)
+ }
+
+ test("edge transform") {
+ def queryTransform(id: Int, transforms: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "transform": $transforms
+ }
+ ]]
+ }""")
+
+ var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdgesSync(queryTransform(0, "[[\"weight\"]]"))
+ (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "weight").map(_.toString).sorted)
+
+ result = getEdgesSync(queryTransform(0, "[[\"_from\"]]"))
+ (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "from").map(_.toString).sorted)
+ }
+
+ test("index") {
+ def queryIndex(ids: Seq[Int], indexName: String) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName))
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps)
+ js
+ }
+
+ // weight order
+ var result = getEdgesSync(queryIndex(Seq(0), "idx_1"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40))
+
+ // timestamp order
+ result = getEdgesSync(queryIndex(Seq(0), "idx_2"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30))
+ }
+
+ // "checkEdges" in {
+ // running(FakeApplication()) {
+ // val json = Json.parse( s"""
+ // [{"from": 0, "to": 1, "label": "$testLabelName"},
+ // {"from": 0, "to": 2, "label": "$testLabelName"}]
+ // """)
+ //
+ // def checkEdges(queryJson: JsValue): JsValue = {
+ // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
+ // contentAsJson(ret)
+ // }
+ //
+ // val res = checkEdges(json)
+ // val typeRes = res.isInstanceOf[JsArray]
+ // typeRes must equalTo(true)
+ //
+ // val fst = res.as[Seq[JsValue]].head \ "to"
+ // fst.as[Int] must equalTo(1)
+ //
+ // val snd = res.as[Seq[JsValue]].last \ "to"
+ // snd.as[Int] must equalTo(2)
+ // }
+ // }
+
+
+
+ test("duration") {
+ def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj(
+ "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100,
+ "duration" -> Json.obj("from" -> from, "to" -> to)))
+
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ Json.obj("srcVertices" -> $from, "steps" -> $steps)
+ }
+
+ // get all
+ var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+ // inclusive, exclusive
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+ insertEdgesSync(bulkEdges: _*)
+
+ // duration test after udpate
+ // get all
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+
+ // inclusive, exclusive
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ }
+
+
+ test("return tree") {
+ def queryParents(id: Long) = Json.parse(
+ s"""
+ {
+ "returnTree": true,
+ "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ }
+ ],[{
+ "label": "$testLabelName",
+ "direction": "in",
+ "offset": 0,
+ "limit": -1
+ }
+ ]]
+ }""".stripMargin)
+
+ val src = 100
+ val tgt = 200
+
+ insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
+
+ val result = TestUtil.getEdgesSync(queryParents(src))
+ val parents = (result \ "results").as[Seq[JsValue]]
+ val ret = parents.forall {
+ edge => (edge \ "parents").as[Seq[JsValue]].size == 1
+ }
+
+ ret should be(true)
+ }
+
+
+
+// test("pagination and _to") {
+// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
+// s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": $offset,
+// "limit": $limit,
+// "_to": $to
+// }
+// ]]
+// }
+// """)
+//
+// val src = System.currentTimeMillis().toInt
+//
+// val bulkEdges = Seq(
+// toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+// toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+// toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
+// toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
+// )
+// insertEdgesSync(bulkEdges: _*)
+//
+// var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
+// var edges = (result \ "results").as[List[JsValue]]
+//
+// edges.size should be(2)
+// (edges(0) \ "to").as[Long] should be(4)
+// (edges(1) \ "to").as[Long] should be(3)
+//
+// result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
+//
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size should be(2)
+// (edges(0) \ "to").as[Long] should be(3)
+// (edges(1) \ "to").as[Long] should be(2)
+//
+// result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size should be(1)
+// }
+
+ test("order by") {
+ def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj(
+ "srcVertices" -> Json.arr(
+ Json.obj(
+ "serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "id" -> id
+ )
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+ def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj(
+ "orderBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ // get edges
+ val edges = getEdgesSync(queryScore(0, Map("weight" -> 1)))
+ val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
+ val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
+
+ val edgesTo = edges \ "results" \\ "to"
+ val orderByTo = orderByScore \ "results" \\ "to"
+ val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
+
+ edgesTo should be(Seq(JsNumber(2), JsNumber(1)))
+ edgesTo should be(orderByTo)
+ ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2)))
+ edgesTo.reverse should be(ascOrderByTo)
+ }
+
+
+ test("query with sampling") {
+ def queryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ },
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50,
+ "sample": $sample
+ },
+ {
+ "label": "$testLabelName2",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50
+ }]
+ }
+ ]
+ }""")
+
+ val sampleSize = 2
+ val ts = "1442985659166"
+ val testId = 22
+
+ val bulkEdges = Seq(
+ toEdge(ts, insert, e, testId, 122, testLabelName),
+ toEdge(ts, insert, e, testId, 222, testLabelName),
+ toEdge(ts, insert, e, testId, 322, testLabelName),
+
+ toEdge(ts, insert, e, testId, 922, testLabelName2),
+ toEdge(ts, insert, e, testId, 222, testLabelName2),
+ toEdge(ts, insert, e, testId, 322, testLabelName2),
+
+ toEdge(ts, insert, e, 122, 1122, testLabelName),
+ toEdge(ts, insert, e, 122, 1222, testLabelName),
+ toEdge(ts, insert, e, 122, 1322, testLabelName),
+ toEdge(ts, insert, e, 222, 2122, testLabelName),
+ toEdge(ts, insert, e, 222, 2222, testLabelName),
+ toEdge(ts, insert, e, 222, 2322, testLabelName),
+ toEdge(ts, insert, e, 322, 3122, testLabelName),
+ toEdge(ts, insert, e, 322, 3222, testLabelName),
+ toEdge(ts, insert, e, 322, 3322, testLabelName)
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val result1 = getEdgesSync(queryWithSampling(testId, sampleSize))
+ (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
+
+ val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
+ (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
+
+ val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
+ (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
+ }
+ test("test query with filterOut query") {
+ def queryWithFilterOut(id1: String, id2: String) = Json.parse(
+ s"""{
+ | "limit": 10,
+ | "filterOut": {
+ | "srcVertices": [{
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }],
+ | "steps": [{
+ | "step": [{
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }]
+ | }]
+ | },
+ | "srcVertices": [{
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }],
+ | "steps": [{
+ | "step": [{
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }]
+ | }]
+ |}
+ """.stripMargin
+ )
+
+ val testId1 = "-23"
+ val testId2 = "-25"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 111, testLabelName, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 333, testLabelName, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 555, testLabelName, Json.obj(weight -> 1))
+ )
+ logger.debug(s"${bulkEdges.mkString("\n")}")
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithFilterOut(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(1)
+ (results(0) \ "to").toString should be("555")
+ }
+
+
+ /** note that this merge two different label result into one */
+ test("weighted union") {
+ def queryWithWeightedUnion(id1: String, id2: String) = Json.parse(
+ s"""
+ |{
+ | "limit": 10,
+ | "weights": [
+ | 10,
+ | 1
+ | ],
+ | "groupBy": ["weight"],
+ | "queries": [
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName2",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
+
+ val testId1 = "1"
+ val testId2 = "2"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 444, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 666, testLabelName2, Json.obj(weight -> 1))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithWeightedUnion(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(2)
+ (results(0) \ "scoreSum").as[Float] should be(30.0)
+ (results(0) \ "agg").as[List[JsValue]].size should be(3)
+ (results(1) \ "scoreSum").as[Float] should be(3.0)
+ (results(1) \ "agg").as[List[JsValue]].size should be(3)
+ }
+
+ test("weighted union with options") {
+ def queryWithWeightedUnionWithOptions(id1: String, id2: String) = Json.parse(
+ s"""
+ |{
+ | "limit": 10,
+ | "weights": [
+ | 10,
+ | 1
+ | ],
+ | "groupBy": ["to"],
+ | "select": ["to", "weight"],
+ | "filterOut": {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | "queries": [
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName2",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
+
+ val testId1 = "-192848"
+ val testId2 = "-193849"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 111, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 333, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithWeightedUnionWithOptions(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(1)
+
+ }
+
+ test("scoreThreshold") {
+ def queryWithScoreThreshold(id: String, scoreThreshold: Int) = Json.parse(
+ s"""{
+ | "limit": 10,
+ | "scoreThreshold": $scoreThreshold,
+ | "groupBy": ["to"],
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | },
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
+
+ val testId = "-23903"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 102, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 104, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 102, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 102, 104, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 103, 105, testLabelName, Json.obj(weight -> 10))
+ )
+ // expected: 104 -> 2, 103 -> 2, 102 -> 1,, 105 -> 1
+ insertEdgesSync(bulkEdges: _*)
+
+ var rs = getEdgesSync(queryWithScoreThreshold(testId, 2))
+ logger.debug(Json.prettyPrint(rs))
+ var results = (rs \ "results").as[List[JsValue]]
+ results.size should be(2)
+
+ rs = getEdgesSync(queryWithScoreThreshold(testId, 1))
+ logger.debug(Json.prettyPrint(rs))
+
+ results = (rs \ "results").as[List[JsValue]]
+ results.size should be(4)
+ }
+
+ def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": $offset,
+ "limit": $limit
+ }
+ ]]
+ }
+ """)
+
+ def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj(
+ "limit" -> limit,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+ override def initTestData(): Unit = {
+ super.initTestData()
+
+ insertEdgesSync(
+ toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
+ toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
+ toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, -1, 1000, testLabelName),
+ toEdge(1, insert, e, -1, 2000, testLabelName),
+ toEdge(1, insert, e, -1, 3000, testLabelName),
+ toEdge(1, insert, e, 1000, 10000, testLabelName),
+ toEdge(1, insert, e, 1000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 13000, testLabelName),
+ toEdge(1, insert, e, 10000, 100000, testLabelName),
+ toEdge(2, insert, e, 11000, 200000, testLabelName),
+ toEdge(3, insert, e, 12000, 300000, testLabelName)
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala
new file mode 100644
index 0000000..1c179bf
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/StrongLabelDeleteTest.scala
@@ -0,0 +1,283 @@
+package org.apache.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, Future}
+import scala.util.Random
+
+class StrongLabelDeleteTest extends IntegrateCommon {
+
+ import StrongDeleteUtil._
+ import TestUtil._
+
+ test("Strong consistency select") {
+ insertEdgesSync(bulkEdges(): _*)
+
+ var result = getEdgesSync(query(0))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ result = getEdgesSync(query(10))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("Strong consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val deleteParam = Json.arr(
+ Json.obj("label" -> testLabelName2,
+ "direction" -> "in",
+ "ids" -> Json.arr("20"),
+ "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteParam)
+
+ result = getEdgesSync(query(11, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, direction = "out"))
+ println(result)
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ test("update delete") {
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val src = (i + 1) * 10000
+// val src = System.currentTimeMillis()
+
+ val (ret, last) = testInner(i, src)
+ ret should be(true)
+ ret
+ }
+
+ ret.forall(identity)
+ }
+
+ test("update delete 2") {
+ val src = System.currentTimeMillis()
+ var ts = 0L
+
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val (ret, lastTs) = testInner(ts, src)
+ val deletedAt = lastTs + 1
+ val deletedAt2 = lastTs + 2
+ ts = deletedAt2 + 1 // nex start ts
+
+ ret should be(true)
+
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+ val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
+
+ val deleteRet = deleteAllSync(deleteAllRequest)
+ val deleteRet2 = deleteAllSync(deleteAllRequest2)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+
+ degreeAfterDeleteAll should be(0)
+ degreeAfterDeleteAll === (0)
+ }
+
+ ret.forall(identity)
+ }
+
+ /** This test stress out test on degree
+ * when contention is low but number of adjacent edges are large
+ * Large set of contention test
+ */
+ test("large degrees") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ // val allRequests = insertRequests ++ deleteRequests
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = insertRequests.size - deleteRequests.size
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ // println(result)
+
+ val ret = resultSize == expectedDegree && resultDegree == resultSize
+ println(s"[MaxSize]: $maxSize")
+ println(s"[DeleteSize]: $deleteSize")
+ println(s"[ResultDegree]: $resultDegree")
+ println(s"[ExpectedDegree]: $expectedDegree")
+ println(s"[ResultSize]: $resultSize")
+ ret should be(true)
+ }
+
+ test("deleteAll") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val deletedAt = System.currentTimeMillis()
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteAllRequest)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+ degreeAfterDeleteAll should be(0)
+ }
+
+ object StrongDeleteUtil {
+
+ val labelName = testLabelName2
+// val labelName = testLabelName
+ val maxTgtId = 10
+ val batchSize = 10
+ val testNum = 100
+ val numOfBatch = 10
+
+ def testInner(startTs: Long, src: Long) = {
+ val lastOps = Array.fill(maxTgtId)("none")
+ var currentTs = startTs
+
+ val allRequests = for {
+ ith <- 0 until numOfBatch
+ jth <- 0 until batchSize
+ } yield {
+ currentTs += 1
+
+ val tgt = Random.nextInt(maxTgtId)
+ val op = if (Random.nextDouble() < 0.5) "delete" else "update"
+
+ lastOps(tgt) = op
+ Seq(currentTs, op, "e", src, tgt, labelName, "{}").mkString("\t")
+ }
+
+ allRequests.foreach(println(_))
+
+ val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ println(lastOps.toList)
+ println(result)
+
+ val ret = resultDegree == expectedDegree && resultSize == resultDegree
+ if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
+
+ (ret, currentTs)
+ }
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", labelName, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", labelName, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", labelName, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", labelName, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", labelName, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", labelName, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", labelName, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", labelName, s"""{"time": 13}""")
+ )
+
+ def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
+ _labelName: String = labelName, direction: String = "out") = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "${_labelName}",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": -1,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+
+ def getDegree(jsValue: JsValue): Long = {
+ ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala
new file mode 100644
index 0000000..b8bfcf6
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/VertexTestHelper.scala
@@ -0,0 +1,71 @@
+package org.apache.s2graph.core.Integrate
+
+import org.apache.s2graph.core.PostProcess
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.Await
+import scala.util.Random
+
+
+class VertexTestHelper extends IntegrateCommon {
+
+ import TestUtil._
+ import VertexTestHelper._
+
+ test("vertex") {
+ val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
+ val (serviceName, columnName) = (testServiceName, testColumnName)
+
+ val data = vertexInsertsPayload(serviceName, columnName, ids)
+ val payload = Json.parse(Json.toJson(data).toString)
+ println(payload)
+
+ val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName))
+ Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime)
+
+ val res = graph.getVertices(vertices).map { vertices =>
+ PostProcess.verticesToJson(vertices)
+ }
+
+ val ret = Await.result(res, HttpRequestWaitingTime)
+ val fetched = ret.as[Seq[JsValue]]
+ for {
+ (d, f) <- data.zip(fetched)
+ } yield {
+ (d \ "id") should be(f \ "id")
+ ((d \ "props") \ "age") should be((f \ "props") \ "age")
+ }
+ }
+
+ object VertexTestHelper {
+ def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
+ Json.parse(
+ s"""
+ |[
+ |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
+ ]}
+ |]
+ """.stripMargin)
+ }
+
+ def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
+ ids.map { id =>
+ Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
+ }
+ }
+
+ val vertexPropsKeys = List(
+ ("age", "int")
+ )
+
+ def randomProps() = {
+ (for {
+ (propKey, propType) <- vertexPropsKeys
+ } yield {
+ propKey -> Random.nextInt(100)
+ }).toMap
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala
new file mode 100644
index 0000000..81c57c6
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/WeakLabelDeleteTest.scala
@@ -0,0 +1,136 @@
+package org.apache.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json.{JsObject, JsValue, Json}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import TestUtil._
+ import WeakLabelDeleteHelper._
+
+ test("test weak consistency select") {
+ var result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(4)
+ result = getEdgesSync(query(10))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("test weak consistency delete") {
+ var result = getEdgesSync(query(0))
+ println(result)
+
+ /** expect 4 edges */
+ (result \ "results").as[List[JsValue]].size should be(4)
+ val edges = (result \ "results").as[List[JsObject]]
+ val edgesToStore = parser.toEdges(Json.toJson(edges), "delete")
+ val rets = graph.mutateEdges(edgesToStore, withWait = true)
+ Await.result(rets, Duration(20, TimeUnit.MINUTES))
+
+ /** expect noting */
+ result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ /** insert should be ignored */
+ /**
+ * I am wondering if this is right test case
+ * This makes sense because hbase think cell is deleted when there are
+ * insert/delete with same timestamp(version) on same cell.
+ * This can be different on different storage system so I think
+ * this test should be removed.
+ */
+// val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert")
+// val rets2 = graph.mutateEdges(edgesToStore2, withWait = true)
+// Await.result(rets2, Duration(20, TimeUnit.MINUTES))
+//
+// result = getEdgesSync(query(0))
+// (result \ "results").as[List[JsValue]].size should be(0)
+ }
+
+
+ test("test weak consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
+ "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
+ println(json)
+ deleteAllSync(json)
+
+ result = getEdgesSync(query(11, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, "out"))
+
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+
+ override def initTestData(): Unit = {
+ super.initTestData()
+ insertEdgesSync(bulkEdges(): _*)
+ }
+
+ object WeakLabelDeleteHelper {
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""")
+ )
+
+ def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$columnName",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelNameWeak}",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+ }
+
+}
+
[27/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
[S2GRAPH-57]: Change package names into org.apahce.s2graph.
refactor package names on all projects.
JIRA:
[S2GRAPH-57] https://issues.apache.org/jira/browse/S2GRAPH-57
Pull Request:
Closes #40
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b6fe32fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b6fe32fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b6fe32fc
Branch: refs/heads/master
Commit: b6fe32fc2c3e15cea544d1af6b7698862e26d0ab
Parents: e71264d
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Mar 15 18:26:19 2016 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Mar 15 18:27:34 2016 +0900
----------------------------------------------------------------------
CHANGES | 2 +
dev_support/docker-compose.yml | 3 -
.../loader/spark/BulkLoadPartitioner.scala | 56 +
.../loader/spark/FamilyHFileWriteOptions.scala | 35 +
.../s2graph/loader/spark/HBaseContext.scala | 849 ++++++++++++
.../loader/spark/HBaseDStreamFunctions.scala | 158 +++
.../loader/spark/HBaseRDDFunctions.scala | 207 +++
.../s2graph/loader/spark/JavaHBaseContext.scala | 342 +++++
.../loader/spark/KeyFamilyQualifier.scala | 46 +
.../loader/subscriber/GraphSubscriber.scala | 335 +++++
.../loader/subscriber/TransferToHFile.scala | 194 +++
.../s2graph/loader/subscriber/WalLogStat.scala | 90 ++
.../loader/subscriber/WalLogToHDFS.scala | 148 +++
.../main/scala/spark/BulkLoadPartitioner.scala | 56 -
.../ColumnFamilyQualifierMapKeyWrapper.scala | 73 --
loader/src/main/scala/spark/DefaultSource.scala | 982 --------------
.../scala/spark/FamilyHFileWriteOptions.scala | 35 -
loader/src/main/scala/spark/HBaseContext.scala | 849 ------------
.../scala/spark/HBaseDStreamFunctions.scala | 158 ---
.../main/scala/spark/HBaseRDDFunctions.scala | 207 ---
.../src/main/scala/spark/JavaHBaseContext.scala | 342 -----
.../main/scala/spark/KeyFamilyQualifier.scala | 46 -
.../main/scala/subscriber/GraphSubscriber.scala | 339 -----
.../subscriber/GraphSubscriberStreaming.scala | 103 --
.../src/main/scala/subscriber/KafkaToHdfs.scala | 111 --
.../main/scala/subscriber/TestEdgeBuilder.scala | 66 -
.../main/scala/subscriber/TransferToHFile.scala | 195 ---
.../scala/subscriber/VertexDegreeBuilder.scala | 102 --
.../src/main/scala/subscriber/WalLogStat.scala | 94 --
.../main/scala/subscriber/WalLogToHDFS.scala | 149 ---
.../loader/subscriber/GraphSubscriberTest.scala | 45 +
.../loader/subscriber/TransferToHFileTest.scala | 169 +++
.../scala/subscriber/GraphSubscriberTest.scala | 45 -
.../scala/subscriber/TransferToHFileTest.scala | 169 ---
.../scala/com/kakao/s2graph/core/Edge.scala | 568 --------
.../kakao/s2graph/core/ExceptionHandler.scala | 162 ---
.../scala/com/kakao/s2graph/core/Graph.scala | 381 ------
.../com/kakao/s2graph/core/GraphElement.scala | 12 -
.../kakao/s2graph/core/GraphExceptions.scala | 29 -
.../com/kakao/s2graph/core/GraphUtil.scala | 138 --
.../com/kakao/s2graph/core/JSONParser.scala | 134 --
.../com/kakao/s2graph/core/Management.scala | 368 ------
.../com/kakao/s2graph/core/OrderingUtil.scala | 146 ---
.../com/kakao/s2graph/core/PostProcess.scala | 554 --------
.../com/kakao/s2graph/core/QueryParam.scala | 583 ---------
.../com/kakao/s2graph/core/QueryResult.scala | 42 -
.../scala/com/kakao/s2graph/core/Vertex.scala | 107 --
.../com/kakao/s2graph/core/mysqls/Bucket.scala | 66 -
.../kakao/s2graph/core/mysqls/ColumnMeta.scala | 110 --
.../kakao/s2graph/core/mysqls/Experiment.scala | 78 --
.../com/kakao/s2graph/core/mysqls/Label.scala | 372 ------
.../kakao/s2graph/core/mysqls/LabelIndex.scala | 143 --
.../kakao/s2graph/core/mysqls/LabelMeta.scala | 151 ---
.../com/kakao/s2graph/core/mysqls/Model.scala | 100 --
.../com/kakao/s2graph/core/mysqls/Service.scala | 96 --
.../s2graph/core/mysqls/ServiceColumn.scala | 82 --
.../s2graph/core/parsers/WhereParser.scala | 207 ---
.../kakao/s2graph/core/rest/RequestParser.scala | 609 ---------
.../kakao/s2graph/core/rest/RestHandler.scala | 245 ----
.../s2graph/core/storage/Deserializable.scala | 25 -
.../kakao/s2graph/core/storage/SKeyValue.scala | 47 -
.../s2graph/core/storage/Serializable.scala | 10 -
.../kakao/s2graph/core/storage/Storage.scala | 1224 ------------------
.../core/storage/StorageDeserializable.scala | 95 --
.../core/storage/StorageSerializable.scala | 41 -
.../core/storage/hbase/AsynchbaseStorage.scala | 533 --------
.../tall/IndexEdgeDeserializable.scala | 132 --
.../indexedge/tall/IndexEdgeSerializable.scala | 53 -
.../wide/IndexEdgeDeserializable.scala | 116 --
.../indexedge/wide/IndexEdgeSerializable.scala | 53 -
.../tall/SnapshotEdgeDeserializable.scala | 84 --
.../tall/SnapshotEdgeSerializable.scala | 47 -
.../wide/SnapshotEdgeDeserializable.scala | 70 -
.../wide/SnapshotEdgeSerializable.scala | 50 -
.../serde/vertex/VertexDeserializable.scala | 46 -
.../serde/vertex/VertexSerializable.scala | 20 -
.../kakao/s2graph/core/types/HBaseType.scala | 163 ---
.../kakao/s2graph/core/types/InnerValLike.scala | 246 ----
.../s2graph/core/types/LabelWithDirection.scala | 65 -
.../com/kakao/s2graph/core/types/VertexId.scala | 145 ---
.../kakao/s2graph/core/types/v1/InnerVal.scala | 226 ----
.../kakao/s2graph/core/types/v2/InnerVal.scala | 162 ---
.../kakao/s2graph/core/utils/DeferCache.scala | 82 --
.../kakao/s2graph/core/utils/Extentions.scala | 72 --
.../kakao/s2graph/core/utils/FutureCache.scala | 82 --
.../com/kakao/s2graph/core/utils/Logger.scala | 44 -
.../s2graph/core/utils/SafeUpdateCache.scala | 62 -
.../scala/org/apache/s2graph/core/Edge.scala | 567 ++++++++
.../apache/s2graph/core/ExceptionHandler.scala | 159 +++
.../scala/org/apache/s2graph/core/Graph.scala | 379 ++++++
.../org/apache/s2graph/core/GraphElement.scala | 10 +
.../apache/s2graph/core/GraphExceptions.scala | 28 +
.../org/apache/s2graph/core/GraphUtil.scala | 139 ++
.../org/apache/s2graph/core/JSONParser.scala | 134 ++
.../org/apache/s2graph/core/Management.scala | 368 ++++++
.../org/apache/s2graph/core/OrderingUtil.scala | 146 +++
.../org/apache/s2graph/core/PostProcess.scala | 553 ++++++++
.../org/apache/s2graph/core/QueryParam.scala | 583 +++++++++
.../org/apache/s2graph/core/QueryResult.scala | 42 +
.../scala/org/apache/s2graph/core/Vertex.scala | 101 ++
.../org/apache/s2graph/core/mysqls/Bucket.scala | 66 +
.../apache/s2graph/core/mysqls/ColumnMeta.scala | 110 ++
.../apache/s2graph/core/mysqls/Experiment.scala | 77 ++
.../org/apache/s2graph/core/mysqls/Label.scala | 372 ++++++
.../apache/s2graph/core/mysqls/LabelIndex.scala | 143 ++
.../apache/s2graph/core/mysqls/LabelMeta.scala | 150 +++
.../org/apache/s2graph/core/mysqls/Model.scala | 100 ++
.../apache/s2graph/core/mysqls/Service.scala | 94 ++
.../s2graph/core/mysqls/ServiceColumn.scala | 82 ++
.../s2graph/core/parsers/WhereParser.scala | 207 +++
.../s2graph/core/rest/RequestParser.scala | 610 +++++++++
.../apache/s2graph/core/rest/RestHandler.scala | 244 ++++
.../s2graph/core/storage/Deserializable.scala | 24 +
.../apache/s2graph/core/storage/SKeyValue.scala | 47 +
.../s2graph/core/storage/Serializable.scala | 8 +
.../apache/s2graph/core/storage/Storage.scala | 1224 ++++++++++++++++++
.../core/storage/StorageDeserializable.scala | 95 ++
.../core/storage/StorageSerializable.scala | 41 +
.../core/storage/hbase/AsynchbaseStorage.scala | 534 ++++++++
.../tall/IndexEdgeDeserializable.scala | 132 ++
.../indexedge/tall/IndexEdgeSerializable.scala | 53 +
.../wide/IndexEdgeDeserializable.scala | 116 ++
.../indexedge/wide/IndexEdgeSerializable.scala | 52 +
.../tall/SnapshotEdgeDeserializable.scala | 84 ++
.../tall/SnapshotEdgeSerializable.scala | 47 +
.../wide/SnapshotEdgeDeserializable.scala | 70 +
.../wide/SnapshotEdgeSerializable.scala | 51 +
.../serde/vertex/VertexDeserializable.scala | 47 +
.../serde/vertex/VertexSerializable.scala | 19 +
.../apache/s2graph/core/types/HBaseType.scala | 160 +++
.../s2graph/core/types/InnerValLike.scala | 243 ++++
.../s2graph/core/types/LabelWithDirection.scala | 61 +
.../apache/s2graph/core/types/VertexId.scala | 142 ++
.../apache/s2graph/core/types/v1/InnerVal.scala | 224 ++++
.../apache/s2graph/core/types/v2/InnerVal.scala | 160 +++
.../apache/s2graph/core/utils/DeferCache.scala | 82 ++
.../apache/s2graph/core/utils/Extentions.scala | 73 ++
.../apache/s2graph/core/utils/FutureCache.scala | 82 ++
.../org/apache/s2graph/core/utils/Logger.scala | 44 +
.../s2graph/core/utils/SafeUpdateCache.scala | 62 +
.../scala/com/kakao/s2graph/core/EdgeTest.scala | 536 --------
.../kakao/s2graph/core/Integrate/CrudTest.scala | 226 ----
.../core/Integrate/IntegrateCommon.scala | 309 -----
.../s2graph/core/Integrate/QueryTest.scala | 909 -------------
.../core/Integrate/StrongLabelDeleteTest.scala | 283 ----
.../core/Integrate/VertexTestHelper.scala | 71 -
.../core/Integrate/WeakLabelDeleteTest.scala | 136 --
.../com/kakao/s2graph/core/JsonParserTest.scala | 66 -
.../kakao/s2graph/core/OrderingUtilTest.scala | 130 --
.../com/kakao/s2graph/core/QueryParamTest.scala | 86 --
.../com/kakao/s2graph/core/TestCommon.scala | 192 ---
.../s2graph/core/TestCommonWithModels.scala | 197 ---
.../com/kakao/s2graph/core/VertexTest.scala | 79 --
.../kakao/s2graph/core/models/ModelTest.scala | 135 --
.../s2graph/core/mysqls/ExperimentSpec.scala | 64 -
.../s2graph/core/parsers/WhereParserTest.scala | 234 ----
.../hbase/AsynchbaseQueryBuilderTest.scala | 53 -
.../storage/hbase/AsynchbaseStorageTest.scala | 36 -
.../core/storage/hbase/IndexEdgeTest.scala | 81 --
.../s2graph/core/types/CompositeIdTest.scala | 109 --
.../kakao/s2graph/core/types/EdgeTypeTest.scala | 69 -
.../kakao/s2graph/core/types/InnerValTest.scala | 130 --
.../s2graph/core/types/SourceVertexIdTest.scala | 52 -
.../s2graph/core/types/TargetVertexIdTest.scala | 53 -
.../kakao/s2graph/core/types/VertexIdTest.scala | 52 -
.../s2graph/core/types/VertexTypeTest.scala | 42 -
.../org/apache/s2graph/core/EdgeTest.scala | 535 ++++++++
.../s2graph/core/Integrate/CrudTest.scala | 226 ++++
.../core/Integrate/IntegrateCommon.scala | 309 +++++
.../s2graph/core/Integrate/QueryTest.scala | 906 +++++++++++++
.../core/Integrate/StrongLabelDeleteTest.scala | 283 ++++
.../core/Integrate/VertexTestHelper.scala | 71 +
.../core/Integrate/WeakLabelDeleteTest.scala | 136 ++
.../apache/s2graph/core/JsonParserTest.scala | 66 +
.../apache/s2graph/core/OrderingUtilTest.scala | 130 ++
.../apache/s2graph/core/QueryParamTest.scala | 86 ++
.../org/apache/s2graph/core/TestCommon.scala | 188 +++
.../s2graph/core/TestCommonWithModels.scala | 194 +++
.../apache/s2graph/core/models/ModelTest.scala | 130 ++
.../s2graph/core/mysqls/ExperimentSpec.scala | 64 +
.../s2graph/core/parsers/WhereParserTest.scala | 234 ++++
.../storage/hbase/AsynchbaseStorageTest.scala | 33 +
.../core/storage/hbase/IndexEdgeTest.scala | 81 ++
.../s2graph/core/types/InnerValTest.scala | 128 ++
.../org/apache/s2graph/counter/TrxLog.scala | 25 +
.../counter/config/ConfigFunctions.scala | 30 +
.../counter/config/S2CounterConfig.scala | 44 +
.../apache/s2graph/counter/core/BytesUtil.scala | 13 +
.../s2graph/counter/core/ExactCounter.scala | 243 ++++
.../apache/s2graph/counter/core/ExactKey.scala | 26 +
.../s2graph/counter/core/ExactQualifier.scala | 73 ++
.../s2graph/counter/core/ExactStorage.scala | 32 +
.../s2graph/counter/core/RankingCounter.scala | 101 ++
.../s2graph/counter/core/RankingKey.scala | 4 +
.../s2graph/counter/core/RankingResult.scala | 3 +
.../s2graph/counter/core/RankingStorage.scala | 17 +
.../s2graph/counter/core/RankingValue.scala | 15 +
.../s2graph/counter/core/RateRankingValue.scala | 15 +
.../s2graph/counter/core/TimedQualifier.scala | 215 +++
.../s2graph/counter/core/v1/BytesUtilV1.scala | 60 +
.../core/v1/ExactStorageAsyncHBase.scala | 317 +++++
.../counter/core/v1/ExactStorageHBase.scala | 326 +++++
.../counter/core/v1/RankingStorageRedis.scala | 185 +++
.../s2graph/counter/core/v2/BytesUtilV2.scala | 88 ++
.../counter/core/v2/ExactStorageGraph.scala | 343 +++++
.../counter/core/v2/GraphOperation.scala | 48 +
.../counter/core/v2/RankingStorageGraph.scala | 395 ++++++
.../s2graph/counter/decay/DecayFormula.scala | 6 +
.../s2graph/counter/decay/ExpDecayFormula.scala | 25 +
.../s2graph/counter/helper/CounterAdmin.scala | 113 ++
.../counter/helper/DistributedScanner.scala | 70 +
.../counter/helper/HashShardingJedis.scala | 153 +++
.../s2graph/counter/helper/Management.scala | 143 ++
.../s2graph/counter/helper/WithHBase.scala | 98 ++
.../s2graph/counter/helper/WithRedis.scala | 59 +
.../s2graph/counter/models/CachedDBModel.scala | 11 +
.../apache/s2graph/counter/models/Counter.scala | 210 +++
.../apache/s2graph/counter/models/DBModel.scala | 24 +
.../org/apache/s2graph/counter/package.scala | 8 +
.../s2graph/counter/util/CartesianProduct.scala | 8 +
.../s2graph/counter/util/CollectionCache.scala | 66 +
.../s2graph/counter/util/FunctionParser.scala | 21 +
.../apache/s2graph/counter/util/Hashes.scala | 21 +
.../s2graph/counter/util/ReduceMapValue.scala | 9 +
.../org/apache/s2graph/counter/util/Retry.scala | 44 +
.../s2graph/counter/util/SplitBytes.scala | 21 +
.../s2graph/counter/util/UnitConverter.scala | 25 +
.../main/scala/s2/config/ConfigFunctions.scala | 33 -
.../main/scala/s2/config/S2CounterConfig.scala | 47 -
.../src/main/scala/s2/counter/TrxLog.scala | 28 -
.../main/scala/s2/counter/core/BytesUtil.scala | 15 -
.../scala/s2/counter/core/ExactCounter.scala | 247 ----
.../main/scala/s2/counter/core/ExactKey.scala | 29 -
.../scala/s2/counter/core/ExactQualifier.scala | 78 --
.../scala/s2/counter/core/ExactStorage.scala | 35 -
.../scala/s2/counter/core/RankingCounter.scala | 105 --
.../main/scala/s2/counter/core/RankingKey.scala | 6 -
.../scala/s2/counter/core/RankingResult.scala | 6 -
.../scala/s2/counter/core/RankingStorage.scala | 19 -
.../scala/s2/counter/core/RankingValue.scala | 19 -
.../s2/counter/core/RateRankingValue.scala | 18 -
.../scala/s2/counter/core/TimedQualifier.scala | 218 ----
.../scala/s2/counter/core/v1/BytesUtilV1.scala | 64 -
.../core/v1/ExactStorageAsyncHBase.scala | 321 -----
.../s2/counter/core/v1/ExactStorageHBase.scala | 327 -----
.../counter/core/v1/RankingStorageRedis.scala | 189 ---
.../scala/s2/counter/core/v2/BytesUtilV2.scala | 91 --
.../s2/counter/core/v2/ExactStorageGraph.scala | 346 -----
.../s2/counter/core/v2/GraphOperation.scala | 52 -
.../counter/core/v2/RankingStorageGraph.scala | 399 ------
.../scala/s2/counter/decay/DecayFormula.scala | 9 -
.../s2/counter/decay/ExpDecayFormula.scala | 28 -
.../src/main/scala/s2/counter/package.scala | 11 -
.../src/main/scala/s2/helper/CounterAdmin.scala | 114 --
.../scala/s2/helper/DistributedScanner.scala | 74 --
.../scala/s2/helper/HashShardingJedis.scala | 156 ---
.../src/main/scala/s2/helper/Management.scala | 146 ---
.../src/main/scala/s2/helper/WithHBase.scala | 101 --
.../src/main/scala/s2/helper/WithRedis.scala | 62 -
.../main/scala/s2/models/CachedDBModel.scala | 14 -
.../src/main/scala/s2/models/Counter.scala | 213 ---
.../src/main/scala/s2/models/DBModel.scala | 27 -
.../main/scala/s2/util/CartesianProduct.scala | 11 -
.../main/scala/s2/util/CollectionCache.scala | 69 -
.../src/main/scala/s2/util/FunctionParser.scala | 24 -
.../src/main/scala/s2/util/Hashes.scala | 24 -
.../src/main/scala/s2/util/ReduceMapValue.scala | 12 -
.../src/main/scala/s2/util/Retry.scala | 47 -
.../src/main/scala/s2/util/SplitBytes.scala | 24 -
.../src/main/scala/s2/util/UnitConverter.scala | 28 -
.../counter/core/RankingCounterSpec.scala | 166 +++
.../counter/models/CounterModelSpec.scala | 50 +
.../s2graph/counter/models/CounterSpec.scala | 33 +
.../s2/counter/core/RankingCounterSpec.scala | 169 ---
.../test/scala/s2/models/CounterModelSpec.scala | 53 -
.../src/test/scala/s2/models/CounterSpec.scala | 36 -
.../counter/loader/CounterBulkLoader.scala | 78 ++
.../counter/loader/EraseDailyCounter.scala | 133 ++
.../counter/loader/config/StreamingConfig.scala | 24 +
.../loader/core/CounterEtlFunctions.scala | 86 ++
.../counter/loader/core/CounterEtlItem.scala | 39 +
.../counter/loader/core/CounterFunctions.scala | 446 +++++++
.../counter/loader/core/DimensionProps.scala | 150 +++
.../loader/models/DefaultCounterModel.scala | 6 +
.../counter/loader/stream/EtlStreaming.scala | 114 ++
.../loader/stream/ExactCounterStreaming.scala | 69 +
.../loader/stream/GraphToETLStreaming.scala | 80 ++
.../loader/stream/RankingCounterStreaming.scala | 72 ++
.../main/scala/s2/config/StreamingConfig.scala | 24 -
.../scala/s2/counter/CounterBulkLoader.scala | 78 --
.../scala/s2/counter/EraseDailyCounter.scala | 133 --
.../s2/counter/core/CounterEtlFunctions.scala | 89 --
.../scala/s2/counter/core/CounterEtlItem.scala | 43 -
.../s2/counter/core/CounterFunctions.scala | 446 -------
.../scala/s2/counter/core/DimensionProps.scala | 154 ---
.../scala/s2/counter/stream/EtlStreaming.scala | 116 --
.../counter/stream/ExactCounterStreaming.scala | 72 --
.../s2/counter/stream/GraphToETLStreaming.scala | 83 --
.../stream/RankingCounterStreaming.scala | 74 --
.../scala/s2/models/DefaultCounterModel.scala | 8 -
.../loader/core/CounterEtlFunctionsSpec.scala | 33 +
.../loader/core/DimensionPropsTest.scala | 46 +
.../stream/ExactCounterStreamingSpec.scala | 198 +++
.../stream/RankingCounterStreamingSpec.scala | 451 +++++++
.../counter/core/CounterEtlFunctionsSpec.scala | 33 -
.../s2/counter/core/DimensionPropsTest.scala | 46 -
.../stream/ExactCounterStreamingSpec.scala | 196 ---
.../stream/RankingCounterStreamingSpec.scala | 448 -------
.../src/main/resources/application.conf | 0
s2rest_netty/src/main/scala/Server.scala | 202 ---
.../org/apache/s2graph/rest/netty/Server.scala | 200 +++
s2rest_play/app/Bootstrap.scala | 81 --
s2rest_play/app/actors/QueueActor.scala | 92 --
s2rest_play/app/config/Config.scala | 41 -
s2rest_play/app/config/CounterConfig.scala | 10 -
.../app/controllers/AdminController.scala | 424 ------
.../app/controllers/ApplicationController.scala | 101 --
.../app/controllers/CounterController.scala | 747 -----------
.../app/controllers/EdgeController.scala | 219 ----
.../app/controllers/ExperimentController.scala | 24 -
.../app/controllers/JsonBodyParser.scala | 86 --
.../app/controllers/PublishController.scala | 54 -
.../app/controllers/QueryController.scala | 52 -
.../app/controllers/TestController.scala | 24 -
.../app/controllers/VertexController.scala | 86 --
s2rest_play/app/models/ExactCounterItem.scala | 38 -
s2rest_play/app/models/RankCounterItem.scala | 40 -
s2rest_play/app/models/package.scala | 8 -
.../apache/s2graph/rest/play/Bootstrap.scala | 81 ++
.../s2graph/rest/play/actors/QueueActor.scala | 89 ++
.../s2graph/rest/play/config/Config.scala | 41 +
.../rest/play/config/CounterConfig.scala | 10 +
.../rest/play/controllers/AdminController.scala | 424 ++++++
.../controllers/ApplicationController.scala | 101 ++
.../play/controllers/CounterController.scala | 744 +++++++++++
.../rest/play/controllers/EdgeController.scala | 219 ++++
.../play/controllers/ExperimentController.scala | 23 +
.../rest/play/controllers/JsonBodyParser.scala | 86 ++
.../play/controllers/PublishController.scala | 54 +
.../rest/play/controllers/QueryController.scala | 52 +
.../play/controllers/VertexController.scala | 85 ++
.../rest/play/models/ExactCounterItem.scala | 38 +
.../rest/play/models/RankCounterItem.scala | 40 +
.../s2graph/rest/play/models/package.scala | 10 +
s2rest_play/app/util/TestDataLoader.scala | 70 -
s2rest_play/conf/reference.conf | 2 +-
s2rest_play/conf/routes | 165 ++-
.../test/benchmark/BenchmarkCommon.scala | 24 -
s2rest_play/test/benchmark/GraphUtilSpec.scala | 125 --
.../test/benchmark/JsonBenchmarkSpec.scala | 46 -
.../benchmark/OrderingUtilBenchmarkSpec.scala | 100 --
.../test/benchmark/SamplingBenchmarkSpec.scala | 85 --
.../test/controllers/PostProcessSpec.scala | 112 --
.../rest/play/benchmark/BenchmarkCommon.scala | 24 +
.../rest/play/benchmark/GraphUtilSpec.scala | 124 ++
.../rest/play/benchmark/JsonBenchmarkSpec.scala | 45 +
.../benchmark/OrderingUtilBenchmarkSpec.scala | 98 ++
.../play/benchmark/SamplingBenchmarkSpec.scala | 87 ++
.../rest/play/controllers/PostProcessSpec.scala | 112 ++
.../s2graph/spark/config/S2ConfigFactory.scala | 20 +
.../s2graph/spark/spark/HashMapParam.scala | 55 +
.../apache/s2graph/spark/spark/RDDUtil.scala | 13 +
.../apache/s2graph/spark/spark/SparkApp.scala | 120 ++
.../spark/spark/SubscriberListener.scala | 20 +
.../apache/s2graph/spark/spark/WithKafka.scala | 69 +
.../streaming/kafka/KafkaRDDFunctions.scala | 3 -
.../spark/streaming/kafka/StreamHelper.scala | 4 -
.../main/scala/s2/config/S2ConfigFactory.scala | 27 -
.../src/main/scala/s2/spark/HashMapParam.scala | 55 -
spark/src/main/scala/s2/spark/RDDUtil.scala | 16 -
spark/src/main/scala/s2/spark/SparkApp.scala | 124 --
.../scala/s2/spark/SubscriberListener.scala | 24 -
spark/src/main/scala/s2/spark/WithKafka.scala | 69 -
.../org/apache/s2graph/spark/SparkAppTest.scala | 17 +
.../s2graph/spark/TestStreamingSpec.scala | 31 +
.../src/test/scala/s2/spark/SparkAppTest.scala | 21 -
.../test/scala/s2/spark/TestStreamingSpec.scala | 34 -
377 files changed, 25150 insertions(+), 27480 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 4c3eadb..8b5fc63 100644
--- a/CHANGES
+++ b/CHANGES
@@ -69,6 +69,8 @@ Release 0.12.1 - unreleased
S2GRAPH-5: Add Apache RAT to valid LICENSE errors. (Committed by DOYUNG YOON).
S2GRAPH-17: Remove unnecessary abstraction layer, Storage. (Committed by DOYUNG YOON).
+
+ S2GRAPH-57: Change package names into org.apahce.s2graph. (Committed by DOYUNG YOON).
SUB TASKS
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/dev_support/docker-compose.yml
----------------------------------------------------------------------
diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml
index 9818251..ba9d489 100644
--- a/dev_support/docker-compose.yml
+++ b/dev_support/docker-compose.yml
@@ -2,9 +2,6 @@ graph:
image: s2rest_play:0.12.1-SNAPSHOT
container_name: graph
net: container:graph_hbase
- links:
- - graph_mysql
- - graph_hbase
graph_mysql:
build: graph_mysql
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/BulkLoadPartitioner.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/BulkLoadPartitioner.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/BulkLoadPartitioner.scala
new file mode 100644
index 0000000..cf73736
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/BulkLoadPartitioner.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import java.util
+import java.util.Comparator
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.spark.Partitioner
+
+/**
+ * A Partitioner implementation that will separate records to different
+ * HBase Regions based on region splits
+ *
+ * @param startKeys The start keys for the given table
+ */
+class BulkLoadPartitioner(startKeys:Array[Array[Byte]])
+ extends Partitioner {
+
+ override def numPartitions: Int = startKeys.length
+
+ override def getPartition(key: Any): Int = {
+
+ val rowKey:Array[Byte] =
+ key match {
+ case qualifier: KeyFamilyQualifier =>
+ qualifier.rowKey
+ case _ =>
+ key.asInstanceOf[Array[Byte]]
+ }
+
+ val comparator: Comparator[Array[Byte]] = new Comparator[Array[Byte]] {
+ override def compare(o1: Array[Byte], o2: Array[Byte]): Int = {
+ Bytes.compareTo(o1, o2)
+ }
+ }
+ val partition = util.Arrays.binarySearch(startKeys, rowKey, comparator)
+ if (partition < 0) partition * -1 + -2
+ else partition
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/FamilyHFileWriteOptions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/FamilyHFileWriteOptions.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/FamilyHFileWriteOptions.scala
new file mode 100644
index 0000000..a9886a2
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/FamilyHFileWriteOptions.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import java.io.Serializable
+
+/**
+ * This object will hold optional data for how a given column family's
+ * writer will work
+ *
+ * @param compression String to define the Compression to be used in the HFile
+ * @param bloomType String to define the bloom type to be used in the HFile
+ * @param blockSize The block size to be used in the HFile
+ * @param dataBlockEncoding String to define the data block encoding to be used
+ * in the HFile
+ */
+class FamilyHFileWriteOptions( val compression:String,
+ val bloomType: String,
+ val blockSize: Int,
+ val dataBlockEncoding: String) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
new file mode 100644
index 0000000..94e12af
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
@@ -0,0 +1,849 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import java.net.InetSocketAddress
+import java.util
+
+import org.apache.hadoop.hbase.fs.HFileSystem
+import org.apache.hadoop.hbase._
+import org.apache.hadoop.hbase.io.compress.Compression
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.io.hfile.{CacheConfig, HFileContextBuilder}
+import org.apache.hadoop.hbase.regionserver.{HStore, StoreFile, BloomType}
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.rdd.RDD
+import org.apache.hadoop.conf.Configuration
+import HBaseRDDFunctions._
+import org.apache.hadoop.hbase.client._
+import scala.reflect.ClassTag
+import org.apache.spark.{Logging, SerializableWritable, SparkContext}
+import org.apache.hadoop.hbase.mapreduce.{TableMapReduceUtil,
+TableInputFormat, IdentityTableMapper}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.mapreduce.Job
+import org.apache.spark.streaming.dstream.DStream
+import java.io._
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
+import org.apache.hadoop.fs.{Path, FileSystem}
+import scala.collection.mutable
+
+/**
+ * HBaseContext is a façade for HBase operations
+ * like bulk put, get, increment, delete, and scan
+ *
+ * HBaseContext will take the responsibilities
+ * of disseminating the configuration information
+ * to the working and managing the life cycle of HConnections.
+ */
+class HBaseContext(@transient sc: SparkContext,
+ @transient config: Configuration,
+ val tmpHdfsConfgFile: String = null)
+ extends Serializable with Logging {
+
+ @transient var credentials = SparkHadoopUtil.get.getCurrentUserCredentials()
+ @transient var tmpHdfsConfiguration:Configuration = config
+ @transient var appliedCredentials = false
+ @transient val job = Job.getInstance(config)
+ TableMapReduceUtil.initCredentials(job)
+ val broadcastedConf = sc.broadcast(new SerializableWritable(config))
+ val credentialsConf = sc.broadcast(new SerializableWritable(job.getCredentials))
+
+ LatestHBaseContextCache.latest = this
+
+ if (tmpHdfsConfgFile != null && config != null) {
+ val fs = FileSystem.newInstance(config)
+ val tmpPath = new Path(tmpHdfsConfgFile)
+ if (!fs.exists(tmpPath)) {
+ val outputStream = fs.create(tmpPath)
+ config.write(outputStream)
+ outputStream.close()
+ } else {
+ logWarning("tmpHdfsConfigDir " + tmpHdfsConfgFile + " exist!!")
+ }
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark RDD foreachPartition.
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * @param rdd Original RDD with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the RDD values and a HConnection object to interact
+ * with HBase
+ */
+ def foreachPartition[T](rdd: RDD[T],
+ f: (Iterator[T], Connection) => Unit):Unit = {
+ rdd.foreachPartition(
+ it => hbaseForeachPartition(broadcastedConf, it, f))
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark Streaming dStream foreach
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * @param dstream Original DStream with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the DStream values and a HConnection object to
+ * interact with HBase
+ */
+ def foreachPartition[T](dstream: DStream[T],
+ f: (Iterator[T], Connection) => Unit):Unit = {
+ dstream.foreachRDD((rdd, time) => {
+ foreachPartition(rdd, f)
+ })
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark RDD mapPartition.
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * @param rdd Original RDD with data to iterate over
+ * @param mp Function to be given a iterator to iterate through
+ * the RDD values and a HConnection object to interact
+ * with HBase
+ * @return Returns a new RDD generated by the user definition
+ * function just like normal mapPartition
+ */
+ def mapPartitions[T, R: ClassTag](rdd: RDD[T],
+ mp: (Iterator[T], Connection) => Iterator[R]): RDD[R] = {
+
+ rdd.mapPartitions[R](it => hbaseMapPartition[T, R](broadcastedConf,
+ it,
+ mp))
+
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark Streaming DStream
+ * foreachPartition.
+ *
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * Note: Make sure to partition correctly to avoid memory issue when
+ * getting data from HBase
+ *
+ * @param dstream Original DStream with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the DStream values and a HConnection object to
+ * interact with HBase
+ * @return Returns a new DStream generated by the user
+ * definition function just like normal mapPartition
+ */
+ def streamForeachPartition[T](dstream: DStream[T],
+ f: (Iterator[T], Connection) => Unit): Unit = {
+
+ dstream.foreachRDD(rdd => this.foreachPartition(rdd, f))
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark Streaming DStream
+ * mapPartition.
+ *
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * Note: Make sure to partition correctly to avoid memory issue when
+ * getting data from HBase
+ *
+ * @param dstream Original DStream with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the DStream values and a HConnection object to
+ * interact with HBase
+ * @return Returns a new DStream generated by the user
+ * definition function just like normal mapPartition
+ */
+ def streamMapPartitions[T, U: ClassTag](dstream: DStream[T],
+ f: (Iterator[T], Connection) => Iterator[U]):
+ DStream[U] = {
+ dstream.mapPartitions(it => hbaseMapPartition[T, U](
+ broadcastedConf,
+ it,
+ f))
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.foreachPartition method.
+ *
+ * It allow addition support for a user to take RDD
+ * and generate puts and send them to HBase.
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param rdd Original RDD with data to iterate over
+ * @param tableName The name of the table to put into
+ * @param f Function to convert a value in the RDD to a HBase Put
+ */
+ def bulkPut[T](rdd: RDD[T], tableName: TableName, f: (T) => Put) {
+
+ val tName = tableName.getName
+ rdd.foreachPartition(
+ it => hbaseForeachPartition[T](
+ broadcastedConf,
+ it,
+ (iterator, connection) => {
+ val m = connection.getBufferedMutator(TableName.valueOf(tName))
+ iterator.foreach(T => m.mutate(f(T)))
+ m.flush()
+ m.close()
+ }))
+ }
+
+ def applyCreds[T] (configBroadcast: Broadcast[SerializableWritable[Configuration]]){
+ credentials = SparkHadoopUtil.get.getCurrentUserCredentials()
+
+ logDebug("appliedCredentials:" + appliedCredentials + ",credentials:" + credentials)
+
+ if (!appliedCredentials && credentials != null) {
+ appliedCredentials = true
+
+ @transient val ugi = UserGroupInformation.getCurrentUser
+ ugi.addCredentials(credentials)
+ // specify that this is a proxy user
+ ugi.setAuthenticationMethod(AuthenticationMethod.PROXY)
+
+ ugi.addCredentials(credentialsConf.value.value)
+ }
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamMapPartition method.
+ *
+ * It allow addition support for a user to take a DStream and
+ * generate puts and send them to HBase.
+ *
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param dstream Original DStream with data to iterate over
+ * @param tableName The name of the table to put into
+ * @param f Function to convert a value in
+ * the DStream to a HBase Put
+ */
+ def streamBulkPut[T](dstream: DStream[T],
+ tableName: TableName,
+ f: (T) => Put) = {
+ val tName = tableName.getName
+ dstream.foreachRDD((rdd, time) => {
+ bulkPut(rdd, TableName.valueOf(tName), f)
+ })
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.foreachPartition method.
+ *
+ * It allow addition support for a user to take a RDD and generate delete
+ * and send them to HBase. The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param rdd Original RDD with data to iterate over
+ * @param tableName The name of the table to delete from
+ * @param f Function to convert a value in the RDD to a
+ * HBase Deletes
+ * @param batchSize The number of delete to batch before sending to HBase
+ */
+ def bulkDelete[T](rdd: RDD[T], tableName: TableName,
+ f: (T) => Delete, batchSize: Integer) {
+ bulkMutation(rdd, tableName, f, batchSize)
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamBulkMutation method.
+ *
+ * It allow addition support for a user to take a DStream and
+ * generate Delete and send them to HBase.
+ *
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param dstream Original DStream with data to iterate over
+ * @param tableName The name of the table to delete from
+ * @param f function to convert a value in the DStream to a
+ * HBase Delete
+ * @param batchSize The number of deletes to batch before sending to HBase
+ */
+ def streamBulkDelete[T](dstream: DStream[T],
+ tableName: TableName,
+ f: (T) => Delete,
+ batchSize: Integer) = {
+ streamBulkMutation(dstream, tableName, f, batchSize)
+ }
+
+ /**
+ * Under lining function to support all bulk mutations
+ *
+ * May be opened up if requested
+ */
+ private def bulkMutation[T](rdd: RDD[T], tableName: TableName,
+ f: (T) => Mutation, batchSize: Integer) {
+
+ val tName = tableName.getName
+ rdd.foreachPartition(
+ it => hbaseForeachPartition[T](
+ broadcastedConf,
+ it,
+ (iterator, connection) => {
+ val table = connection.getTable(TableName.valueOf(tName))
+ val mutationList = new java.util.ArrayList[Mutation]
+ iterator.foreach(T => {
+ mutationList.add(f(T))
+ if (mutationList.size >= batchSize) {
+ table.batch(mutationList, null)
+ mutationList.clear()
+ }
+ })
+ if (mutationList.size() > 0) {
+ table.batch(mutationList, null)
+ mutationList.clear()
+ }
+ table.close()
+ }))
+ }
+
+ /**
+ * Under lining function to support all bulk streaming mutations
+ *
+ * May be opened up if requested
+ */
+ private def streamBulkMutation[T](dstream: DStream[T],
+ tableName: TableName,
+ f: (T) => Mutation,
+ batchSize: Integer) = {
+ val tName = tableName.getName
+ dstream.foreachRDD((rdd, time) => {
+ bulkMutation(rdd, TableName.valueOf(tName), f, batchSize)
+ })
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.mapPartition method.
+ *
+ * It allow addition support for a user to take a RDD and generates a
+ * new RDD based on Gets and the results they bring back from HBase
+ *
+ * @param rdd Original RDD with data to iterate over
+ * @param tableName The name of the table to get from
+ * @param makeGet function to convert a value in the RDD to a
+ * HBase Get
+ * @param convertResult This will convert the HBase Result object to
+ * what ever the user wants to put in the resulting
+ * RDD
+ * return new RDD that is created by the Get to HBase
+ */
+ def bulkGet[T, U: ClassTag](tableName: TableName,
+ batchSize: Integer,
+ rdd: RDD[T],
+ makeGet: (T) => Get,
+ convertResult: (Result) => U): RDD[U] = {
+
+ val getMapPartition = new GetMapPartition(tableName,
+ batchSize,
+ makeGet,
+ convertResult)
+
+ rdd.mapPartitions[U](it =>
+ hbaseMapPartition[T, U](
+ broadcastedConf,
+ it,
+ getMapPartition.run))
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamMap method.
+ *
+ * It allow addition support for a user to take a DStream and
+ * generates a new DStream based on Gets and the results
+ * they bring back from HBase
+ *
+ * @param tableName The name of the table to get from
+ * @param batchSize The number of Gets to be sent in a single batch
+ * @param dStream Original DStream with data to iterate over
+ * @param makeGet Function to convert a value in the DStream to a
+ * HBase Get
+ * @param convertResult This will convert the HBase Result object to
+ * what ever the user wants to put in the resulting
+ * DStream
+ * @return A new DStream that is created by the Get to HBase
+ */
+ def streamBulkGet[T, U: ClassTag](tableName: TableName,
+ batchSize: Integer,
+ dStream: DStream[T],
+ makeGet: (T) => Get,
+ convertResult: (Result) => U): DStream[U] = {
+
+ val getMapPartition = new GetMapPartition(tableName,
+ batchSize,
+ makeGet,
+ convertResult)
+
+ dStream.mapPartitions[U](it => hbaseMapPartition[T, U](
+ broadcastedConf,
+ it,
+ getMapPartition.run))
+ }
+
+ /**
+ * This function will use the native HBase TableInputFormat with the
+ * given scan object to generate a new RDD
+ *
+ * @param tableName the name of the table to scan
+ * @param scan the HBase scan object to use to read data from HBase
+ * @param f function to convert a Result object from HBase into
+ * what the user wants in the final generated RDD
+ * @return new RDD with results from scan
+ */
+ def hbaseRDD[U: ClassTag](tableName: TableName, scan: Scan,
+ f: ((ImmutableBytesWritable, Result)) => U): RDD[U] = {
+
+ val job: Job = Job.getInstance(getConf(broadcastedConf))
+
+ TableMapReduceUtil.initCredentials(job)
+ TableMapReduceUtil.initTableMapperJob(tableName, scan,
+ classOf[IdentityTableMapper], null, null, job)
+
+ sc.newAPIHadoopRDD(job.getConfiguration,
+ classOf[TableInputFormat],
+ classOf[ImmutableBytesWritable],
+ classOf[Result]).map(f)
+ }
+
+ /**
+ * A overloaded version of HBaseContext hbaseRDD that defines the
+ * type of the resulting RDD
+ *
+ * @param tableName the name of the table to scan
+ * @param scans the HBase scan object to use to read data from HBase
+ * @return New RDD with results from scan
+ *
+ */
+ def hbaseRDD(tableName: TableName, scans: Scan):
+ RDD[(ImmutableBytesWritable, Result)] = {
+
+ hbaseRDD[(ImmutableBytesWritable, Result)](
+ tableName,
+ scans,
+ (r: (ImmutableBytesWritable, Result)) => r)
+ }
+
+ /**
+ * underlining wrapper all foreach functions in HBaseContext
+ */
+ private def hbaseForeachPartition[T](configBroadcast:
+ Broadcast[SerializableWritable[Configuration]],
+ it: Iterator[T],
+ f: (Iterator[T], Connection) => Unit) = {
+
+ val config = getConf(configBroadcast)
+
+ applyCreds(configBroadcast)
+ // specify that this is a proxy user
+ val connection = ConnectionFactory.createConnection(config)
+ f(it, connection)
+ connection.close()
+ }
+
+ private def getConf(configBroadcast: Broadcast[SerializableWritable[Configuration]]):
+ Configuration = {
+
+ if (tmpHdfsConfiguration == null && tmpHdfsConfgFile != null) {
+ val fs = FileSystem.newInstance(SparkHadoopUtil.get.conf)
+ val inputStream = fs.open(new Path(tmpHdfsConfgFile))
+ tmpHdfsConfiguration = new Configuration(false)
+ tmpHdfsConfiguration.readFields(inputStream)
+ inputStream.close()
+ }
+
+ if (tmpHdfsConfiguration == null) {
+ try {
+ tmpHdfsConfiguration = configBroadcast.value.value
+ } catch {
+ case ex: Exception => logError("Unable to getConfig from broadcast", ex)
+ }
+ }
+ tmpHdfsConfiguration
+ }
+
+ /**
+ * underlining wrapper all mapPartition functions in HBaseContext
+ *
+ */
+ private def hbaseMapPartition[K, U](
+ configBroadcast:
+ Broadcast[SerializableWritable[Configuration]],
+ it: Iterator[K],
+ mp: (Iterator[K], Connection) =>
+ Iterator[U]): Iterator[U] = {
+
+ val config = getConf(configBroadcast)
+ applyCreds(configBroadcast)
+
+ val connection = ConnectionFactory.createConnection(config)
+ val res = mp(it, connection)
+ connection.close()
+ res
+
+ }
+
+ /**
+ * underlining wrapper all get mapPartition functions in HBaseContext
+ */
+ private class GetMapPartition[T, U](tableName: TableName,
+ batchSize: Integer,
+ makeGet: (T) => Get,
+ convertResult: (Result) => U)
+ extends Serializable {
+
+ val tName = tableName.getName
+
+ def run(iterator: Iterator[T], connection: Connection): Iterator[U] = {
+ val table = connection.getTable(TableName.valueOf(tName))
+
+ val gets = new java.util.ArrayList[Get]()
+ var res = List[U]()
+
+ while (iterator.hasNext) {
+ gets.add(makeGet(iterator.next()))
+
+ if (gets.size() == batchSize) {
+ val results = table.get(gets)
+ res = res ++ results.map(convertResult)
+ gets.clear()
+ }
+ }
+ if (gets.size() > 0) {
+ val results = table.get(gets)
+ res = res ++ results.map(convertResult)
+ gets.clear()
+ }
+ table.close()
+ res.iterator
+ }
+ }
+
+ /**
+ * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+ *
+ * This method is used to keep ClassTags out of the external Java API, as
+ * the Java compiler cannot produce them automatically. While this
+ * ClassTag-faking does please the compiler, it can cause problems at runtime
+ * if the Scala API relies on ClassTags for correctness.
+ *
+ * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
+ * just worse performance or security issues.
+ * For instance, an Array of AnyRef can hold any type T, but may lose primitive
+ * specialization.
+ */
+ private[spark]
+ def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
+
+ /**
+ * A Spark Implementation of HBase Bulk load
+ *
+ * This will take the content from an existing RDD then sort and shuffle
+ * it with respect to region splits. The result of that sort and shuffle
+ * will be written to HFiles.
+ *
+ * After this function is executed the user will have to call
+ * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
+ *
+ * Also note this version of bulk load is different from past versions in
+ * that it includes the qualifier as part of the sort process. The
+ * reason for this is to be able to support rows will very large number
+ * of columns.
+ *
+ * @param rdd The RDD we are bulk loading from
+ * @param tableName The HBase table we are loading into
+ * @param flatMap A flapMap function that will make every
+ * row in the RDD
+ * into N cells for the bulk load
+ * @param stagingDir The location on the FileSystem to bulk load into
+ * @param familyHFileWriteOptionsMap Options that will define how the HFile for a
+ * column family is written
+ * @param compactionExclude Compaction excluded for the HFiles
+ * @param maxSize Max size for the HFiles before they roll
+ * @tparam T The Type of values in the original RDD
+ */
+ def bulkLoad[T](rdd:RDD[T],
+ tableName: TableName,
+ flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
+ stagingDir:String,
+ familyHFileWriteOptionsMap:
+ util.Map[Array[Byte], FamilyHFileWriteOptions] =
+ new util.HashMap[Array[Byte], FamilyHFileWriteOptions],
+ compactionExclude: Boolean = false,
+ maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):
+ Unit = {
+ val conn = ConnectionFactory.createConnection(config)
+ val regionLocator = conn.getRegionLocator(tableName)
+ val startKeys = regionLocator.getStartKeys
+ val defaultCompressionStr = config.get("hfile.compression",
+ Compression.Algorithm.NONE.getName)
+ val defaultCompression = Compression.getCompressionAlgorithmByName(defaultCompressionStr)
+// HFileWriterImpl
+// .compressionByName(defaultCompressionStr)
+ val now = System.currentTimeMillis()
+ val tableNameByteArray = tableName.getName
+
+ val familyHFileWriteOptionsMapInternal =
+ new util.HashMap[ByteArrayWrapper, FamilyHFileWriteOptions]
+
+ val entrySetIt = familyHFileWriteOptionsMap.entrySet().iterator()
+
+ while (entrySetIt.hasNext) {
+ val entry = entrySetIt.next()
+ familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(entry.getKey), entry.getValue)
+ }
+
+ /**
+ * This will return a new HFile writer when requested
+ *
+ * @param family column family
+ * @param conf configuration to connect to HBase
+ * @param favoredNodes nodes that we would like to write too
+ * @param fs FileSystem object where we will be writing the HFiles to
+ * @return WriterLength object
+ */
+ def getNewWriter(family: Array[Byte], conf: Configuration,
+ favoredNodes: Array[InetSocketAddress],
+ fs:FileSystem,
+ familydir:Path): WriterLength = {
+
+
+ var familyOptions = familyHFileWriteOptionsMapInternal.get(new ByteArrayWrapper(family))
+
+ if (familyOptions == null) {
+ familyOptions = new FamilyHFileWriteOptions(defaultCompression.toString,
+ BloomType.NONE.toString, HConstants.DEFAULT_BLOCKSIZE, DataBlockEncoding.NONE.toString)
+ familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(family), familyOptions)
+ }
+
+ val tempConf = new Configuration(conf)
+ tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f)
+ val contextBuilder = new HFileContextBuilder()
+ .withCompression(Algorithm.valueOf(familyOptions.compression))
+ .withChecksumType(HStore.getChecksumType(conf))
+ .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
+ .withBlockSize(familyOptions.blockSize)
+ contextBuilder.withDataBlockEncoding(DataBlockEncoding.
+ valueOf(familyOptions.dataBlockEncoding))
+ val hFileContext = contextBuilder.build()
+
+ if (null == favoredNodes) {
+ new WriterLength(0, new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs)
+ .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
+ .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext).build())
+// .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext).build())
+ } else {
+ new WriterLength(0,
+ new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), new HFileSystem(fs))
+ .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
+ .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext)
+// .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext)
+ .withFavoredNodes(favoredNodes).build())
+ }
+ }
+
+ val regionSplitPartitioner =
+ new BulkLoadPartitioner(startKeys)
+
+ //This is where all the magic happens
+ //Here we are going to do the following things
+ // 1. FlapMap every row in the RDD into key column value tuples
+ // 2. Then we are going to repartition sort and shuffle
+ // 3. Finally we are going to write out our HFiles
+ rdd.flatMap( r => flatMap(r)).
+ repartitionAndSortWithinPartitions(regionSplitPartitioner).
+ hbaseForeachPartition(this, (it, conn) => {
+
+ val conf = broadcastedConf.value.value
+ val fs = FileSystem.get(conf)
+ val writerMap = new mutable.HashMap[ByteArrayWrapper, WriterLength]
+ var previousRow:Array[Byte] = HConstants.EMPTY_BYTE_ARRAY
+ var rollOverRequested = false
+
+ /**
+ * This will roll all writers
+ */
+ def rollWriters(): Unit = {
+ writerMap.values.foreach( wl => {
+ if (wl.writer != null) {
+ logDebug("Writer=" + wl.writer.getPath +
+ (if (wl.written == 0) "" else ", wrote=" + wl.written))
+ close(wl.writer)
+ }
+ })
+ writerMap.clear()
+ rollOverRequested = false
+ }
+
+ /**
+ * This function will close a given HFile writer
+ * @param w The writer to close
+ */
+ def close(w:StoreFile.Writer): Unit = {
+ if (w != null) {
+ w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
+ Bytes.toBytes(System.currentTimeMillis()))
+ w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
+ Bytes.toBytes(regionSplitPartitioner.getPartition(previousRow)))
+ w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY,
+ Bytes.toBytes(true))
+ w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
+ Bytes.toBytes(compactionExclude))
+ w.appendTrackedTimestampsToMetadata()
+ w.close()
+ }
+ }
+
+ //Here is where we finally iterate through the data in this partition of the
+ //RDD that has been sorted and partitioned
+ it.foreach{ case (keyFamilyQualifier, cellValue:Array[Byte]) =>
+
+ //This will get a writer for the column family
+ //If there is no writer for a given column family then
+ //it will get created here.
+ val wl = writerMap.getOrElseUpdate(new ByteArrayWrapper(keyFamilyQualifier.family), {
+
+ val familyDir = new Path(stagingDir, Bytes.toString(keyFamilyQualifier.family))
+
+ fs.mkdirs(familyDir)
+
+ val loc:HRegionLocation = {
+ try {
+ val locator =
+ conn.getRegionLocator(TableName.valueOf(tableNameByteArray))
+ locator.getRegionLocation(keyFamilyQualifier.rowKey)
+ } catch {
+ case e: Throwable =>
+ logWarning("there's something wrong when locating rowkey: " +
+ Bytes.toString(keyFamilyQualifier.rowKey))
+ null
+ }
+ }
+ if (null == loc) {
+ if (log.isTraceEnabled) {
+ logTrace("failed to get region location, so use default writer: " +
+ Bytes.toString(keyFamilyQualifier.rowKey))
+ }
+ getNewWriter(family = keyFamilyQualifier.family, conf = conf, favoredNodes = null,
+ fs = fs, familydir = familyDir)
+ } else {
+ if (log.isDebugEnabled) {
+ logDebug("first rowkey: [" + Bytes.toString(keyFamilyQualifier.rowKey) + "]")
+ }
+ val initialIsa =
+ new InetSocketAddress(loc.getHostname, loc.getPort)
+ if (initialIsa.isUnresolved) {
+ if (log.isTraceEnabled) {
+ logTrace("failed to resolve bind address: " + loc.getHostname + ":"
+ + loc.getPort + ", so use default writer")
+ }
+ getNewWriter(keyFamilyQualifier.family, conf, null, fs, familyDir)
+ } else {
+ if(log.isDebugEnabled) {
+ logDebug("use favored nodes writer: " + initialIsa.getHostString)
+ }
+ getNewWriter(keyFamilyQualifier.family, conf,
+ Array[InetSocketAddress](initialIsa), fs, familyDir)
+ }
+ }
+ })
+
+ val keyValue =new KeyValue(keyFamilyQualifier.rowKey,
+ keyFamilyQualifier.family,
+ keyFamilyQualifier.qualifier,
+ now,cellValue)
+
+ wl.writer.append(keyValue)
+ wl.written += keyValue.getLength
+
+ rollOverRequested = rollOverRequested || wl.written > maxSize
+
+ //This will only roll if we have at least one column family file that is
+ //bigger then maxSize and we have finished a given row key
+ if (rollOverRequested && Bytes.compareTo(previousRow, keyFamilyQualifier.rowKey) != 0) {
+ rollWriters()
+ }
+
+ previousRow = keyFamilyQualifier.rowKey
+ }
+ //We have finished all the data so lets close up the writers
+ rollWriters()
+ })
+ }
+
+ /**
+ * This is a wrapper class around StoreFile.Writer. The reason for the
+ * wrapper is to keep the length of the file along side the writer
+ *
+ * @param written The writer to be wrapped
+ * @param writer The number of bytes written to the writer
+ */
+ class WriterLength(var written:Long, val writer:StoreFile.Writer)
+
+ /**
+ * This is a wrapper over a byte array so it can work as
+ * a key in a hashMap
+ *
+ * @param o1 The Byte Array value
+ */
+ class ByteArrayWrapper (val o1:Array[Byte])
+ extends Comparable[ByteArrayWrapper] with Serializable {
+ override def compareTo(o2: ByteArrayWrapper): Int = {
+ Bytes.compareTo(o1,o2.o1)
+ }
+ override def equals(o2: Any): Boolean = {
+ o2 match {
+ case wrapper: ByteArrayWrapper =>
+ Bytes.equals(o1, wrapper.o1)
+ case _ =>
+ false
+ }
+ }
+ override def hashCode():Int = {
+ Bytes.hashCode(o1)
+ }
+ }
+}
+
+object LatestHBaseContextCache {
+ var latest:HBaseContext = null
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseDStreamFunctions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseDStreamFunctions.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseDStreamFunctions.scala
new file mode 100644
index 0000000..80f289b
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseDStreamFunctions.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.s2graph.loader.spark
+
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.spark.streaming.dstream.DStream
+
+import scala.reflect.ClassTag
+
+/**
+ * HBaseDStreamFunctions contains a set of implicit functions that can be
+ * applied to a Spark DStream so that we can easily interact with HBase
+ */
+object HBaseDStreamFunctions {
+
+ /**
+ * These are implicit methods for a DStream that contains any type of
+ * data.
+ *
+ * @param dStream This is for dStreams of any type
+ * @tparam T Type T
+ */
+ implicit class GenericHBaseDStreamFunctions[T](val dStream: DStream[T]) {
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * put. This will not return a new Stream. Think of it like a foreach
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param f The function that will turn the DStream values
+ * into HBase Put objects.
+ */
+ def hbaseBulkPut(hc: HBaseContext,
+ tableName: TableName,
+ f: (T) => Put): Unit = {
+ hc.streamBulkPut(dStream, tableName, f)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * get. This will return a new DStream. Think about it as a DStream map
+ * function. In that every DStream value will get a new value out of
+ * HBase. That new value will populate the newly generated DStream.
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param batchSize How many gets to execute in a single batch
+ * @param f The function that will turn the RDD values
+ * in HBase Get objects
+ * @param convertResult The function that will convert a HBase
+ * Result object into a value that will go
+ * into the resulting DStream
+ * @tparam R The type of Object that will be coming
+ * out of the resulting DStream
+ * @return A resulting DStream with type R objects
+ */
+ def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
+ tableName: TableName,
+ batchSize:Int, f: (T) => Get, convertResult: (Result) => R):
+ DStream[R] = {
+ hc.streamBulkGet[T, R](tableName, batchSize, dStream, f, convertResult)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * get. This will return a new DStream. Think about it as a DStream map
+ * function. In that every DStream value will get a new value out of
+ * HBase. That new value will populate the newly generated DStream.
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param batchSize How many gets to execute in a single batch
+ * @param f The function that will turn the RDD values
+ * in HBase Get objects
+ * @return A resulting DStream with type R objects
+ */
+ def hbaseBulkGet(hc: HBaseContext,
+ tableName: TableName, batchSize:Int,
+ f: (T) => Get): DStream[(ImmutableBytesWritable, Result)] = {
+ hc.streamBulkGet[T, (ImmutableBytesWritable, Result)](
+ tableName, batchSize, dStream, f,
+ result => (new ImmutableBytesWritable(result.getRow), result))
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * Delete. This will not return a new DStream.
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param tableName The tableName that the deletes will be sent to
+ * @param f The function that will convert the DStream value into
+ * a HBase Delete Object
+ * @param batchSize The number of Deletes to be sent in a single batch
+ */
+ def hbaseBulkDelete(hc: HBaseContext,
+ tableName: TableName,
+ f:(T) => Delete, batchSize:Int): Unit = {
+ hc.streamBulkDelete(dStream, tableName, f, batchSize)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's
+ * foreachPartition method. This will ack very much like a normal DStream
+ * foreach method but for the fact that you will now have a HBase connection
+ * while iterating through the values.
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param f This function will get an iterator for a Partition of an
+ * DStream along with a connection object to HBase
+ */
+ def hbaseForeachPartition(hc: HBaseContext,
+ f: (Iterator[T], Connection) => Unit): Unit = {
+ hc.streamForeachPartition(dStream, f)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's
+ * mapPartitions method. This will ask very much like a normal DStream
+ * map partitions method but for the fact that you will now have a
+ * HBase connection while iterating through the values
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param f This function will get an iterator for a Partition of an
+ * DStream along with a connection object to HBase
+ * @tparam R This is the type of objects that will go into the resulting
+ * DStream
+ * @return A resulting DStream of type R
+ */
+ def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
+ f: (Iterator[T], Connection) => Iterator[R]):
+ DStream[R] = {
+ hc.streamMapPartitions(dStream, f)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
new file mode 100644
index 0000000..b818a3c
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import java.util
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.hbase.{HConstants, TableName}
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.spark.rdd.RDD
+
+import scala.collection.immutable.HashMap
+import scala.reflect.ClassTag
+
+/**
+ * HBaseRDDFunctions contains a set of implicit functions that can be
+ * applied to a Spark RDD so that we can easily interact with HBase
+ */
+object HBaseRDDFunctions
+{
+
+ /**
+ * These are implicit methods for a RDD that contains any type of
+ * data.
+ *
+ * @param rdd This is for rdd of any type
+ * @tparam T This is any type
+ */
+ implicit class GenericHBaseRDDFunctions[T](val rdd: RDD[T]) {
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * put. This will not return a new RDD. Think of it like a foreach
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param f The function that will turn the RDD values
+ * into HBase Put objects.
+ */
+ def hbaseBulkPut(hc: HBaseContext,
+ tableName: TableName,
+ f: (T) => Put): Unit = {
+ hc.bulkPut(rdd, tableName, f)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * get. This will return a new RDD. Think about it as a RDD map
+ * function. In that every RDD value will get a new value out of
+ * HBase. That new value will populate the newly generated RDD.
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param batchSize How many gets to execute in a single batch
+ * @param f The function that will turn the RDD values
+ * in HBase Get objects
+ * @param convertResult The function that will convert a HBase
+ * Result object into a value that will go
+ * into the resulting RDD
+ * @tparam R The type of Object that will be coming
+ * out of the resulting RDD
+ * @return A resulting RDD with type R objects
+ */
+ def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
+ tableName: TableName, batchSize:Int,
+ f: (T) => Get, convertResult: (Result) => R): RDD[R] = {
+ hc.bulkGet[T, R](tableName, batchSize, rdd, f, convertResult)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * get. This will return a new RDD. Think about it as a RDD map
+ * function. In that every RDD value will get a new value out of
+ * HBase. That new value will populate the newly generated RDD.
+ *
+ * @param hc The hbaseContext object to identify which
+ * HBase cluster connection to use
+ * @param tableName The tableName that the put will be sent to
+ * @param batchSize How many gets to execute in a single batch
+ * @param f The function that will turn the RDD values
+ * in HBase Get objects
+ * @return A resulting RDD with type R objects
+ */
+ def hbaseBulkGet(hc: HBaseContext,
+ tableName: TableName, batchSize:Int,
+ f: (T) => Get): RDD[(ImmutableBytesWritable, Result)] = {
+ hc.bulkGet[T, (ImmutableBytesWritable, Result)](tableName,
+ batchSize, rdd, f,
+ result => if (result != null && result.getRow != null) {
+ (new ImmutableBytesWritable(result.getRow), result)
+ } else {
+ null
+ })
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's bulk
+ * Delete. This will not return a new RDD.
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param tableName The tableName that the deletes will be sent to
+ * @param f The function that will convert the RDD value into
+ * a HBase Delete Object
+ * @param batchSize The number of Deletes to be sent in a single batch
+ */
+ def hbaseBulkDelete(hc: HBaseContext,
+ tableName: TableName, f:(T) => Delete, batchSize:Int): Unit = {
+ hc.bulkDelete(rdd, tableName, f, batchSize)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's
+ * foreachPartition method. This will ack very much like a normal RDD
+ * foreach method but for the fact that you will now have a HBase connection
+ * while iterating through the values.
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param f This function will get an iterator for a Partition of an
+ * RDD along with a connection object to HBase
+ */
+ def hbaseForeachPartition(hc: HBaseContext,
+ f: (Iterator[T], Connection) => Unit): Unit = {
+ hc.foreachPartition(rdd, f)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's
+ * mapPartitions method. This will ask very much like a normal RDD
+ * map partitions method but for the fact that you will now have a
+ * HBase connection while iterating through the values
+ *
+ * @param hc The hbaseContext object to identify which HBase
+ * cluster connection to use
+ * @param f This function will get an iterator for a Partition of an
+ * RDD along with a connection object to HBase
+ * @tparam R This is the type of objects that will go into the resulting
+ * RDD
+ * @return A resulting RDD of type R
+ */
+ def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
+ f: (Iterator[T], Connection) => Iterator[R]):
+ RDD[R] = {
+ hc.mapPartitions[T,R](rdd, f)
+ }
+
+ /**
+ * Implicit method that gives easy access to HBaseContext's
+ * bulkLoad method.
+ *
+ * A Spark Implementation of HBase Bulk load
+ *
+ * This will take the content from an existing RDD then sort and shuffle
+ * it with respect to region splits. The result of that sort and shuffle
+ * will be written to HFiles.
+ *
+ * After this function is executed the user will have to call
+ * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
+ *
+ * Also note this version of bulk load is different from past versions in
+ * that it includes the qualifier as part of the sort process. The
+ * reason for this is to be able to support rows will very large number
+ * of columns.
+ *
+ * @param tableName The HBase table we are loading into
+ * @param flatMap A flapMap function that will make every row in the RDD
+ * into N cells for the bulk load
+ * @param stagingDir The location on the FileSystem to bulk load into
+ * @param familyHFileWriteOptionsMap Options that will define how the HFile for a
+ * column family is written
+ * @param compactionExclude Compaction excluded for the HFiles
+ * @param maxSize Max size for the HFiles before they roll
+ */
+ def hbaseBulkLoad(hc: HBaseContext,
+ tableName: TableName,
+ flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
+ stagingDir:String,
+ familyHFileWriteOptionsMap:
+ util.Map[Array[Byte], FamilyHFileWriteOptions] =
+ new util.HashMap[Array[Byte], FamilyHFileWriteOptions](),
+ compactionExclude: Boolean = false,
+ maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):Unit = {
+ hc.bulkLoad(rdd, tableName,
+ flatMap, stagingDir, familyHFileWriteOptionsMap,
+ compactionExclude, maxSize)
+ }
+ }
+}
[06/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/FunctionParser.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/FunctionParser.scala b/s2counter_core/src/main/scala/s2/util/FunctionParser.scala
deleted file mode 100644
index 2454b0f..0000000
--- a/s2counter_core/src/main/scala/s2/util/FunctionParser.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package s2.util
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 29..
- */
-object FunctionParser {
- val funcRe = """([a-zA-Z_]+)(\((\d+)?\))?""".r
-
- def apply(str: String): Option[(String, String)] = {
- str match {
- case funcRe(funcName, funcParam, funcArg) =>
- funcName match {
- case x: String =>
- Some((funcName, funcArg match {
- case x: String => funcArg
- case null => ""
- }))
- case null => None
- }
- case _ =>
- None
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/Hashes.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/Hashes.scala b/s2counter_core/src/main/scala/s2/util/Hashes.scala
deleted file mode 100644
index 2edbcd8..0000000
--- a/s2counter_core/src/main/scala/s2/util/Hashes.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package s2.util
-
-import org.apache.hadoop.hbase.util.Bytes
-
-import scala.util.hashing.MurmurHash3
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 27..
- */
-object Hashes {
- def sha1(s: String): String = {
- val md = java.security.MessageDigest.getInstance("SHA-1")
- Bytes.toHex(md.digest(s.getBytes("UTF-8")))
- }
-
- private def positiveHash(h: Int): Int = {
- if (h < 0) -1 * (h + 1) else h
- }
-
- def murmur3(s: String): Int = {
- val hash = MurmurHash3.stringHash(s)
- positiveHash(hash)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/ReduceMapValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/ReduceMapValue.scala b/s2counter_core/src/main/scala/s2/util/ReduceMapValue.scala
deleted file mode 100644
index 9c13fa0..0000000
--- a/s2counter_core/src/main/scala/s2/util/ReduceMapValue.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package s2.util
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 7. 20..
- */
-class ReduceMapValue[T, U](op: (U, U) => U, default: U) {
- def apply(m1: Map[T, U], m2: Map[T, U]): Map[T, U] = {
- m1 ++ m2.map { case (k, v) =>
- k -> op(m1.getOrElse(k, default), v)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/Retry.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/Retry.scala b/s2counter_core/src/main/scala/s2/util/Retry.scala
deleted file mode 100644
index d1f7213..0000000
--- a/s2counter_core/src/main/scala/s2/util/Retry.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-package s2.util
-
-import scala.annotation.tailrec
-import scala.concurrent.{ExecutionContext, Future, Promise}
-import scala.util.{Failure, Success, Try}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 1. 6..
- */
-object Retry {
- @tailrec
- def apply[T](n: Int, withSleep: Boolean = true, tryCount: Int = 0)(fn: => T): T = {
- Try { fn } match {
- case Success(x) => x
- case Failure(e) if e.isInstanceOf[RetryStopException] => throw e.getCause
- case _ if n > 1 =>
- // backoff
- if (withSleep) Thread.sleep(tryCount * 1000)
- apply(n - 1, withSleep, tryCount + 1)(fn)
- case Failure(e) => throw e
- }
- }
-}
-
-object RetryAsync {
- def apply[T](n: Int, withSleep: Boolean = true, tryCount: Int = 0)(fn: => Future[T])(implicit ex: ExecutionContext): Future[T] = {
- val promise = Promise[T]()
- fn onComplete {
- case Success(x) => promise.success(x)
- case Failure(e) if e.isInstanceOf[RetryStopException] => promise.failure(e.getCause)
- case _ if n > 1 =>
- // backoff
- if (withSleep) Thread.sleep(tryCount * 1000)
- apply(n - 1, withSleep, tryCount + 1)(fn)
- case Failure(e) => promise.failure(e)
- }
- promise.future
- }
-}
-
-class RetryStopException(message: String, cause: Throwable)
- extends Exception(message, cause) {
-
- def this(message: String) = this(message, null)
-
- def this(cause: Throwable) = this(cause.toString, cause)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/SplitBytes.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/SplitBytes.scala b/s2counter_core/src/main/scala/s2/util/SplitBytes.scala
deleted file mode 100644
index a5ca998..0000000
--- a/s2counter_core/src/main/scala/s2/util/SplitBytes.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package s2.util
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 12..
- */
-object SplitBytes {
- def apply(bytes: Array[Byte], sizes: Seq[Int]): Seq[Array[Byte]] = {
- if (sizes.sum > bytes.length) {
- throw new Exception(s"sizes.sum bigger than bytes.length ${sizes.sum} > ${bytes.length}} ")
- }
-
- var position = 0
- val rtn = {
- for {
- size <- sizes
- } yield {
- val slice = bytes.slice(position, position + size)
- position += size
- slice
- }
- }
- rtn ++ Seq(bytes.drop(position))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/UnitConverter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/UnitConverter.scala b/s2counter_core/src/main/scala/s2/util/UnitConverter.scala
deleted file mode 100644
index fb0b0d0..0000000
--- a/s2counter_core/src/main/scala/s2/util/UnitConverter.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-package s2.util
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 4. 3..
- */
-object UnitConverter {
- def toMillis(ts: Int): Long = {
- ts * 1000L
- }
-
- def toMillis(ts: Long): Long = {
- if (ts <= Int.MaxValue) {
- ts * 1000
- } else {
- ts
- }
- }
-
- def toMillis(s: String): Long = {
- toMillis(s.toLong)
- }
-
- def toHours(ts: Long): Long = {
- toMillis(ts) / HOUR_MILLIS * HOUR_MILLIS
- }
-
- val HOUR_MILLIS = 60 * 60 * 1000
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/org/apache/s2graph/counter/core/RankingCounterSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/org/apache/s2graph/counter/core/RankingCounterSpec.scala b/s2counter_core/src/test/scala/org/apache/s2graph/counter/core/RankingCounterSpec.scala
new file mode 100644
index 0000000..1ae7b12
--- /dev/null
+++ b/s2counter_core/src/test/scala/org/apache/s2graph/counter/core/RankingCounterSpec.scala
@@ -0,0 +1,166 @@
+package org.apache.s2graph.counter.core
+
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.core.{Graph, Management}
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core.v2.{GraphOperation, RankingStorageGraph}
+import org.apache.s2graph.counter.helper.CounterAdmin
+import org.apache.s2graph.counter.models.{Counter, CounterModel, DBModel}
+import org.apache.s2graph.counter.util.Retry
+import org.specs2.mutable.Specification
+import org.specs2.specification.BeforeAfterAll
+import play.api.libs.json.Json
+
+import scala.util.{Failure, Random, Success, Try}
+
+class RankingCounterSpec extends Specification with BeforeAfterAll {
+ val config = ConfigFactory.load()
+ DBModel.initialize(config)
+ val counterModel = new CounterModel(config)
+ val admin = new CounterAdmin(config)
+
+ val s2config = new S2CounterConfig(config)
+
+// val rankingCounterV1 = new RankingCounter(config, new RankingStorageV1(config))
+// val rankingCounterV2 = new RankingCounter(config, new RankingStorageV2(config))
+ val rankingCounterV2 = new RankingCounter(config, new RankingStorageGraph(config))
+
+// "RankingCounterV1" >> {
+// val policy = counterModel.findByServiceAction("test", "test_action", useCache = false).get
+// val rankingKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier(IntervalUnit.TOTAL, 0L), Map.empty[String, String]))
+// "get top k" >> {
+// val result = rankingCounterV1.getTopK(rankingKey, 100)
+//
+// println(result)
+//
+// result must not be empty
+// }
+//
+// "get and increment" >> {
+// val result = rankingCounterV1.getTopK(rankingKey, 100).get
+//
+// val value = 2d
+// val contents = {
+// for {
+// (item, score) <- result.values
+// } yield {
+// item -> RankingValue(score + value, value)
+// }
+// }.toMap
+// rankingCounterV1.update(rankingKey, contents, 100)
+//
+// val result2 = rankingCounterV1.getTopK(rankingKey, 100).get
+//
+// result2.totalScore must_== result.totalScore + contents.values.map(_.increment).sum
+// result2.values must containTheSameElementsAs(result.values.map { case (k, v) => (k, v + value) })
+// }
+// }
+
+ val service = "test"
+ val action = "test_case"
+
+ override def beforeAll: Unit = {
+ Try {
+ Retry(3) {
+ admin.setupCounterOnGraph
+ }
+
+ val graphOp = new GraphOperation(config)
+ val graph = new Graph(config)(scala.concurrent.ExecutionContext.global)
+ val management = new Management(graph)
+ management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
+ val strJs =
+ s"""
+ |{
+ | "label": "$action",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "src",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "$action",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | ],
+ | "props": [
+ | ]
+ |}
+ """.stripMargin
+ Retry(3) {
+ if (Label.findByName(action).isEmpty) {
+ graphOp.createLabel(Json.parse(strJs))
+ }
+ }
+
+ admin.deleteCounter(service, action).foreach {
+ case Failure(ex) =>
+ println(s"$ex")
+ throw ex
+ case Success(v) =>
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
+ } match {
+ case Failure(ex) =>
+ println(s"$ex")
+ case Success(_) =>
+ }
+ }
+
+ override def afterAll: Unit = {
+ admin.deleteCounter(service, action)
+ }
+
+ "RankingCounterV2" >> {
+ "get top k" >> {
+ val policy = counterModel.findByServiceAction(service, action, useCache = true).get
+
+ val rankingKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier(IntervalUnit.TOTAL, 0L), Map.empty[String, String]))
+
+ val orgMap = Map(
+ "1" -> 1d,
+ "2" -> 2d,
+ "3" -> 3d,
+ "4" -> 4d,
+ "5" -> 5d,
+ "6" -> 6d,
+ "7" -> 7d,
+ "8" -> 8d,
+ "9" -> 9d,
+ "10" -> 10d,
+ "11" -> 11d,
+ "12" -> 12d,
+ "13" -> 13d,
+ "14" -> 14d,
+ "15" -> 15d,
+ "16" -> 16d,
+ "17" -> 17d,
+ "18" -> 18d,
+ "19" -> 19d,
+ "20" -> 20d,
+ "100" -> 100d
+ )
+
+ val valueMap = Random.shuffle(orgMap).toMap
+
+ val predictResult = valueMap.toSeq.sortBy(-_._2)
+
+ val rvMap = valueMap.map { case (k, score) =>
+ k -> RankingValue(score, 0)
+ }
+ Try {
+ rankingCounterV2.update(rankingKey, rvMap, 100)
+ }.isSuccess must_== true
+
+ Thread.sleep(1000)
+
+ val result : RankingResult = rankingCounterV2.getTopK(rankingKey, 10).get
+
+ println(result.values)
+
+ result must not be empty
+ result.values must have size 10
+ result.values must_== predictResult.take(10)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterModelSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterModelSpec.scala b/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterModelSpec.scala
new file mode 100644
index 0000000..a52b8f9
--- /dev/null
+++ b/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterModelSpec.scala
@@ -0,0 +1,50 @@
+package org.apache.s2graph.counter.models
+
+import com.typesafe.config.ConfigFactory
+import org.specs2.mutable.Specification
+
+class CounterModelSpec extends Specification {
+ val config = ConfigFactory.load()
+
+ DBModel.initialize(config)
+
+ "CounterModel" should {
+ val model = new CounterModel(config)
+ "findById" in {
+ model.findById(0, useCache = false) must beNone
+ }
+
+ "findByServiceAction using cache" in {
+ val service = "test"
+ val action = "test_action"
+ val counter = Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING,
+ autoComb = true, "", useProfile = true, None, useRank = true, 0, None, None, None, None, None, None)
+ model.createServiceAction(counter)
+ model.findByServiceAction(service, action, useCache = false) must beSome
+ val opt = model.findByServiceAction(service, action, useCache = true)
+ opt must beSome
+ model.findById(opt.get.id) must beSome
+ model.deleteServiceAction(opt.get)
+ model.findById(opt.get.id) must beSome
+ model.findById(opt.get.id, useCache = false) must beNone
+ }
+
+ "create and delete policy" in {
+ val (service, action) = ("test", "test_case")
+ for {
+ policy <- model.findByServiceAction(service, action, useCache = false)
+ } {
+ model.deleteServiceAction(policy)
+ }
+ model.createServiceAction(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING,
+ autoComb = true, "", useProfile = true, None, useRank = true, 0, None, None, None, None, None, None))
+ model.findByServiceAction(service, action, useCache = false).map { policy =>
+ policy.service mustEqual service
+ policy.action mustEqual action
+ model.deleteServiceAction(policy)
+ policy
+ } must beSome
+ model.findByServiceAction(service, action, useCache = false) must beNone
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterSpec.scala b/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterSpec.scala
new file mode 100644
index 0000000..220c30f
--- /dev/null
+++ b/s2counter_core/src/test/scala/org/apache/s2graph/counter/models/CounterSpec.scala
@@ -0,0 +1,33 @@
+package org.apache.s2graph.counter.models
+
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.specs2.mutable.Specification
+
+class CounterSpec extends Specification {
+ "Counter" should {
+ "dimension auto combination" in {
+ val policy = Counter(
+ useFlag = true,
+ 2,
+ "test",
+ "test_case",
+ ItemType.LONG,
+ autoComb = true,
+ "p1,p2,p3",
+ useProfile = false,
+ None,
+ useRank = true,
+ 0,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None
+ )
+
+ policy.dimensionSp mustEqual Array("p1", "p2", "p3")
+ policy.dimensionList.map { arr => arr.toSeq }.toSet -- Set(Seq.empty[String], Seq("p1"), Seq("p2"), Seq("p3"), Seq("p1", "p2"), Seq("p1", "p3"), Seq("p2", "p3"), Seq("p1", "p2", "p3")) must beEmpty
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/s2/counter/core/RankingCounterSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/s2/counter/core/RankingCounterSpec.scala b/s2counter_core/src/test/scala/s2/counter/core/RankingCounterSpec.scala
deleted file mode 100644
index ea67a2a..0000000
--- a/s2counter_core/src/test/scala/s2/counter/core/RankingCounterSpec.scala
+++ /dev/null
@@ -1,169 +0,0 @@
-package s2.counter.core
-
-import com.kakao.s2graph.core.{Management, Graph}
-import com.kakao.s2graph.core.mysqls.Label
-import com.typesafe.config.ConfigFactory
-import org.specs2.mutable.Specification
-import org.specs2.specification.BeforeAfterAll
-import play.api.libs.json.Json
-import s2.config.S2CounterConfig
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core.v2.{GraphOperation, RankingStorageGraph}
-import s2.helper.CounterAdmin
-import s2.models.{Counter, CounterModel, DBModel}
-import s2.util.Retry
-
-import scala.util.{Failure, Random, Success, Try}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-class RankingCounterSpec extends Specification with BeforeAfterAll {
- val config = ConfigFactory.load()
- DBModel.initialize(config)
- val counterModel = new CounterModel(config)
- val admin = new CounterAdmin(config)
-
- val s2config = new S2CounterConfig(config)
-
-// val rankingCounterV1 = new RankingCounter(config, new RankingStorageV1(config))
-// val rankingCounterV2 = new RankingCounter(config, new RankingStorageV2(config))
- val rankingCounterV2 = new RankingCounter(config, new RankingStorageGraph(config))
-
-// "RankingCounterV1" >> {
-// val policy = counterModel.findByServiceAction("test", "test_action", useCache = false).get
-// val rankingKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier(IntervalUnit.TOTAL, 0L), Map.empty[String, String]))
-// "get top k" >> {
-// val result = rankingCounterV1.getTopK(rankingKey, 100)
-//
-// println(result)
-//
-// result must not be empty
-// }
-//
-// "get and increment" >> {
-// val result = rankingCounterV1.getTopK(rankingKey, 100).get
-//
-// val value = 2d
-// val contents = {
-// for {
-// (item, score) <- result.values
-// } yield {
-// item -> RankingValue(score + value, value)
-// }
-// }.toMap
-// rankingCounterV1.update(rankingKey, contents, 100)
-//
-// val result2 = rankingCounterV1.getTopK(rankingKey, 100).get
-//
-// result2.totalScore must_== result.totalScore + contents.values.map(_.increment).sum
-// result2.values must containTheSameElementsAs(result.values.map { case (k, v) => (k, v + value) })
-// }
-// }
-
- val service = "test"
- val action = "test_case"
-
- override def beforeAll: Unit = {
- Try {
- Retry(3) {
- admin.setupCounterOnGraph
- }
-
- val graphOp = new GraphOperation(config)
- val graph = new Graph(config)(scala.concurrent.ExecutionContext.global)
- val management = new Management(graph)
- management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
- val strJs =
- s"""
- |{
- | "label": "$action",
- | "srcServiceName": "$service",
- | "srcColumnName": "src",
- | "srcColumnType": "string",
- | "tgtServiceName": "$service",
- | "tgtColumnName": "$action",
- | "tgtColumnType": "string",
- | "indices": [
- | ],
- | "props": [
- | ]
- |}
- """.stripMargin
- Retry(3) {
- if (Label.findByName(action).isEmpty) {
- graphOp.createLabel(Json.parse(strJs))
- }
- }
-
- admin.deleteCounter(service, action).foreach {
- case Failure(ex) =>
- println(s"$ex")
- throw ex
- case Success(v) =>
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
- } match {
- case Failure(ex) =>
- println(s"$ex")
- case Success(_) =>
- }
- }
-
- override def afterAll: Unit = {
- admin.deleteCounter(service, action)
- }
-
- "RankingCounterV2" >> {
- "get top k" >> {
- val policy = counterModel.findByServiceAction(service, action, useCache = true).get
-
- val rankingKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier(IntervalUnit.TOTAL, 0L), Map.empty[String, String]))
-
- val orgMap = Map(
- "1" -> 1d,
- "2" -> 2d,
- "3" -> 3d,
- "4" -> 4d,
- "5" -> 5d,
- "6" -> 6d,
- "7" -> 7d,
- "8" -> 8d,
- "9" -> 9d,
- "10" -> 10d,
- "11" -> 11d,
- "12" -> 12d,
- "13" -> 13d,
- "14" -> 14d,
- "15" -> 15d,
- "16" -> 16d,
- "17" -> 17d,
- "18" -> 18d,
- "19" -> 19d,
- "20" -> 20d,
- "100" -> 100d
- )
-
- val valueMap = Random.shuffle(orgMap).toMap
-
- val predictResult = valueMap.toSeq.sortBy(-_._2)
-
- val rvMap = valueMap.map { case (k, score) =>
- k -> RankingValue(score, 0)
- }
- Try {
- rankingCounterV2.update(rankingKey, rvMap, 100)
- }.isSuccess must_== true
-
- Thread.sleep(1000)
-
- val result : RankingResult = rankingCounterV2.getTopK(rankingKey, 10).get
-
- println(result.values)
-
- result must not be empty
- result.values must have size 10
- result.values must_== predictResult.take(10)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/s2/models/CounterModelSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/s2/models/CounterModelSpec.scala b/s2counter_core/src/test/scala/s2/models/CounterModelSpec.scala
deleted file mode 100644
index dff55ef..0000000
--- a/s2counter_core/src/test/scala/s2/models/CounterModelSpec.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package s2.models
-
-import com.typesafe.config.ConfigFactory
-import org.specs2.mutable.Specification
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 26..
- */
-class CounterModelSpec extends Specification {
- val config = ConfigFactory.load()
-
- DBModel.initialize(config)
-
- "CounterModel" should {
- val model = new CounterModel(config)
- "findById" in {
- model.findById(0, useCache = false) must beNone
- }
-
- "findByServiceAction using cache" in {
- val service = "test"
- val action = "test_action"
- val counter = Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING,
- autoComb = true, "", useProfile = true, None, useRank = true, 0, None, None, None, None, None, None)
- model.createServiceAction(counter)
- model.findByServiceAction(service, action, useCache = false) must beSome
- val opt = model.findByServiceAction(service, action, useCache = true)
- opt must beSome
- model.findById(opt.get.id) must beSome
- model.deleteServiceAction(opt.get)
- model.findById(opt.get.id) must beSome
- model.findById(opt.get.id, useCache = false) must beNone
- }
-
- "create and delete policy" in {
- val (service, action) = ("test", "test_case")
- for {
- policy <- model.findByServiceAction(service, action, useCache = false)
- } {
- model.deleteServiceAction(policy)
- }
- model.createServiceAction(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING,
- autoComb = true, "", useProfile = true, None, useRank = true, 0, None, None, None, None, None, None))
- model.findByServiceAction(service, action, useCache = false).map { policy =>
- policy.service mustEqual service
- policy.action mustEqual action
- model.deleteServiceAction(policy)
- policy
- } must beSome
- model.findByServiceAction(service, action, useCache = false) must beNone
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/test/scala/s2/models/CounterSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/test/scala/s2/models/CounterSpec.scala b/s2counter_core/src/test/scala/s2/models/CounterSpec.scala
deleted file mode 100644
index a03c70e..0000000
--- a/s2counter_core/src/test/scala/s2/models/CounterSpec.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-package s2.models
-
-import org.specs2.mutable.Specification
-import s2.models.Counter.ItemType
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 11..
- */
-class CounterSpec extends Specification {
- "Counter" should {
- "dimension auto combination" in {
- val policy = Counter(
- useFlag = true,
- 2,
- "test",
- "test_case",
- ItemType.LONG,
- autoComb = true,
- "p1,p2,p3",
- useProfile = false,
- None,
- useRank = true,
- 0,
- None,
- None,
- None,
- None,
- None,
- None
- )
-
- policy.dimensionSp mustEqual Array("p1", "p2", "p3")
- policy.dimensionList.map { arr => arr.toSeq }.toSet -- Set(Seq.empty[String], Seq("p1"), Seq("p2"), Seq("p3"), Seq("p1", "p2"), Seq("p1", "p3"), Seq("p2", "p3"), Seq("p1", "p2", "p3")) must beEmpty
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/CounterBulkLoader.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/CounterBulkLoader.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/CounterBulkLoader.scala
new file mode 100644
index 0000000..64da217
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/CounterBulkLoader.scala
@@ -0,0 +1,78 @@
+package org.apache.s2graph.counter.loader
+
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.BlobExactKey
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.core.{CounterFunctions, CounterEtlFunctions}
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.apache.s2graph.counter.models.{DBModel, CounterModel}
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{HashMapParam, SparkApp, WithKafka}
+import org.apache.spark.SparkContext
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext
+
+object CounterBulkLoader extends SparkApp with WithKafka {
+ lazy val config = S2ConfigFactory.config
+ lazy val s2Config = new S2CounterConfig(config)
+ lazy val counterModel = new CounterModel(config)
+ lazy val className = getClass.getName.stripSuffix("$")
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ implicit val ec = ExecutionContext.Implicits.global
+
+ val initialize = {
+ println("initialize")
+ // Graph(config)
+ DBModel.initialize(config)
+ true
+ }
+
+ override def run(): Unit = {
+ val hdfsPath = args(0)
+ val blockSize = args(1).toInt
+ val minPartitions = args(2).toInt
+ val conf = sparkConf(s"$hdfsPath: CounterBulkLoader")
+
+ val sc = new SparkContext(conf)
+ val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ val msgs = sc.textFile(hdfsPath)
+
+ val etlRdd = msgs.repartition(minPartitions).mapPartitions { part =>
+ // parse and etl
+ assert(initialize)
+ val items = {
+ for {
+ msg <- part
+ line <- GraphUtil.parseString(msg)
+ sp = GraphUtil.split(line) if sp.size <= 7 || GraphUtil.split(line)(7) != "in"
+ item <- CounterEtlFunctions.parseEdgeFormat(line)
+ } yield {
+ acc +=("Edges", 1)
+ item
+ }
+ }
+ items.grouped(blockSize).flatMap { grouped =>
+ grouped.groupBy(e => (e.service, e.action)).flatMap { case ((service, action), v) =>
+ CounterEtlFunctions.checkPolicyAndMergeDimension(service, action, v.toList)
+ }
+ }
+ }
+
+ val exactRdd = CounterFunctions.exactCountFromEtl(etlRdd, etlRdd.partitions.length)
+ val logRdd = exactRdd.mapPartitions { part =>
+ val seq = part.toSeq
+ CounterFunctions.insertBlobValue(seq.map(_._1).filter(_.itemType == ItemType.BLOB).map(_.asInstanceOf[BlobExactKey]), acc)
+ // update exact counter
+ CounterFunctions.updateExactCounter(seq, acc).toIterator
+ }
+
+ val rankRdd = CounterFunctions.makeRankingRddFromTrxLog(logRdd, logRdd.partitions.length)
+ rankRdd.foreachPartition { part =>
+ CounterFunctions.updateRankingCounter(part, acc)
+ }
+ }
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/EraseDailyCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/EraseDailyCounter.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/EraseDailyCounter.scala
new file mode 100644
index 0000000..8bdc5ba
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/EraseDailyCounter.scala
@@ -0,0 +1,133 @@
+package org.apache.s2graph.counter.loader
+
+import java.text.SimpleDateFormat
+
+import kafka.producer.KeyedMessage
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.core.v1.ExactStorageHBase
+import org.apache.s2graph.counter.core.v2.ExactStorageGraph
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter._
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.core.CounterEtlItem
+import org.apache.s2graph.counter.models.{CounterModel, DBModel, Counter}
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{SparkApp, WithKafka}
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import play.api.libs.json.Json
+import scala.collection.mutable
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext
+
+object EraseDailyCounter extends SparkApp with WithKafka {
+ implicit val ec = ExecutionContext.Implicits.global
+
+
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+ def valueToEtlItem(policy: Counter, key: ExactKeyTrait, values: ExactValueMap): Seq[CounterEtlItem] = {
+ if (values.nonEmpty) {
+ for {
+ (eq, value) <- filter(values.toList)
+ } yield {
+ CounterEtlItem(eq.tq.ts, policy.service, policy.action, key.itemKey, Json.toJson(eq.dimKeyValues), Json.toJson(Map("value" -> -value)))
+ }
+ } else {
+ Nil
+ }
+ }
+
+ def filter(values: List[(ExactQualifier, Long)]): List[(ExactQualifier, Long)] = {
+ val sorted = values.sortBy(_._1.dimKeyValues.size).reverse
+ val (eq, value) = sorted.head
+ val dimKeys = eq.dimKeyValues.toSeq
+ val flat = {
+ for {
+ i <- 0 to dimKeys.length
+ comb <- dimKeys.combinations(i)
+ } yield {
+ ExactQualifier(eq.tq, comb.toMap) -> value
+ }
+ }.toMap
+
+ // println("flat >>>", flat)
+
+ val valuesMap = values.toMap
+ val remain = (valuesMap ++ flat.map { case (k, v) =>
+ k -> (valuesMap(k) - v)
+ }).filter(_._2 > 0).toList
+
+ // println("remain >>>", remain)
+
+ if (remain.isEmpty) {
+ List((eq, value))
+ } else {
+ (eq, value) :: filter(remain)
+ }
+ }
+
+ def produce(policy: Counter, exactRdd: RDD[(ExactKeyTrait, ExactValueMap)]): Unit = {
+ exactRdd.mapPartitions { part =>
+ for {
+ (key, values) <- part
+ item <- valueToEtlItem(policy, key, values)
+ } yield {
+ item
+ }
+ }.foreachPartition { part =>
+ val m = MutableHashMap.empty[Int, mutable.MutableList[CounterEtlItem]]
+ part.foreach { item =>
+ val k = getPartKey(item.item, 20)
+ val values = m.getOrElse(k, mutable.MutableList.empty[CounterEtlItem])
+ values += item
+ m.update(k, values)
+ }
+ m.foreach { case (k, v) =>
+ v.map(_.toKafkaMessage).grouped(1000).foreach { grouped =>
+ // println(grouped)
+ producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_COUNTER, null, k, grouped.mkString("\n")))
+ }
+ }
+ }
+ }
+
+ def rddToExactRdd(policy: Counter, date: String, rdd: RDD[String]): RDD[(ExactKeyTrait, ExactValueMap)] = {
+ val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
+ val fromTs = dateFormat.parse(date).getTime
+ val toTs = fromTs + 23 * 60 * 60 * 1000
+
+ rdd.mapPartitions { part =>
+ val exactCounter = policy.version match {
+ case VERSION_1 => new ExactCounter(S2ConfigFactory.config, new ExactStorageHBase(S2ConfigFactory.config))
+ case VERSION_2 => new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
+ }
+
+ for {
+ line <- part
+ FetchedCounts(exactKey, qualifierWithCountMap) <- exactCounter.getCount(policy, line, Array(TimedQualifier.IntervalUnit.DAILY), fromTs, toTs)
+ } yield {
+ (exactKey, qualifierWithCountMap)
+ }
+ }
+ }
+
+ lazy val className = getClass.getName.stripSuffix("$")
+
+ override def run(): Unit = {
+ validateArgument("service", "action", "date", "file", "op")
+ DBModel.initialize(S2ConfigFactory.config)
+
+ val (service, action, date, file, op) = (args(0), args(1), args(2), args(3), args(4))
+ val conf = sparkConf(s"$className: $service.$action")
+
+ val ctx = new SparkContext(conf)
+
+ val rdd = ctx.textFile(file, 20)
+
+ val counterModel = new CounterModel(S2ConfigFactory.config)
+
+ val policy = counterModel.findByServiceAction(service, action).get
+ val exactRdd = rddToExactRdd(policy, date, rdd)
+ produce(policy, exactRdd)
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/config/StreamingConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/config/StreamingConfig.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/config/StreamingConfig.scala
new file mode 100644
index 0000000..8c9a8dd
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/config/StreamingConfig.scala
@@ -0,0 +1,24 @@
+package org.apache.s2graph.counter.loader.config
+
+import org.apache.s2graph.counter.config.ConfigFunctions
+import org.apache.s2graph.spark.config.S2ConfigFactory
+
+object StreamingConfig extends ConfigFunctions(S2ConfigFactory.config) {
+ // kafka
+ val KAFKA_ZOOKEEPER = getOrElse("kafka.zookeeper", "localhost")
+ val KAFKA_BROKERS = getOrElse("kafka.brokers", "localhost")
+ val KAFKA_TOPIC_GRAPH = getOrElse("kafka.topic.graph", "s2graphInalpha")
+ val KAFKA_TOPIC_ETL = getOrElse("kafka.topic.etl", "s2counter-etl-alpha")
+ val KAFKA_TOPIC_COUNTER = getOrElse("kafka.topic.counter", "s2counter-alpha")
+ val KAFKA_TOPIC_COUNTER_TRX = getOrElse("kafka.topic.counter-trx", "s2counter-trx-alpha")
+ val KAFKA_TOPIC_COUNTER_FAIL = getOrElse("kafka.topic.counter-fail", "s2counter-fail-alpha")
+
+ // profile cache
+ val PROFILE_CACHE_TTL_SECONDS = getOrElse("profile.cache.ttl.seconds", 60 * 60 * 24) // default 1 day
+ val PROFILE_CACHE_MAX_SIZE = getOrElse("profile.cache.max.size", 10000)
+ val PROFILE_PREFETCH_SIZE = getOrElse("profile.prefetch.size", 10)
+
+ // graph url
+ val GRAPH_URL = getOrElse("s2graph.url", "")
+ val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
new file mode 100644
index 0000000..3e80e25
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctions.scala
@@ -0,0 +1,86 @@
+package org.apache.s2graph.counter.loader.core
+
+import org.apache.s2graph.core.{Edge, Graph, GraphUtil}
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.models.CounterModel
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.spark.Logging
+import play.api.libs.json._
+import scala.collection.mutable.{HashMap => MutableHashMap}
+
+object CounterEtlFunctions extends Logging {
+ lazy val filterOps = Seq("insert", "insertBulk", "update", "increment").map(op => GraphUtil.operations(op))
+ lazy val preFetchSize = StreamingConfig.PROFILE_PREFETCH_SIZE
+ lazy val config = S2ConfigFactory.config
+ lazy val counterModel = new CounterModel(config)
+
+ def logToEdge(line: String): Option[Edge] = {
+ for {
+ elem <- Graph.toGraphElement(line) if elem.isInstanceOf[Edge]
+ edge <- Some(elem.asInstanceOf[Edge]).filter { x =>
+ filterOps.contains(x.op)
+ }
+ } yield {
+ edge
+ }
+ }
+
+ def parseEdgeFormat(line: String): Option[CounterEtlItem] = {
+ /**
+ * 1427082276804 insert edge 19073318 52453027_93524145648511699 story_user_ch_doc_view {"doc_type" : "l", "channel_subscribing" : "y", "view_from" : "feed"}
+ */
+ for {
+ elem <- Graph.toGraphElement(line) if elem.isInstanceOf[Edge]
+ edge <- Some(elem.asInstanceOf[Edge]).filter { x =>
+ filterOps.contains(x.op)
+ }
+ } yield {
+ val label = edge.label
+ val labelName = label.label
+ val tgtService = label.tgtColumn.service.serviceName
+ val tgtId = edge.tgtVertex.innerId.toString()
+ val srcId = edge.srcVertex.innerId.toString()
+
+ // make empty property if no exist edge property
+ val dimension = Json.parse(Some(GraphUtil.split(line)).filter(_.length >= 7).map(_(6)).getOrElse("{}"))
+ val bucketKeys = Seq("_from")
+ val bucketKeyValues = {
+ for {
+ variable <- bucketKeys
+ } yield {
+ val jsValue = variable match {
+ case "_from" => JsString(srcId)
+ case s => dimension \ s
+ }
+ s"[[$variable]]" -> jsValue
+ }
+ }
+ val property = Json.toJson(bucketKeyValues :+ ("value" -> JsString("1")) toMap)
+// val property = Json.toJson(Map("_from" -> srcId, "_to" -> tgtId, "value" -> "1"))
+
+ CounterEtlItem(edge.ts, tgtService, labelName, tgtId, dimension, property)
+ }
+ }
+
+ def parseEdgeFormat(lines: List[String]): List[CounterEtlItem] = {
+ for {
+ line <- lines
+ item <- parseEdgeFormat(line)
+ } yield {
+ item
+ }
+ }
+
+ def checkPolicyAndMergeDimension(service: String, action: String, items: List[CounterEtlItem]): List[CounterEtlItem] = {
+ counterModel.findByServiceAction(service, action).map { policy =>
+ if (policy.useProfile) {
+ policy.bucketImpId match {
+ case Some(_) => DimensionProps.mergeDimension(policy, items)
+ case None => Nil
+ }
+ } else {
+ items
+ }
+ }.getOrElse(Nil)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
new file mode 100644
index 0000000..7a1ebb7
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterEtlItem.scala
@@ -0,0 +1,39 @@
+package org.apache.s2graph.counter.loader.core
+
+import org.apache.s2graph.counter.util.UnitConverter
+import org.slf4j.LoggerFactory
+import play.api.libs.json._
+import scala.util.{Failure, Success, Try}
+
+case class CounterEtlItem(ts: Long, service: String, action: String, item: String, dimension: JsValue, property: JsValue, useProfile: Boolean = false) {
+ def toKafkaMessage: String = {
+ s"$ts\t$service\t$action\t$item\t${dimension.toString()}\t${property.toString()}"
+ }
+
+ lazy val value = {
+ property \ "value" match {
+ case JsNumber(n) => n.longValue()
+ case JsString(s) => s.toLong
+ case _: JsUndefined => 1L
+ case _ => throw new Exception("wrong type")
+ }
+ }
+ }
+
+object CounterEtlItem {
+ val log = LoggerFactory.getLogger(this.getClass)
+
+ def apply(line: String): Option[CounterEtlItem] = {
+ Try {
+ val Array(ts, service, action, item, dimension, property) = line.split('\t')
+ CounterEtlItem(UnitConverter.toMillis(ts.toLong), service, action, item, Json.parse(dimension), Json.parse(property))
+ } match {
+ case Success(item) =>
+ Some(item)
+ case Failure(ex) =>
+ log.error(">>> failed")
+ log.error(s"${ex.toString}: $line")
+ None
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
new file mode 100644
index 0000000..59ac841
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/CounterFunctions.scala
@@ -0,0 +1,446 @@
+package org.apache.s2graph.counter.loader.core
+
+import kafka.producer.KeyedMessage
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.counter.TrxLog
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter.core.v2.{RankingStorageGraph, ExactStorageGraph}
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.models.DefaultCounterModel
+import org.apache.s2graph.counter.models.{Counter, DBModel}
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.WithKafka
+import org.apache.spark.rdd.RDD
+import org.apache.spark.{Accumulable, Logging}
+import play.api.libs.json.{JsNumber, JsString, JsValue, Json}
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext
+import scala.language.postfixOps
+import scala.util.Try
+
+object CounterFunctions extends Logging with WithKafka {
+
+ private val K_MAX = 500
+ implicit val ec = ExecutionContext.Implicits.global
+
+ val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
+ val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
+
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ type HashMapAccumulable = Accumulable[MutableHashMap[String, Long], (String, Long)]
+
+ val initialize = {
+ logInfo("initialize CounterFunctions")
+ DBModel.initialize(S2ConfigFactory.config)
+ true
+ }
+
+ def getCountValue(policy: Counter, item: CounterEtlItem): ExactValueMap = {
+ for {
+ dimKeys <- policy.dimensionList
+ dimValues <- getDimensionValues(item.dimension, dimKeys).toSeq
+ eq <- ExactQualifier.getQualifiers(policy.intervals.map(IntervalUnit.withName), item.ts, dimKeys.zip(dimValues).toMap)
+ } yield {
+ eq -> item.value
+ }
+ }.toMap
+
+ def getDimensionValues(dimension: JsValue, keys: Array[String]): Option[Array[String]] = {
+ Try {
+ for {
+ k <- keys
+ jsValue = dimension \ k
+ } yield {
+ jsValue match {
+ case JsNumber(n) => n.toString()
+ case JsString(s) => s
+ case _ => throw new Exception()
+ }
+ }
+ }.toOption
+ }
+
+ def exactMapper(item: CounterEtlItem): Option[(ExactKeyTrait, ExactValueMap)] = {
+ DefaultCounterModel.findByServiceAction(item.service, item.action).map { policy =>
+ (ExactKey(policy, item.item, checkItemType = true), getCountValue(policy, item))
+ }
+ }
+
+ def rankingMapper(row: ItemRankingRow): Seq[(RankingKey, RankingValueMap)] = {
+ // normal ranking
+ for {
+ (eq, rv) <- row.value
+ } yield {
+ (RankingKey(row.key.policyId, row.key.version, eq), Map(row.key.itemKey -> rv))
+ }
+ }.toSeq
+
+ def logToRankValue(log: TrxLog): Option[(ExactKeyTrait, Map[ExactQualifier, RankingValue])] = {
+ DefaultCounterModel.findById(log.policyId).map { policy =>
+ val key = ExactKey(policy, log.item, checkItemType = false)
+ val value = {
+ for {
+ result <- log.results
+ } yield {
+ ExactQualifier(TimedQualifier(result.interval, result.ts), result.dimension) -> RankingValue(result.result, result.value)
+ }
+ }.toMap
+ key -> value
+ }
+ }
+
+ def reduceValue[T, U](op: (U, U) => U, default: U)(m1: Map[T, U], m2: Map[T, U]): Map[T, U] = {
+ m1 ++ m2.map { case (k, v) =>
+ k -> op(m1.getOrElse(k, default), v)
+ }
+ }
+
+ def makeExactRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[(ExactKeyTrait, ExactValueMap)] = {
+ rdd.mapPartitions { part =>
+ assert(initialize)
+ for {
+ (k, v) <- part
+ line <- GraphUtil.parseString(v)
+ item <- CounterEtlItem(line).toSeq
+ ev <- exactMapper(item).toSeq
+ } yield {
+ ev
+ }
+ }.reduceByKey(reduceValue[ExactQualifier, Long](_ + _, 0L)(_, _), numPartitions)
+ }
+
+ def makeRankingRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
+ val logRdd = makeTrxLogRdd(rdd, numPartitions)
+ makeRankingRddFromTrxLog(logRdd, numPartitions)
+ }
+
+ def makeRankingRddFromTrxLog(rdd: RDD[TrxLog], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
+ val itemRankingRdd = makeItemRankingRdd(rdd, numPartitions).cache()
+ try {
+ rankingCount(itemRankingRdd, numPartitions) union
+ rateRankingCount(itemRankingRdd, numPartitions) union
+ trendRankingCount(itemRankingRdd, numPartitions) coalesce numPartitions
+ } finally {
+ itemRankingRdd.unpersist(false)
+ }
+ }
+
+ def makeTrxLogRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[TrxLog] = {
+ rdd.mapPartitions { part =>
+ assert(initialize)
+ for {
+ (k, v) <- part
+ line <- GraphUtil.parseString(v)
+ trxLog = Json.parse(line).as[TrxLog] if trxLog.success
+ } yield {
+ trxLog
+ }
+ }
+ }
+
+ def rankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
+ rdd.mapPartitions { part =>
+ for {
+ row <- part
+ rv <- rankingMapper(row)
+ } yield {
+ rv
+ }
+ }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
+ }
+
+ case class ItemRankingRow(key: ExactKeyTrait, value: Map[ExactQualifier, RankingValue])
+
+ def makeItemRankingRdd(rdd: RDD[TrxLog], numPartitions: Int): RDD[ItemRankingRow] = {
+ rdd.mapPartitions { part =>
+ for {
+ log <- part
+ rv <- logToRankValue(log)
+ } yield {
+ rv
+ }
+ }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions).mapPartitions { part =>
+ for {
+ (key, value) <- part
+ } yield {
+ ItemRankingRow(key, value)
+ }
+ }
+ }
+
+ def mapTrendRankingValue(rows: Seq[ItemRankingRow]): Seq[(ExactKeyTrait, Map[ExactQualifier, RateRankingValue])] = {
+ for {
+ row <- rows
+ trendPolicy <- DefaultCounterModel.findByTrendActionId(row.key.policyId)
+ } yield {
+ val key = ExactKey(trendPolicy, row.key.itemKey, checkItemType = false)
+ val value = row.value.filter { case (eq, rv) =>
+ // eq filter by rate policy dimension
+ trendPolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
+ }.map { case (eq, rv) =>
+ eq -> RateRankingValue(rv.score, -1)
+ }
+ (key, value)
+ }
+ }
+
+ def mapRateRankingValue(rows: Seq[ItemRankingRow]): Seq[(ExactKeyTrait, Map[ExactQualifier, RateRankingValue])] = {
+ val actionPart = {
+ for {
+ row <- rows
+ ratePolicy <- DefaultCounterModel.findByRateActionId(row.key.policyId)
+ } yield {
+ val key = ExactKey(ratePolicy, row.key.itemKey, checkItemType = false)
+ val value = row.value.filter { case (eq, rv) =>
+ // eq filter by rate policy dimension
+ ratePolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
+ }.map { case (eq, rv) =>
+ eq -> RateRankingValue(rv.score, -1)
+ }
+ (key, value)
+ }
+ }
+
+ val basePart = {
+ for {
+ row <- rows
+ ratePolicy <- DefaultCounterModel.findByRateBaseId(row.key.policyId)
+ } yield {
+ val key = ExactKey(ratePolicy, row.key.itemKey, checkItemType = false)
+ val value = row.value.filter { case (eq, rv) =>
+ // eq filter by rate policy dimension
+ ratePolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
+ }.map { case (eq, rv) =>
+ eq -> RateRankingValue(-1, rv.score)
+ }
+ (key, value)
+ }
+ }
+
+ actionPart ++ basePart
+ }
+
+ def trendRankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
+ rdd.mapPartitions { part =>
+ mapTrendRankingValue(part.toSeq) toIterator
+ }.reduceByKey(reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(_, _), numPartitions).mapPartitions { part =>
+ val missingByPolicy = {
+ for {
+ (key, value) <- part.toSeq
+ trendPolicy <- DefaultCounterModel.findById(key.policyId).toSeq
+ actionId <- trendPolicy.rateActionId.toSeq
+ actionPolicy <- DefaultCounterModel.findById(actionId).toSeq
+ } yield {
+ // filter total eq
+ val missingQualifiersWithRRV = value.filterKeys { eq => eq.tq.q != IntervalUnit.TOTAL }
+ (actionPolicy, key, missingQualifiersWithRRV)
+ }
+ }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
+
+ val filled = {
+ for {
+ (policy, missing) <- missingByPolicy.toSeq
+ keyWithPast = exactCounter.getPastCounts(policy, missing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
+ (key, current) <- missing
+ } yield {
+ val past = keyWithPast.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
+ val base = past.mapValues(l => RateRankingValue(-1, l))
+// log.warn(s"trend: $policy $key -> $current $base")
+ key -> reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, base)
+ }
+ }
+
+// filled.foreach(println)
+
+ {
+ // filter by rate threshold
+ for {
+ (key, value) <- filled
+ ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
+ (eq, rrv) <- value if rrv.baseScore >= ratePolicy.rateThreshold.getOrElse(Int.MinValue)
+ } yield {
+ (RankingKey(key.policyId, key.version, eq), Map(key.itemKey -> rrv.rankingValue))
+ }
+ } toIterator
+ }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
+ }
+
+ def rateRankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
+ rdd.mapPartitions { part =>
+ mapRateRankingValue(part.toSeq) toIterator
+ }.reduceByKey(reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(_, _), numPartitions).mapPartitions { part =>
+ val seq = part.toSeq
+// seq.foreach(x => println(s"item ranking row>> $x"))
+
+ // find and evaluate action value is -1
+ val actionMissingByPolicy = {
+ for {
+ (key, value) <- seq if value.exists { case (eq, rrv) => rrv.actionScore == -1 }
+ ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
+ actionId <- ratePolicy.rateActionId.toSeq
+ actionPolicy <- DefaultCounterModel.findById(actionId)
+ } yield {
+ (actionPolicy, key, value.filter { case (eq, rrv) => rrv.actionScore == -1 })
+ }
+ }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
+
+ val actionFilled = {
+ for {
+ (actionPolicy, actionMissing) <- actionMissingByPolicy.toSeq
+ keyWithRelated = exactCounter.getRelatedCounts(actionPolicy, actionMissing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
+ (key, current) <- actionMissing
+ } yield {
+ val related = keyWithRelated.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
+ val found = related.mapValues(l => RateRankingValue(l, -1))
+ val filled = reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, found)
+// log.warn(s"action: $key -> $found $filled")
+ key -> filled
+ }
+ }
+
+// actionFilled.foreach(x => println(s"action filled>> $x"))
+
+ // find and evaluate base value is -1
+ val baseMissingByPolicy = {
+ for {
+ (key, value) <- seq if value.exists { case (eq, rrv) => rrv.baseScore == -1 }
+ ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
+ baseId <- ratePolicy.rateBaseId.toSeq
+ basePolicy <- DefaultCounterModel.findById(baseId)
+ } yield {
+ (basePolicy, key, value.filter { case (eq, rrv) => rrv.baseScore == -1 })
+ }
+ }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
+
+ val baseFilled = {
+ for {
+ (basePolicy, baseMissing) <- baseMissingByPolicy.toSeq
+ keyWithRelated = exactCounter.getRelatedCounts(basePolicy, baseMissing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
+ (key, current) <- baseMissing
+ } yield {
+ val related = keyWithRelated.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
+ val found = related.mapValues(l => RateRankingValue(-1, l))
+ val filled = reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, found)
+// log.warn(s"base: $basePolicy $key -> $found $filled")
+ key -> filled
+ }
+ }
+
+// baseFilled.foreach(x => println(s"base filled>> $x"))
+
+ val alreadyFilled = {
+ for {
+ (key, value) <- seq if value.exists { case (eq, rrv) => rrv.actionScore != -1 && rrv.baseScore != -1 }
+ } yield {
+ key -> value.filter { case (eq, rrv) => rrv.actionScore != -1 && rrv.baseScore != -1 }
+ }
+ }
+
+ val rtn = {
+ // filter by rate threshold
+ for {
+ (key, value) <- actionFilled ++ baseFilled ++ alreadyFilled
+ ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
+ (eq, rrv) <- value if rrv.baseScore >= ratePolicy.rateThreshold.getOrElse(Int.MinValue)
+ } yield {
+ (RankingKey(key.policyId, key.version, eq), Map(key.itemKey -> rrv.rankingValue))
+ }
+ }
+ rtn.toIterator
+ }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
+ }
+
+ def insertBlobValue(keys: Seq[BlobExactKey], acc: HashMapAccumulable): Unit = {
+ val keyByPolicy = {
+ for {
+ key <- keys
+ policy <- DefaultCounterModel.findById(key.policyId)
+ } yield {
+ (policy, key)
+ }
+ }.groupBy(_._1).mapValues(values => values.map(_._2))
+
+ for {
+ (policy, allKeys) <- keyByPolicy
+ keys <- allKeys.grouped(10)
+ success <- exactCounter.insertBlobValue(policy, keys)
+ } yield {
+ success match {
+ case true => acc += ("BLOB", 1)
+ case false => acc += ("BLOBFailed", 1)
+ }
+ }
+ }
+
+ def updateExactCounter(counts: Seq[(ExactKeyTrait, ExactValueMap)], acc: HashMapAccumulable): Seq[TrxLog] = {
+ val countsByPolicy = {
+ for {
+ (key, count) <- counts
+ policy <- DefaultCounterModel.findById(key.policyId)
+ } yield {
+ (policy, (key, count))
+ }
+ }.groupBy { case (policy, _) => policy }.mapValues(values => values.map(_._2))
+
+ for {
+ (policy, allCounts) <- countsByPolicy
+ counts <- allCounts.grouped(10)
+ trxLog <- exactCounter.updateCount(policy, counts)
+ } yield {
+ trxLog.success match {
+ case true => acc += (s"ExactV${policy.version}", 1)
+ case false => acc += (s"ExactFailedV${policy.version}", 1)
+ }
+ trxLog
+ }
+ }.toSeq
+
+ def exactCountFromEtl(rdd: RDD[CounterEtlItem], numPartitions: Int): RDD[(ExactKeyTrait, ExactValueMap)] = {
+ rdd.mapPartitions { part =>
+ for {
+ item <- part
+ ev <- exactMapper(item).toSeq
+ } yield {
+ ev
+ }
+ }.reduceByKey(reduceValue[ExactQualifier, Long](_ + _, 0L)(_, _), numPartitions)
+ }
+
+ def updateRankingCounter(values: TraversableOnce[(RankingKey, RankingValueMap)], acc: HashMapAccumulable): Unit = {
+ assert(initialize)
+ val valuesByPolicy = {
+ for {
+ (key, value) <- values.toSeq
+ policy <- DefaultCounterModel.findById(key.policyId)
+ if policy.useRank && rankingCounter.ready(policy) // update only rank counter enabled and ready
+ } yield {
+ (policy, (key, value))
+ }
+ }.groupBy { case (policy, _) => policy }.mapValues(values => values.map(_._2))
+
+ for {
+ (policy, allValues) <- valuesByPolicy
+ groupedValues <- allValues.grouped(10)
+ } {
+ rankingCounter.update(groupedValues, K_MAX)
+ acc += (s"RankingV${policy.version}", groupedValues.length)
+ }
+ }
+
+ def produceTrxLog(trxLogs: TraversableOnce[TrxLog]): Unit = {
+ for {
+ trxLog <- trxLogs
+ } {
+ val topic = trxLog.success match {
+ case true => StreamingConfig.KAFKA_TOPIC_COUNTER_TRX
+ case false => StreamingConfig.KAFKA_TOPIC_COUNTER_FAIL
+ }
+ val msg = new KeyedMessage[String, String](topic, s"${trxLog.policyId}${trxLog.item}", Json.toJson(trxLog).toString())
+ producer.send(msg)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
new file mode 100644
index 0000000..b1ebe50
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
@@ -0,0 +1,150 @@
+package org.apache.s2graph.counter.loader.core
+
+import org.apache.commons.httpclient.HttpStatus
+import org.apache.s2graph.core.mysqls.{Bucket, Experiment, Service}
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.util.{RetryAsync, CollectionCache, CollectionCacheConfig}
+import org.slf4j.LoggerFactory
+import play.api.libs.json._
+import scala.annotation.tailrec
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Future}
+import scala.util.Try
+
+object DimensionProps {
+ // using play-ws without play app
+ private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ private val client = new play.api.libs.ws.ning.NingWSClient(builder.build)
+ private val log = LoggerFactory.getLogger(this.getClass)
+
+ private val retryCnt = 3
+
+ val cacheConfig = CollectionCacheConfig(StreamingConfig.PROFILE_CACHE_MAX_SIZE,
+ StreamingConfig.PROFILE_CACHE_TTL_SECONDS,
+ negativeCache = true,
+ 3600 // negative ttl 1 hour
+ )
+ val cache: CollectionCache[Option[JsValue]] = new CollectionCache[Option[JsValue]](cacheConfig)
+
+ @tailrec
+ private[counter] def makeRequestBody(requestBody: String, keyValues: List[(String, String)]): String = {
+ keyValues match {
+ case head :: tail =>
+ makeRequestBody(requestBody.replace(head._1, head._2), tail)
+ case Nil => requestBody
+ }
+ }
+
+ private[counter] def query(bucket: Bucket, item: CounterEtlItem): Future[Option[JsValue]] = {
+ val keyValues = (item.dimension.as[JsObject] ++ item.property.as[JsObject] fields)
+ .filter { case (key, _) => key.startsWith("[[") && key.endsWith("]]") }
+ .map { case (key, jsValue) =>
+ val replacement = jsValue match {
+ case JsString(s) => s
+ case value => value.toString()
+ }
+ key -> replacement
+ }.toList
+
+ val cacheKey = s"${bucket.impressionId}=" + keyValues.flatMap(x => Seq(x._1, x._2)).mkString("_")
+
+ cache.withCacheAsync(cacheKey) {
+ val retryFuture = RetryAsync(retryCnt, withSleep = false) {
+ val future = bucket.httpVerb.toUpperCase match {
+ case "GET" =>
+ client.url(bucket.apiPath).get()
+ case "POST" =>
+ val newBody = makeRequestBody(bucket.requestBody, keyValues)
+ client.url(bucket.apiPath).post(Json.parse(newBody))
+ }
+
+ future.map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val json = Json.parse(resp.body)
+ for {
+ results <- (json \ "results").asOpt[Seq[JsValue]]
+ result <- results.headOption
+ props <- (result \ "props").asOpt[JsValue]
+ } yield {
+ props
+ }
+ case _ =>
+ log.error(s"${resp.body}(${resp.status}}) item: $item")
+ None
+ }
+ }
+ }
+
+ // if fail to retry
+ retryFuture onFailure { case t => log.error(s"${t.getMessage} item: $item") }
+
+ retryFuture
+ }
+ }
+
+ private[counter] def query(service: Service, experiment: Experiment, item: CounterEtlItem): Future[Option[JsValue]] = {
+ val keyValues = (item.dimension.as[JsObject] ++ item.property.as[JsObject] fields)
+ .filter { case (key, _) => key.startsWith("[[") && key.endsWith("]]") }.toMap
+
+ val cacheKey = s"${experiment.name}=" + keyValues.flatMap(x => Seq(x._1, x._2)).mkString("_")
+
+ cache.withCacheAsync(cacheKey) {
+ val retryFuture = RetryAsync(retryCnt, withSleep = false) {
+ val url = s"${StreamingConfig.GRAPH_URL}/graphs/experiment/${service.accessToken}/${experiment.name}/0"
+ val future = client.url(url).post(Json.toJson(keyValues))
+
+ future.map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val json = Json.parse(resp.body)
+ for {
+ results <- (json \ "results").asOpt[Seq[JsValue]]
+ result <- results.headOption
+ props <- (result \ "props").asOpt[JsValue]
+ } yield {
+ props
+ }
+ case _ =>
+ log.error(s"${resp.body}(${resp.status}}) item: $item")
+ None
+ }
+ }
+ }
+
+ // if fail to retry
+ retryFuture onFailure { case t => log.error(s"${t.getMessage} item: $item") }
+
+ retryFuture
+ }
+ }
+
+ def mergeDimension(policy: Counter, items: List[CounterEtlItem]): List[CounterEtlItem] = {
+ for {
+ impId <- policy.bucketImpId
+ bucket <- Bucket.findByImpressionId(impId)
+ experiment <- Experiment.findById(bucket.experimentId)
+ service <- Try { Service.findById(experiment.serviceId) }.toOption
+ } yield {
+ val futures = {
+ for {
+ item <- items
+ } yield {
+ query(service, experiment, item).map {
+ case Some(jsValue) =>
+ val newDimension = item.dimension.as[JsObject] ++ jsValue.as[JsObject]
+ item.copy(dimension = newDimension)
+ case None => item
+ }
+ }
+ }
+ Await.result(Future.sequence(futures), 10 seconds)
+ }
+ }.getOrElse(items)
+
+ def getCacheStatsString: String = {
+ cache.getStatsString
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/models/DefaultCounterModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/models/DefaultCounterModel.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/models/DefaultCounterModel.scala
new file mode 100644
index 0000000..5908e1c
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/models/DefaultCounterModel.scala
@@ -0,0 +1,6 @@
+package org.apache.s2graph.counter.loader.models
+
+import org.apache.s2graph.counter.models.CounterModel
+import org.apache.s2graph.spark.config.S2ConfigFactory
+
+case object DefaultCounterModel extends CounterModel(S2ConfigFactory.config)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/EtlStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/EtlStreaming.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/EtlStreaming.scala
new file mode 100644
index 0000000..12f5f73
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/EtlStreaming.scala
@@ -0,0 +1,114 @@
+package org.apache.s2graph.counter.loader.stream
+
+import kafka.producer.KeyedMessage
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.core.{DimensionProps, CounterEtlItem, CounterEtlFunctions}
+import org.apache.s2graph.counter.models.{DBModel, CounterModel}
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
+import org.apache.spark.streaming.kafka.StreamHelper
+import scala.collection.mutable
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext
+
+object EtlStreaming extends SparkApp with WithKafka {
+ lazy val config = S2ConfigFactory.config
+ lazy val s2Config = new S2CounterConfig(config)
+ lazy val counterModel = new CounterModel(config)
+ lazy val className = getClass.getName.stripSuffix("$")
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ implicit val graphEx = ExecutionContext.Implicits.global
+
+ val initialize = {
+ println("streaming initialize")
+// Graph(config)
+ DBModel.initialize(config)
+ true
+ }
+
+ val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_ETL)
+ val strInputTopics = inputTopics.mkString(",")
+ val groupId = buildKafkaGroupId(strInputTopics, "etl_to_counter")
+ val kafkaParam = Map(
+ "group.id" -> groupId,
+ "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
+ "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
+ "zookeeper.connection.timeout.ms" -> "10000"
+ )
+ val streamHelper = StreamHelper(kafkaParam)
+
+ override def run(): Unit = {
+ validateArgument("interval")
+ val (intervalInSec) = seconds(args(0).toLong)
+
+ val conf = sparkConf(s"$strInputTopics: $className")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ /**
+ * read message from etl topic and join user profile from graph and then produce whole message to counter topic
+ */
+ val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
+
+ // etl logic
+ stream.foreachRDD { (rdd, ts) =>
+ rdd.foreachPartitionWithOffsetRange { case (osr, part) =>
+ assert(initialize)
+
+ // convert to edge format
+ val items = {
+ for {
+ (k, v) <- part
+ line <- GraphUtil.parseString(v)
+ item <- CounterEtlFunctions.parseEdgeFormat(line)
+ } yield {
+ acc += ("Edges", 1)
+ item
+ }
+ }
+
+ // join user profile
+ val joinItems = items.toList.groupBy { e =>
+ (e.service, e.action)
+ }.flatMap { case ((service, action), v) =>
+ CounterEtlFunctions.checkPolicyAndMergeDimension(service, action, v)
+ }
+
+ // group by kafka partition key and send to kafka
+ val m = MutableHashMap.empty[Int, mutable.MutableList[CounterEtlItem]]
+ joinItems.foreach { item =>
+ if (item.useProfile) {
+ acc += ("ETL", 1)
+ }
+ val k = getPartKey(item.item, 20)
+ val values: mutable.MutableList[CounterEtlItem] = m.getOrElse(k, mutable.MutableList.empty[CounterEtlItem])
+ values += item
+ m.update(k, values)
+ }
+ m.foreach { case (k, v) =>
+ v.map(_.toKafkaMessage).grouped(1000).foreach { grouped =>
+ acc += ("Produce", grouped.size)
+ producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_COUNTER, null, k, grouped.mkString("\n")))
+ }
+ }
+
+ streamHelper.commitConsumerOffset(osr)
+ }
+
+ if (ts.milliseconds / 1000 % 60 == 0) {
+ log.warn(DimensionProps.getCacheStatsString)
+ }
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreaming.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreaming.scala
new file mode 100644
index 0000000..3eea406
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreaming.scala
@@ -0,0 +1,69 @@
+package org.apache.s2graph.counter.loader.stream
+
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.core.CounterFunctions
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
+import org.apache.spark.streaming.kafka.{HasOffsetRanges, StreamHelper}
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.language.postfixOps
+
+object ExactCounterStreaming extends SparkApp with WithKafka {
+ lazy val config = S2ConfigFactory.config
+ lazy val s2Config = new S2CounterConfig(config)
+ lazy val className = getClass.getName.stripSuffix("$")
+
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_COUNTER)
+ val strInputTopics = inputTopics.mkString(",")
+ val groupId = buildKafkaGroupId(strInputTopics, "counter_v2")
+ val kafkaParam = Map(
+// "auto.offset.reset" -> "smallest",
+ "group.id" -> groupId,
+ "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
+ "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
+ "zookeeper.connection.timeout.ms" -> "10000"
+ )
+ val streamHelper = StreamHelper(kafkaParam)
+
+ override def run() = {
+ validateArgument("interval", "clear")
+ val (intervalInSec, clear) = (seconds(args(0).toLong), args(1).toBoolean)
+
+ if (clear) {
+ streamHelper.kafkaHelper.consumerGroupCleanup()
+ }
+
+ val conf = sparkConf(s"$strInputTopics: $className")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ implicit val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ // make stream
+ val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
+ stream.foreachRDD { (rdd, ts) =>
+ val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
+
+ val exactRDD = CounterFunctions.makeExactRdd(rdd, offsets.length)
+
+ // for at-least once semantic
+ exactRDD.foreachPartitionWithIndex { (i, part) =>
+ // update exact counter
+ val trxLogs = CounterFunctions.updateExactCounter(part.toSeq, acc)
+ CounterFunctions.produceTrxLog(trxLogs)
+
+ // commit offset range
+ streamHelper.commitConsumerOffset(offsets(i))
+ }
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/GraphToETLStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/GraphToETLStreaming.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/GraphToETLStreaming.scala
new file mode 100644
index 0000000..9e6d6be
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/GraphToETLStreaming.scala
@@ -0,0 +1,80 @@
+package org.apache.s2graph.counter.loader.stream
+
+import kafka.producer.KeyedMessage
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
+import scala.collection.mutable
+import scala.collection.mutable.{HashMap => MutableHashMap}
+
+object GraphToETLStreaming extends SparkApp with WithKafka {
+ lazy val config = S2ConfigFactory.config
+ lazy val s2Config = new S2CounterConfig(config)
+ lazy val className = getClass.getName.stripSuffix("$")
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ override def run(): Unit = {
+ validateArgument("interval", "topic")
+ val (intervalInSec, topic) = (seconds(args(0).toLong), args(1))
+
+ val groupId = buildKafkaGroupId(topic, "graph_to_etl")
+ val kafkaParam = Map(
+// "auto.offset.reset" -> "smallest",
+ "group.id" -> groupId,
+ "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
+ "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
+ "zookeeper.connection.timeout.ms" -> "10000"
+ )
+
+ val conf = sparkConf(s"$topic: $className")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ /**
+ * consume graphIn topic and produce messages to etl topic
+ * two purpose
+ * 1. partition by target vertex id
+ * 2. expand kafka partition count
+ */
+ val stream = getStreamHelper(kafkaParam).createStream[String, String, StringDecoder, StringDecoder](ssc, topic.split(',').toSet)
+ stream.foreachRDD { rdd =>
+ rdd.foreachPartitionWithOffsetRange { case (osr, part) =>
+ val m = MutableHashMap.empty[Int, mutable.MutableList[String]]
+ for {
+ (k, v) <- part
+ line <- GraphUtil.parseString(v)
+ } {
+ try {
+ val sp = GraphUtil.split(line)
+ // get partition key by target vertex id
+ val partKey = getPartKey(sp(4), 20)
+ val values = m.getOrElse(partKey, mutable.MutableList.empty[String])
+ values += line
+ m.update(partKey, values)
+ } catch {
+ case ex: Throwable =>
+ log.error(s"$ex: $line")
+ }
+ }
+
+ m.foreach { case (k, v) =>
+ v.grouped(1000).foreach { grouped =>
+ producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_ETL, null, k, grouped.mkString("\n")))
+ }
+ }
+
+ getStreamHelper(kafkaParam).commitConsumerOffset(osr)
+ }
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreaming.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreaming.scala
new file mode 100644
index 0000000..2c2335b
--- /dev/null
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreaming.scala
@@ -0,0 +1,72 @@
+package org.apache.s2graph.counter.loader.stream
+
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.loader.config.StreamingConfig
+import org.apache.s2graph.counter.loader.core.CounterFunctions
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
+import org.apache.spark.streaming.kafka.{HasOffsetRanges, StreamHelper}
+import scala.collection.mutable.{HashMap => MutableHashMap}
+
+object RankingCounterStreaming extends SparkApp with WithKafka {
+ lazy val config = S2ConfigFactory.config
+ lazy val s2Config = new S2CounterConfig(config)
+ lazy val className = getClass.getName.stripSuffix("$")
+
+ lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
+
+ val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_COUNTER_TRX)
+ val strInputTopics = inputTopics.mkString(",")
+ val groupId = buildKafkaGroupId(strInputTopics, "ranking_v2")
+ val kafkaParam = Map(
+// "auto.offset.reset" -> "smallest",
+ "group.id" -> groupId,
+ "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
+ "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
+ "zookeeper.connection.timeout.ms" -> "10000"
+ )
+ val streamHelper = StreamHelper(kafkaParam)
+
+ override def run() = {
+ validateArgument("interval", "clear")
+ val (intervalInSec, clear) = (seconds(args(0).toLong), args(1).toBoolean)
+
+ if (clear) {
+ streamHelper.kafkaHelper.consumerGroupCleanup()
+ }
+
+ val conf = sparkConf(s"$strInputTopics: $className")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ implicit val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ // make stream
+ val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
+ stream.foreachRDD { (rdd, ts) =>
+ // for at-least once semantic
+ val nextRdd = {
+ CounterFunctions.makeRankingRdd(rdd, sc.defaultParallelism).foreachPartition { part =>
+ // update ranking counter
+ CounterFunctions.updateRankingCounter(part, acc)
+ }
+ rdd
+ }
+
+ streamHelper.commitConsumerOffsets(nextRdd.asInstanceOf[HasOffsetRanges])
+// CounterFunctions.makeRankingRdd(rdd, offsets.length).foreachPartitionWithIndex { (i, part) =>
+// // update ranking counter
+// CounterFunctions.updateRankingCounter(part, acc)
+//
+// // commit offset range
+// streamHelper.commitConsumerOffset(offsets(i))
+// }
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
deleted file mode 100644
index ba5e863..0000000
--- a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package s2.config
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 4. 7..
- */
-object StreamingConfig extends ConfigFunctions(S2ConfigFactory.config) {
- // kafka
- val KAFKA_ZOOKEEPER = getOrElse("kafka.zookeeper", "localhost")
- val KAFKA_BROKERS = getOrElse("kafka.brokers", "localhost")
- val KAFKA_TOPIC_GRAPH = getOrElse("kafka.topic.graph", "s2graphInalpha")
- val KAFKA_TOPIC_ETL = getOrElse("kafka.topic.etl", "s2counter-etl-alpha")
- val KAFKA_TOPIC_COUNTER = getOrElse("kafka.topic.counter", "s2counter-alpha")
- val KAFKA_TOPIC_COUNTER_TRX = getOrElse("kafka.topic.counter-trx", "s2counter-trx-alpha")
- val KAFKA_TOPIC_COUNTER_FAIL = getOrElse("kafka.topic.counter-fail", "s2counter-fail-alpha")
-
- // profile cache
- val PROFILE_CACHE_TTL_SECONDS = getOrElse("profile.cache.ttl.seconds", 60 * 60 * 24) // default 1 day
- val PROFILE_CACHE_MAX_SIZE = getOrElse("profile.cache.max.size", 10000)
- val PROFILE_PREFETCH_SIZE = getOrElse("profile.prefetch.size", 10)
-
- // graph url
- val GRAPH_URL = getOrElse("s2graph.url", "")
- val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/CounterBulkLoader.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/CounterBulkLoader.scala b/s2counter_loader/src/main/scala/s2/counter/CounterBulkLoader.scala
deleted file mode 100644
index 2843022..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/CounterBulkLoader.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package s2.counter
-
-import com.kakao.s2graph.core.{Graph, GraphUtil}
-import org.apache.spark.SparkContext
-import s2.config.{S2ConfigFactory, S2CounterConfig, StreamingConfig}
-import s2.counter.core.{BlobExactKey, CounterEtlFunctions, CounterFunctions}
-import s2.models.Counter.ItemType
-import s2.models.{CounterModel, DBModel}
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.concurrent.ExecutionContext
-
-/**
- * Created by rain on 7/1/15.
- */
-object CounterBulkLoader extends SparkApp with WithKafka {
- lazy val config = S2ConfigFactory.config
- lazy val s2Config = new S2CounterConfig(config)
- lazy val counterModel = new CounterModel(config)
- lazy val className = getClass.getName.stripSuffix("$")
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- implicit val graphEx = ExecutionContext.Implicits.global
-
- val initialize = {
- println("initialize")
-// Graph(config)
- DBModel.initialize(config)
- true
- }
-
- override def run(): Unit = {
- val hdfsPath = args(0)
- val blockSize = args(1).toInt
- val minPartitions = args(2).toInt
- val conf = sparkConf(s"$hdfsPath: CounterBulkLoader")
-
- val sc = new SparkContext(conf)
- val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- val msgs = sc.textFile(hdfsPath)
-
- val etlRdd = msgs.repartition(minPartitions).mapPartitions { part =>
- // parse and etl
- assert(initialize)
- val items = {
- for {
- msg <- part
- line <- GraphUtil.parseString(msg)
- sp = GraphUtil.split(line) if sp.size <= 7 || GraphUtil.split(line)(7) != "in"
- item <- CounterEtlFunctions.parseEdgeFormat(line)
- } yield {
- acc +=("Edges", 1)
- item
- }
- }
- items.grouped(blockSize).flatMap { grouped =>
- grouped.groupBy(e => (e.service, e.action)).flatMap { case ((service, action), v) =>
- CounterEtlFunctions.checkPolicyAndMergeDimension(service, action, v.toList)
- }
- }
- }
-
- val exactRdd = CounterFunctions.exactCountFromEtl(etlRdd, etlRdd.partitions.length)
- val logRdd = exactRdd.mapPartitions { part =>
- val seq = part.toSeq
- CounterFunctions.insertBlobValue(seq.map(_._1).filter(_.itemType == ItemType.BLOB).map(_.asInstanceOf[BlobExactKey]), acc)
- // update exact counter
- CounterFunctions.updateExactCounter(seq, acc).toIterator
- }
-
- val rankRdd = CounterFunctions.makeRankingRddFromTrxLog(logRdd, logRdd.partitions.length)
- rankRdd.foreachPartition { part =>
- CounterFunctions.updateRankingCounter(part, acc)
- }
- }
-}
\ No newline at end of file
[20/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
deleted file mode 100644
index 8789502..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
+++ /dev/null
@@ -1,1224 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core.ExceptionHandler.{Key, Val, KafkaMessage}
-import com.kakao.s2graph.core.GraphExceptions.FetchTimeoutException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.storage.serde._
-import com.kakao.s2graph.core.storage.serde.snapshotedge.tall
-import com.kakao.s2graph.core.storage.serde.snapshotedge.wide.SnapshotEdgeDeserializable
-import com.kakao.s2graph.core.storage.serde.vertex._
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.{Extensions, logger}
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.kafka.clients.producer.ProducerRecord
-import scala.annotation.tailrec
-import scala.collection.Seq
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Random, Try}
-
-abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) {
- import HBaseType._
-
- /** storage dependent configurations */
- val MaxRetryNum = config.getInt("max.retry.number")
- val MaxBackOff = config.getInt("max.back.off")
- val DeleteAllFetchSize = config.getInt("delete.all.fetch.size")
- val FailProb = config.getDouble("hbase.fail.prob")
- val LockExpireDuration = Math.max(MaxRetryNum * MaxBackOff * 2, 10000)
- val maxSize = config.getInt("future.cache.max.size")
- val expireAfterWrite = config.getInt("future.cache.expire.after.write")
- val expireAfterAccess = config.getInt("future.cache.expire.after.access")
-
- /**
- * Compatibility table
- * | label schema version | snapshot edge | index edge | vertex | note |
- * | v1 | serde.snapshotedge.wide | serde.indexedge.wide | serde.vertex | do not use this. this exist only for backward compatibility issue |
- * | v2 | serde.snapshotedge.wide | serde.indexedge.wide | serde.vertex | do not use this. this exist only for backward compatibility issue |
- * | v3 | serde.snapshotedge.tall | serde.indexedge.wide | serde.vertex | recommended with HBase. current stable schema |
- * | v4 | serde.snapshotedge.tall | serde.indexedge.tall | serde.vertex | experimental schema. use scanner instead of get |
- *
- */
-
- /**
- * create serializer that knows how to convert given snapshotEdge into kvs: Seq[SKeyValue]
- * so we can store this kvs.
- * @param snapshotEdge: snapshotEdge to serialize
- * @return serializer implementation for StorageSerializable which has toKeyValues return Seq[SKeyValue]
- */
- def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): Serializable[SnapshotEdge] = {
- snapshotEdge.schemaVer match {
- case VERSION1 | VERSION2 => new serde.snapshotedge.wide.SnapshotEdgeSerializable(snapshotEdge)
- case VERSION3 | VERSION4 => new serde.snapshotedge.tall.SnapshotEdgeSerializable(snapshotEdge)
- case _ => throw new RuntimeException(s"not supported version: ${snapshotEdge.schemaVer}")
- }
- }
-
- /**
- * create serializer that knows how to convert given indexEdge into kvs: Seq[SKeyValue]
- * @param indexEdge: indexEdge to serialize
- * @return serializer implementation
- */
- def indexEdgeSerializer(indexEdge: IndexEdge): Serializable[IndexEdge] = {
- indexEdge.schemaVer match {
- case VERSION1 | VERSION2 | VERSION3 => new indexedge.wide.IndexEdgeSerializable(indexEdge)
- case VERSION4 => new indexedge.tall.IndexEdgeSerializable(indexEdge)
- case _ => throw new RuntimeException(s"not supported version: ${indexEdge.schemaVer}")
-
- }
- }
-
- /**
- * create serializer that knows how to convert given vertex into kvs: Seq[SKeyValue]
- * @param vertex: vertex to serialize
- * @return serializer implementation
- */
- def vertexSerializer(vertex: Vertex) = new VertexSerializable(vertex)
-
- /**
- * create deserializer that can parse stored CanSKeyValue into snapshotEdge.
- * note that each storage implementation should implement implicit type class
- * to convert storage dependent dataType into common SKeyValue type by implementing CanSKeyValue
- *
- * ex) Asynchbase use it's KeyValue class and CanSKeyValue object has implicit type conversion method.
- * if any storaage use different class to represent stored byte array,
- * then that storage implementation is responsible to provide implicit type conversion method on CanSKeyValue.
- * */
-
- val snapshotEdgeDeserializers: Map[String, Deserializable[SnapshotEdge]] = Map(
- VERSION1 -> new snapshotedge.wide.SnapshotEdgeDeserializable,
- VERSION2 -> new snapshotedge.wide.SnapshotEdgeDeserializable,
- VERSION3 -> new tall.SnapshotEdgeDeserializable,
- VERSION4 -> new tall.SnapshotEdgeDeserializable
- )
- def snapshotEdgeDeserializer(schemaVer: String) =
- snapshotEdgeDeserializers.get(schemaVer).getOrElse(throw new RuntimeException(s"not supported version: ${schemaVer}"))
-
- /** create deserializer that can parse stored CanSKeyValue into indexEdge. */
- val indexEdgeDeserializers: Map[String, Deserializable[IndexEdge]] = Map(
- VERSION1 -> new indexedge.wide.IndexEdgeDeserializable,
- VERSION2 -> new indexedge.wide.IndexEdgeDeserializable,
- VERSION3 -> new indexedge.wide.IndexEdgeDeserializable,
- VERSION4 -> new indexedge.tall.IndexEdgeDeserializable
- )
-
- def indexEdgeDeserializer(schemaVer: String) =
- indexEdgeDeserializers.get(schemaVer).getOrElse(throw new RuntimeException(s"not supported version: ${schemaVer}"))
-
- /** create deserializer that can parser stored CanSKeyValue into vertex. */
- val vertexDeserializer = new VertexDeserializable
-
-
- /**
- * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
- * note that this should be return true on all success.
- * we assumes that each storage implementation has client as member variable.
- *
- *
- * @param cluster: where this key values should be stored.
- * @param kvs: sequence of SKeyValue that need to be stored in storage.
- * @param withWait: flag to control wait ack from storage.
- * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
- * it never block thread, but rather submit work and notified by event loop when storage send ack back.
- * @return ack message from storage.
- */
- def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean]
-
-// def writeToStorage(kv: SKeyValue, withWait: Boolean): Future[Boolean]
-
- /**
- * fetch SnapshotEdge for given request from storage.
- * also storage datatype should be converted into SKeyValue.
- * note that return type is Sequence rather than single SKeyValue for simplicity,
- * even though there is assertions sequence.length == 1.
- * @param request
- * @return
- */
- def fetchSnapshotEdgeKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
-
- /**
- * write requestKeyValue into storage if the current value in storage that is stored matches.
- * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
- *
- * Most important thing is this have to be 'atomic' operation.
- * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
- * either blocked or failed on write-write conflict case.
- *
- * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
- * prevent wrong data for read.
- *
- * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
- * compareAndSet to synchronize.
- *
- * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
- * for storage that does not support concurrency control, then storage implementation
- * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
- * and write(writeLock).
- * @param requestKeyValue
- * @param expectedOpt
- * @return
- */
- def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean]
-
- /**
- * build proper request which is specific into storage to call fetchIndexEdgeKeyValues or fetchSnapshotEdgeKeyValues.
- * for example, Asynchbase use GetRequest, Scanner so this method is responsible to build
- * client request(GetRequest, Scanner) based on user provided query.
- * @param queryRequest
- * @return
- */
- def buildRequest(queryRequest: QueryRequest): AnyRef
-
- /**
- * fetch IndexEdges for given queryParam in queryRequest.
- * this expect previous step starting score to propagate score into next step.
- * also parentEdges is necessary to return full bfs tree when query require it.
- *
- * note that return type is general type.
- * for example, currently we wanted to use Asynchbase
- * so single I/O return type should be Deferred[T].
- *
- * if we use native hbase client, then this return type can be Future[T] or just T.
- * @param queryRequest
- * @param prevStepScore
- * @param isInnerCall
- * @param parentEdges
- * @return
- */
- def fetch(queryRequest: QueryRequest,
- prevStepScore: Double,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): R
-
- /**
- * responsible to fire parallel fetch call into storage and create future that will return merged result.
- * @param queryRequestWithScoreLs
- * @param prevStepEdges
- * @return
- */
- def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]]
-
- /**
- * fetch Vertex for given request from storage.
- * @param request
- * @return
- */
- def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
-
- /**
- * decide how to apply given edges(indexProps values + Map(_count -> countVal)) into storage.
- * @param edges
- * @param withWait
- * @return
- */
- def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]]
-
- /**
- * this method need to be called when client shutdown. this is responsible to cleanUp the resources
- * such as client into storage.
- */
- def flush(): Unit
-
- /**
- * create table on storage.
- * if storage implementation does not support namespace or table, then there is nothing to be done
- * @param zkAddr
- * @param tableName
- * @param cfs
- * @param regionMultiplier
- * @param ttl
- * @param compressionAlgorithm
- */
- def createTable(zkAddr: String,
- tableName: String,
- cfs: List[String],
- regionMultiplier: Int,
- ttl: Option[Int],
- compressionAlgorithm: String): Unit
-
-
-
-
-
- /** Public Interface */
-
- def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
- def fromResult(queryParam: QueryParam,
- kvs: Seq[SKeyValue],
- version: String): Option[Vertex] = {
- if (kvs.isEmpty) None
- else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
- }
-
- val futures = vertices.map { vertex =>
- val queryParam = QueryParam.Empty
- val q = Query.toQuery(Seq(vertex), queryParam)
- val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
- fetchVertexKeyValues(buildRequest(queryRequest)).map { kvs =>
- fromResult(queryParam, kvs, vertex.serviceColumn.schemaVersion)
- } recoverWith { case ex: Throwable =>
- Future.successful(None)
- }
- }
-
- Future.sequence(futures).map { result => result.toList.flatten }
- }
-
- def mutateElements(elements: Seq[GraphElement],
- withWait: Boolean = false): Future[Seq[Boolean]] = {
-
- val edgeBuffer = ArrayBuffer[Edge]()
- val vertexBuffer = ArrayBuffer[Vertex]()
-
- elements.foreach {
- case e: Edge => edgeBuffer += e
- case v: Vertex => vertexBuffer += v
- case any@_ => logger.error(s"Unknown type: ${any}")
- }
-
- val edgeFuture = mutateEdges(edgeBuffer, withWait)
- val vertexFuture = mutateVertices(vertexBuffer, withWait)
-
- val graphFuture = for {
- edgesMutated <- edgeFuture
- verticesMutated <- vertexFuture
- } yield edgesMutated ++ verticesMutated
-
- graphFuture
- }
-
- def mutateEdges(edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
- val (strongEdges, weakEdges) =
- (edges.partition(e => e.label.consistencyLevel == "strong" || e.op == GraphUtil.operations("insertBulk")))
-
- val weakEdgesFutures = weakEdges.groupBy { e => e.label.hbaseZkAddr }.map { case (zkQuorum, edges) =>
- val mutations = edges.flatMap { edge =>
- val (_, edgeUpdate) =
- if (edge.op == GraphUtil.operations("delete")) Edge.buildDeleteBulk(None, edge)
- else Edge.buildOperation(None, Seq(edge))
- buildVertexPutsAsync(edge) ++ indexedEdgeMutations(edgeUpdate) ++
- snapshotEdgeMutations(edgeUpdate) ++ increments(edgeUpdate)
- }
- writeToStorage(zkQuorum, mutations, withWait)
- }
- val strongEdgesFutures = mutateStrongEdges(strongEdges, withWait)
- for {
- weak <- Future.sequence(weakEdgesFutures)
- strong <- strongEdgesFutures
- } yield {
- strong ++ weak
- }
- }
- def mutateStrongEdges(_edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
-
- val grouped = _edges.groupBy { edge => (edge.label, edge.srcVertex.innerId, edge.tgtVertex.innerId) } toSeq
-
- val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
- val (deleteAllEdges, edges) = edgeGroup.partition(_.op == GraphUtil.operations("deleteAll"))
-
- // DeleteAll first
- val deleteAllFutures = deleteAllEdges.map { edge =>
- deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.label), edge.labelWithDir.dir, edge.ts)
- }
-
- // After deleteAll, process others
- lazy val mutateEdgeFutures = edges.toList match {
- case head :: tail =>
- // val strongConsistency = edges.head.label.consistencyLevel == "strong"
- // if (strongConsistency) {
- val edgeFuture = mutateEdgesInner(edges, checkConsistency = true , withWait)(Edge.buildOperation)
-
- //TODO: decide what we will do on failure on vertex put
- val puts = buildVertexPutsAsync(head)
- val vertexFuture = writeToStorage(head.label.hbaseZkAddr, puts, withWait)
- Seq(edgeFuture, vertexFuture)
- // } else {
- // edges.map { edge => mutateEdge(edge, withWait = withWait) }
- // }
- case Nil => Nil
- }
-
- val composed = for {
- deleteRet <- Future.sequence(deleteAllFutures)
- mutateRet <- Future.sequence(mutateEdgeFutures)
- } yield deleteRet ++ mutateRet
-
- composed.map(_.forall(identity))
- }
-
- Future.sequence(mutateEdges)
- }
-
- def mutateVertex(vertex: Vertex, withWait: Boolean): Future[Boolean] = {
- if (vertex.op == GraphUtil.operations("delete")) {
- writeToStorage(vertex.hbaseZkAddr,
- vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
- } else if (vertex.op == GraphUtil.operations("deleteAll")) {
- logger.info(s"deleteAll for vertex is truncated. $vertex")
- Future.successful(true) // Ignore withWait parameter, because deleteAll operation may takes long time
- } else {
- writeToStorage(vertex.hbaseZkAddr, buildPutsAll(vertex), withWait)
- }
- }
-
- def mutateVertices(vertices: Seq[Vertex],
- withWait: Boolean = false): Future[Seq[Boolean]] = {
- val futures = vertices.map { vertex => mutateVertex(vertex, withWait) }
- Future.sequence(futures)
- }
-
-
- def mutateEdgesInner(edges: Seq[Edge],
- checkConsistency: Boolean,
- withWait: Boolean)(f: (Option[Edge], Seq[Edge]) => (Edge, EdgeMutate)): Future[Boolean] = {
- if (!checkConsistency) {
- val zkQuorum = edges.head.label.hbaseZkAddr
- val futures = edges.map { edge =>
- val (_, edgeUpdate) = f(None, Seq(edge))
- val mutations =
- indexedEdgeMutations(edgeUpdate) ++
- snapshotEdgeMutations(edgeUpdate) ++
- increments(edgeUpdate)
- writeToStorage(zkQuorum, mutations, withWait)
- }
- Future.sequence(futures).map { rets => rets.forall(identity) }
- } else {
- def commit(_edges: Seq[Edge], statusCode: Byte): Future[Boolean] = {
-
- fetchSnapshotEdge(_edges.head) flatMap { case (queryParam, snapshotEdgeOpt, kvOpt) =>
-
- val (newEdge, edgeUpdate) = f(snapshotEdgeOpt, _edges)
- logger.debug(s"${snapshotEdgeOpt}\n${edgeUpdate.toLogString}")
- //shouldReplace false.
- if (edgeUpdate.newSnapshotEdge.isEmpty && statusCode <= 0) {
- logger.debug(s"${newEdge.toLogString} drop.")
- Future.successful(true)
- } else {
- commitUpdate(newEdge, statusCode)(snapshotEdgeOpt, kvOpt, edgeUpdate).map { ret =>
- if (ret) {
- logger.info(s"[Success] commit: \n${_edges.map(_.toLogString).mkString("\n")}")
- } else {
- throw new PartialFailureException(newEdge, 3, "commit failed.")
- }
- true
- }
- }
- }
- }
- def retry(tryNum: Int)(edges: Seq[Edge], statusCode: Byte)(fn: (Seq[Edge], Byte) => Future[Boolean]): Future[Boolean] = {
- if (tryNum >= MaxRetryNum) {
- edges.foreach { edge =>
- logger.error(s"commit failed after $MaxRetryNum\n${edge.toLogString}")
- ExceptionHandler.enqueue(ExceptionHandler.toKafkaMessage(element = edge))
- }
- Future.successful(false)
- } else {
- val future = fn(edges, statusCode)
- future.onSuccess {
- case success =>
- logger.debug(s"Finished. [$tryNum]\n${edges.head.toLogString}\n")
- }
- future recoverWith {
- case FetchTimeoutException(retryEdge) =>
- logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}")
- retry(tryNum + 1)(edges, statusCode)(fn)
-
- case PartialFailureException(retryEdge, failedStatusCode, faileReason) =>
- val status = failedStatusCode match {
- case 0 => "AcquireLock failed."
- case 1 => "Mutation failed."
- case 2 => "Increment failed."
- case 3 => "ReleaseLock failed."
- case 4 => "Unknown"
- }
-
- Thread.sleep(Random.nextInt(MaxBackOff))
- logger.info(s"[Try: $tryNum], [Status: $status] partial fail.\n${retryEdge.toLogString}\nFailReason: ${faileReason}")
- retry(tryNum + 1)(Seq(retryEdge), failedStatusCode)(fn)
- case ex: Exception =>
- logger.error("Unknown exception", ex)
- Future.successful(false)
- }
- }
- }
- retry(1)(edges, 0)(commit)
- }
- }
-
- def mutateLog(snapshotEdgeOpt: Option[Edge], edges: Seq[Edge],
- newEdge: Edge, edgeMutate: EdgeMutate) =
- Seq("----------------------------------------------",
- s"SnapshotEdge: ${snapshotEdgeOpt.map(_.toLogString)}",
- s"requestEdges: ${edges.map(_.toLogString).mkString("\n")}",
- s"newEdge: ${newEdge.toLogString}",
- s"mutation: \n${edgeMutate.toLogString}",
- "----------------------------------------------").mkString("\n")
-
-
- /** Delete All */
- protected def deleteAllFetchedEdgesAsyncOld(queryRequest: QueryRequest,
- queryResult: QueryResult,
- requestTs: Long,
- retryNum: Int): Future[Boolean] = {
- val queryParam = queryRequest.queryParam
- val zkQuorum = queryParam.label.hbaseZkAddr
- val futures = for {
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- } yield {
- /** reverted direction */
- val reversedIndexedEdgesMutations = edge.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
- indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- buildIncrementsAsync(indexEdge, -1L)
- }
- val reversedSnapshotEdgeMutations = snapshotEdgeSerializer(edge.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
- val forwardIndexedEdgeMutations = edge.edgesWithIndex.flatMap { indexEdge =>
- indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- buildIncrementsAsync(indexEdge, -1L)
- }
- val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
- writeToStorage(zkQuorum, mutations, withWait = true)
- }
-
- Future.sequence(futures).map { rets => rets.forall(identity) }
- }
-
- protected def buildEdgesToDelete(queryRequestWithResultLs: QueryRequestWithResult, requestTs: Long): QueryResult = {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs).get
- val edgeWithScoreLs = queryResult.edgeWithScoreLs.filter { edgeWithScore =>
- (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
- }.map { edgeWithScore =>
- val label = queryRequest.queryParam.label
- val (newOp, newVersion, newPropsWithTs) = label.consistencyLevel match {
- case "strong" =>
- val _newPropsWithTs = edgeWithScore.edge.propsWithTs ++
- Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(requestTs, requestTs, label.schemaVersion))
- (GraphUtil.operations("delete"), requestTs, _newPropsWithTs)
- case _ =>
- val oldEdge = edgeWithScore.edge
- (oldEdge.op, oldEdge.version, oldEdge.propsWithTs)
- }
-
- val copiedEdge =
- edgeWithScore.edge.copy(op = newOp, version = newVersion, propsWithTs = newPropsWithTs)
-
- val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
-// logger.debug(s"delete edge from deleteAll: ${edgeToDelete.edge.toLogString}")
- edgeToDelete
- }
-
- queryResult.copy(edgeWithScoreLs = edgeWithScoreLs)
- }
-
- protected def deleteAllFetchedEdgesLs(queryRequestWithResultLs: Seq[QueryRequestWithResult], requestTs: Long): Future[(Boolean, Boolean)] = {
- val queryResultLs = queryRequestWithResultLs.map(_.queryResult)
- queryResultLs.foreach { queryResult =>
- if (queryResult.isFailure) throw new RuntimeException("fetched result is fallback.")
- }
- val futures = for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- deleteQueryResult = buildEdgesToDelete(queryRequestWithResult, requestTs)
- if deleteQueryResult.edgeWithScoreLs.nonEmpty
- } yield {
- val label = queryRequest.queryParam.label
- label.schemaVersion match {
- case HBaseType.VERSION3 | HBaseType.VERSION4 =>
- if (label.consistencyLevel == "strong") {
- /**
- * read: snapshotEdge on queryResult = O(N)
- * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
- */
- mutateEdges(deleteQueryResult.edgeWithScoreLs.map(_.edge), withWait = true).map(_.forall(identity))
- } else {
- deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
- }
- case _ =>
-
- /**
- * read: x
- * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
- */
- deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
- }
- }
-
- if (futures.isEmpty) {
- // all deleted.
- Future.successful(true -> true)
- } else {
- Future.sequence(futures).map { rets => false -> rets.forall(identity) }
- }
- }
-
- protected def fetchAndDeleteAll(query: Query, requestTs: Long): Future[(Boolean, Boolean)] = {
- val future = for {
- queryRequestWithResultLs <- getEdges(query)
- (allDeleted, ret) <- deleteAllFetchedEdgesLs(queryRequestWithResultLs, requestTs)
- } yield {
-// logger.debug(s"fetchAndDeleteAll: ${allDeleted}, ${ret}")
- (allDeleted, ret)
- }
-
- Extensions.retryOnFailure(MaxRetryNum) {
- future
- } {
- logger.error(s"fetch and deleteAll failed.")
- (true, false)
- }
-
- }
-
- def deleteAllAdjacentEdges(srcVertices: Seq[Vertex],
- labels: Seq[Label],
- dir: Int,
- ts: Long): Future[Boolean] = {
-
- def enqueueLogMessage() = {
- val kafkaMessages = for {
- vertice <- srcVertices
- id = vertice.innerId.toIdString()
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", GraphUtil.fromOp(dir.toByte)).mkString("\t")
- val topic = ExceptionHandler.failTopic
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
-
- ExceptionHandler.enqueues(kafkaMessages)
- }
-
- val requestTs = ts
- val queryParams = for {
- label <- labels
- } yield {
- val labelWithDir = LabelWithDirection(label.id.get, dir)
- QueryParam(labelWithDir).limit(0, DeleteAllFetchSize).duplicatePolicy(Option(Query.DuplicatePolicy.Raw))
- }
-
- val step = Step(queryParams.toList)
- val q = Query(srcVertices, Vector(step))
-
- // Extensions.retryOnSuccessWithBackoff(MaxRetryNum, Random.nextInt(MaxBackOff) + 1) {
- val retryFuture = Extensions.retryOnSuccess(MaxRetryNum) {
- fetchAndDeleteAll(q, requestTs)
- } { case (allDeleted, deleteSuccess) =>
- allDeleted && deleteSuccess
- }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
-
- retryFuture onFailure {
- case ex =>
- logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
- enqueueLogMessage()
- }
-
- retryFuture
- }
-
- /** End Of Delete All */
-
-
-
-
- /** Parsing Logic: parse from kv from Storage into Edge */
- def toEdge[K: CanSKeyValue](kv: K,
- queryParam: QueryParam,
- cacheElementOpt: Option[IndexEdge],
- parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
-// logger.debug(s"toEdge: $kv")
- try {
- val schemaVer = queryParam.label.schemaVersion
- val indexEdgeOpt = indexEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
- indexEdgeOpt.map(indexEdge => indexEdge.toEdge.copy(parentEdges = parentEdges))
- } catch {
- case ex: Exception =>
- logger.error(s"Fail on toEdge: ${kv.toString}, ${queryParam}", ex)
- None
- }
- }
-
- def toSnapshotEdge[K: CanSKeyValue](kv: K,
- queryParam: QueryParam,
- cacheElementOpt: Option[SnapshotEdge] = None,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
-// logger.debug(s"SnapshottoEdge: $kv")
- val schemaVer = queryParam.label.schemaVersion
- val snapshotEdgeOpt = snapshotEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
-
- if (isInnerCall) {
- snapshotEdgeOpt.flatMap { snapshotEdge =>
- val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
- if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
- else None
- }
- } else {
- snapshotEdgeOpt.flatMap { snapshotEdge =>
- if (Edge.allPropsDeleted(snapshotEdge.props)) None
- else {
- val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
- if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
- else None
- }
- }
- }
- }
-
- def toEdges[K: CanSKeyValue](kvs: Seq[K],
- queryParam: QueryParam,
- prevScore: Double = 1.0,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
- if (kvs.isEmpty) Seq.empty
- else {
- val first = kvs.head
- val kv = first
- val schemaVer = queryParam.label.schemaVersion
- val cacheElementOpt =
- if (queryParam.isSnapshotEdge) None
- else indexEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, None)
-
- for {
- kv <- kvs
- edge <-
- if (queryParam.isSnapshotEdge) toSnapshotEdge(kv, queryParam, None, isInnerCall, parentEdges)
- else toEdge(kv, queryParam, cacheElementOpt, parentEdges)
- } yield {
- //TODO: Refactor this.
- val currentScore =
- queryParam.scorePropagateOp match {
- case "plus" => edge.rank(queryParam.rank) + prevScore
- case _ => edge.rank(queryParam.rank) * prevScore
- }
- EdgeWithScore(edge, currentScore)
- }
- }
- }
-
- /** End Of Parse Logic */
-
-// /** methods for consistency */
-// protected def writeAsyncSimple(zkQuorum: String, elementRpcs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
-// if (elementRpcs.isEmpty) {
-// Future.successful(true)
-// } else {
-// val futures = elementRpcs.map { rpc => writeToStorage(rpc, withWait) }
-// Future.sequence(futures).map(_.forall(identity))
-// }
-// }
-
- case class PartialFailureException(edge: Edge, statusCode: Byte, failReason: String) extends Exception
-
- protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge) = {
- val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}").mkString("\n")
- logger.debug(msg)
- }
-
- protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge, edgeMutate: EdgeMutate) = {
- val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}",
- s"${edgeMutate.toLogString}").mkString("\n")
- logger.debug(msg)
- }
-
- protected def buildLockEdge(snapshotEdgeOpt: Option[Edge], edge: Edge, kvOpt: Option[SKeyValue]) = {
- val currentTs = System.currentTimeMillis()
- val lockTs = snapshotEdgeOpt match {
- case None => Option(currentTs)
- case Some(snapshotEdge) =>
- snapshotEdge.pendingEdgeOpt match {
- case None => Option(currentTs)
- case Some(pendingEdge) => pendingEdge.lockTs
- }
- }
- val newVersion = kvOpt.map(_.timestamp).getOrElse(edge.ts) + 1
- // snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
- val pendingEdge = edge.copy(version = newVersion, statusCode = 1, lockTs = lockTs)
- val base = snapshotEdgeOpt match {
- case None =>
- // no one ever mutated on this snapshotEdge.
- edge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
- case Some(snapshotEdge) =>
- // there is at least one mutation have been succeed.
- snapshotEdgeOpt.get.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
- }
- base.copy(version = newVersion, statusCode = 1, lockTs = None)
- }
-
- protected def buildReleaseLockEdge(snapshotEdgeOpt: Option[Edge], lockEdge: SnapshotEdge,
- edgeMutate: EdgeMutate) = {
- val newVersion = lockEdge.version + 1
- val base = edgeMutate.newSnapshotEdge match {
- case None =>
- // shouldReplace false
- assert(snapshotEdgeOpt.isDefined)
- snapshotEdgeOpt.get.toSnapshotEdge
- case Some(newSnapshotEdge) => newSnapshotEdge
- }
- base.copy(version = newVersion, statusCode = 0, pendingEdgeOpt = None)
- }
-
- protected def acquireLock(statusCode: Byte,
- edge: Edge,
- oldSnapshotEdgeOpt: Option[Edge],
- lockEdge: SnapshotEdge,
- oldBytes: Array[Byte]): Future[Boolean] = {
- if (statusCode >= 1) {
- logger.debug(s"skip acquireLock: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 0, s"$p")
- else {
- val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
- val oldPut = oldSnapshotEdgeOpt.map(e => snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head)
-// val lockEdgePut = buildPutAsync(lockEdge).head
-// val oldPut = oldSnapshotEdgeOpt.map(e => buildPutAsync(e.toSnapshotEdge).head)
- writeLock(lockEdgePut, oldPut).recoverWith { case ex: Exception =>
- logger.error(s"AcquireLock RPC Failed.")
- throw new PartialFailureException(edge, 0, "AcquireLock RPC Failed")
- }.map { ret =>
- if (ret) {
- val log = Seq(
- "\n",
- "=" * 50,
- s"[Success]: acquireLock",
- s"[RequestEdge]: ${edge.toLogString}",
- s"[LockEdge]: ${lockEdge.toLogString()}",
- s"[PendingEdge]: ${lockEdge.pendingEdgeOpt.map(_.toLogString).getOrElse("")}",
- "=" * 50, "\n").mkString("\n")
-
- logger.debug(log)
- // debug(ret, "acquireLock", edge.toSnapshotEdge)
- } else {
- throw new PartialFailureException(edge, 0, "hbase fail.")
- }
- true
- }
- }
- }
- }
-
-
-
- protected def releaseLock(predicate: Boolean,
- edge: Edge,
- lockEdge: SnapshotEdge,
- releaseLockEdge: SnapshotEdge,
- _edgeMutate: EdgeMutate,
- oldBytes: Array[Byte]): Future[Boolean] = {
- if (!predicate) {
- throw new PartialFailureException(edge, 3, "predicate failed.")
- }
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 3, s"$p")
- else {
- val releaseLockEdgePut = snapshotEdgeSerializer(releaseLockEdge).toKeyValues.head
- val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
- writeLock(releaseLockEdgePut, Option(lockEdgePut)).recoverWith {
- case ex: Exception =>
- logger.error(s"ReleaseLock RPC Failed.")
- throw new PartialFailureException(edge, 3, "ReleaseLock RPC Failed")
- }.map { ret =>
- if (ret) {
- debug(ret, "releaseLock", edge.toSnapshotEdge)
- } else {
- val msg = Seq("\nFATAL ERROR\n",
- "=" * 50,
- oldBytes.toList,
- lockEdgePut,
- releaseLockEdgePut,
- // lockEdgePut.value.toList,
- // releaseLockEdgePut.value().toList,
- "=" * 50,
- "\n"
- )
- logger.error(msg.mkString("\n"))
- // error(ret, "releaseLock", edge.toSnapshotEdge)
- throw new PartialFailureException(edge, 3, "hbase fail.")
- }
- true
- }
- }
- Future.successful(true)
- }
-
-
- protected def mutate(predicate: Boolean,
- edge: Edge,
- statusCode: Byte,
- _edgeMutate: EdgeMutate): Future[Boolean] = {
- if (!predicate) throw new PartialFailureException(edge, 1, "predicate failed.")
-
- if (statusCode >= 2) {
- logger.debug(s"skip mutate: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 1, s"$p")
- else
- writeToStorage(edge.label.hbaseZkAddr, indexedEdgeMutations(_edgeMutate), withWait = true).map { ret =>
- if (ret) {
- debug(ret, "mutate", edge.toSnapshotEdge, _edgeMutate)
- } else {
- throw new PartialFailureException(edge, 1, "hbase fail.")
- }
- true
- }
- }
- }
-
- protected def increment(predicate: Boolean,
- edge: Edge,
- statusCode: Byte, _edgeMutate: EdgeMutate): Future[Boolean] = {
- if (!predicate) throw new PartialFailureException(edge, 2, "predicate failed.")
- if (statusCode >= 3) {
- logger.debug(s"skip increment: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 2, s"$p")
- else
- writeToStorage(edge.label.hbaseZkAddr, increments(_edgeMutate), withWait = true).map { ret =>
- if (ret) {
- debug(ret, "increment", edge.toSnapshotEdge, _edgeMutate)
- } else {
- throw new PartialFailureException(edge, 2, "hbase fail.")
- }
- true
- }
- }
- }
-
-
- /** this may be overrided by specific storage implementation */
- protected def commitProcess(edge: Edge, statusCode: Byte)
- (snapshotEdgeOpt: Option[Edge], kvOpt: Option[SKeyValue])
- (lockEdge: SnapshotEdge, releaseLockEdge: SnapshotEdge, _edgeMutate: EdgeMutate): Future[Boolean] = {
- val oldBytes = kvOpt.map(kv => kv.value).getOrElse(Array.empty[Byte])
- for {
- locked <- acquireLock(statusCode, edge, snapshotEdgeOpt, lockEdge, oldBytes)
- mutated <- mutate(locked, edge, statusCode, _edgeMutate)
- incremented <- increment(mutated, edge, statusCode, _edgeMutate)
- released <- releaseLock(incremented, edge, lockEdge, releaseLockEdge, _edgeMutate, oldBytes)
- } yield {
- released
- }
- }
-
- protected def commitUpdate(edge: Edge,
- statusCode: Byte)(snapshotEdgeOpt: Option[Edge],
- kvOpt: Option[SKeyValue],
- edgeUpdate: EdgeMutate): Future[Boolean] = {
- val label = edge.label
- def oldBytes = kvOpt.map(_.value).getOrElse(Array.empty)
-
- val lockEdge = buildLockEdge(snapshotEdgeOpt, edge, kvOpt)
- val releaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, edgeUpdate)
- val _process = commitProcess(edge, statusCode)(snapshotEdgeOpt, kvOpt)_
- snapshotEdgeOpt match {
- case None =>
- // no one ever did success on acquire lock.
- _process(lockEdge, releaseLockEdge, edgeUpdate)
- // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
- case Some(snapshotEdge) =>
- // someone did success on acquire lock at least one.
- snapshotEdge.pendingEdgeOpt match {
- case None =>
- // not locked
- _process(lockEdge, releaseLockEdge, edgeUpdate)
- // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
- case Some(pendingEdge) =>
- def isLockExpired = pendingEdge.lockTs.get + LockExpireDuration < System.currentTimeMillis()
- if (isLockExpired) {
- val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
- val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(pendingEdge))
- val newLockEdge = buildLockEdge(snapshotEdgeOpt, pendingEdge, kvOpt)
- val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, newLockEdge, newEdgeUpdate)
- commitProcess(edge, statusCode = 0)(snapshotEdgeOpt, kvOpt)(newLockEdge, newReleaseLockEdge, newEdgeUpdate).flatMap { ret =>
- // process(newLockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode = 0).flatMap { ret =>
- val log = s"[Success]: Resolving expired pending edge.\n${pendingEdge.toLogString}"
- throw new PartialFailureException(edge, 0, log)
- }
- } else {
- // locked
- if (pendingEdge.ts == edge.ts && statusCode > 0) {
- // self locked
- val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
- val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(edge))
- val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, newEdgeUpdate)
-
- /** lockEdge will be ignored */
- _process(lockEdge, newReleaseLockEdge, newEdgeUpdate)
- // process(lockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode)
- } else {
- throw new PartialFailureException(edge, statusCode, s"others[${pendingEdge.ts}] is mutating. me[${edge.ts}]")
- }
- }
- }
- }
- }
-
- /** end of methods for consistency */
-
-
- // def futureCache[T] = Cache[Long, (Long, T)]
-
- protected def toRequestEdge(queryRequest: QueryRequest): Edge = {
- val srcVertex = queryRequest.vertex
- // val tgtVertexOpt = queryRequest.tgtVertexOpt
- val edgeCf = Serializable.edgeCf
- val queryParam = queryRequest.queryParam
- val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
- val label = queryParam.label
- val labelWithDir = queryParam.labelWithDir
- val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
- val (srcInnerId, tgtInnerId) = tgtVertexIdOpt match {
- case Some(tgtVertexId) => // _to is given.
- /** we use toSnapshotEdge so dont need to swap src, tgt */
- val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- val tgt = InnerVal.convertVersion(tgtVertexId, tgtColumn.columnType, label.schemaVersion)
- (src, tgt)
- case None =>
- val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- (src, src)
- }
-
- val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
- val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
- val currentTs = System.currentTimeMillis()
- val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs)).toMap
- Edge(srcV, tgtV, labelWithDir, propsWithTs = propsWithTs)
- }
-
-
-
- protected def fetchSnapshotEdge(edge: Edge): Future[(QueryParam, Option[Edge], Option[SKeyValue])] = {
- val labelWithDir = edge.labelWithDir
- val queryParam = QueryParam(labelWithDir)
- val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
- val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
- val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
-
- fetchSnapshotEdgeKeyValues(buildRequest(queryRequest)).map { kvs =>
- val (edgeOpt, kvOpt) =
- if (kvs.isEmpty) (None, None)
- else {
- val _edgeOpt = toEdges(kvs, queryParam, 1.0, isInnerCall = true, parentEdges = Nil).headOption.map(_.edge)
- val _kvOpt = kvs.headOption
- (_edgeOpt, _kvOpt)
- }
- (queryParam, edgeOpt, kvOpt)
- } recoverWith { case ex: Throwable =>
- logger.error(s"fetchQueryParam failed. fallback return.", ex)
- throw new FetchTimeoutException(s"${edge.toLogString}")
- }
- }
-
- protected def fetchStep(orgQuery: Query, queryRequestWithResultsLs: Seq[QueryRequestWithResult]): Future[Seq[QueryRequestWithResult]] = {
- if (queryRequestWithResultsLs.isEmpty) Future.successful(Nil)
- else {
- val queryRequest = queryRequestWithResultsLs.head.queryRequest
- val q = orgQuery
- val queryResultsLs = queryRequestWithResultsLs.map(_.queryResult)
-
- val stepIdx = queryRequest.stepIdx + 1
-
- val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
- val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
- val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
- val step = q.steps(stepIdx)
- val alreadyVisited =
- if (stepIdx == 0) Map.empty[(LabelWithDirection, Vertex), Boolean]
- else Graph.alreadyVisitedVertices(queryResultsLs)
-
- val groupedBy = queryResultsLs.flatMap { queryResult =>
- queryResult.edgeWithScoreLs.map { case edgeWithScore =>
- edgeWithScore.edge.tgtVertex -> edgeWithScore
- }
- }.groupBy { case (vertex, edgeWithScore) => vertex }
-
- val groupedByFiltered = for {
- (vertex, edgesWithScore) <- groupedBy
- aggregatedScore = edgesWithScore.map(_._2.score).sum if aggregatedScore >= prevStepThreshold
- } yield vertex -> aggregatedScore
-
- val prevStepTgtVertexIdEdges = for {
- (vertex, edgesWithScore) <- groupedBy
- } yield vertex.id -> edgesWithScore.map { case (vertex, edgeWithScore) => edgeWithScore }
-
- val nextStepSrcVertices = if (prevStepLimit >= 0) {
- groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
- } else {
- groupedByFiltered.toSeq
- }
-
- val queryRequests = for {
- (vertex, prevStepScore) <- nextStepSrcVertices
- queryParam <- step.queryParams
- } yield (QueryRequest(q, stepIdx, vertex, queryParam), prevStepScore)
-
- Graph.filterEdges(fetches(queryRequests, prevStepTgtVertexIdEdges), alreadyVisited)(ec)
- }
- }
-
- protected def fetchStepFuture(orgQuery: Query, queryRequestWithResultLsFuture: Future[Seq[QueryRequestWithResult]]): Future[Seq[QueryRequestWithResult]] = {
- for {
- queryRequestWithResultLs <- queryRequestWithResultLsFuture
- ret <- fetchStep(orgQuery, queryRequestWithResultLs)
- } yield ret
- }
-
- def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = {
- val fallback = {
- val queryRequest = QueryRequest(query = q, stepIdx = 0, q.vertices.head, queryParam = QueryParam.Empty)
- Future.successful(q.vertices.map(v => QueryRequestWithResult(queryRequest, QueryResult())))
- }
- Try {
-
- if (q.steps.isEmpty) {
- // TODO: this should be get vertex query.
- fallback
- } else {
- // current stepIdx = -1
- val startQueryResultLs = QueryResult.fromVertices(q)
- q.steps.foldLeft(Future.successful(startQueryResultLs)) { case (acc, step) =>
- fetchStepFuture(q, acc)
-// fetchStepFuture(q, acc).map { stepResults =>
-// step.queryParams.zip(stepResults).foreach { case (qParam, queryRequestWithResult) =>
-// val cursor = Base64.getEncoder.encodeToString(queryRequestWithResult.queryResult.tailCursor)
-// qParam.cursorOpt = Option(cursor)
-// }
-// stepResults
-// }
- }
- }
- } recover {
- case e: Exception =>
- logger.error(s"getEdgesAsync: $e", e)
- fallback
- } get
- }
-
- def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = {
- val ts = System.currentTimeMillis()
- val futures = for {
- (srcVertex, tgtVertex, queryParam) <- params
- propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, queryParam.label.schemaVersion))
- edge = Edge(srcVertex, tgtVertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
- } yield {
- fetchSnapshotEdge(edge).map { case (queryParam, edgeOpt, kvOpt) =>
- val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
- val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
- val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
- val queryResult = QueryResult(edgeOpt.toSeq.map(e => EdgeWithScore(e, 1.0)))
- QueryRequestWithResult(queryRequest, queryResult)
- }
- }
-
- Future.sequence(futures)
- }
-
-
-
- @tailrec
- final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (range < sampleNumber || set.size == sampleNumber) set
- else randomInt(sampleNumber, range, set + Random.nextInt(range))
- }
-
- protected def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
- if (edges.size <= n){
- edges
- }else{
- val plainEdges = if (queryRequest.queryParam.offset == 0) {
- edges.tail
- } else edges
-
- val randoms = randomInt(n, plainEdges.size)
- var samples = List.empty[EdgeWithScore]
- var idx = 0
- plainEdges.foreach { e =>
- if (randoms.contains(idx)) samples = e :: samples
- idx += 1
- }
- samples.toSeq
- }
-
- }
- /** end of query */
-
- /** Mutation Builder */
-
-
- /** EdgeMutate */
- def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] = {
- val deleteMutations = edgeMutate.edgesToDelete.flatMap { indexEdge =>
- indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
- }
- val insertMutations = edgeMutate.edgesToInsert.flatMap { indexEdge =>
- indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
- }
-
- deleteMutations ++ insertMutations
- }
-
- def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
- edgeMutate.newSnapshotEdge.map(e => snapshotEdgeSerializer(e).toKeyValues).getOrElse(Nil)
-
- def increments(edgeMutate: EdgeMutate): Seq[SKeyValue] =
- (edgeMutate.edgesToDelete.isEmpty, edgeMutate.edgesToInsert.isEmpty) match {
- case (true, true) =>
-
- /** when there is no need to update. shouldUpdate == false */
- List.empty
- case (true, false) =>
-
- /** no edges to delete but there is new edges to insert so increase degree by 1 */
- edgeMutate.edgesToInsert.flatMap { e => buildIncrementsAsync(e) }
- case (false, true) =>
-
- /** no edges to insert but there is old edges to delete so decrease degree by 1 */
- edgeMutate.edgesToDelete.flatMap { e => buildIncrementsAsync(e, -1L) }
- case (false, false) =>
-
- /** update on existing edges so no change on degree */
- List.empty
- }
-
- /** IndexEdge */
- def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
- val newProps = indexedEdge.props ++ Map(LabelMeta.degreeSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
- val _indexedEdge = indexedEdge.copy(props = newProps)
- indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
- }
-
- def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
- val newProps = indexedEdge.props ++ Map(LabelMeta.countSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
- val _indexedEdge = indexedEdge.copy(props = newProps)
- indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
- }
- def buildDeleteBelongsToId(vertex: Vertex): Seq[SKeyValue] = {
- val kvs = vertexSerializer(vertex).toKeyValues
- val kv = kvs.head
- vertex.belongLabelIds.map { id =>
- kv.copy(qualifier = Bytes.toBytes(Vertex.toPropKey(id)), operation = SKeyValue.Delete)
- }
- }
-
- def buildVertexPutsAsync(edge: Edge): Seq[SKeyValue] =
- if (edge.op == GraphUtil.operations("delete"))
- buildDeleteBelongsToId(edge.srcForVertex) ++ buildDeleteBelongsToId(edge.tgtForVertex)
- else
- vertexSerializer(edge.srcForVertex).toKeyValues ++ vertexSerializer(edge.tgtForVertex).toKeyValues
-
- def buildPutsAll(vertex: Vertex): Seq[SKeyValue] = {
- vertex.op match {
- case d: Byte if d == GraphUtil.operations("delete") => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
- case _ => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Put))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
deleted file mode 100644
index 4b3300a..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
+++ /dev/null
@@ -1,95 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.QueryParam
-import com.kakao.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, InnerValLikeWithTs}
-import org.apache.hadoop.hbase.util.Bytes
-
-object StorageDeserializable {
- /** Deserializer */
- def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
- val byte = bytes(offset)
- val isInverted = if ((byte & 1) != 0) true else false
- val labelOrderSeq = byte >> 1
- (labelOrderSeq.toByte, isInverted)
- }
-
- def bytesToKeyValues(bytes: Array[Byte],
- offset: Int,
- length: Int,
- version: String): (Array[(Byte, InnerValLike)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLike)](len)
- var i = 0
- while (i < len) {
- val k = bytes(pos)
- pos += 1
- val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- val ret = (kvs, pos)
- // logger.debug(s"bytesToProps: $ret")
- ret
- }
-
- def bytesToKeyValuesWithTs(bytes: Array[Byte],
- offset: Int,
- version: String): (Array[(Byte, InnerValLikeWithTs)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLikeWithTs)](len)
- var i = 0
- while (i < len) {
- val k = bytes(pos)
- pos += 1
- val (v, numOfBytesUsed) = InnerValLikeWithTs.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- val ret = (kvs, pos)
- // logger.debug(s"bytesToProps: $ret")
- ret
- }
-
- def bytesToProps(bytes: Array[Byte],
- offset: Int,
- version: String): (Array[(Byte, InnerValLike)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLike)](len)
- var i = 0
- while (i < len) {
- val k = HBaseType.EMPTY_SEQ_BYTE
- val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- // logger.error(s"bytesToProps: $kvs")
- val ret = (kvs, pos)
-
- ret
- }
-
- def bytesToLong(bytes: Array[Byte], offset: Int): Long = Bytes.toLong(bytes, offset)
-}
-
-trait StorageDeserializable[E] {
- def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): Option[E] = {
- try {
- Option(fromKeyValuesInner(queryParam, kvs, version, cacheElementOpt))
- } catch {
- case e: Exception =>
- logger.error(s"${this.getClass.getName} fromKeyValues failed.", e)
- None
- }
- }
- def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): E
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageSerializable.scala
deleted file mode 100644
index 575f4ab..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageSerializable.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core.types.{InnerValLikeWithTs, InnerValLike}
-import org.apache.hadoop.hbase.util.Bytes
-
-object StorageSerializable {
- /** serializer */
- def propsToBytes(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, v.bytes)
- bytes
- }
-
- def propsToKeyValues(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
- bytes
- }
-
- def propsToKeyValuesWithTs(props: Seq[(Byte, InnerValLikeWithTs)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
- bytes
- }
-
- def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
- assert(labelOrderSeq < (1 << 6))
- val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
- Array.fill(1)(byte.toByte)
- }
-}
-
-trait StorageSerializable[E] {
- def toKeyValues: Seq[SKeyValue]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
deleted file mode 100644
index 8441c6b..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ /dev/null
@@ -1,533 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.storage._
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.{FutureCache, DeferCache, Extensions, logger}
-import com.stumbleupon.async.Deferred
-import com.typesafe.config.{ConfigFactory, Config}
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.client.{ConnectionFactory, Durability}
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
-import org.apache.hadoop.hbase.regionserver.BloomType
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
-import org.apache.hadoop.security.UserGroupInformation
-import org.hbase.async._
-import scala.collection.JavaConversions._
-import scala.collection.{Map, Seq}
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext, Future, duration}
-import scala.util.hashing.MurmurHash3
-import java.util
-import java.util.Base64
-
-
-object AsynchbaseStorage {
- val vertexCf = Serializable.vertexCf
- val edgeCf = Serializable.edgeCf
- val emptyKVs = new util.ArrayList[KeyValue]()
-
-
- def makeClient(config: Config, overrideKv: (String, String)*) = {
- val asyncConfig: org.hbase.async.Config =
- if (config.hasPath("hbase.security.auth.enable") && config.getBoolean("hbase.security.auth.enable")) {
- val krb5Conf = config.getString("java.security.krb5.conf")
- val jaas = config.getString("java.security.auth.login.config")
-
- System.setProperty("java.security.krb5.conf", krb5Conf)
- System.setProperty("java.security.auth.login.config", jaas)
- new org.hbase.async.Config()
- } else {
- new org.hbase.async.Config()
- }
-
- for (entry <- config.entrySet() if entry.getKey.contains("hbase")) {
- asyncConfig.overrideConfig(entry.getKey, entry.getValue.unwrapped().toString)
- }
-
- for ((k, v) <- overrideKv) {
- asyncConfig.overrideConfig(k, v)
- }
-
- val client = new HBaseClient(asyncConfig)
- logger.info(s"Asynchbase: ${client.getConfig.dumpConfiguration()}")
- client
- }
-}
-
-
-class AsynchbaseStorage(override val config: Config)(implicit ec: ExecutionContext)
- extends Storage[Deferred[QueryRequestWithResult]](config) {
-
- import Extensions.DeferOps
-
- /**
- * Asynchbase client setup.
- * note that we need two client, one for bulk(withWait=false) and another for withWait=true
- */
- val configWithFlush = config.withFallback(ConfigFactory.parseMap(Map("hbase.rpcs.buffered_flush_interval" -> "0")))
- val client = AsynchbaseStorage.makeClient(config)
-
- private val clientWithFlush = AsynchbaseStorage.makeClient(config, "hbase.rpcs.buffered_flush_interval" -> "0")
- private val clients = Seq(client, clientWithFlush)
- private val clientFlushInterval = config.getInt("hbase.rpcs.buffered_flush_interval").toString().toShort
- private val emptyKeyValues = new util.ArrayList[KeyValue]()
- private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
-
- /** Future Cache to squash request */
- private val futureCache = new DeferCache[QueryResult](config)(ec)
-
- /** Simple Vertex Cache */
- private val vertexCache = new FutureCache[Seq[SKeyValue]](config)(ec)
-
-
- /**
- * fire rpcs into proper hbase cluster using client and
- * return true on all mutation success. otherwise return false.
- */
- override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
- if (kvs.isEmpty) Future.successful(true)
- else {
- val _client = client(withWait)
- val futures = kvs.map { kv =>
- val _defer = kv.operation match {
- case SKeyValue.Put => _client.put(new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp))
- case SKeyValue.Delete =>
- if (kv.qualifier == null) _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.timestamp))
- else _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.timestamp))
- case SKeyValue.Increment =>
- _client.atomicIncrement(new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, Bytes.toLong(kv.value)))
- }
- val future = _defer.withCallback { ret => true }.recoverWith { ex =>
- logger.error(s"mutation failed. $kv", ex)
- false
- }.toFuture
-
- if (withWait) future else Future.successful(true)
- }
-
- Future.sequence(futures).map(_.forall(identity))
- }
- }
-
-
- override def fetchSnapshotEdgeKeyValues(hbaseRpc: AnyRef): Future[Seq[SKeyValue]] = {
- val defer = fetchKeyValuesInner(hbaseRpc)
- defer.toFuture.map { kvsArr =>
- kvsArr.map { kv =>
- implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
- } toSeq
- }
- }
-
- /**
- * since HBase natively provide CheckAndSet on storage level, implementation becomes simple.
- * @param rpc: key value that is need to be stored on storage.
- * @param expectedOpt: last valid value for rpc's KeyValue.value from fetching.
- * @return return true if expected value matches and our rpc is successfully applied, otherwise false.
- * note that when some other thread modified same cell and have different value on this KeyValue,
- * then HBase atomically return false.
- */
- override def writeLock(rpc: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean] = {
- val put = new PutRequest(rpc.table, rpc.row, rpc.cf, rpc.qualifier, rpc.value, rpc.timestamp)
- val expected = expectedOpt.map(_.value).getOrElse(Array.empty)
- client(withWait = true).compareAndSet(put, expected).withCallback(ret => ret.booleanValue()).toFuture
- }
-
-
- /**
- * given queryRequest, build storage specific RPC Request.
- * In HBase case, we either build Scanner or GetRequest.
- *
- * IndexEdge layer:
- * Tall schema(v4): use scanner.
- * Wide schema(label's schema version in v1, v2, v3): use GetRequest with columnRangeFilter
- * when query is given with itnerval option.
- * SnapshotEdge layer:
- * Tall schema(v3, v4): use GetRequest without column filter.
- * Wide schema(label's schema version in v1, v2): use GetRequest with columnRangeFilter.
- * Vertex layer:
- * all version: use GetRequest without column filter.
- * @param queryRequest
- * @return Scanner or GetRequest with proper setup with StartKey, EndKey, RangeFilter.
- */
- override def buildRequest(queryRequest: QueryRequest): AnyRef = {
- import Serializable._
- val queryParam = queryRequest.queryParam
- val label = queryParam.label
- val edge = toRequestEdge(queryRequest)
-
- val kv = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
- val snapshotEdge = edge.toSnapshotEdge
- snapshotEdgeSerializer(snapshotEdge).toKeyValues.head
- // new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
- } else {
- val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == queryParam.labelOrderSeq)
- assert(indexedEdgeOpt.isDefined)
-
- val indexedEdge = indexedEdgeOpt.get
- indexEdgeSerializer(indexedEdge).toKeyValues.head
- }
-
- val (minTs, maxTs) = queryParam.duration.getOrElse((0L, Long.MaxValue))
-
- label.schemaVersion match {
- case HBaseType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty =>
- val scanner = client.newScanner(label.hbaseTableName.getBytes)
- scanner.setFamily(edgeCf)
-
- /**
- * TODO: remove this part.
- */
- val indexEdgeOpt = edge.edgesWithIndex.filter(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq).headOption
- val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
-
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
- // val labelIndexSeqWithIsInvertedStopBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = true)
- val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, Bytes.add(labelIndexSeqWithIsInvertedBytes, Array.fill(1)(edge.op)))
- val (startKey, stopKey) =
- if (queryParam.columnRangeFilter != null) {
- // interval is set.
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Bytes.add(Base64.getDecoder.decode(cursor), Array.fill(1)(0))
- case None => Bytes.add(baseKey, queryParam.columnRangeFilterMinBytes)
- }
- (_startKey, Bytes.add(baseKey, queryParam.columnRangeFilterMaxBytes))
- } else {
- /**
- * note: since propsToBytes encode size of property map at first byte, we are sure about max value here
- */
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Bytes.add(Base64.getDecoder.decode(cursor), Array.fill(1)(0))
- case None => baseKey
- }
- (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
- }
-// logger.debug(s"[StartKey]: ${startKey.toList}")
-// logger.debug(s"[StopKey]: ${stopKey.toList}")
-
- scanner.setStartKey(startKey)
- scanner.setStopKey(stopKey)
-
- if (queryParam.limit == Int.MinValue) logger.debug(s"MinValue: $queryParam")
-
- scanner.setMaxVersions(1)
- scanner.setMaxNumRows(queryParam.limit)
- scanner.setMaxTimestamp(maxTs)
- scanner.setMinTimestamp(minTs)
- scanner.setRpcTimeout(queryParam.rpcTimeoutInMillis)
- // SET option for this rpc properly.
- scanner
- case _ =>
- val get =
- if (queryParam.tgtVertexInnerIdOpt.isDefined) new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
- else new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf)
-
- get.maxVersions(1)
- get.setFailfast(true)
- get.setMaxResultsPerColumnFamily(queryParam.limit)
- get.setRowOffsetPerColumnFamily(queryParam.offset)
- get.setMinTimestamp(minTs)
- get.setMaxTimestamp(maxTs)
- get.setTimeout(queryParam.rpcTimeoutInMillis)
-
- if (queryParam.columnRangeFilter != null) get.setFilter(queryParam.columnRangeFilter)
-
- get
- }
- }
-
- /**
- * we are using future cache to squash requests into same key on storage.
- *
- * @param queryRequest
- * @param prevStepScore
- * @param isInnerCall
- * @param parentEdges
- * @return we use Deferred here since it has much better performrance compared to scala.concurrent.Future.
- * seems like map, flatMap on scala.concurrent.Future is slower than Deferred's addCallback
- */
- override def fetch(queryRequest: QueryRequest,
- prevStepScore: Double,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): Deferred[QueryRequestWithResult] = {
-
- def fetchInner(hbaseRpc: AnyRef): Deferred[QueryResult] = {
- fetchKeyValuesInner(hbaseRpc).withCallback { kvs =>
- val edgeWithScores = toEdges(kvs, queryRequest.queryParam, prevStepScore, isInnerCall, parentEdges)
- val resultEdgesWithScores = if (queryRequest.queryParam.sample >= 0) {
- sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
- } else edgeWithScores
- QueryResult(resultEdgesWithScores, tailCursor = kvs.lastOption.map(_.key).getOrElse(Array.empty[Byte]))
-// QueryRequestWithResult(queryRequest, QueryResult(resultEdgesWithScores, tailCursor = kvs.lastOption.map(_.key).getOrElse(Array.empty)))
-
- } recoverWith { ex =>
- logger.error(s"fetchInner failed. fallback return. $hbaseRpc}", ex)
- QueryResult(isFailure = true)
-// QueryRequestWithResult(queryRequest, QueryResult(isFailure = true))
- }
- }
-
- val queryParam = queryRequest.queryParam
- val cacheTTL = queryParam.cacheTTLInMillis
- val request = buildRequest(queryRequest)
-
- val defer =
- if (cacheTTL <= 0) fetchInner(request)
- else {
- val cacheKeyBytes = Bytes.add(queryRequest.query.cacheKeyBytes, toCacheKeyBytes(request))
- val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
- futureCache.getOrElseUpdate(cacheKey, cacheTTL)(fetchInner(request))
- }
- defer withCallback { queryResult => QueryRequestWithResult(queryRequest, queryResult)}
- }
-
-
- override def fetches(queryRequestWithScoreLs: scala.Seq[(QueryRequest, Double)],
- prevStepEdges: Predef.Map[VertexId, scala.Seq[EdgeWithScore]]): Future[scala.Seq[QueryRequestWithResult]] = {
- val defers: Seq[Deferred[QueryRequestWithResult]] = for {
- (queryRequest, prevStepScore) <- queryRequestWithScoreLs
- parentEdges <- prevStepEdges.get(queryRequest.vertex.id)
- } yield fetch(queryRequest, prevStepScore, isInnerCall = false, parentEdges)
-
- val grouped: Deferred[util.ArrayList[QueryRequestWithResult]] = Deferred.group(defers)
- grouped withCallback {
- queryResults: util.ArrayList[QueryRequestWithResult] =>
- queryResults.toIndexedSeq
- } toFuture
- }
-
-
- def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]] = fetchSnapshotEdgeKeyValues(request)
-
-
- /**
- * when withWait is given, we use client with flushInterval set to 0.
- * if we are not using this, then we are adding extra wait time as much as flushInterval in worst case.
- *
- * @param edges
- * @param withWait
- * @return
- */
- override def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = {
- val _client = client(withWait)
- val defers: Seq[Deferred[(Boolean, Long)]] = for {
- edge <- edges
- } yield {
- val edgeWithIndex = edge.edgesWithIndex.head
- val countWithTs = edge.propsWithTs(LabelMeta.countSeq)
- val countVal = countWithTs.innerVal.toString().toLong
- val incr = buildIncrementsCountAsync(edgeWithIndex, countVal).head
- val request = incr.asInstanceOf[AtomicIncrementRequest]
- _client.bufferAtomicIncrement(request) withCallback { resultCount: java.lang.Long =>
- (true, resultCount.longValue())
- } recoverWith { ex =>
- logger.error(s"mutation failed. $request", ex)
- (false, -1L)
- }
- }
-
- val grouped: Deferred[util.ArrayList[(Boolean, Long)]] = Deferred.groupInOrder(defers)
- grouped.toFuture.map(_.toSeq)
- }
-
-
- override def flush(): Unit = clients.foreach { client =>
- val timeout = Duration((clientFlushInterval + 10) * 20, duration.MILLISECONDS)
- Await.result(client.flush().toFuture, timeout)
- }
-
-
- override def createTable(zkAddr: String,
- tableName: String,
- cfs: List[String],
- regionMultiplier: Int,
- ttl: Option[Int],
- compressionAlgorithm: String): Unit = {
- logger.info(s"create table: $tableName on $zkAddr, $cfs, $regionMultiplier, $compressionAlgorithm")
- val admin = getAdmin(zkAddr)
- val regionCount = admin.getClusterStatus.getServersSize * regionMultiplier
- if (!admin.tableExists(TableName.valueOf(tableName))) {
- try {
- val desc = new HTableDescriptor(TableName.valueOf(tableName))
- desc.setDurability(Durability.ASYNC_WAL)
- for (cf <- cfs) {
- val columnDesc = new HColumnDescriptor(cf)
- .setCompressionType(Algorithm.valueOf(compressionAlgorithm.toUpperCase))
- .setBloomFilterType(BloomType.ROW)
- .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)
- .setMaxVersions(1)
- .setTimeToLive(2147483647)
- .setMinVersions(0)
- .setBlocksize(32768)
- .setBlockCacheEnabled(true)
- if (ttl.isDefined) columnDesc.setTimeToLive(ttl.get)
- desc.addFamily(columnDesc)
- }
-
- if (regionCount <= 1) admin.createTable(desc)
- else admin.createTable(desc, getStartKey(regionCount), getEndKey(regionCount), regionCount)
- } catch {
- case e: Throwable =>
- logger.error(s"$zkAddr, $tableName failed with $e", e)
- throw e
- }
- } else {
- logger.info(s"$zkAddr, $tableName, $cfs already exist.")
- }
- }
-
-
- /** Asynchbase implementation override default getVertices to use future Cache */
- override def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
- def fromResult(queryParam: QueryParam,
- kvs: Seq[SKeyValue],
- version: String): Option[Vertex] = {
-
- if (kvs.isEmpty) None
- else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
- }
-
- val futures = vertices.map { vertex =>
- val kvs = vertexSerializer(vertex).toKeyValues
- val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
- // get.setTimeout(this.singleGetTimeout.toShort)
- get.setFailfast(true)
- get.maxVersions(1)
-
- val cacheKey = MurmurHash3.stringHash(get.toString)
- vertexCache.getOrElseUpdate(cacheKey, cacheTTL = 10000)(fetchVertexKeyValues(get)).map { kvs =>
- fromResult(QueryParam.Empty, kvs, vertex.serviceColumn.schemaVersion)
- }
- }
-
- Future.sequence(futures).map { result => result.toList.flatten }
- }
-
-
-
-
-
- /**
- * Private Methods which is specific to Asynchbase implementation.
- */
- private def fetchKeyValuesInner(rpc: AnyRef): Deferred[util.ArrayList[KeyValue]] = {
- rpc match {
- case getRequest: GetRequest => client.get(getRequest)
- case scanner: Scanner =>
- scanner.nextRows().withCallback { kvsLs =>
- val ls = new util.ArrayList[KeyValue]
- if (kvsLs == null) {
-
- } else {
- kvsLs.foreach { kvs =>
- if (kvs != null) kvs.foreach { kv => ls.add(kv) }
- else {
-
- }
- }
- }
- scanner.close()
- ls
- }.recoverWith { ex =>
- logger.error(s"fetchKeyValuesInner failed.", ex)
- scanner.close()
- emptyKeyValues
- }
- case _ => Deferred.fromError(new RuntimeException(s"fetchKeyValues failed. $rpc"))
- }
- }
-
- private def toCacheKeyBytes(hbaseRpc: AnyRef): Array[Byte] = {
- hbaseRpc match {
- case getRequest: GetRequest => getRequest.key()
- case scanner: Scanner => scanner.getCurrentKey()
- case _ =>
- logger.error(s"toCacheKeyBytes failed. not supported class type. $hbaseRpc")
- Array.empty[Byte]
- }
- }
-
- private def getSecureClusterAdmin(zkAddr: String) = {
- val jaas = config.getString("java.security.auth.login.config")
- val krb5Conf = config.getString("java.security.krb5.conf")
- val realm = config.getString("realm")
- val principal = config.getString("principal")
- val keytab = config.getString("keytab")
-
-
-
- System.setProperty("java.security.auth.login.config", jaas)
- System.setProperty("java.security.krb5.conf", krb5Conf)
- // System.setProperty("sun.security.krb5.debug", "true")
- // System.setProperty("sun.security.spnego.debug", "true")
- val conf = new Configuration(true)
- val hConf = HBaseConfiguration.create(conf)
-
- hConf.set("hbase.zookeeper.quorum", zkAddr)
-
- hConf.set("hadoop.security.authentication", "Kerberos")
- hConf.set("hbase.security.authentication", "Kerberos")
- hConf.set("hbase.master.kerberos.principal", "hbase/_HOST@" + realm)
- hConf.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@" + realm)
-
- System.out.println("Connecting secure cluster, using keytab\n")
- UserGroupInformation.setConfiguration(hConf)
- UserGroupInformation.loginUserFromKeytab(principal, keytab)
- val currentUser = UserGroupInformation.getCurrentUser()
- System.out.println("current user : " + currentUser + "\n")
-
- // get table list
- val conn = ConnectionFactory.createConnection(hConf)
- conn.getAdmin
- }
-
- /**
- * following configuration need to come together to use secured hbase cluster.
- * 1. set hbase.security.auth.enable = true
- * 2. set file path to jaas file java.security.auth.login.config
- * 3. set file path to kerberos file java.security.krb5.conf
- * 4. set realm
- * 5. set principal
- * 6. set file path to keytab
- * @param zkAddr
- * @return
- */
- private def getAdmin(zkAddr: String) = {
- if (config.hasPath("hbase.security.auth.enable") && config.getBoolean("hbase.security.auth.enable")) {
- getSecureClusterAdmin(zkAddr)
- } else {
- val conf = HBaseConfiguration.create()
- conf.set("hbase.zookeeper.quorum", zkAddr)
- val conn = ConnectionFactory.createConnection(conf)
- conn.getAdmin
- }
- }
-
- private def enableTable(zkAddr: String, tableName: String) = {
- getAdmin(zkAddr).enableTable(TableName.valueOf(tableName))
- }
-
- private def disableTable(zkAddr: String, tableName: String) = {
- getAdmin(zkAddr).disableTable(TableName.valueOf(tableName))
- }
-
- private def dropTable(zkAddr: String, tableName: String) = {
- getAdmin(zkAddr).disableTable(TableName.valueOf(tableName))
- getAdmin(zkAddr).deleteTable(TableName.valueOf(tableName))
- }
-
- private def getStartKey(regionCount: Int): Array[Byte] = {
- Bytes.toBytes((Int.MaxValue / regionCount))
- }
-
- private def getEndKey(regionCount: Int): Array[Byte] = {
- Bytes.toBytes((Int.MaxValue / regionCount * (regionCount - 1)))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
deleted file mode 100644
index 014a5c9..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
+++ /dev/null
@@ -1,132 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.indexedge.tall
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.StorageDeserializable._
-import com.kakao.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue, StorageDeserializable}
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.{GraphUtil, IndexEdge, QueryParam, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-
-class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] {
- import StorageDeserializable._
-
- type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
- type ValueRaw = (Array[(Byte, InnerValLike)], Int)
-
- private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- // val degree = Bytes.toLong(kv.value)
- val degree = bytesToLongFunc(kv.value, 0)
- val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
- val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
- (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
- }
-
- private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- var qualifierLen = 0
- var pos = 0
- val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
- val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
- pos = endAt
- qualifierLen += endAt
- val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
- (HBaseType.defaultTgtVertexId, 0)
- } else {
- TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
- }
- qualifierLen += tgtVertexIdLen
- (props, endAt, tgtVertexId, tgtVertexIdLen)
- }
- val (op, opLen) =
- if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
- else (kv.qualifier(qualifierLen), 1)
-
- qualifierLen += opLen
-
- (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
- }
-
- private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
- val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
- (props, endAt)
- }
-
- private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
- (Array.empty[(Byte, InnerValLike)], 0)
- }
-
- override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[IndexEdge]): IndexEdge = {
-
- assert(_kvs.size == 1)
-
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-
- val kv = kvs.head
-
- // logger.debug(s"[Des]: ${kv.row.toList}, ${kv.qualifier.toList}, ${kv.value.toList}")
- var pos = 0
- val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
- pos += srcIdLen
- val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
- pos += 4
- val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
- pos += 1
-
- val op = kv.row(pos)
- pos += 1
-
- if (pos == kv.row.length) {
- // degree
- // val degreeVal = Bytes.toLong(kv.value)
- val degreeVal = bytesToLongFunc(kv.value, 0)
- val ts = kv.timestamp
- val props = Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, version),
- LabelMeta.degreeSeq -> InnerVal.withLong(degreeVal, version))
- val tgtVertexId = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
- IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, props)
- } else {
- // not degree edge
- val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${queryParam.label.id.get}, ${labelIdxSeq}"))
-
- val (idxPropsRaw, endAt) = bytesToProps(kv.row, pos, version)
- pos = endAt
- val (tgtVertexIdRaw, tgtVertexIdLen) = if (endAt == kv.row.length) {
- (HBaseType.defaultTgtVertexId, 0)
- } else {
- TargetVertexId.fromBytes(kv.row, endAt, kv.row.length, version)
- }
-
- val idxProps = for {
- (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
- } yield if (k == LabelMeta.degreeSeq) k -> v else seq -> v
-
- val idxPropsMap = idxProps.toMap
-
- val tgtVertexId =
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => tgtVertexIdRaw
- case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
- }
-
- val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
- // val countVal = Bytes.toLong(kv.value)
- val countVal = bytesToLongFunc(kv.value, 0)
- val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
- (dummyProps, 8)
- } else {
- bytesToKeyValues(kv.value, 0, kv.value.length, version)
- }
-
- val _mergedProps = (idxProps ++ props).toMap
- val mergedProps =
- if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
- else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
-
- val ts = kv.timestamp
- IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
-
- }
- }
- }
[25/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/DefaultSource.scala b/loader/src/main/scala/spark/DefaultSource.scala
deleted file mode 100644
index cc36a15..0000000
--- a/loader/src/main/scala/spark/DefaultSource.scala
+++ /dev/null
@@ -1,982 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package spark
-//
-//import java.util
-//import java.util.concurrent.ConcurrentLinkedQueue
-//
-//import org.apache.hadoop.hbase.client.{ConnectionFactory, Get, Result, Scan}
-//import org.apache.hadoop.hbase.types._
-//import org.apache.hadoop.hbase.util.{SimplePositionedMutableByteRange, PositionedByteRange, Bytes}
-//import org.apache.hadoop.hbase.{TableName, HBaseConfiguration}
-//import org.apache.spark.Logging
-//import org.apache.spark.rdd.RDD
-//import org.apache.spark.sql.types.DataType
-//import org.apache.spark.sql.{Row, SQLContext}
-//import org.apache.spark.sql.sources._
-//import org.apache.spark.sql.types._
-//
-//import scala.collection.mutable
-//
-///**
-// * DefaultSource for integration with Spark's dataframe datasources.
-// * This class will produce a relationProvider based on input given to it from spark
-// *
-// * In all this DefaultSource support the following datasource functionality
-// * - Scan range pruning through filter push down logic based on rowKeys
-// * - Filter push down logic on HBase Cells
-// * - Qualifier filtering based on columns used in the SparkSQL statement
-// * - Type conversions of basic SQL types. All conversions will be
-// * Through the HBase Bytes object commands.
-// */
-//class DefaultSource extends RelationProvider {
-//
-// val TABLE_KEY:String = "hbase.table"
-// val SCHEMA_COLUMNS_MAPPING_KEY:String = "hbase.columns.mapping"
-// val BATCHING_NUM_KEY:String = "hbase.batching.num"
-// val CACHING_NUM_KEY:String = "hbase.caching.num"
-// val HBASE_CONFIG_RESOURCES_LOCATIONS:String = "hbase.config.resources"
-// val USE_HBASE_CONTEXT:String = "hbase.use.hbase.context"
-//
-// /**
-// * Is given input from SparkSQL to construct a BaseRelation
-// * @param sqlContext SparkSQL context
-// * @param parameters Parameters given to us from SparkSQL
-// * @return A BaseRelation Object
-// */
-// override def createRelation(sqlContext: SQLContext,
-// parameters: Map[String, String]):
-// BaseRelation = {
-//
-//
-// val tableName = parameters.get(TABLE_KEY)
-// if (tableName.isEmpty)
-// new IllegalArgumentException("Invalid value for " + TABLE_KEY +" '" + tableName + "'")
-//
-// val schemaMappingString = parameters.getOrElse(SCHEMA_COLUMNS_MAPPING_KEY, "")
-// val batchingNumStr = parameters.getOrElse(BATCHING_NUM_KEY, "1000")
-// val cachingNumStr = parameters.getOrElse(CACHING_NUM_KEY, "1000")
-// val hbaseConfigResources = parameters.getOrElse(HBASE_CONFIG_RESOURCES_LOCATIONS, "")
-// val useHBaseReources = parameters.getOrElse(USE_HBASE_CONTEXT, "true")
-//
-// val batchingNum:Int = try {
-// batchingNumStr.toInt
-// } catch {
-// case e:NumberFormatException => throw
-// new IllegalArgumentException("Invalid value for " + BATCHING_NUM_KEY +" '"
-// + batchingNumStr + "'", e)
-// }
-//
-// val cachingNum:Int = try {
-// cachingNumStr.toInt
-// } catch {
-// case e:NumberFormatException => throw
-// new IllegalArgumentException("Invalid value for " + CACHING_NUM_KEY +" '"
-// + cachingNumStr + "'", e)
-// }
-//
-// new HBaseRelation(tableName.get,
-// generateSchemaMappingMap(schemaMappingString),
-// batchingNum.toInt,
-// cachingNum.toInt,
-// hbaseConfigResources,
-// useHBaseReources.equalsIgnoreCase("true"))(sqlContext)
-// }
-//
-// /**
-// * Reads the SCHEMA_COLUMNS_MAPPING_KEY and converts it to a map of
-// * SchemaQualifierDefinitions with the original sql column name as the key
-// * @param schemaMappingString The schema mapping string from the SparkSQL map
-// * @return A map of definitions keyed by the SparkSQL column name
-// */
-// def generateSchemaMappingMap(schemaMappingString:String):
-// java.util.HashMap[String, SchemaQualifierDefinition] = {
-// try {
-// val columnDefinitions = schemaMappingString.split(',')
-// val resultingMap = new java.util.HashMap[String, SchemaQualifierDefinition]()
-// columnDefinitions.map(cd => {
-// val parts = cd.trim.split(' ')
-//
-// //Make sure we get three parts
-// //<ColumnName> <ColumnType> <ColumnFamily:Qualifier>
-// if (parts.length == 3) {
-// val hbaseDefinitionParts = if (parts(2).charAt(0) == ':') {
-// Array[String]("", "key")
-// } else {
-// parts(2).split(':')
-// }
-// resultingMap.put(parts(0), new SchemaQualifierDefinition(parts(0),
-// parts(1), hbaseDefinitionParts(0), hbaseDefinitionParts(1)))
-// } else {
-// throw new IllegalArgumentException("Invalid value for schema mapping '" + cd +
-// "' should be '<columnName> <columnType> <columnFamily>:<qualifier>' " +
-// "for columns and '<columnName> <columnType> :<qualifier>' for rowKeys")
-// }
-// })
-// resultingMap
-// } catch {
-// case e:Exception => throw
-// new IllegalArgumentException("Invalid value for " + SCHEMA_COLUMNS_MAPPING_KEY +
-// " '" + schemaMappingString + "'", e )
-// }
-// }
-//}
-//
-///**
-// * Implementation of Spark BaseRelation that will build up our scan logic
-// * , do the scan pruning, filter push down, and value conversions
-// *
-// * @param tableName HBase table that we plan to read from
-// * @param schemaMappingDefinition SchemaMapping information to map HBase
-// * Qualifiers to SparkSQL columns
-// * @param batchingNum The batching number to be applied to the
-// * scan object
-// * @param cachingNum The caching number to be applied to the
-// * scan object
-// * @param configResources Optional comma separated list of config resources
-// * to get based on their URI
-// * @param useHBaseContext If true this will look to see if
-// * HBaseContext.latest is populated to use that
-// * connection information
-// * @param sqlContext SparkSQL context
-// */
-//class HBaseRelation (val tableName:String,
-// val schemaMappingDefinition:
-// java.util.HashMap[String, SchemaQualifierDefinition],
-// val batchingNum:Int,
-// val cachingNum:Int,
-// val configResources:String,
-// val useHBaseContext:Boolean) (
-// @transient val sqlContext:SQLContext)
-// extends BaseRelation with PrunedFilteredScan with Logging {
-//
-// //create or get latest HBaseContext
-// @transient val hbaseContext:HBaseContext = if (useHBaseContext) {
-// LatestHBaseContextCache.latest
-// } else {
-// val config = HBaseConfiguration.create()
-// configResources.split(",").foreach( r => config.addResource(r))
-// new HBaseContext(sqlContext.sparkContext, config)
-// }
-//
-// /**
-// * Generates a Spark SQL schema object so Spark SQL knows what is being
-// * provided by this BaseRelation
-// *
-// * @return schema generated from the SCHEMA_COLUMNS_MAPPING_KEY value
-// */
-// override def schema: StructType = {
-//
-// val metadataBuilder = new MetadataBuilder()
-//
-// val structFieldArray = new Array[StructField](schemaMappingDefinition.size())
-//
-// val schemaMappingDefinitionIt = schemaMappingDefinition.values().iterator()
-// var indexCounter = 0
-// while (schemaMappingDefinitionIt.hasNext) {
-// val c = schemaMappingDefinitionIt.next()
-//
-// val metadata = metadataBuilder.putString("name", c.columnName).build()
-// val structField =
-// new StructField(c.columnName, c.columnSparkSqlType, nullable = true, metadata)
-//
-// structFieldArray(indexCounter) = structField
-// indexCounter += 1
-// }
-//
-// val result = new StructType(structFieldArray)
-// result
-// }
-//
-// /**
-// * Here we are building the functionality to populate the resulting RDD[Row]
-// * Here is where we will do the following:
-// * - Filter push down
-// * - Scan or GetList pruning
-// * - Executing our scan(s) or/and GetList to generate result
-// *
-// * @param requiredColumns The columns that are being requested by the requesting query
-// * @param filters The filters that are being applied by the requesting query
-// * @return RDD will all the results from HBase needed for SparkSQL to
-// * execute the query on
-// */
-// override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = {
-//
-// val columnFilterCollection = buildColumnFilterCollection(filters)
-//
-// val requiredQualifierDefinitionArray = new mutable.MutableList[SchemaQualifierDefinition]
-// requiredColumns.foreach( c => {
-// val definition = schemaMappingDefinition.get(c)
-// if (definition.columnFamilyBytes.length > 0) {
-// requiredQualifierDefinitionArray += definition
-// }
-// })
-//
-// //Create a local variable so that scala doesn't have to
-// // serialize the whole HBaseRelation Object
-// val serializableDefinitionMap = schemaMappingDefinition
-//
-//
-// //retain the information for unit testing checks
-// DefaultSourceStaticUtils.populateLatestExecutionRules(columnFilterCollection,
-// requiredQualifierDefinitionArray)
-// var resultRDD: RDD[Row] = null
-//
-// if (columnFilterCollection != null) {
-// val pushDownFilterJava =
-// new SparkSQLPushDownFilter(
-// columnFilterCollection.generateFamilyQualifiterFilterMap(schemaMappingDefinition))
-//
-// val getList = new util.ArrayList[Get]()
-// val rddList = new util.ArrayList[RDD[Row]]()
-//
-// val it = columnFilterCollection.columnFilterMap.iterator
-// while (it.hasNext) {
-// val e = it.next()
-// val columnDefinition = schemaMappingDefinition.get(e._1)
-// //check is a rowKey
-// if (columnDefinition != null && columnDefinition.columnFamily.isEmpty) {
-// //add points to getList
-// e._2.points.foreach(p => {
-// val get = new Get(p)
-// requiredQualifierDefinitionArray.foreach( d =>
-// get.addColumn(d.columnFamilyBytes, d.qualifierBytes))
-// getList.add(get)
-// })
-//
-// val rangeIt = e._2.ranges.iterator
-//
-// while (rangeIt.hasNext) {
-// val r = rangeIt.next()
-//
-// val scan = new Scan()
-// scan.setBatch(batchingNum)
-// scan.setCaching(cachingNum)
-// requiredQualifierDefinitionArray.foreach( d =>
-// scan.addColumn(d.columnFamilyBytes, d.qualifierBytes))
-//
-// if (pushDownFilterJava.columnFamilyQualifierFilterMap.size() > 0) {
-// scan.setFilter(pushDownFilterJava)
-// }
-//
-// //Check if there is a lower bound
-// if (r.lowerBound != null && r.lowerBound.length > 0) {
-//
-// if (r.isLowerBoundEqualTo) {
-// //HBase startRow is inclusive: Therefore it acts like isLowerBoundEqualTo
-// // by default
-// scan.setStartRow(r.lowerBound)
-// } else {
-// //Since we don't equalTo we want the next value we need
-// // to add another byte to the start key. That new byte will be
-// // the min byte value.
-// val newArray = new Array[Byte](r.lowerBound.length + 1)
-// System.arraycopy(r.lowerBound, 0, newArray, 0, r.lowerBound.length)
-//
-// //new Min Byte
-// newArray(r.lowerBound.length) = Byte.MinValue
-// scan.setStartRow(newArray)
-// }
-// }
-//
-// //Check if there is a upperBound
-// if (r.upperBound != null && r.upperBound.length > 0) {
-// if (r.isUpperBoundEqualTo) {
-// //HBase stopRow is exclusive: therefore it DOESN'T ast like isUpperBoundEqualTo
-// // by default. So we need to add a new max byte to the stopRow key
-// val newArray = new Array[Byte](r.upperBound.length + 1)
-// System.arraycopy(r.upperBound, 0, newArray, 0, r.upperBound.length)
-//
-// //New Max Bytes
-// newArray(r.upperBound.length) = Byte.MaxValue
-//
-// scan.setStopRow(newArray)
-// } else {
-// //Here equalTo is false for Upper bound which is exclusive and
-// // HBase stopRow acts like that by default so no need to mutate the
-// // rowKey
-// scan.setStopRow(r.upperBound)
-// }
-// }
-//
-// val rdd = hbaseContext.hbaseRDD(TableName.valueOf(tableName), scan).map(r => {
-// Row.fromSeq(requiredColumns.map(c =>
-// DefaultSourceStaticUtils.getValue(c, serializableDefinitionMap, r._2)))
-// })
-// rddList.add(rdd)
-// }
-// }
-// }
-//
-// //If there is more then one RDD then we have to union them together
-// for (i <- 0 until rddList.size()) {
-// if (resultRDD == null) resultRDD = rddList.get(i)
-// else resultRDD = resultRDD.union(rddList.get(i))
-//
-// }
-//
-// //If there are gets then we can get them from the driver and union that rdd in
-// // with the rest of the values.
-// if (getList.size() > 0) {
-// val connection = ConnectionFactory.createConnection(hbaseContext.tmpHdfsConfiguration)
-// try {
-// val table = connection.getTable(TableName.valueOf(tableName))
-// try {
-// val results = table.get(getList)
-// val rowList = mutable.MutableList[Row]()
-// for (i <- 0 until results.length) {
-// val rowArray = requiredColumns.map(c =>
-// DefaultSourceStaticUtils.getValue(c, schemaMappingDefinition, results(i)))
-// rowList += Row.fromSeq(rowArray)
-// }
-// val getRDD = sqlContext.sparkContext.parallelize(rowList)
-// if (resultRDD == null) resultRDD = getRDD
-// else {
-// resultRDD = resultRDD.union(getRDD)
-// }
-// } finally {
-// table.close()
-// }
-// } finally {
-// connection.close()
-// }
-// }
-// }
-// if (resultRDD == null) {
-// val scan = new Scan()
-// scan.setBatch(batchingNum)
-// scan.setCaching(cachingNum)
-// requiredQualifierDefinitionArray.foreach( d =>
-// scan.addColumn(d.columnFamilyBytes, d.qualifierBytes))
-//
-// val rdd = hbaseContext.hbaseRDD(TableName.valueOf(tableName), scan).map(r => {
-// Row.fromSeq(requiredColumns.map(c => DefaultSourceStaticUtils.getValue(c,
-// serializableDefinitionMap, r._2)))
-// })
-// resultRDD=rdd
-// }
-// resultRDD
-// }
-//
-// /**
-// * Root recursive function that will loop over the filters provided by
-// * SparkSQL. Some filters are AND or OR functions and contain additional filters
-// * hence the need for recursion.
-// *
-// * @param filters Filters provided by SparkSQL.
-// * Filters are joined with the AND operater
-// * @return A ColumnFilterCollection whish is a consolidated construct to
-// * hold the high level filter information
-// */
-// def buildColumnFilterCollection(filters: Array[Filter]): ColumnFilterCollection = {
-// var superCollection: ColumnFilterCollection = null
-//
-// filters.foreach( f => {
-// val parentCollection = new ColumnFilterCollection
-// buildColumnFilterCollection(parentCollection, f)
-// if (superCollection == null)
-// superCollection = parentCollection
-// else
-// superCollection.mergeIntersect(parentCollection)
-// })
-// superCollection
-// }
-//
-// /**
-// * Recursive function that will work to convert Spark Filter
-// * objects to ColumnFilterCollection
-// *
-// * @param parentFilterCollection Parent ColumnFilterCollection
-// * @param filter Current given filter from SparkSQL
-// */
-// def buildColumnFilterCollection(parentFilterCollection:ColumnFilterCollection,
-// filter:Filter): Unit = {
-// filter match {
-//
-// case EqualTo(attr, value) =>
-// parentFilterCollection.mergeUnion(attr,
-// new ColumnFilter(DefaultSourceStaticUtils.getByteValue(attr,
-// schemaMappingDefinition, value.toString)))
-//
-// case LessThan(attr, value) =>
-// parentFilterCollection.mergeUnion(attr, new ColumnFilter(null,
-// new ScanRange(DefaultSourceStaticUtils.getByteValue(attr,
-// schemaMappingDefinition, value.toString), false,
-// new Array[Byte](0), true)))
-//
-// case GreaterThan(attr, value) =>
-// parentFilterCollection.mergeUnion(attr, new ColumnFilter(null,
-// new ScanRange(null, true, DefaultSourceStaticUtils.getByteValue(attr,
-// schemaMappingDefinition, value.toString), false)))
-//
-// case LessThanOrEqual(attr, value) =>
-// parentFilterCollection.mergeUnion(attr, new ColumnFilter(null,
-// new ScanRange(DefaultSourceStaticUtils.getByteValue(attr,
-// schemaMappingDefinition, value.toString), true,
-// new Array[Byte](0), true)))
-//
-// case GreaterThanOrEqual(attr, value) =>
-// parentFilterCollection.mergeUnion(attr, new ColumnFilter(null,
-// new ScanRange(null, true, DefaultSourceStaticUtils.getByteValue(attr,
-// schemaMappingDefinition, value.toString), true)))
-//
-// case Or(left, right) =>
-// buildColumnFilterCollection(parentFilterCollection, left)
-// val rightSideCollection = new ColumnFilterCollection
-// buildColumnFilterCollection(rightSideCollection, right)
-// parentFilterCollection.mergeUnion(rightSideCollection)
-// case And(left, right) =>
-// buildColumnFilterCollection(parentFilterCollection, left)
-// val rightSideCollection = new ColumnFilterCollection
-// buildColumnFilterCollection(rightSideCollection, right)
-// parentFilterCollection.mergeIntersect(rightSideCollection)
-// case _ => //nothing
-// }
-// }
-//}
-//
-///**
-// * Construct to contains column data that spend SparkSQL and HBase
-// *
-// * @param columnName SparkSQL column name
-// * @param colType SparkSQL column type
-// * @param columnFamily HBase column family
-// * @param qualifier HBase qualifier name
-// */
-//case class SchemaQualifierDefinition(columnName:String,
-// colType:String,
-// columnFamily:String,
-// qualifier:String) extends Serializable {
-// val columnFamilyBytes = Bytes.toBytes(columnFamily)
-// val qualifierBytes = Bytes.toBytes(qualifier)
-// val columnSparkSqlType:DataType = if (colType.equals("BOOLEAN")) BooleanType
-// else if (colType.equals("TINYINT")) IntegerType
-// else if (colType.equals("INT")) IntegerType
-// else if (colType.equals("BIGINT")) LongType
-// else if (colType.equals("FLOAT")) FloatType
-// else if (colType.equals("DOUBLE")) DoubleType
-// else if (colType.equals("STRING")) StringType
-// else if (colType.equals("TIMESTAMP")) TimestampType
-// else if (colType.equals("DECIMAL")) StringType //DataTypes.createDecimalType(precision, scale)
-// else throw new IllegalArgumentException("Unsupported column type :" + colType)
-//}
-//
-///**
-// * Construct to contain a single scan ranges information. Also
-// * provide functions to merge with other scan ranges through AND
-// * or OR operators
-// *
-// * @param upperBound Upper bound of scan
-// * @param isUpperBoundEqualTo Include upper bound value in the results
-// * @param lowerBound Lower bound of scan
-// * @param isLowerBoundEqualTo Include lower bound value in the results
-// */
-//class ScanRange(var upperBound:Array[Byte], var isUpperBoundEqualTo:Boolean,
-// var lowerBound:Array[Byte], var isLowerBoundEqualTo:Boolean)
-// extends Serializable {
-//
-// /**
-// * Function to merge another scan object through a AND operation
-// * @param other Other scan object
-// */
-// def mergeIntersect(other:ScanRange): Unit = {
-// val upperBoundCompare = compareRange(upperBound, other.upperBound)
-// val lowerBoundCompare = compareRange(lowerBound, other.lowerBound)
-//
-// upperBound = if (upperBoundCompare <0) upperBound else other.upperBound
-// lowerBound = if (lowerBoundCompare >0) lowerBound else other.lowerBound
-//
-// isLowerBoundEqualTo = if (lowerBoundCompare == 0)
-// isLowerBoundEqualTo && other.isLowerBoundEqualTo
-// else isLowerBoundEqualTo
-//
-// isUpperBoundEqualTo = if (upperBoundCompare == 0)
-// isUpperBoundEqualTo && other.isUpperBoundEqualTo
-// else isUpperBoundEqualTo
-// }
-//
-// /**
-// * Function to merge another scan object through a OR operation
-// * @param other Other scan object
-// */
-// def mergeUnion(other:ScanRange): Unit = {
-//
-// val upperBoundCompare = compareRange(upperBound, other.upperBound)
-// val lowerBoundCompare = compareRange(lowerBound, other.lowerBound)
-//
-// upperBound = if (upperBoundCompare >0) upperBound else other.upperBound
-// lowerBound = if (lowerBoundCompare <0) lowerBound else other.lowerBound
-//
-// isLowerBoundEqualTo = if (lowerBoundCompare == 0)
-// isLowerBoundEqualTo || other.isLowerBoundEqualTo
-// else isLowerBoundEqualTo
-//
-// isUpperBoundEqualTo = if (upperBoundCompare == 0)
-// isUpperBoundEqualTo || other.isUpperBoundEqualTo
-// else isUpperBoundEqualTo
-// }
-//
-// /**
-// * Common function to see if this scan over laps with another
-// *
-// * Reference Visual
-// *
-// * A B
-// * |---------------------------|
-// * LL--------------LU
-// * RL--------------RU
-// *
-// * A = lowest value is byte[0]
-// * B = highest value is null
-// * LL = Left Lower Bound
-// * LU = Left Upper Bound
-// * RL = Right Lower Bound
-// * RU = Right Upper Bound
-// *
-// * @param other Other scan object
-// * @return True is overlap false is not overlap
-// */
-// def doesOverLap(other:ScanRange): Boolean = {
-//
-// var leftRange:ScanRange = null
-// var rightRange:ScanRange = null
-//
-// //First identify the Left range
-// // Also lower bound can't be null
-// if (Bytes.compareTo(lowerBound, other.lowerBound) <=0) {
-// leftRange = this
-// rightRange = other
-// } else {
-// leftRange = other
-// rightRange = this
-// }
-//
-// //Then see if leftRange goes to null or if leftRange.upperBound
-// // upper is greater or equals to rightRange.lowerBound
-// leftRange.upperBound == null ||
-// Bytes.compareTo(leftRange.upperBound, rightRange.lowerBound) >= 0
-// }
-//
-// /**
-// * Special compare logic because we can have null values
-// * for left or right bound
-// *
-// * @param left Left byte array
-// * @param right Right byte array
-// * @return 0 for equals 1 is left is greater and -1 is right is greater
-// */
-// def compareRange(left:Array[Byte], right:Array[Byte]): Int = {
-// if (left == null && right == null) 0
-// else if (left == null && right != null) 1
-// else if (left != null && right == null) -1
-// else Bytes.compareTo(left, right)
-// }
-// override def toString:String = {
-// "ScanRange:(" + Bytes.toString(upperBound) + "," + isUpperBoundEqualTo + "," +
-// Bytes.toString(lowerBound) + "," + isLowerBoundEqualTo + ")"
-// }
-//}
-//
-///**
-// * Contains information related to a filters for a given column.
-// * This can contain many ranges or points.
-// *
-// * @param currentPoint the initial point when the filter is created
-// * @param currentRange the initial scanRange when the filter is created
-// */
-//class ColumnFilter (currentPoint:Array[Byte] = null,
-// currentRange:ScanRange = null,
-// var points:mutable.MutableList[Array[Byte]] =
-// new mutable.MutableList[Array[Byte]](),
-// var ranges:mutable.MutableList[ScanRange] =
-// new mutable.MutableList[ScanRange]() ) extends Serializable {
-// //Collection of ranges
-// if (currentRange != null ) ranges.+=(currentRange)
-//
-// //Collection of points
-// if (currentPoint != null) points.+=(currentPoint)
-//
-// /**
-// * This will validate a give value through the filter's points and/or ranges
-// * the result will be if the value passed the filter
-// *
-// * @param value Value to be validated
-// * @param valueOffSet The offset of the value
-// * @param valueLength The length of the value
-// * @return True is the value passes the filter false if not
-// */
-// def validate(value:Array[Byte], valueOffSet:Int, valueLength:Int):Boolean = {
-// var result = false
-//
-// points.foreach( p => {
-// if (Bytes.equals(p, 0, p.length, value, valueOffSet, valueLength)) {
-// result = true
-// }
-// })
-//
-// ranges.foreach( r => {
-// val upperBoundPass = r.upperBound == null ||
-// (r.isUpperBoundEqualTo &&
-// Bytes.compareTo(r.upperBound, 0, r.upperBound.length,
-// value, valueOffSet, valueLength) >= 0) ||
-// (!r.isUpperBoundEqualTo &&
-// Bytes.compareTo(r.upperBound, 0, r.upperBound.length,
-// value, valueOffSet, valueLength) > 0)
-//
-// val lowerBoundPass = r.lowerBound == null || r.lowerBound.length == 0
-// (r.isLowerBoundEqualTo &&
-// Bytes.compareTo(r.lowerBound, 0, r.lowerBound.length,
-// value, valueOffSet, valueLength) <= 0) ||
-// (!r.isLowerBoundEqualTo &&
-// Bytes.compareTo(r.lowerBound, 0, r.lowerBound.length,
-// value, valueOffSet, valueLength) < 0)
-//
-// result = result || (upperBoundPass && lowerBoundPass)
-// })
-// result
-// }
-//
-// /**
-// * This will allow us to merge filter logic that is joined to the existing filter
-// * through a OR operator
-// *
-// * @param other Filter to merge
-// */
-// def mergeUnion(other:ColumnFilter): Unit = {
-// other.points.foreach( p => points += p)
-//
-// other.ranges.foreach( otherR => {
-// var doesOverLap = false
-// ranges.foreach{ r =>
-// if (r.doesOverLap(otherR)) {
-// r.mergeUnion(otherR)
-// doesOverLap = true
-// }}
-// if (!doesOverLap) ranges.+=(otherR)
-// })
-// }
-//
-// /**
-// * This will allow us to merge filter logic that is joined to the existing filter
-// * through a AND operator
-// *
-// * @param other Filter to merge
-// */
-// def mergeIntersect(other:ColumnFilter): Unit = {
-// val survivingPoints = new mutable.MutableList[Array[Byte]]()
-// points.foreach( p => {
-// other.points.foreach( otherP => {
-// if (Bytes.equals(p, otherP)) {
-// survivingPoints.+=(p)
-// }
-// })
-// })
-// points = survivingPoints
-//
-// val survivingRanges = new mutable.MutableList[ScanRange]()
-//
-// other.ranges.foreach( otherR => {
-// ranges.foreach( r => {
-// if (r.doesOverLap(otherR)) {
-// r.mergeIntersect(otherR)
-// survivingRanges += r
-// }
-// })
-// })
-// ranges = survivingRanges
-// }
-//
-// override def toString:String = {
-// val strBuilder = new StringBuilder
-// strBuilder.append("(points:(")
-// var isFirst = true
-// points.foreach( p => {
-// if (isFirst) isFirst = false
-// else strBuilder.append(",")
-// strBuilder.append(Bytes.toString(p))
-// })
-// strBuilder.append("),ranges:")
-// isFirst = true
-// ranges.foreach( r => {
-// if (isFirst) isFirst = false
-// else strBuilder.append(",")
-// strBuilder.append(r)
-// })
-// strBuilder.append("))")
-// strBuilder.toString()
-// }
-//}
-//
-///**
-// * A collection of ColumnFilters indexed by column names.
-// *
-// * Also contains merge commends that will consolidate the filters
-// * per column name
-// */
-//class ColumnFilterCollection {
-// val columnFilterMap = new mutable.HashMap[String, ColumnFilter]
-//
-// def clear(): Unit = {
-// columnFilterMap.clear()
-// }
-//
-// /**
-// * This will allow us to merge filter logic that is joined to the existing filter
-// * through a OR operator. This will merge a single columns filter
-// *
-// * @param column The column to be merged
-// * @param other The other ColumnFilter object to merge
-// */
-// def mergeUnion(column:String, other:ColumnFilter): Unit = {
-// val existingFilter = columnFilterMap.get(column)
-// if (existingFilter.isEmpty) {
-// columnFilterMap.+=((column, other))
-// } else {
-// existingFilter.get.mergeUnion(other)
-// }
-// }
-//
-// /**
-// * This will allow us to merge all filters in the existing collection
-// * to the filters in the other collection. All merges are done as a result
-// * of a OR operator
-// *
-// * @param other The other Column Filter Collection to be merged
-// */
-// def mergeUnion(other:ColumnFilterCollection): Unit = {
-// other.columnFilterMap.foreach( e => {
-// mergeUnion(e._1, e._2)
-// })
-// }
-//
-// /**
-// * This will allow us to merge all filters in the existing collection
-// * to the filters in the other collection. All merges are done as a result
-// * of a AND operator
-// *
-// * @param other The column filter from the other collection
-// */
-// def mergeIntersect(other:ColumnFilterCollection): Unit = {
-// other.columnFilterMap.foreach( e => {
-// val existingColumnFilter = columnFilterMap.get(e._1)
-// if (existingColumnFilter.isEmpty) {
-// columnFilterMap += e
-// } else {
-// existingColumnFilter.get.mergeIntersect(e._2)
-// }
-// })
-// }
-//
-// /**
-// * This will collect all the filter information in a way that is optimized
-// * for the HBase filter commend. Allowing the filter to be accessed
-// * with columnFamily and qualifier information
-// *
-// * @param schemaDefinitionMap Schema Map that will help us map the right filters
-// * to the correct columns
-// * @return HashMap oc column filters
-// */
-// def generateFamilyQualifiterFilterMap(schemaDefinitionMap:
-// java.util.HashMap[String,
-// SchemaQualifierDefinition]):
-// util.HashMap[ColumnFamilyQualifierMapKeyWrapper, ColumnFilter] = {
-// val familyQualifierFilterMap =
-// new util.HashMap[ColumnFamilyQualifierMapKeyWrapper, ColumnFilter]()
-//
-// columnFilterMap.foreach( e => {
-// val definition = schemaDefinitionMap.get(e._1)
-// //Don't add rowKeyFilter
-// if (definition.columnFamilyBytes.size > 0) {
-// familyQualifierFilterMap.put(
-// new ColumnFamilyQualifierMapKeyWrapper(
-// definition.columnFamilyBytes, 0, definition.columnFamilyBytes.length,
-// definition.qualifierBytes, 0, definition.qualifierBytes.length), e._2)
-// }
-// })
-// familyQualifierFilterMap
-// }
-//
-// override def toString:String = {
-// val strBuilder = new StringBuilder
-// columnFilterMap.foreach( e => strBuilder.append(e))
-// strBuilder.toString()
-// }
-//}
-//
-///**
-// * Status object to store static functions but also to hold last executed
-// * information that can be used for unit testing.
-// */
-//object DefaultSourceStaticUtils {
-//
-// val rawInteger = new RawInteger
-// val rawLong = new RawLong
-// val rawFloat = new RawFloat
-// val rawDouble = new RawDouble
-// val rawString = RawString.ASCENDING
-//
-// val byteRange = new ThreadLocal[PositionedByteRange]{
-// override def initialValue(): PositionedByteRange = {
-// val range = new SimplePositionedMutableByteRange()
-// range.setOffset(0)
-// range.setPosition(0)
-// }
-// }
-//
-// def getFreshByteRange(bytes:Array[Byte]): PositionedByteRange = {
-// getFreshByteRange(bytes, 0, bytes.length)
-// }
-//
-// def getFreshByteRange(bytes:Array[Byte], offset:Int = 0, length:Int): PositionedByteRange = {
-// byteRange.get().set(bytes).setLength(length).setOffset(offset)
-// }
-//
-// //This will contain the last 5 filters and required fields used in buildScan
-// // These values can be used in unit testing to make sure we are converting
-// // The Spark SQL input correctly
-// val lastFiveExecutionRules =
-// new ConcurrentLinkedQueue[ExecutionRuleForUnitTesting]()
-//
-// /**
-// * This method is to populate the lastFiveExecutionRules for unit test perposes
-// * This method is not thread safe.
-// *
-// * @param columnFilterCollection The filters in the last job
-// * @param requiredQualifierDefinitionArray The required columns in the last job
-// */
-// def populateLatestExecutionRules(columnFilterCollection: ColumnFilterCollection,
-// requiredQualifierDefinitionArray:
-// mutable.MutableList[SchemaQualifierDefinition]):Unit = {
-// lastFiveExecutionRules.add(new ExecutionRuleForUnitTesting(
-// columnFilterCollection, requiredQualifierDefinitionArray))
-// while (lastFiveExecutionRules.size() > 5) {
-// lastFiveExecutionRules.poll()
-// }
-// }
-//
-// /**
-// * This method will convert the result content from HBase into the
-// * SQL value type that is requested by the Spark SQL schema definition
-// *
-// * @param columnName The name of the SparkSQL Column
-// * @param schemaMappingDefinition The schema definition map
-// * @param r The result object from HBase
-// * @return The converted object type
-// */
-// def getValue(columnName: String,
-// schemaMappingDefinition:
-// java.util.HashMap[String, SchemaQualifierDefinition],
-// r: Result): Any = {
-//
-// val columnDef = schemaMappingDefinition.get(columnName)
-//
-// if (columnDef == null) throw new IllegalArgumentException("Unknown column:" + columnName)
-//
-//
-// if (columnDef.columnFamilyBytes.isEmpty) {
-// val row = r.getRow
-//
-// columnDef.columnSparkSqlType match {
-// case IntegerType => rawInteger.decode(getFreshByteRange(row))
-// case LongType => rawLong.decode(getFreshByteRange(row))
-// case FloatType => rawFloat.decode(getFreshByteRange(row))
-// case DoubleType => rawDouble.decode(getFreshByteRange(row))
-// case StringType => rawString.decode(getFreshByteRange(row))
-// case TimestampType => rawLong.decode(getFreshByteRange(row))
-// case _ => Bytes.toString(row)
-// }
-// } else {
-// val cellByteValue =
-// r.getColumnLatestCell(columnDef.columnFamilyBytes, columnDef.qualifierBytes)
-// if (cellByteValue == null) null
-// else columnDef.columnSparkSqlType match {
-// case IntegerType => rawInteger.decode(getFreshByteRange(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength))
-// case LongType => rawLong.decode(getFreshByteRange(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength))
-// case FloatType => rawFloat.decode(getFreshByteRange(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength))
-// case DoubleType => rawDouble.decode(getFreshByteRange(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength))
-// case StringType => Bytes.toString(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength)
-// case TimestampType => rawLong.decode(getFreshByteRange(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength))
-// case _ => Bytes.toString(cellByteValue.getValueArray,
-// cellByteValue.getValueOffset, cellByteValue.getValueLength)
-// }
-// }
-// }
-//
-// /**
-// * This will convert the value from SparkSQL to be stored into HBase using the
-// * right byte Type
-// *
-// * @param columnName SparkSQL column name
-// * @param schemaMappingDefinition Schema definition map
-// * @param value String value from SparkSQL
-// * @return Returns the byte array to go into HBase
-// */
-// def getByteValue(columnName: String,
-// schemaMappingDefinition:
-// java.util.HashMap[String, SchemaQualifierDefinition],
-// value: String): Array[Byte] = {
-//
-// val columnDef = schemaMappingDefinition.get(columnName)
-//
-// if (columnDef == null) {
-// throw new IllegalArgumentException("Unknown column:" + columnName)
-// } else {
-// columnDef.columnSparkSqlType match {
-// case IntegerType =>
-// val result = new Array[Byte](Bytes.SIZEOF_INT)
-// val localDataRange = getFreshByteRange(result)
-// rawInteger.encode(localDataRange, value.toInt)
-// localDataRange.getBytes
-// case LongType =>
-// val result = new Array[Byte](Bytes.SIZEOF_LONG)
-// val localDataRange = getFreshByteRange(result)
-// rawLong.encode(localDataRange, value.toLong)
-// localDataRange.getBytes
-// case FloatType =>
-// val result = new Array[Byte](Bytes.SIZEOF_FLOAT)
-// val localDataRange = getFreshByteRange(result)
-// rawFloat.encode(localDataRange, value.toFloat)
-// localDataRange.getBytes
-// case DoubleType =>
-// val result = new Array[Byte](Bytes.SIZEOF_DOUBLE)
-// val localDataRange = getFreshByteRange(result)
-// rawDouble.encode(localDataRange, value.toDouble)
-// localDataRange.getBytes
-// case StringType =>
-// Bytes.toBytes(value)
-// case TimestampType =>
-// val result = new Array[Byte](Bytes.SIZEOF_LONG)
-// val localDataRange = getFreshByteRange(result)
-// rawLong.encode(localDataRange, value.toLong)
-// localDataRange.getBytes
-//
-// case _ => Bytes.toBytes(value)
-// }
-// }
-// }
-//}
-//
-//class ExecutionRuleForUnitTesting(val columnFilterCollection: ColumnFilterCollection,
-// val requiredQualifierDefinitionArray:
-// mutable.MutableList[SchemaQualifierDefinition])
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/FamilyHFileWriteOptions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/FamilyHFileWriteOptions.scala b/loader/src/main/scala/spark/FamilyHFileWriteOptions.scala
deleted file mode 100644
index 2e4023c..0000000
--- a/loader/src/main/scala/spark/FamilyHFileWriteOptions.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import java.io.Serializable
-
-/**
- * This object will hold optional data for how a given column family's
- * writer will work
- *
- * @param compression String to define the Compression to be used in the HFile
- * @param bloomType String to define the bloom type to be used in the HFile
- * @param blockSize The block size to be used in the HFile
- * @param dataBlockEncoding String to define the data block encoding to be used
- * in the HFile
- */
-class FamilyHFileWriteOptions( val compression:String,
- val bloomType: String,
- val blockSize: Int,
- val dataBlockEncoding: String) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/HBaseContext.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/HBaseContext.scala b/loader/src/main/scala/spark/HBaseContext.scala
deleted file mode 100644
index 4ea58f1..0000000
--- a/loader/src/main/scala/spark/HBaseContext.scala
+++ /dev/null
@@ -1,849 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import java.net.InetSocketAddress
-import java.util
-
-import org.apache.hadoop.hbase.fs.HFileSystem
-import org.apache.hadoop.hbase._
-import org.apache.hadoop.hbase.io.compress.Compression
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
-import org.apache.hadoop.hbase.io.hfile.{CacheConfig, HFileContextBuilder}
-import org.apache.hadoop.hbase.regionserver.{HStore, StoreFile, BloomType}
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.rdd.RDD
-import org.apache.hadoop.conf.Configuration
-import spark.HBaseRDDFunctions._
-import org.apache.hadoop.hbase.client._
-import scala.reflect.ClassTag
-import org.apache.spark.{Logging, SerializableWritable, SparkContext}
-import org.apache.hadoop.hbase.mapreduce.{TableMapReduceUtil,
-TableInputFormat, IdentityTableMapper}
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.hadoop.mapreduce.Job
-import org.apache.spark.streaming.dstream.DStream
-import java.io._
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
-import org.apache.hadoop.fs.{Path, FileSystem}
-import scala.collection.mutable
-
-/**
- * HBaseContext is a façade for HBase operations
- * like bulk put, get, increment, delete, and scan
- *
- * HBaseContext will take the responsibilities
- * of disseminating the configuration information
- * to the working and managing the life cycle of HConnections.
- */
-class HBaseContext(@transient sc: SparkContext,
- @transient config: Configuration,
- val tmpHdfsConfgFile: String = null)
- extends Serializable with Logging {
-
- @transient var credentials = SparkHadoopUtil.get.getCurrentUserCredentials()
- @transient var tmpHdfsConfiguration:Configuration = config
- @transient var appliedCredentials = false
- @transient val job = Job.getInstance(config)
- TableMapReduceUtil.initCredentials(job)
- val broadcastedConf = sc.broadcast(new SerializableWritable(config))
- val credentialsConf = sc.broadcast(new SerializableWritable(job.getCredentials))
-
- LatestHBaseContextCache.latest = this
-
- if (tmpHdfsConfgFile != null && config != null) {
- val fs = FileSystem.newInstance(config)
- val tmpPath = new Path(tmpHdfsConfgFile)
- if (!fs.exists(tmpPath)) {
- val outputStream = fs.create(tmpPath)
- config.write(outputStream)
- outputStream.close()
- } else {
- logWarning("tmpHdfsConfigDir " + tmpHdfsConfgFile + " exist!!")
- }
- }
-
- /**
- * A simple enrichment of the traditional Spark RDD foreachPartition.
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * @param rdd Original RDD with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the RDD values and a HConnection object to interact
- * with HBase
- */
- def foreachPartition[T](rdd: RDD[T],
- f: (Iterator[T], Connection) => Unit):Unit = {
- rdd.foreachPartition(
- it => hbaseForeachPartition(broadcastedConf, it, f))
- }
-
- /**
- * A simple enrichment of the traditional Spark Streaming dStream foreach
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * @param dstream Original DStream with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the DStream values and a HConnection object to
- * interact with HBase
- */
- def foreachPartition[T](dstream: DStream[T],
- f: (Iterator[T], Connection) => Unit):Unit = {
- dstream.foreachRDD((rdd, time) => {
- foreachPartition(rdd, f)
- })
- }
-
- /**
- * A simple enrichment of the traditional Spark RDD mapPartition.
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * @param rdd Original RDD with data to iterate over
- * @param mp Function to be given a iterator to iterate through
- * the RDD values and a HConnection object to interact
- * with HBase
- * @return Returns a new RDD generated by the user definition
- * function just like normal mapPartition
- */
- def mapPartitions[T, R: ClassTag](rdd: RDD[T],
- mp: (Iterator[T], Connection) => Iterator[R]): RDD[R] = {
-
- rdd.mapPartitions[R](it => hbaseMapPartition[T, R](broadcastedConf,
- it,
- mp))
-
- }
-
- /**
- * A simple enrichment of the traditional Spark Streaming DStream
- * foreachPartition.
- *
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * Note: Make sure to partition correctly to avoid memory issue when
- * getting data from HBase
- *
- * @param dstream Original DStream with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the DStream values and a HConnection object to
- * interact with HBase
- * @return Returns a new DStream generated by the user
- * definition function just like normal mapPartition
- */
- def streamForeachPartition[T](dstream: DStream[T],
- f: (Iterator[T], Connection) => Unit): Unit = {
-
- dstream.foreachRDD(rdd => this.foreachPartition(rdd, f))
- }
-
- /**
- * A simple enrichment of the traditional Spark Streaming DStream
- * mapPartition.
- *
- * This function differs from the original in that it offers the
- * developer access to a already connected HConnection object
- *
- * Note: Do not close the HConnection object. All HConnection
- * management is handled outside this method
- *
- * Note: Make sure to partition correctly to avoid memory issue when
- * getting data from HBase
- *
- * @param dstream Original DStream with data to iterate over
- * @param f Function to be given a iterator to iterate through
- * the DStream values and a HConnection object to
- * interact with HBase
- * @return Returns a new DStream generated by the user
- * definition function just like normal mapPartition
- */
- def streamMapPartitions[T, U: ClassTag](dstream: DStream[T],
- f: (Iterator[T], Connection) => Iterator[U]):
- DStream[U] = {
- dstream.mapPartitions(it => hbaseMapPartition[T, U](
- broadcastedConf,
- it,
- f))
- }
-
- /**
- * A simple abstraction over the HBaseContext.foreachPartition method.
- *
- * It allow addition support for a user to take RDD
- * and generate puts and send them to HBase.
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param rdd Original RDD with data to iterate over
- * @param tableName The name of the table to put into
- * @param f Function to convert a value in the RDD to a HBase Put
- */
- def bulkPut[T](rdd: RDD[T], tableName: TableName, f: (T) => Put) {
-
- val tName = tableName.getName
- rdd.foreachPartition(
- it => hbaseForeachPartition[T](
- broadcastedConf,
- it,
- (iterator, connection) => {
- val m = connection.getBufferedMutator(TableName.valueOf(tName))
- iterator.foreach(T => m.mutate(f(T)))
- m.flush()
- m.close()
- }))
- }
-
- def applyCreds[T] (configBroadcast: Broadcast[SerializableWritable[Configuration]]){
- credentials = SparkHadoopUtil.get.getCurrentUserCredentials()
-
- logDebug("appliedCredentials:" + appliedCredentials + ",credentials:" + credentials)
-
- if (!appliedCredentials && credentials != null) {
- appliedCredentials = true
-
- @transient val ugi = UserGroupInformation.getCurrentUser
- ugi.addCredentials(credentials)
- // specify that this is a proxy user
- ugi.setAuthenticationMethod(AuthenticationMethod.PROXY)
-
- ugi.addCredentials(credentialsConf.value.value)
- }
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamMapPartition method.
- *
- * It allow addition support for a user to take a DStream and
- * generate puts and send them to HBase.
- *
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param dstream Original DStream with data to iterate over
- * @param tableName The name of the table to put into
- * @param f Function to convert a value in
- * the DStream to a HBase Put
- */
- def streamBulkPut[T](dstream: DStream[T],
- tableName: TableName,
- f: (T) => Put) = {
- val tName = tableName.getName
- dstream.foreachRDD((rdd, time) => {
- bulkPut(rdd, TableName.valueOf(tName), f)
- })
- }
-
- /**
- * A simple abstraction over the HBaseContext.foreachPartition method.
- *
- * It allow addition support for a user to take a RDD and generate delete
- * and send them to HBase. The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param rdd Original RDD with data to iterate over
- * @param tableName The name of the table to delete from
- * @param f Function to convert a value in the RDD to a
- * HBase Deletes
- * @param batchSize The number of delete to batch before sending to HBase
- */
- def bulkDelete[T](rdd: RDD[T], tableName: TableName,
- f: (T) => Delete, batchSize: Integer) {
- bulkMutation(rdd, tableName, f, batchSize)
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamBulkMutation method.
- *
- * It allow addition support for a user to take a DStream and
- * generate Delete and send them to HBase.
- *
- * The complexity of managing the HConnection is
- * removed from the developer
- *
- * @param dstream Original DStream with data to iterate over
- * @param tableName The name of the table to delete from
- * @param f function to convert a value in the DStream to a
- * HBase Delete
- * @param batchSize The number of deletes to batch before sending to HBase
- */
- def streamBulkDelete[T](dstream: DStream[T],
- tableName: TableName,
- f: (T) => Delete,
- batchSize: Integer) = {
- streamBulkMutation(dstream, tableName, f, batchSize)
- }
-
- /**
- * Under lining function to support all bulk mutations
- *
- * May be opened up if requested
- */
- private def bulkMutation[T](rdd: RDD[T], tableName: TableName,
- f: (T) => Mutation, batchSize: Integer) {
-
- val tName = tableName.getName
- rdd.foreachPartition(
- it => hbaseForeachPartition[T](
- broadcastedConf,
- it,
- (iterator, connection) => {
- val table = connection.getTable(TableName.valueOf(tName))
- val mutationList = new java.util.ArrayList[Mutation]
- iterator.foreach(T => {
- mutationList.add(f(T))
- if (mutationList.size >= batchSize) {
- table.batch(mutationList, null)
- mutationList.clear()
- }
- })
- if (mutationList.size() > 0) {
- table.batch(mutationList, null)
- mutationList.clear()
- }
- table.close()
- }))
- }
-
- /**
- * Under lining function to support all bulk streaming mutations
- *
- * May be opened up if requested
- */
- private def streamBulkMutation[T](dstream: DStream[T],
- tableName: TableName,
- f: (T) => Mutation,
- batchSize: Integer) = {
- val tName = tableName.getName
- dstream.foreachRDD((rdd, time) => {
- bulkMutation(rdd, TableName.valueOf(tName), f, batchSize)
- })
- }
-
- /**
- * A simple abstraction over the HBaseContext.mapPartition method.
- *
- * It allow addition support for a user to take a RDD and generates a
- * new RDD based on Gets and the results they bring back from HBase
- *
- * @param rdd Original RDD with data to iterate over
- * @param tableName The name of the table to get from
- * @param makeGet function to convert a value in the RDD to a
- * HBase Get
- * @param convertResult This will convert the HBase Result object to
- * what ever the user wants to put in the resulting
- * RDD
- * return new RDD that is created by the Get to HBase
- */
- def bulkGet[T, U: ClassTag](tableName: TableName,
- batchSize: Integer,
- rdd: RDD[T],
- makeGet: (T) => Get,
- convertResult: (Result) => U): RDD[U] = {
-
- val getMapPartition = new GetMapPartition(tableName,
- batchSize,
- makeGet,
- convertResult)
-
- rdd.mapPartitions[U](it =>
- hbaseMapPartition[T, U](
- broadcastedConf,
- it,
- getMapPartition.run))
- }
-
- /**
- * A simple abstraction over the HBaseContext.streamMap method.
- *
- * It allow addition support for a user to take a DStream and
- * generates a new DStream based on Gets and the results
- * they bring back from HBase
- *
- * @param tableName The name of the table to get from
- * @param batchSize The number of Gets to be sent in a single batch
- * @param dStream Original DStream with data to iterate over
- * @param makeGet Function to convert a value in the DStream to a
- * HBase Get
- * @param convertResult This will convert the HBase Result object to
- * what ever the user wants to put in the resulting
- * DStream
- * @return A new DStream that is created by the Get to HBase
- */
- def streamBulkGet[T, U: ClassTag](tableName: TableName,
- batchSize: Integer,
- dStream: DStream[T],
- makeGet: (T) => Get,
- convertResult: (Result) => U): DStream[U] = {
-
- val getMapPartition = new GetMapPartition(tableName,
- batchSize,
- makeGet,
- convertResult)
-
- dStream.mapPartitions[U](it => hbaseMapPartition[T, U](
- broadcastedConf,
- it,
- getMapPartition.run))
- }
-
- /**
- * This function will use the native HBase TableInputFormat with the
- * given scan object to generate a new RDD
- *
- * @param tableName the name of the table to scan
- * @param scan the HBase scan object to use to read data from HBase
- * @param f function to convert a Result object from HBase into
- * what the user wants in the final generated RDD
- * @return new RDD with results from scan
- */
- def hbaseRDD[U: ClassTag](tableName: TableName, scan: Scan,
- f: ((ImmutableBytesWritable, Result)) => U): RDD[U] = {
-
- val job: Job = Job.getInstance(getConf(broadcastedConf))
-
- TableMapReduceUtil.initCredentials(job)
- TableMapReduceUtil.initTableMapperJob(tableName, scan,
- classOf[IdentityTableMapper], null, null, job)
-
- sc.newAPIHadoopRDD(job.getConfiguration,
- classOf[TableInputFormat],
- classOf[ImmutableBytesWritable],
- classOf[Result]).map(f)
- }
-
- /**
- * A overloaded version of HBaseContext hbaseRDD that defines the
- * type of the resulting RDD
- *
- * @param tableName the name of the table to scan
- * @param scans the HBase scan object to use to read data from HBase
- * @return New RDD with results from scan
- *
- */
- def hbaseRDD(tableName: TableName, scans: Scan):
- RDD[(ImmutableBytesWritable, Result)] = {
-
- hbaseRDD[(ImmutableBytesWritable, Result)](
- tableName,
- scans,
- (r: (ImmutableBytesWritable, Result)) => r)
- }
-
- /**
- * underlining wrapper all foreach functions in HBaseContext
- */
- private def hbaseForeachPartition[T](configBroadcast:
- Broadcast[SerializableWritable[Configuration]],
- it: Iterator[T],
- f: (Iterator[T], Connection) => Unit) = {
-
- val config = getConf(configBroadcast)
-
- applyCreds(configBroadcast)
- // specify that this is a proxy user
- val connection = ConnectionFactory.createConnection(config)
- f(it, connection)
- connection.close()
- }
-
- private def getConf(configBroadcast: Broadcast[SerializableWritable[Configuration]]):
- Configuration = {
-
- if (tmpHdfsConfiguration == null && tmpHdfsConfgFile != null) {
- val fs = FileSystem.newInstance(SparkHadoopUtil.get.conf)
- val inputStream = fs.open(new Path(tmpHdfsConfgFile))
- tmpHdfsConfiguration = new Configuration(false)
- tmpHdfsConfiguration.readFields(inputStream)
- inputStream.close()
- }
-
- if (tmpHdfsConfiguration == null) {
- try {
- tmpHdfsConfiguration = configBroadcast.value.value
- } catch {
- case ex: Exception => logError("Unable to getConfig from broadcast", ex)
- }
- }
- tmpHdfsConfiguration
- }
-
- /**
- * underlining wrapper all mapPartition functions in HBaseContext
- *
- */
- private def hbaseMapPartition[K, U](
- configBroadcast:
- Broadcast[SerializableWritable[Configuration]],
- it: Iterator[K],
- mp: (Iterator[K], Connection) =>
- Iterator[U]): Iterator[U] = {
-
- val config = getConf(configBroadcast)
- applyCreds(configBroadcast)
-
- val connection = ConnectionFactory.createConnection(config)
- val res = mp(it, connection)
- connection.close()
- res
-
- }
-
- /**
- * underlining wrapper all get mapPartition functions in HBaseContext
- */
- private class GetMapPartition[T, U](tableName: TableName,
- batchSize: Integer,
- makeGet: (T) => Get,
- convertResult: (Result) => U)
- extends Serializable {
-
- val tName = tableName.getName
-
- def run(iterator: Iterator[T], connection: Connection): Iterator[U] = {
- val table = connection.getTable(TableName.valueOf(tName))
-
- val gets = new java.util.ArrayList[Get]()
- var res = List[U]()
-
- while (iterator.hasNext) {
- gets.add(makeGet(iterator.next()))
-
- if (gets.size() == batchSize) {
- val results = table.get(gets)
- res = res ++ results.map(convertResult)
- gets.clear()
- }
- }
- if (gets.size() > 0) {
- val results = table.get(gets)
- res = res ++ results.map(convertResult)
- gets.clear()
- }
- table.close()
- res.iterator
- }
- }
-
- /**
- * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
- *
- * This method is used to keep ClassTags out of the external Java API, as
- * the Java compiler cannot produce them automatically. While this
- * ClassTag-faking does please the compiler, it can cause problems at runtime
- * if the Scala API relies on ClassTags for correctness.
- *
- * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
- * just worse performance or security issues.
- * For instance, an Array of AnyRef can hold any type T, but may lose primitive
- * specialization.
- */
- private[spark]
- def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
-
- /**
- * A Spark Implementation of HBase Bulk load
- *
- * This will take the content from an existing RDD then sort and shuffle
- * it with respect to region splits. The result of that sort and shuffle
- * will be written to HFiles.
- *
- * After this function is executed the user will have to call
- * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
- *
- * Also note this version of bulk load is different from past versions in
- * that it includes the qualifier as part of the sort process. The
- * reason for this is to be able to support rows will very large number
- * of columns.
- *
- * @param rdd The RDD we are bulk loading from
- * @param tableName The HBase table we are loading into
- * @param flatMap A flapMap function that will make every
- * row in the RDD
- * into N cells for the bulk load
- * @param stagingDir The location on the FileSystem to bulk load into
- * @param familyHFileWriteOptionsMap Options that will define how the HFile for a
- * column family is written
- * @param compactionExclude Compaction excluded for the HFiles
- * @param maxSize Max size for the HFiles before they roll
- * @tparam T The Type of values in the original RDD
- */
- def bulkLoad[T](rdd:RDD[T],
- tableName: TableName,
- flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
- stagingDir:String,
- familyHFileWriteOptionsMap:
- util.Map[Array[Byte], FamilyHFileWriteOptions] =
- new util.HashMap[Array[Byte], FamilyHFileWriteOptions],
- compactionExclude: Boolean = false,
- maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):
- Unit = {
- val conn = ConnectionFactory.createConnection(config)
- val regionLocator = conn.getRegionLocator(tableName)
- val startKeys = regionLocator.getStartKeys
- val defaultCompressionStr = config.get("hfile.compression",
- Compression.Algorithm.NONE.getName)
- val defaultCompression = Compression.getCompressionAlgorithmByName(defaultCompressionStr)
-// HFileWriterImpl
-// .compressionByName(defaultCompressionStr)
- val now = System.currentTimeMillis()
- val tableNameByteArray = tableName.getName
-
- val familyHFileWriteOptionsMapInternal =
- new util.HashMap[ByteArrayWrapper, FamilyHFileWriteOptions]
-
- val entrySetIt = familyHFileWriteOptionsMap.entrySet().iterator()
-
- while (entrySetIt.hasNext) {
- val entry = entrySetIt.next()
- familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(entry.getKey), entry.getValue)
- }
-
- /**
- * This will return a new HFile writer when requested
- *
- * @param family column family
- * @param conf configuration to connect to HBase
- * @param favoredNodes nodes that we would like to write too
- * @param fs FileSystem object where we will be writing the HFiles to
- * @return WriterLength object
- */
- def getNewWriter(family: Array[Byte], conf: Configuration,
- favoredNodes: Array[InetSocketAddress],
- fs:FileSystem,
- familydir:Path): WriterLength = {
-
-
- var familyOptions = familyHFileWriteOptionsMapInternal.get(new ByteArrayWrapper(family))
-
- if (familyOptions == null) {
- familyOptions = new FamilyHFileWriteOptions(defaultCompression.toString,
- BloomType.NONE.toString, HConstants.DEFAULT_BLOCKSIZE, DataBlockEncoding.NONE.toString)
- familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(family), familyOptions)
- }
-
- val tempConf = new Configuration(conf)
- tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f)
- val contextBuilder = new HFileContextBuilder()
- .withCompression(Algorithm.valueOf(familyOptions.compression))
- .withChecksumType(HStore.getChecksumType(conf))
- .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
- .withBlockSize(familyOptions.blockSize)
- contextBuilder.withDataBlockEncoding(DataBlockEncoding.
- valueOf(familyOptions.dataBlockEncoding))
- val hFileContext = contextBuilder.build()
-
- if (null == favoredNodes) {
- new WriterLength(0, new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs)
- .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
- .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext).build())
-// .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext).build())
- } else {
- new WriterLength(0,
- new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), new HFileSystem(fs))
- .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
- .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext)
-// .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext)
- .withFavoredNodes(favoredNodes).build())
- }
- }
-
- val regionSplitPartitioner =
- new BulkLoadPartitioner(startKeys)
-
- //This is where all the magic happens
- //Here we are going to do the following things
- // 1. FlapMap every row in the RDD into key column value tuples
- // 2. Then we are going to repartition sort and shuffle
- // 3. Finally we are going to write out our HFiles
- rdd.flatMap( r => flatMap(r)).
- repartitionAndSortWithinPartitions(regionSplitPartitioner).
- hbaseForeachPartition(this, (it, conn) => {
-
- val conf = broadcastedConf.value.value
- val fs = FileSystem.get(conf)
- val writerMap = new mutable.HashMap[ByteArrayWrapper, WriterLength]
- var previousRow:Array[Byte] = HConstants.EMPTY_BYTE_ARRAY
- var rollOverRequested = false
-
- /**
- * This will roll all writers
- */
- def rollWriters(): Unit = {
- writerMap.values.foreach( wl => {
- if (wl.writer != null) {
- logDebug("Writer=" + wl.writer.getPath +
- (if (wl.written == 0) "" else ", wrote=" + wl.written))
- close(wl.writer)
- }
- })
- writerMap.clear()
- rollOverRequested = false
- }
-
- /**
- * This function will close a given HFile writer
- * @param w The writer to close
- */
- def close(w:StoreFile.Writer): Unit = {
- if (w != null) {
- w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
- Bytes.toBytes(System.currentTimeMillis()))
- w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
- Bytes.toBytes(regionSplitPartitioner.getPartition(previousRow)))
- w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY,
- Bytes.toBytes(true))
- w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
- Bytes.toBytes(compactionExclude))
- w.appendTrackedTimestampsToMetadata()
- w.close()
- }
- }
-
- //Here is where we finally iterate through the data in this partition of the
- //RDD that has been sorted and partitioned
- it.foreach{ case (keyFamilyQualifier, cellValue:Array[Byte]) =>
-
- //This will get a writer for the column family
- //If there is no writer for a given column family then
- //it will get created here.
- val wl = writerMap.getOrElseUpdate(new ByteArrayWrapper(keyFamilyQualifier.family), {
-
- val familyDir = new Path(stagingDir, Bytes.toString(keyFamilyQualifier.family))
-
- fs.mkdirs(familyDir)
-
- val loc:HRegionLocation = {
- try {
- val locator =
- conn.getRegionLocator(TableName.valueOf(tableNameByteArray))
- locator.getRegionLocation(keyFamilyQualifier.rowKey)
- } catch {
- case e: Throwable =>
- logWarning("there's something wrong when locating rowkey: " +
- Bytes.toString(keyFamilyQualifier.rowKey))
- null
- }
- }
- if (null == loc) {
- if (log.isTraceEnabled) {
- logTrace("failed to get region location, so use default writer: " +
- Bytes.toString(keyFamilyQualifier.rowKey))
- }
- getNewWriter(family = keyFamilyQualifier.family, conf = conf, favoredNodes = null,
- fs = fs, familydir = familyDir)
- } else {
- if (log.isDebugEnabled) {
- logDebug("first rowkey: [" + Bytes.toString(keyFamilyQualifier.rowKey) + "]")
- }
- val initialIsa =
- new InetSocketAddress(loc.getHostname, loc.getPort)
- if (initialIsa.isUnresolved) {
- if (log.isTraceEnabled) {
- logTrace("failed to resolve bind address: " + loc.getHostname + ":"
- + loc.getPort + ", so use default writer")
- }
- getNewWriter(keyFamilyQualifier.family, conf, null, fs, familyDir)
- } else {
- if(log.isDebugEnabled) {
- logDebug("use favored nodes writer: " + initialIsa.getHostString)
- }
- getNewWriter(keyFamilyQualifier.family, conf,
- Array[InetSocketAddress](initialIsa), fs, familyDir)
- }
- }
- })
-
- val keyValue =new KeyValue(keyFamilyQualifier.rowKey,
- keyFamilyQualifier.family,
- keyFamilyQualifier.qualifier,
- now,cellValue)
-
- wl.writer.append(keyValue)
- wl.written += keyValue.getLength
-
- rollOverRequested = rollOverRequested || wl.written > maxSize
-
- //This will only roll if we have at least one column family file that is
- //bigger then maxSize and we have finished a given row key
- if (rollOverRequested && Bytes.compareTo(previousRow, keyFamilyQualifier.rowKey) != 0) {
- rollWriters()
- }
-
- previousRow = keyFamilyQualifier.rowKey
- }
- //We have finished all the data so lets close up the writers
- rollWriters()
- })
- }
-
- /**
- * This is a wrapper class around StoreFile.Writer. The reason for the
- * wrapper is to keep the length of the file along side the writer
- *
- * @param written The writer to be wrapped
- * @param writer The number of bytes written to the writer
- */
- class WriterLength(var written:Long, val writer:StoreFile.Writer)
-
- /**
- * This is a wrapper over a byte array so it can work as
- * a key in a hashMap
- *
- * @param o1 The Byte Array value
- */
- class ByteArrayWrapper (val o1:Array[Byte])
- extends Comparable[ByteArrayWrapper] with Serializable {
- override def compareTo(o2: ByteArrayWrapper): Int = {
- Bytes.compareTo(o1,o2.o1)
- }
- override def equals(o2: Any): Boolean = {
- o2 match {
- case wrapper: ByteArrayWrapper =>
- Bytes.equals(o1, wrapper.o1)
- case _ =>
- false
- }
- }
- override def hashCode():Int = {
- Bytes.hashCode(o1)
- }
- }
-}
-
-object LatestHBaseContextCache {
- var latest:HBaseContext = null
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/HBaseDStreamFunctions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/HBaseDStreamFunctions.scala b/loader/src/main/scala/spark/HBaseDStreamFunctions.scala
deleted file mode 100644
index fc45865..0000000
--- a/loader/src/main/scala/spark/HBaseDStreamFunctions.scala
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package spark
-
-import org.apache.hadoop.hbase.TableName
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.spark.streaming.dstream.DStream
-
-import scala.reflect.ClassTag
-
-/**
- * HBaseDStreamFunctions contains a set of implicit functions that can be
- * applied to a Spark DStream so that we can easily interact with HBase
- */
-object HBaseDStreamFunctions {
-
- /**
- * These are implicit methods for a DStream that contains any type of
- * data.
- *
- * @param dStream This is for dStreams of any type
- * @tparam T Type T
- */
- implicit class GenericHBaseDStreamFunctions[T](val dStream: DStream[T]) {
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * put. This will not return a new Stream. Think of it like a foreach
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param f The function that will turn the DStream values
- * into HBase Put objects.
- */
- def hbaseBulkPut(hc: HBaseContext,
- tableName: TableName,
- f: (T) => Put): Unit = {
- hc.streamBulkPut(dStream, tableName, f)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * get. This will return a new DStream. Think about it as a DStream map
- * function. In that every DStream value will get a new value out of
- * HBase. That new value will populate the newly generated DStream.
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param batchSize How many gets to execute in a single batch
- * @param f The function that will turn the RDD values
- * in HBase Get objects
- * @param convertResult The function that will convert a HBase
- * Result object into a value that will go
- * into the resulting DStream
- * @tparam R The type of Object that will be coming
- * out of the resulting DStream
- * @return A resulting DStream with type R objects
- */
- def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
- tableName: TableName,
- batchSize:Int, f: (T) => Get, convertResult: (Result) => R):
- DStream[R] = {
- hc.streamBulkGet[T, R](tableName, batchSize, dStream, f, convertResult)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * get. This will return a new DStream. Think about it as a DStream map
- * function. In that every DStream value will get a new value out of
- * HBase. That new value will populate the newly generated DStream.
- *
- * @param hc The hbaseContext object to identify which
- * HBase cluster connection to use
- * @param tableName The tableName that the put will be sent to
- * @param batchSize How many gets to execute in a single batch
- * @param f The function that will turn the RDD values
- * in HBase Get objects
- * @return A resulting DStream with type R objects
- */
- def hbaseBulkGet(hc: HBaseContext,
- tableName: TableName, batchSize:Int,
- f: (T) => Get): DStream[(ImmutableBytesWritable, Result)] = {
- hc.streamBulkGet[T, (ImmutableBytesWritable, Result)](
- tableName, batchSize, dStream, f,
- result => (new ImmutableBytesWritable(result.getRow), result))
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's bulk
- * Delete. This will not return a new DStream.
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param tableName The tableName that the deletes will be sent to
- * @param f The function that will convert the DStream value into
- * a HBase Delete Object
- * @param batchSize The number of Deletes to be sent in a single batch
- */
- def hbaseBulkDelete(hc: HBaseContext,
- tableName: TableName,
- f:(T) => Delete, batchSize:Int): Unit = {
- hc.streamBulkDelete(dStream, tableName, f, batchSize)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's
- * foreachPartition method. This will ack very much like a normal DStream
- * foreach method but for the fact that you will now have a HBase connection
- * while iterating through the values.
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param f This function will get an iterator for a Partition of an
- * DStream along with a connection object to HBase
- */
- def hbaseForeachPartition(hc: HBaseContext,
- f: (Iterator[T], Connection) => Unit): Unit = {
- hc.streamForeachPartition(dStream, f)
- }
-
- /**
- * Implicit method that gives easy access to HBaseContext's
- * mapPartitions method. This will ask very much like a normal DStream
- * map partitions method but for the fact that you will now have a
- * HBase connection while iterating through the values
- *
- * @param hc The hbaseContext object to identify which HBase
- * cluster connection to use
- * @param f This function will get an iterator for a Partition of an
- * DStream along with a connection object to HBase
- * @tparam R This is the type of objects that will go into the resulting
- * DStream
- * @return A resulting DStream of type R
- */
- def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
- f: (Iterator[T], Connection) => Iterator[R]):
- DStream[R] = {
- hc.streamMapPartitions(dStream, f)
- }
- }
-}
[21/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelMeta.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelMeta.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelMeta.scala
deleted file mode 100644
index 3cd7995..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelMeta.scala
+++ /dev/null
@@ -1,151 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-/**
- * Created by shon on 6/3/15.
- */
-
-
-import com.kakao.s2graph.core.JSONParser
-import com.kakao.s2graph.core.GraphExceptions.MaxPropSizeReachedException
-import play.api.libs.json.Json
-import scalikejdbc._
-
-object LabelMeta extends Model[LabelMeta] with JSONParser {
-
- /** dummy sequences */
-
- val fromSeq = -4.toByte
- val toSeq = -5.toByte
- val lastOpSeq = -3.toByte
- val lastDeletedAt = -2.toByte
- val timeStampSeq = 0.toByte
- val countSeq = (Byte.MaxValue - 2).toByte
- val degreeSeq = (Byte.MaxValue - 1).toByte
- val maxValue = Byte.MaxValue
- val emptyValue = Byte.MaxValue
-
- /** reserved sequences */
- // val deleted = LabelMeta(id = Some(lastDeletedAt), labelId = lastDeletedAt, name = "lastDeletedAt",
- // seq = lastDeletedAt, defaultValue = "", dataType = "long")
- val from = LabelMeta(id = Some(fromSeq), labelId = fromSeq, name = "_from",
- seq = fromSeq, defaultValue = fromSeq.toString, dataType = "long")
- val to = LabelMeta(id = Some(toSeq), labelId = toSeq, name = "_to",
- seq = toSeq, defaultValue = toSeq.toString, dataType = "long")
- val timestamp = LabelMeta(id = Some(-1), labelId = -1, name = "_timestamp",
- seq = timeStampSeq, defaultValue = "0", dataType = "long")
- val degree = LabelMeta(id = Some(-1), labelId = -1, name = "_degree",
- seq = degreeSeq, defaultValue = "0", dataType = "long")
- val count = LabelMeta(id = Some(-1), labelId = -1, name = "_count",
- seq = countSeq, defaultValue = "-1", dataType = "long")
-
- // Each reserved column(_timestamp, timestamp) has same seq number, starts with '_' has high priority
- val reservedMetas = List(from, to, degree, timestamp, count).flatMap { lm => List(lm, lm.copy(name = lm.name.drop(1))) }.reverse
- val reservedMetasInner = List(from, to, degree, timestamp, count)
-
- def apply(rs: WrappedResultSet): LabelMeta = {
- LabelMeta(Some(rs.int("id")), rs.int("label_id"), rs.string("name"), rs.byte("seq"), rs.string("default_value"), rs.string("data_type").toLowerCase)
- }
-
- def isValidSeq(seq: Byte): Boolean = seq >= 0 && seq <= countSeq
- def isValidSeqForAdmin(seq: Byte): Boolean = seq > 0 && seq < countSeq
-
- def findById(id: Int)(implicit session: DBSession = AutoSession): LabelMeta = {
- val cacheKey = "id=" + id
-
- withCache(cacheKey) {
- sql"""select * from label_metas where id = ${id}""".map { rs => LabelMeta(rs) }.single.apply
- }.get
- }
-
- def findAllByLabelId(labelId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[LabelMeta] = {
- val cacheKey = "labelId=" + labelId
- lazy val labelMetas = sql"""select *
- from label_metas
- where label_id = ${labelId} order by seq ASC""".map(LabelMeta(_)).list.apply()
-
- if (useCache) withCaches(cacheKey)(labelMetas)
- else labelMetas
- }
-
- def findByName(labelId: Int, name: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[LabelMeta] = {
- name match {
- case timestamp.name => Some(timestamp)
- case from.name => Some(from)
- case to.name => Some(to)
- case _ =>
- val cacheKey = "labelId=" + labelId + ":name=" + name
- lazy val labelMeta = sql"""
- select *
- from label_metas where label_id = ${labelId} and name = ${name}"""
- .map { rs => LabelMeta(rs) }.single.apply()
-
- if (useCache) withCache(cacheKey)(labelMeta)
- else labelMeta
- }
- }
-
- def insert(labelId: Int, name: String, defaultValue: String, dataType: String)(implicit session: DBSession = AutoSession) = {
- val ls = findAllByLabelId(labelId, useCache = false)
- val seq = ls.size + 1
-
- if (seq < maxValue) {
- sql"""insert into label_metas(label_id, name, seq, default_value, data_type)
- select ${labelId}, ${name}, ${seq}, ${defaultValue}, ${dataType}""".updateAndReturnGeneratedKey.apply()
- } else {
- throw MaxPropSizeReachedException("max property size reached")
- }
- }
-
- def findOrInsert(labelId: Int,
- name: String,
- defaultValue: String,
- dataType: String)(implicit session: DBSession = AutoSession): LabelMeta = {
-
- findByName(labelId, name) match {
- case Some(c) => c
- case None =>
- insert(labelId, name, defaultValue, dataType)
- val cacheKey = "labelId=" + labelId + ":name=" + name
- val cacheKeys = "labelId=" + labelId
- expireCache(cacheKey)
- expireCaches(cacheKeys)
- findByName(labelId, name, useCache = false).get
- }
- }
-
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val labelMeta = findById(id)
- val (labelId, name) = (labelMeta.labelId, labelMeta.name)
- sql"""delete from label_metas where id = ${id}""".execute.apply()
- val cacheKeys = List(s"id=$id", s"labelId=$labelId", s"labelId=$labelId:name=$name")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- }
-
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from label_metas""".map { rs => LabelMeta(rs) }.list.apply
- putsToCache(ls.map { x =>
- val cacheKey = s"id=${x.id.get}"
- cacheKey -> x
- })
- putsToCache(ls.map { x =>
- val cacheKey = s"labelId=${x.labelId}:name=${x.name}"
- cacheKey -> x
- })
- putsToCache(ls.map { x =>
- val cacheKey = s"labelId=${x.labelId}:seq=${x.seq}"
- cacheKey -> x
- })
-
- putsToCaches(ls.groupBy(x => x.labelId).map { case (labelId, ls) =>
- val cacheKey = s"labelId=${labelId}"
- cacheKey -> ls
- }.toList)
- }
-}
-
-case class LabelMeta(id: Option[Int], labelId: Int, name: String, seq: Byte, defaultValue: String, dataType: String) extends JSONParser {
- lazy val toJson = Json.obj("name" -> name, "defaultValue" -> defaultValue, "dataType" -> dataType)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
deleted file mode 100644
index d41e9fa..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Model.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-import java.util.concurrent.Executors
-
-import com.kakao.s2graph.core.utils.{SafeUpdateCache, logger}
-import com.typesafe.config.Config
-import scalikejdbc._
-
-import scala.concurrent.ExecutionContext
-import scala.language.{higherKinds, implicitConversions}
-import scala.util.{Failure, Try}
-
-object Model {
- var maxSize = 10000
- var ttl = 60
- val numOfThread = Runtime.getRuntime.availableProcessors()
- val threadPool = Executors.newFixedThreadPool(numOfThread)
- val ec = ExecutionContext.fromExecutor(threadPool)
-
- def apply(config: Config) = {
- maxSize = config.getInt("cache.max.size")
- ttl = config.getInt("cache.ttl.seconds")
- Class.forName(config.getString("db.default.driver"))
-
- val settings = ConnectionPoolSettings(
- initialSize = 10,
- maxSize = 10,
- connectionTimeoutMillis = 30000L,
- validationQuery = "select 1;")
-
- ConnectionPool.singleton(
- config.getString("db.default.url"),
- config.getString("db.default.user"),
- config.getString("db.default.password"),
- settings)
- }
-
- def withTx[T](block: DBSession => T): Try[T] = {
- using(DB(ConnectionPool.borrow())) { conn =>
- val res = Try {
- conn.begin()
- val session = conn.withinTxSession()
- val result = block(session)
-
- conn.commit()
-
- result
- } recoverWith {
- case e: Exception =>
- conn.rollbackIfActive()
- Failure(e)
- }
-
- res
- }
- }
-
- def shutdown() = {
- ConnectionPool.closeAll()
- }
-
- def loadCache() = {
- Service.findAll()
- ServiceColumn.findAll()
- Label.findAll()
- LabelMeta.findAll()
- LabelIndex.findAll()
- ColumnMeta.findAll()
- }
-}
-
-trait Model[V] extends SQLSyntaxSupport[V] {
-
- import Model._
-
- implicit val ec: ExecutionContext = Model.ec
-
- val cName = this.getClass.getSimpleName()
- logger.info(s"LocalCache[$cName]: TTL[$ttl], MaxSize[$maxSize]")
-
- val optionCache = new SafeUpdateCache[Option[V]](cName, maxSize, ttl)
- val listCache = new SafeUpdateCache[List[V]](cName, maxSize, ttl)
-
- val withCache = optionCache.withCache _
-
- val withCaches = listCache.withCache _
-
- val expireCache = optionCache.invalidate _
-
- val expireCaches = listCache.invalidate _
-
- def putsToCache(kvs: List[(String, V)]) = kvs.foreach {
- case (key, value) => optionCache.put(key, Option(value))
- }
-
- def putsToCaches(kvs: List[(String, List[V])]) = kvs.foreach {
- case (key, values) => listCache.put(key, values)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Service.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Service.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Service.scala
deleted file mode 100644
index 2840db8..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Service.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-
-import java.util.UUID
-
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{Management}
-import play.api.libs.json.Json
-import scalikejdbc._
-
-object Service extends Model[Service] {
- def apply(rs: WrappedResultSet): Service = {
- Service(rs.intOpt("id"), rs.string("service_name"), rs.string("access_token"),
- rs.string("cluster"), rs.string("hbase_table_name"), rs.int("pre_split_size"), rs.intOpt("hbase_table_ttl"))
- }
-
- def findByAccessToken(accessToken: String)(implicit session: DBSession = AutoSession): Option[Service] = {
- val cacheKey = s"accessToken=$accessToken"
- withCache(cacheKey)( sql"""select * from services where access_token = ${accessToken}""".map { rs => Service(rs) }.single.apply)
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession): Service = {
- val cacheKey = "id=" + id
- withCache(cacheKey)( sql"""select * from services where id = ${id}""".map { rs => Service(rs) }.single.apply).get
- }
-
- def findByName(serviceName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Service] = {
- val cacheKey = "serviceName=" + serviceName
- lazy val serviceOpt = sql"""
- select * from services where service_name = ${serviceName}
- """.map { rs => Service(rs) }.single.apply()
-
- if (useCache) withCache(cacheKey)(serviceOpt)
- else serviceOpt
- }
-
- def insert(serviceName: String, cluster: String,
- hTableName: String, preSplitSize: Int, hTableTTL: Option[Int],
- compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Unit = {
- logger.info(s"$serviceName, $cluster, $hTableName, $preSplitSize, $hTableTTL, $compressionAlgorithm")
- val accessToken = UUID.randomUUID().toString()
- sql"""insert into services(service_name, access_token, cluster, hbase_table_name, pre_split_size, hbase_table_ttl)
- values(${serviceName}, ${accessToken}, ${cluster}, ${hTableName}, ${preSplitSize}, ${hTableTTL})""".execute.apply()
- }
-
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val service = findById(id)
- val serviceName = service.serviceName
- sql"""delete from service_columns where id = ${id}""".execute.apply()
- val cacheKeys = List(s"id=$id", s"serviceName=$serviceName")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- }
-
- def findOrInsert(serviceName: String, cluster: String, hTableName: String,
- preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Service = {
- findByName(serviceName) match {
- case Some(s) => s
- case None =>
- insert(serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
- val cacheKey = "serviceName=" + serviceName
- expireCache(cacheKey)
- findByName(serviceName).get
- }
- }
-
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from services""".map { rs => Service(rs) }.list.apply
- putsToCache(ls.map { x =>
- val cacheKey = s"id=${x.id.get}"
- (cacheKey -> x)
- })
-
- putsToCache(ls.map { x =>
- val cacheKey = s"serviceName=${x.serviceName}"
- (cacheKey -> x)
- })
- }
-
- def findAllConn()(implicit session: DBSession = AutoSession): List[String] = {
- sql"""select distinct(cluster) from services""".map { rs => rs.string("cluster") }.list.apply
- }
-}
-
-case class Service(id: Option[Int], serviceName: String, accessToken: String, cluster: String, hTableName: String, preSplitSize: Int, hTableTTL: Option[Int]) {
- lazy val toJson =
- id match {
- case Some(_id) =>
- Json.obj("id" -> _id, "name" -> serviceName, "accessToken" -> accessToken, "cluster" -> cluster,
- "hTableName" -> hTableName, "preSplitSize" -> preSplitSize, "hTableTTL" -> hTableTTL)
- case None =>
- Json.parse("{}")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ServiceColumn.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ServiceColumn.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ServiceColumn.scala
deleted file mode 100644
index 9ca32a0..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ServiceColumn.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-/**
- * Created by shon on 6/3/15.
- */
-
-import com.kakao.s2graph.core.JSONParser
-import play.api.libs.json.Json
-import scalikejdbc._
-object ServiceColumn extends Model[ServiceColumn] {
-
- def apply(rs: WrappedResultSet): ServiceColumn = {
- ServiceColumn(rs.intOpt("id"), rs.int("service_id"), rs.string("column_name"), rs.string("column_type").toLowerCase(), rs.string("schema_version"))
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession): ServiceColumn = {
-// val cacheKey = s"id=$id"
- val cacheKey = "id=" + id
- withCache(cacheKey)(sql"""select * from service_columns where id = ${id}""".map { x => ServiceColumn(x) }.single.apply).get
- }
- def find(serviceId: Int, columnName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[ServiceColumn] = {
-// val cacheKey = s"serviceId=$serviceId:columnName=$columnName"
- val cacheKey = "serviceId=" + serviceId + ":columnName=" + columnName
- if (useCache) {
- withCache(cacheKey) {
- sql"""
- select * from service_columns where service_id = ${serviceId} and column_name = ${columnName}
- """.map { rs => ServiceColumn(rs) }.single.apply()
- }
- } else {
- sql"""
- select * from service_columns where service_id = ${serviceId} and column_name = ${columnName}
- """.map { rs => ServiceColumn(rs) }.single.apply()
- }
- }
- def insert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String)(implicit session: DBSession = AutoSession) = {
- sql"""insert into service_columns(service_id, column_name, column_type, schema_version)
- values(${serviceId}, ${columnName}, ${columnType}, ${schemaVersion})""".execute.apply()
- }
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val serviceColumn = findById(id)
- val (serviceId, columnName) = (serviceColumn.serviceId, serviceColumn.columnName)
- sql"""delete from service_columns where id = ${id}""".execute.apply()
- val cacheKeys = List(s"id=$id", s"serviceId=$serviceId:columnName=$columnName")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- }
- def findOrInsert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String)(implicit session: DBSession = AutoSession): ServiceColumn = {
- find(serviceId, columnName) match {
- case Some(sc) => sc
- case None =>
- insert(serviceId, columnName, columnType, schemaVersion)
-// val cacheKey = s"serviceId=$serviceId:columnName=$columnName"
- val cacheKey = "serviceId=" + serviceId + ":columnName=" + columnName
- expireCache(cacheKey)
- find(serviceId, columnName).get
- }
- }
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from service_columns""".map { rs => ServiceColumn(rs) }.list.apply
- putsToCache(ls.map { x =>
- var cacheKey = s"id=${x.id.get}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- var cacheKey = s"serviceId=${x.serviceId}:columnName=${x.columnName}"
- (cacheKey -> x)
- })
- }
-}
-case class ServiceColumn(id: Option[Int], serviceId: Int, columnName: String, columnType: String, schemaVersion: String) extends JSONParser {
-
- lazy val service = Service.findById(serviceId)
- lazy val metas = ColumnMeta.findAllByColumn(id.get)
- lazy val metasInvMap = metas.map { meta => meta.name -> meta} toMap
- lazy val metaNamesMap = (ColumnMeta.lastModifiedAtColumn :: metas).map(x => (x.seq.toInt, x.name)) toMap
- lazy val toJson = Json.obj("serviceName" -> service.serviceName, "columnName" -> columnName, "columnType" -> columnType)
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
deleted file mode 100644
index 2a62b3b..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
+++ /dev/null
@@ -1,207 +0,0 @@
-package com.kakao.s2graph.core.parsers
-
-import com.kakao.s2graph.core.GraphExceptions.WhereParserException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.types.InnerValLike
-
-import scala.annotation.tailrec
-import scala.util.Try
-import scala.util.parsing.combinator.JavaTokenParsers
-
-trait ExtractValue extends JSONParser {
- val parent = "_parent."
-
- def propToInnerVal(edge: Edge, key: String) = {
- val (propKey, parentEdge) = findParentEdge(edge, key)
-
- val label = parentEdge.label
- val metaPropInvMap = label.metaPropsInvMap
- val labelMeta = metaPropInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
- val metaSeq = labelMeta.seq
-
- metaSeq match {
- case LabelMeta.from.seq => parentEdge.srcVertex.innerId
- case LabelMeta.to.seq => parentEdge.tgtVertex.innerId
- case _ => parentEdge.propsWithTs.get(metaSeq) match {
- case None => toInnerVal(labelMeta.defaultValue, labelMeta.dataType, label.schemaVersion)
- case Some(edgeVal) => edgeVal.innerVal
- }
- }
- }
-
- def valueToCompare(edge: Edge, key: String, value: String) = {
- val label = edge.label
- if (value.startsWith(parent) || label.metaPropsInvMap.contains(value)) propToInnerVal(edge, value)
- else {
- val (propKey, _) = findParentEdge(edge, key)
-
- val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
- val (srcColumn, tgtColumn) = label.srcTgtColumn(edge.labelWithDir.dir)
- val dataType = propKey match {
- case "_to" | "to" => tgtColumn.columnType
- case "_from" | "from" => srcColumn.columnType
- case _ => labelMeta.dataType
- }
- toInnerVal(value, dataType, label.schemaVersion)
- }
- }
-
- @tailrec
- private def findParent(edge: Edge, depth: Int): Edge =
- if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1)
- else edge
-
- private def findParentEdge(edge: Edge, key: String): (String, Edge) = {
- if (!key.startsWith(parent)) (key, edge)
- else {
- val split = key.split(parent)
- val depth = split.length - 1
- val propKey = split.last
-
- val parentEdge = findParent(edge, depth)
-
- (propKey, parentEdge)
- }
- }
-}
-
-trait Clause extends ExtractValue {
- def and(otherField: Clause): Clause = And(this, otherField)
-
- def or(otherField: Clause): Clause = Or(this, otherField)
-
- def filter(edge: Edge): Boolean
-
- def binaryOp(binOp: (InnerValLike, InnerValLike) => Boolean)(propKey: String, value: String)(edge: Edge): Boolean = {
- val propValue = propToInnerVal(edge, propKey)
- val compValue = valueToCompare(edge, propKey, value)
-
- binOp(propValue, compValue)
- }
-}
-
-case class Where(clauses: Seq[Clause] = Seq.empty[Clause]) {
- def filter(edge: Edge) =
- if (clauses.isEmpty) true else clauses.map(_.filter(edge)).forall(identity)
-}
-
-case class Gt(propKey: String, value: String) extends Clause {
- override def filter(edge: Edge): Boolean = binaryOp(_ > _)(propKey, value)(edge)
-}
-
-case class Lt(propKey: String, value: String) extends Clause {
- override def filter(edge: Edge): Boolean = binaryOp(_ < _)(propKey, value)(edge)
-}
-
-case class Eq(propKey: String, value: String) extends Clause {
- override def filter(edge: Edge): Boolean = binaryOp(_ == _)(propKey, value)(edge)
-}
-
-case class InWithoutParent(label: Label, propKey: String, values: Set[String]) extends Clause {
- val innerValLikeLs = values.map { value =>
- val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
- val dataType = propKey match {
- case "_to" | "to" => label.tgtColumn.columnType
- case "_from" | "from" => label.srcColumn.columnType
- case _ => labelMeta.dataType
- }
- toInnerVal(value, dataType, label.schemaVersion)
- }
- override def filter(edge: Edge): Boolean = {
- val propVal = propToInnerVal(edge, propKey)
- innerValLikeLs.contains(propVal)
- }
-}
-
-case class IN(propKey: String, values: Set[String]) extends Clause {
- override def filter(edge: Edge): Boolean = {
- val propVal = propToInnerVal(edge, propKey)
- values.exists { value =>
- valueToCompare(edge, propKey, value) == propVal
- }
- }
-}
-
-case class Between(propKey: String, minValue: String, maxValue: String) extends Clause {
- override def filter(edge: Edge): Boolean = {
- val propVal = propToInnerVal(edge, propKey)
- val minVal = valueToCompare(edge, propKey, minValue)
- val maxVal = valueToCompare(edge, propKey, maxValue)
-
- minVal <= propVal && propVal <= maxVal
- }
-}
-
-case class Not(self: Clause) extends Clause {
- override def filter(edge: Edge) = !self.filter(edge)
-}
-
-case class And(left: Clause, right: Clause) extends Clause {
- override def filter(edge: Edge) = left.filter(edge) && right.filter(edge)
-}
-
-case class Or(left: Clause, right: Clause) extends Clause {
- override def filter(edge: Edge) = left.filter(edge) || right.filter(edge)
-}
-
-object WhereParser {
- val success = Where()
-}
-
-case class WhereParser(label: Label) extends JavaTokenParsers with JSONParser {
-
- val anyStr = "[^\\s(),]+".r
-
- val and = "and|AND".r
-
- val or = "or|OR".r
-
- val between = "between|BETWEEN".r
-
- val in = "in|IN".r
-
- val notIn = "not in|NOT IN".r
-
- def where: Parser[Where] = rep(clause) ^^ (Where(_))
-
- def paren: Parser[Clause] = "(" ~> clause <~ ")"
-
- def clause: Parser[Clause] = (predicate | paren) * (and ^^^ { (a: Clause, b: Clause) => And(a, b) } | or ^^^ { (a: Clause, b: Clause) => Or(a, b) })
-
- def identWithDot: Parser[String] = repsep(ident, ".") ^^ { case values => values.mkString(".") }
-
- def predicate = {
- identWithDot ~ ("!=" | "=") ~ anyStr ^^ {
- case f ~ op ~ s =>
- if (op == "=") Eq(f, s)
- else Not(Eq(f, s))
- } | identWithDot ~ (">=" | "<=" | ">" | "<") ~ anyStr ^^ {
- case f ~ op ~ s => op match {
- case ">" => Gt(f, s)
- case ">=" => Or(Gt(f, s), Eq(f, s))
- case "<" => Lt(f, s)
- case "<=" => Or(Lt(f, s), Eq(f, s))
- }
- } | identWithDot ~ (between ~> anyStr <~ and) ~ anyStr ^^ {
- case f ~ minV ~ maxV => Between(f, minV, maxV)
- } | identWithDot ~ (notIn | in) ~ ("(" ~> repsep(anyStr, ",") <~ ")") ^^ {
- case f ~ op ~ values =>
- val inClause =
- if (f.startsWith("_parent")) IN(f, values.toSet)
- else InWithoutParent(label, f, values.toSet)
- if (op.toLowerCase == "in") inClause
- else Not(inClause)
-
-
- case _ => throw WhereParserException(s"Failed to parse where clause. ")
- }
- }
-
- def parse(sql: String): Try[Where] = Try {
- parseAll(where, sql) match {
- case Success(r, q) => r
- case fail => throw WhereParserException(s"Where parsing error: ${fail.toString}")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
deleted file mode 100644
index f8129db..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ /dev/null
@@ -1,609 +0,0 @@
-package com.kakao.s2graph.core.rest
-
-import java.util.concurrent.{Callable, TimeUnit}
-
-import com.google.common.cache.{CacheLoader, CacheBuilder}
-import com.kakao.s2graph.core.GraphExceptions.{BadQueryException, ModelNotFoundException}
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.parsers.{Where, WhereParser}
-import com.kakao.s2graph.core.types._
-import com.typesafe.config.Config
-import play.api.libs.json._
-import scala.util.{Failure, Success, Try}
-
-object TemplateHelper {
- val findVar = """\"?\$\{(.*?)\}\"?""".r
- val num = """(next_day|next_hour|next_week|now)?\s*(-?\s*[0-9]+)?\s*(hour|day|week)?""".r
- val hour = 60 * 60 * 1000L
- val day = hour * 24L
- val week = day * 7L
- def calculate(now: Long, n: Int, unit: String): Long = {
- val duration = unit match {
- case "hour" | "HOUR" => n * hour
- case "day" | "DAY" => n * day
- case "week" | "WEEK" => n * week
- case _ => n * day
- }
-
- duration + now
- }
-
- def replaceVariable(now: Long, body: String): String = {
- findVar.replaceAllIn(body, m => {
- val matched = m group 1
-
- num.replaceSomeIn(matched, m => {
- val (_pivot, n, unit) = (m.group(1), m.group(2), m.group(3))
- val ts = _pivot match {
- case null => now
- case "now" | "NOW" => now
- case "next_week" | "NEXT_WEEK" => now / week * week + week
- case "next_day" | "NEXT_DAY" => now / day * day + day
- case "next_hour" | "NEXT_HOUR" => now / hour * hour + hour
- }
-
- if (_pivot == null && n == null && unit == null) None
- else if (n == null || unit == null) Option(ts.toString)
- else Option(calculate(ts, n.replaceAll(" ", "").toInt, unit).toString)
- })
- })
- }
-}
-
-class RequestParser(config: Config) extends JSONParser {
-
- import Management.JsonModel._
-
- val hardLimit = 100000
- val defaultLimit = 100
- val maxLimit = Int.MaxValue - 1
- val DefaultRpcTimeout = config.getInt("hbase.rpc.timeout")
- val DefaultMaxAttempt = config.getInt("hbase.client.retries.number")
- val DefaultCluster = config.getString("hbase.zookeeper.quorum")
- val DefaultCompressionAlgorithm = config.getString("hbase.table.compression.algorithm")
- val DefaultPhase = config.getString("phase")
- val parserCache = CacheBuilder.newBuilder()
- .expireAfterAccess(10000, TimeUnit.MILLISECONDS)
- .expireAfterWrite(10000, TimeUnit.MILLISECONDS)
- .maximumSize(10000)
- .initialCapacity(1000)
- .build[String, Try[Where]]
-
- private def extractScoring(labelId: Int, value: JsValue) = {
- val ret = for {
- js <- parse[Option[JsObject]](value, "scoring")
- } yield {
- for {
- (k, v) <- js.fields
- labelOrderType <- LabelMeta.findByName(labelId, k)
- } yield {
- val value = v match {
- case n: JsNumber => n.as[Double]
- case _ => throw new Exception("scoring weight should be double.")
- }
- (labelOrderType.seq, value)
- }
- }
- ret
- }
-
- def extractInterval(label: Label, _jsValue: JsValue) = {
- val replaced = TemplateHelper.replaceVariable(System.currentTimeMillis(), _jsValue.toString())
- val jsValue = Json.parse(replaced)
-
- def extractKv(js: JsValue) = js match {
- case JsObject(obj) => obj
- case JsArray(arr) => arr.flatMap {
- case JsObject(obj) => obj
- case _ => throw new RuntimeException(s"cannot support json type $js")
- }
- case _ => throw new RuntimeException(s"cannot support json type: $js")
- }
-
- val ret = for {
- js <- parse[Option[JsObject]](jsValue, "interval")
- fromJs <- (js \ "from").asOpt[JsValue]
- toJs <- (js \ "to").asOpt[JsValue]
- } yield {
- val from = Management.toProps(label, extractKv(fromJs))
- val to = Management.toProps(label, extractKv(toJs))
- (from, to)
- }
-
- ret
- }
-
- def extractDuration(label: Label, _jsValue: JsValue) = {
- val replaced = TemplateHelper.replaceVariable(System.currentTimeMillis(), _jsValue.toString())
- val jsValue = Json.parse(replaced)
-
- for {
- js <- parse[Option[JsObject]](jsValue, "duration")
- } yield {
- val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue)
- val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue)
-
- if (minTs > maxTs) {
- throw new BadQueryException("Duration error. Timestamp of From cannot be larger than To.")
- }
-
- (minTs, maxTs)
- }
- }
-
- def extractHas(label: Label, jsValue: JsValue) = {
- val ret = for {
- js <- parse[Option[JsObject]](jsValue, "has")
- } yield {
- for {
- (k, v) <- js.fields
- labelMeta <- LabelMeta.findByName(label.id.get, k)
- value <- jsValueToInnerVal(v, labelMeta.dataType, label.schemaVersion)
- } yield {
- labelMeta.seq -> value
- }
- }
- ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
- }
-
- def extractWhere(label: Label, whereClauseOpt: Option[String]): Try[Where] = {
- whereClauseOpt match {
- case None => Success(WhereParser.success)
- case Some(_where) =>
- val where = TemplateHelper.replaceVariable(System.currentTimeMillis(), _where)
- val whereParserKey = s"${label.label}_${where}"
- parserCache.get(whereParserKey, new Callable[Try[Where]] {
- override def call(): Try[Where] = {
- WhereParser(label).parse(where) match {
- case s@Success(_) => s
- case Failure(ex) => throw BadQueryException(ex.getMessage, ex)
- }
- }
- })
- }
- }
-
- def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[Vertex] = {
- val vertices = for {
- label <- Label.findByName(labelName).toSeq
- serviceColumn = if (direction == "out") label.srcColumn else label.tgtColumn
- id <- ids
- innerId <- jsValueToInnerVal(id, serviceColumn.columnType, label.schemaVersion)
- } yield {
- Vertex(SourceVertexId(serviceColumn.id.get, innerId), System.currentTimeMillis())
- }
- vertices.toSeq
- }
-
- def toMultiQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): MultiQuery = {
- val queries = for {
- queryJson <- (jsValue \ "queries").asOpt[Seq[JsValue]].getOrElse(Seq.empty)
- } yield {
- toQuery(queryJson, isEdgeQuery)
- }
- val weights = (jsValue \ "weights").asOpt[Seq[Double]].getOrElse(queries.map(_ => 1.0))
- MultiQuery(queries = queries, weights = weights,
- queryOption = toQueryOption(jsValue), jsonQuery = jsValue)
- }
-
- def toQueryOption(jsValue: JsValue): QueryOption = {
- val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name))
- val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q =>
- q.copy(queryOption = q.queryOption.copy(filterOutFields = filterOutFields))
- }
- val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true)
- val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty)
- val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty)
- val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs =>
- for {
- js <- jsLs
- (column, orderJs) <- js.fields
- } yield {
- val ascending = orderJs.as[String].toUpperCase match {
- case "ASC" => true
- case "DESC" => false
- }
- column -> ascending
- }
- }.getOrElse(List("score" -> false, "timestamp" -> false))
- val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true)
- val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false)
- //TODO: Refactor this
- val limitOpt = (jsValue \ "limit").asOpt[Int]
- val returnAgg = (jsValue \ "returnAgg").asOpt[Boolean].getOrElse(true)
- val scoreThreshold = (jsValue \ "scoreThreshold").asOpt[Double].getOrElse(Double.MinValue)
- val returnDegree = (jsValue \ "returnDegree").asOpt[Boolean].getOrElse(true)
-
- QueryOption(removeCycle = removeCycle,
- selectColumns = selectColumns,
- groupByColumns = groupByColumns,
- orderByColumns = orderByColumns,
- filterOutQuery = filterOutQuery,
- filterOutFields = filterOutFields,
- withScore = withScore,
- returnTree = returnTree,
- limitOpt = limitOpt,
- returnAgg = returnAgg,
- scoreThreshold = scoreThreshold,
- returnDegree = returnDegree
- )
- }
- def toQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): Query = {
- try {
- val vertices =
- (for {
- value <- parse[List[JsValue]](jsValue, "srcVertices")
- serviceName = parse[String](value, "serviceName")
- column = parse[String](value, "columnName")
- } yield {
- val service = Service.findByName(serviceName).getOrElse(throw BadQueryException("service not found"))
- val col = ServiceColumn.find(service.id.get, column).getOrElse(throw BadQueryException("bad column name"))
- val (idOpt, idsOpt) = ((value \ "id").asOpt[JsValue], (value \ "ids").asOpt[List[JsValue]])
- for {
- idVal <- idOpt ++ idsOpt.toSeq.flatten
-
- /** bug, need to use labels schemaVersion */
- innerVal <- jsValueToInnerVal(idVal, col.columnType, col.schemaVersion)
- } yield {
- Vertex(SourceVertexId(col.id.get, innerVal), System.currentTimeMillis())
- }
- }).flatten
-
- if (vertices.isEmpty) throw BadQueryException("srcVertices`s id is empty")
- val steps = parse[Vector[JsValue]](jsValue, "steps")
-
- val queryOption = toQueryOption(jsValue)
-
- val querySteps =
- steps.zipWithIndex.map { case (step, stepIdx) =>
- val labelWeights = step match {
- case obj: JsObject =>
- val converted = for {
- (k, v) <- (obj \ "weights").asOpt[JsObject].getOrElse(Json.obj()).fields
- l <- Label.findByName(k)
- } yield {
- l.id.get -> v.toString().toDouble
- }
- converted.toMap
- case _ => Map.empty[Int, Double]
- }
- val queryParamJsVals = step match {
- case arr: JsArray => arr.as[List[JsValue]]
- case obj: JsObject => (obj \ "step").as[List[JsValue]]
- case _ => List.empty[JsValue]
- }
- val nextStepScoreThreshold = step match {
- case obj: JsObject => (obj \ "nextStepThreshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
- case _ => QueryParam.DefaultThreshold
- }
- val nextStepLimit = step match {
- case obj: JsObject => (obj \ "nextStepLimit").asOpt[Int].getOrElse(-1)
- case _ => -1
- }
- val cacheTTL = step match {
- case obj: JsObject => (obj \ "cacheTTL").asOpt[Int].getOrElse(-1)
- case _ => -1
- }
- val queryParams =
- for {
- labelGroup <- queryParamJsVals
- queryParam <- parseQueryParam(labelGroup)
- } yield {
- val (_, columnName) =
- if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) {
- (queryParam.label.srcService.serviceName, queryParam.label.srcColumnName)
- } else {
- (queryParam.label.tgtService.serviceName, queryParam.label.tgtColumnName)
- }
- //FIXME:
- if (stepIdx == 0 && vertices.nonEmpty && !vertices.exists(v => v.serviceColumn.columnName == columnName)) {
- throw BadQueryException("srcVertices contains incompatiable serviceName or columnName with first step.")
- }
-
- queryParam
- }
- Step(queryParams.toList, labelWeights = labelWeights,
- // scoreThreshold = stepThreshold,
- nextStepScoreThreshold = nextStepScoreThreshold,
- nextStepLimit = nextStepLimit,
- cacheTTL = cacheTTL)
-
- }
-
- val ret = Query(vertices, querySteps, queryOption, jsValue)
- // logger.debug(ret.toString)
- ret
- } catch {
- case e: BadQueryException =>
- throw e
- case e: ModelNotFoundException =>
- throw BadQueryException(e.getMessage, e)
- case e: Exception =>
- throw BadQueryException(s"$jsValue, $e", e)
- }
- }
-
- private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = {
- for {
- labelName <- parse[Option[String]](labelGroup, "label")
- } yield {
- val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found"))
- val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
- val limit = {
- parse[Option[Int]](labelGroup, "limit") match {
- case None => defaultLimit
- case Some(l) if l < 0 => maxLimit
- case Some(l) if l >= 0 =>
- val default = hardLimit
- Math.min(l, default)
- }
- }
- val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0)
- val interval = extractInterval(label, labelGroup)
- val duration = extractDuration(label, labelGroup)
- val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList
- val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false)
- val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false)
- val hasFilter = extractHas(label, labelGroup)
- val labelWithDir = LabelWithDirection(label.id.get, direction)
- val indexNameOpt = (labelGroup \ "index").asOpt[String]
- val indexSeq = indexNameOpt match {
- case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq)
- case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
- }
- val whereClauseOpt = (labelGroup \ "where").asOpt[String]
- val where = extractWhere(label, whereClauseOpt)
- val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
- val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout)
- val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt)
- val tgtVertexInnerIdOpt = (labelGroup \ "_to").asOpt[JsValue].flatMap { jsVal =>
- jsValueToInnerVal(jsVal, label.tgtColumnWithDir(direction).columnType, label.schemaVersion)
- }
- val cacheTTL = (labelGroup \ "cacheTTL").asOpt[Long].getOrElse(-1L)
- val timeDecayFactor = (labelGroup \ "timeDecay").asOpt[JsObject].map { jsVal =>
- val propName = (jsVal \ "propName").asOpt[String].getOrElse(LabelMeta.timestamp.name)
- val propNameSeq = label.metaPropsInvMap.get(propName).map(_.seq).getOrElse(LabelMeta.timeStampSeq)
- val initial = (jsVal \ "initial").asOpt[Double].getOrElse(1.0)
- val decayRate = (jsVal \ "decayRate").asOpt[Double].getOrElse(0.1)
- if (decayRate >= 1.0 || decayRate <= 0.0) throw new BadQueryException("decay rate should be 0.0 ~ 1.0")
- val timeUnit = (jsVal \ "timeUnit").asOpt[Double].getOrElse(60 * 60 * 24.0)
- TimeDecay(initial, decayRate, timeUnit, propNameSeq)
- }
- val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
- // TODO: refactor this. dirty
- val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
-
- val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s)))
- val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue]
- val scorePropagateOp = (labelGroup \ "scorePropagateOp").asOpt[String].getOrElse("multiply")
- val sample = (labelGroup \ "sample").asOpt[Int].getOrElse(-1)
- val shouldNormalize = (labelGroup \ "normalize").asOpt[Boolean].getOrElse(false)
- val cursorOpt = (labelGroup \ "cursor").asOpt[String]
- // FIXME: Order of command matter
- QueryParam(labelWithDir)
- .sample(sample)
- .limit(offset, limit)
- .rank(RankParam(label.id.get, scoring))
- .exclude(exclude)
- .include(include)
- .duration(duration)
- .has(hasFilter)
- .labelOrderSeq(indexSeq)
- .interval(interval)
- .where(where)
- .duplicatePolicy(duplicate)
- .includeDegree(includeDegree)
- .rpcTimeout(rpcTimeout)
- .maxAttempt(maxAttempt)
- .tgtVertexInnerIdOpt(tgtVertexInnerIdOpt)
- .cacheTTLInMillis(cacheTTL)
- .timeDecay(timeDecayFactor)
- .threshold(threshold)
- .transformer(transformer)
- .scorePropagateOp(scorePropagateOp)
- .shouldNormalize(shouldNormalize)
- .cursorOpt(cursorOpt)
- }
- }
-
- private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = {
- (js \ key).validate[R]
- .fold(
- errors => {
- val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg")
- val e = Json.obj("args" -> key, "error" -> msg)
- throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
- },
- r => {
- r
- })
- }
-
- def toJsValues(jsValue: JsValue): List[JsValue] = {
- jsValue match {
- case obj: JsObject => List(obj)
- case arr: JsArray => arr.as[List[JsValue]]
- case _ => List.empty[JsValue]
- }
-
- }
-
- def toEdgesWithOrg(jsValue: JsValue, operation: String): (List[Edge], List[JsValue]) = {
- val jsValues = toJsValues(jsValue)
- val edges = jsValues.flatMap(toEdge(_, operation))
-
- (edges, jsValues)
- }
-
- def toEdges(jsValue: JsValue, operation: String): List[Edge] = {
- toJsValues(jsValue).flatMap { edgeJson =>
- toEdge(edgeJson, operation)
- }
- }
-
-
- private def toEdge(jsValue: JsValue, operation: String): List[Edge] = {
-
- def parseId(js: JsValue) = js match {
- case s: JsString => s.as[String]
- case o@_ => s"${o}"
- }
- val srcId = (jsValue \ "from").asOpt[JsValue].toList.map(parseId(_))
- val tgtId = (jsValue \ "to").asOpt[JsValue].toList.map(parseId(_))
- val srcIds = (jsValue \ "froms").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ srcId
- val tgtIds = (jsValue \ "tos").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ tgtId
-
- val label = parse[String](jsValue, "label")
- val timestamp = parse[Long](jsValue, "timestamp")
- val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
- val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
- for {
- srcId <- srcIds
- tgtId <- tgtIds
- } yield {
- Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
- }
- }
-
- def toVertices(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None) = {
- toJsValues(jsValue).map(toVertex(_, operation, serviceName, columnName))
- }
-
- def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = {
- val id = parse[JsValue](jsValue, "id")
- val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
- val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
- val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get
- val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
- Management.toVertex(ts, operation, id.toString, sName, cName, props.toString)
- }
-
- def toPropElements(jsObj: JsValue) = Try {
- val propName = (jsObj \ "name").as[String]
- val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
- val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
- case JsString(s) => s
- case _@js => js.toString
- }
- Prop(propName, defaultValue, dataType)
- }
-
- def toPropsElements(jsValue: JsValue): Seq[Prop] = for {
- jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil)
- } yield {
- val propName = (jsObj \ "name").as[String]
- val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
- val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
- case JsString(s) => s
- case _@js => js.toString
- }
- Prop(propName, defaultValue, dataType)
- }
-
- def toIndicesElements(jsValue: JsValue): Seq[Index] = for {
- jsObj <- jsValue.as[Seq[JsValue]]
- indexName = (jsObj \ "name").as[String]
- propNames = (jsObj \ "propNames").as[Seq[String]]
- } yield Index(indexName, propNames)
-
- def toLabelElements(jsValue: JsValue) = Try {
- val labelName = parse[String](jsValue, "label")
- val srcServiceName = parse[String](jsValue, "srcServiceName")
- val tgtServiceName = parse[String](jsValue, "tgtServiceName")
- val srcColumnName = parse[String](jsValue, "srcColumnName")
- val tgtColumnName = parse[String](jsValue, "tgtColumnName")
- val srcColumnType = parse[String](jsValue, "srcColumnType")
- val tgtColumnType = parse[String](jsValue, "tgtColumnType")
- val serviceName = (jsValue \ "serviceName").asOpt[String].getOrElse(tgtServiceName)
- val isDirected = (jsValue \ "isDirected").asOpt[Boolean].getOrElse(true)
-
- val allProps = toPropsElements(jsValue \ "props")
- val indices = toIndicesElements(jsValue \ "indices")
-
- val consistencyLevel = (jsValue \ "consistencyLevel").asOpt[String].getOrElse("weak")
-
- // expect new label don`t provide hTableName
- val hTableName = (jsValue \ "hTableName").asOpt[String]
- val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
- val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(HBaseType.DEFAULT_VERSION)
- val isAsync = (jsValue \ "isAsync").asOpt[Boolean].getOrElse(false)
- val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
-
- (labelName, srcServiceName, srcColumnName, srcColumnType,
- tgtServiceName, tgtColumnName, tgtColumnType, isDirected, serviceName,
- indices, allProps, consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
- }
-
- def toIndexElements(jsValue: JsValue) = Try {
- val labelName = parse[String](jsValue, "label")
- val indices = toIndicesElements(jsValue \ "indices")
- (labelName, indices)
- }
-
- def toServiceElements(jsValue: JsValue) = {
- val serviceName = parse[String](jsValue, "serviceName")
- val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(DefaultCluster)
- val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${DefaultPhase}")
- val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1)
- val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
- val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
- (serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
- }
-
- def toServiceColumnElements(jsValue: JsValue) = Try {
- val serviceName = parse[String](jsValue, "serviceName")
- val columnName = parse[String](jsValue, "columnName")
- val columnType = parse[String](jsValue, "columnType")
- val props = toPropsElements(jsValue \ "props")
- (serviceName, columnName, columnType, props)
- }
-
- def toCheckEdgeParam(jsValue: JsValue) = {
- val params = jsValue.as[List[JsValue]]
- var isReverted = false
- val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]()
- val quads = for {
- param <- params
- labelName <- (param \ "label").asOpt[String]
- direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out"))
- label <- Label.findByName(labelName)
- srcId <- jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion)
- tgtId <- jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion)
- } yield {
- val labelWithDir = LabelWithDirection(label.id.get, direction)
- labelWithDirs += labelWithDir
- val (src, tgt, dir) = if (direction == 1) {
- isReverted = true
- (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)),
- Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0)
- } else {
- (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)),
- Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0)
- }
- (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir)))
- }
- (quads, isReverted)
- }
-
- def toGraphElements(str: String): Seq[GraphElement] = {
- val edgeStrs = str.split("\\n")
-
- for {
- edgeStr <- edgeStrs
- str <- GraphUtil.parseString(edgeStr)
- element <- Graph.toGraphElement(str)
- } yield element
- }
-
- def toDeleteParam(json: JsValue) = {
- val labelName = (json \ "label").as[String]
- val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil).filterNot(_.isAsync)
- val direction = (json \ "direction").asOpt[String].getOrElse("out")
-
- val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
- val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
- val vertices = toVertices(labelName, direction, ids)
- (labels, direction, ids, ts, vertices)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala
deleted file mode 100644
index b130854..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala
+++ /dev/null
@@ -1,245 +0,0 @@
-package com.kakao.s2graph.core.rest
-
-import java.net.URL
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{Bucket, Experiment, Service}
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json._
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Try
-
-
-object RestHandler {
- case class HandlerResult(body: Future[JsValue], headers: (String, String)*)
-}
-
-/**
- * Public API, only return Future.successful or Future.failed
- * Don't throw exception
- */
-class RestHandler(graph: Graph)(implicit ec: ExecutionContext) {
-
- import RestHandler._
- val requestParser = new RequestParser(graph.config)
-
- /**
- * Public APIS
- */
- def doPost(uri: String, body: String, impKeyOpt: => Option[String] = None): HandlerResult = {
- try {
- val jsQuery = Json.parse(body)
-
- uri match {
- case "/graphs/getEdges" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.toSimpleVertexArrJson))
- case "/graphs/getEdges/grouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithListFormatted))
- case "/graphs/getEdgesExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.toSimpleVertexArrJson))
- case "/graphs/getEdgesExcluded/grouped" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted))
- case "/graphs/checkEdges" => checkEdges(jsQuery)
- case "/graphs/getEdgesGrouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithList))
- case "/graphs/getEdgesGroupedExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExclude))
- case "/graphs/getEdgesGroupedExcludedFormatted" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted))
- case "/graphs/getVertices" => HandlerResult(getVertices(jsQuery))
- case uri if uri.startsWith("/graphs/experiment") =>
- val Array(accessToken, experimentName, uuid) = uri.split("/").takeRight(3)
- experiment(jsQuery, accessToken, experimentName, uuid, impKeyOpt)
- case _ => throw new RuntimeException("route is not found")
- }
- } catch {
- case e: Exception => HandlerResult(Future.failed(e))
- }
- }
-
- // TODO: Refactor to doGet
- def checkEdges(jsValue: JsValue): HandlerResult = {
- try {
- val (quads, isReverted) = requestParser.toCheckEdgeParam(jsValue)
-
- HandlerResult(graph.checkEdges(quads).map { case queryRequestWithResultLs =>
- val edgeJsons = for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- convertedEdge = if (isReverted) edge.duplicateEdge else edge
- edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam)
- } yield Json.toJson(edgeJson)
-
- Json.toJson(edgeJsons)
- })
- } catch {
- case e: Exception => HandlerResult(Future.failed(e))
- }
- }
-
-
- /**
- * Private APIS
- */
- private def experiment(contentsBody: JsValue, accessToken: String, experimentName: String, uuid: String, impKeyOpt: => Option[String]): HandlerResult = {
-
- try {
- val bucketOpt = for {
- service <- Service.findByAccessToken(accessToken)
- experiment <- Experiment.findBy(service.id.get, experimentName)
- bucket <- experiment.findBucket(uuid, impKeyOpt)
- } yield bucket
-
- val bucket = bucketOpt.getOrElse(throw new RuntimeException("bucket is not found"))
- if (bucket.isGraphQuery) {
- val ret = buildRequestInner(contentsBody, bucket, uuid)
- HandlerResult(ret.body, Experiment.impressionKey -> bucket.impressionId)
- }
- else throw new RuntimeException("not supported yet")
- } catch {
- case e: Exception => HandlerResult(Future.failed(e))
- }
- }
-
- private def buildRequestInner(contentsBody: JsValue, bucket: Bucket, uuid: String): HandlerResult = {
- if (bucket.isEmpty) HandlerResult(Future.successful(PostProcess.emptyResults))
- else {
- val body = buildRequestBody(Option(contentsBody), bucket, uuid)
- val url = new URL(bucket.apiPath)
- val path = url.getPath
-
- // dummy log for sampling
- val experimentLog = s"POST $path took -1 ms 200 -1 $body"
- logger.debug(experimentLog)
-
- doPost(path, body)
- }
- }
-
- private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = {
- val filterOutQueryResultsLs = q.filterOutQuery match {
- case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
- case None => Future.successful(Seq.empty)
- }
-
- for {
- queryResultsLs <- graph.getEdges(q)
- filterOutResultsLs <- filterOutQueryResultsLs
- } yield {
- val json = post(queryResultsLs, filterOutResultsLs)
- json
- }
- }
-
- def getEdgesAsync(jsonQuery: JsValue)
- (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
-
- val fetch = eachQuery(post) _
- jsonQuery match {
- case JsArray(arr) => Future.traverse(arr.map(requestParser.toQuery(_)))(fetch).map(JsArray)
- case obj@JsObject(_) =>
- (obj \ "queries").asOpt[JsValue] match {
- case None => fetch(requestParser.toQuery(obj))
- case _ =>
- val multiQuery = requestParser.toMultiQuery(obj)
- val filterOutFuture = multiQuery.queryOption.filterOutQuery match {
- case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
- case None => Future.successful(Seq.empty)
- }
- val futures = multiQuery.queries.zip(multiQuery.weights).map { case (query, weight) =>
- val filterOutQueryResultsLs = query.queryOption.filterOutQuery match {
- case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
- case None => Future.successful(Seq.empty)
- }
- for {
- queryRequestWithResultLs <- graph.getEdges(query)
- filterOutResultsLs <- filterOutQueryResultsLs
- } yield {
- val newQueryRequestWithResult = for {
- queryRequestWithResult <- queryRequestWithResultLs
- queryResult = queryRequestWithResult.queryResult
- } yield {
- val newEdgesWithScores = for {
- edgeWithScore <- queryRequestWithResult.queryResult.edgeWithScoreLs
- } yield {
- edgeWithScore.copy(score = edgeWithScore.score * weight)
- }
- queryRequestWithResult.copy(queryResult = queryResult.copy(edgeWithScoreLs = newEdgesWithScores))
- }
- logger.debug(s"[Size]: ${newQueryRequestWithResult.map(_.queryResult.edgeWithScoreLs.size).sum}")
- (newQueryRequestWithResult, filterOutResultsLs)
- }
- }
- for {
- filterOut <- filterOutFuture
- resultWithExcludeLs <- Future.sequence(futures)
- } yield {
- PostProcess.toSimpleVertexArrJsonMulti(multiQuery.queryOption, resultWithExcludeLs, filterOut)
- // val initial = (ListBuffer.empty[QueryRequestWithResult], ListBuffer.empty[QueryRequestWithResult])
- // val (results, excludes) = resultWithExcludeLs.foldLeft(initial) { case ((prevResults, prevExcludes), (results, excludes)) =>
- // (prevResults ++= results, prevExcludes ++= excludes)
- // }
- // PostProcess.toSimpleVertexArrJson(multiQuery.queryOption, results, excludes ++ filterOut)
- }
- }
- case _ => throw BadQueryException("Cannot support")
- }
- }
-
- private def getEdgesExcludedAsync(jsonQuery: JsValue)
- (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
- val q = requestParser.toQuery(jsonQuery)
- val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
-
- val fetchFuture = graph.getEdges(q)
- val excludeFuture = graph.getEdges(filterOutQuery)
-
- for {
- queryResultLs <- fetchFuture
- exclude <- excludeFuture
- } yield {
- post(queryResultLs, exclude)
- }
- }
-
- private def getVertices(jsValue: JsValue) = {
- val jsonQuery = jsValue
- val ts = System.currentTimeMillis()
- val props = "{}"
-
- val vertices = jsonQuery.as[List[JsValue]].flatMap { js =>
- val serviceName = (js \ "serviceName").as[String]
- val columnName = (js \ "columnName").as[String]
- for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield {
- Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props)
- }
- }
-
- graph.getVertices(vertices) map { vertices => PostProcess.verticesToJson(vertices) }
- }
-
- private def buildRequestBody(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): String = {
- var body = bucket.requestBody.replace("#uuid", uuid)
-
- // // replace variable
- // body = TemplateHelper.replaceVariable(System.currentTimeMillis(), body)
-
- // replace param
- for {
- requestKeyJson <- requestKeyJsonOpt
- jsObj <- requestKeyJson.asOpt[JsObject]
- (key, value) <- jsObj.fieldSet
- } {
- val replacement = value match {
- case JsString(s) => s
- case _ => value.toString
- }
- body = body.replace(key, replacement)
- }
-
- body
- }
-
- def calcSize(js: JsValue): Int = js match {
- case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0)
- case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum
- case _ => 0
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
deleted file mode 100644
index cff968f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core.storage.{SKeyValue, StorageDeserializable}
-import com.kakao.s2graph.core.types.{LabelWithDirection, SourceVertexId, VertexId}
-import org.apache.hadoop.hbase.util.Bytes
-
-
-trait Deserializable[E] extends StorageDeserializable[E] {
- import StorageDeserializable._
-
- type RowKeyRaw = (VertexId, LabelWithDirection, Byte, Boolean, Int)
-
- /** version 1 and version 2 share same code for parsing row key part */
- def parseRow(kv: SKeyValue, version: String): RowKeyRaw = {
- var pos = 0
- val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
- pos += srcIdLen
- val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
- pos += 4
- val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
-
- val rowLen = srcIdLen + 4 + 1
- (srcVertexId, labelWithDir, labelIdxSeq, isInverted, rowLen)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
deleted file mode 100644
index a0aa261..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.KeyValue
-
-
-object SKeyValue {
- val Put = 1
- val Delete = 2
- val Increment = 3
- val Default = Put
-}
-case class SKeyValue(table: Array[Byte],
- row: Array[Byte],
- cf: Array[Byte],
- qualifier: Array[Byte],
- value: Array[Byte],
- timestamp: Long,
- operation: Int = SKeyValue.Default) {
- def toLogString = {
- Map("table" -> table.toList, "row" -> row.toList, "cf" -> Bytes.toString(cf),
- "qualifier" -> qualifier.toList, "value" -> value.toList, "timestamp" -> timestamp,
- "operation" -> operation).toString
- }
- override def toString(): String = toLogString
-}
-
-trait CanSKeyValue[T] {
- def toSKeyValue(from: T): SKeyValue
-}
-
-object CanSKeyValue {
-
- // For asyncbase KeyValues
- implicit val asyncKeyValue = new CanSKeyValue[KeyValue] {
- def toSKeyValue(kv: KeyValue): SKeyValue = {
- SKeyValue(Array.empty[Byte], kv.key(), kv.family(), kv.qualifier(), kv.value(), kv.timestamp())
- }
- }
-
- // For asyncbase KeyValues
- implicit val sKeyValue = new CanSKeyValue[SKeyValue] {
- def toSKeyValue(kv: SKeyValue): SKeyValue = kv
- }
-
- // For hbase KeyValues
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
deleted file mode 100644
index bee064b..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core.storage.StorageSerializable
-
-object Serializable {
- val vertexCf = "v".getBytes()
- val edgeCf = "e".getBytes()
-}
-
-trait Serializable[E] extends StorageSerializable[E]
[17/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
new file mode 100644
index 0000000..46b3255
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -0,0 +1,553 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.GraphExceptions.BadQueryException
+import org.apache.s2graph.core.mysqls.{ColumnMeta, Label, LabelMeta, ServiceColumn}
+import org.apache.s2graph.core.types.{InnerVal, InnerValLike}
+import play.api.libs.json.{Json, _}
+
+import scala.collection.mutable.ListBuffer
+
+object PostProcess extends JSONParser {
+
+
+ type EDGE_VALUES = Map[String, JsValue]
+ type ORDER_BY_VALUES = (Any, Any, Any, Any)
+ type RAW_EDGE = (EDGE_VALUES, Double, ORDER_BY_VALUES)
+
+ /**
+ * Result Entity score field name
+ */
+ val emptyDegrees = Seq.empty[JsValue]
+ val timeoutResults = Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr(), "isTimeout" -> true)
+ val emptyResults = Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr(), "isEmpty" -> true)
+ def badRequestResults(ex: => Exception) = ex match {
+ case ex: BadQueryException => Json.obj("message" -> ex.msg)
+ case _ => Json.obj("message" -> ex.getMessage)
+ }
+
+ val SCORE_FIELD_NAME = "scoreSum"
+ val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
+
+ def groupEdgeResult(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+ // filterNot {case (edge, score) => edge.props.contains(LabelMeta.degreeSeq)}
+ val groupedEdgesWithRank = (for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
+ } yield {
+ (queryRequest.queryParam, edge, score)
+ }).groupBy {
+ case (queryParam, edge, rank) if edge.labelWithDir.dir == GraphUtil.directions("in") =>
+ (queryParam.label.srcColumn, queryParam.label.label, queryParam.label.tgtColumn, edge.tgtVertex.innerId, edge.isDegree)
+ case (queryParam, edge, rank) =>
+ (queryParam.label.tgtColumn, queryParam.label.label, queryParam.label.srcColumn, edge.tgtVertex.innerId, edge.isDegree)
+ }
+
+ val ret = for {
+ ((tgtColumn, labelName, srcColumn, target, isDegreeEdge), edgesAndRanks) <- groupedEdgesWithRank if !isDegreeEdge
+ edgesWithRanks = edgesAndRanks.groupBy(x => x._2.srcVertex).map(_._2.head)
+ id <- innerValToJsValue(target, tgtColumn.columnType)
+ } yield {
+ Json.obj("name" -> tgtColumn.columnName, "id" -> id,
+ SCORE_FIELD_NAME -> edgesWithRanks.map(_._3).sum,
+ "label" -> labelName,
+ "aggr" -> Json.obj(
+ "name" -> srcColumn.columnName,
+ "ids" -> edgesWithRanks.flatMap { case (queryParam, edge, rank) =>
+ innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)
+ },
+ "edges" -> edgesWithRanks.map { case (queryParam, edge, rank) =>
+ Json.obj("id" -> innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType),
+ "props" -> propsToJson(edge),
+ "score" -> rank
+ )
+ }
+ )
+ )
+ }
+
+ ret.toList
+ }
+
+ def sortWithFormatted(jsons: Seq[JsObject],
+ scoreField: String = "scoreSum",
+ queryRequestWithResultLs: Seq[QueryRequestWithResult],
+ decrease: Boolean = true): JsObject = {
+ val ordering = if (decrease) -1 else 1
+ val sortedJsons = jsons.sortBy { jsObject => (jsObject \ scoreField).as[Double] * ordering }
+
+ if (queryRequestWithResultLs.isEmpty) Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
+ else Json.obj(
+ "size" -> sortedJsons.size,
+ "results" -> sortedJsons,
+ "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId()
+ )
+ }
+
+ private def toHashKey(edge: Edge, queryParam: QueryParam, fields: Seq[String], delimiter: String = ","): Int = {
+ val ls = for {
+ field <- fields
+ } yield {
+ field match {
+ case "from" | "_from" => edge.srcVertex.innerId.toIdString()
+ case "to" | "_to" => edge.tgtVertex.innerId.toIdString()
+ case "label" => edge.labelWithDir.labelId
+ case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
+ case "_timestamp" | "timestamp" => edge.ts
+ case _ =>
+ queryParam.label.metaPropsInvMap.get(field) match {
+ case None => throw new RuntimeException(s"unknow column: $field")
+ case Some(labelMeta) => edge.propsWithTs.get(labelMeta.seq) match {
+ case None => labelMeta.defaultValue
+ case Some(propVal) => propVal
+ }
+ }
+ }
+ }
+ val ret = ls.hashCode()
+ ret
+ }
+
+ def resultInnerIds(queryRequestWithResultLs: Seq[QueryRequestWithResult], isSrcVertex: Boolean = false): Seq[Int] = {
+ for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ q = queryRequest.query
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ } yield toHashKey(edge, queryRequest.queryParam, q.filterOutFields)
+ }
+
+ def summarizeWithListExcludeFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs, decrease = true)
+ }
+
+ def summarizeWithList(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
+ }
+
+ def summarizeWithListFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
+ val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
+ sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
+ }
+
+ def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult]): JsValue = {
+ toSimpleVertexArrJson(queryRequestWithResultLs, Seq.empty[QueryRequestWithResult])
+ }
+
+ private def orderBy(queryOption: QueryOption,
+ rawEdges: ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]): ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)] = {
+ import OrderingUtil._
+
+ if (queryOption.withScore && queryOption.orderByColumns.nonEmpty) {
+ val ascendingLs = queryOption.orderByColumns.map(_._2)
+ rawEdges.sortBy(_._3)(TupleMultiOrdering[Any](ascendingLs))
+ } else {
+ rawEdges
+ }
+ }
+
+ private def getColumnValue(keyWithJs: Map[String, JsValue], score: Double, edge: Edge, column: String): Any = {
+ column match {
+ case "score" => score
+ case "timestamp" | "_timestamp" => edge.ts
+ case _ =>
+ keyWithJs.get(column) match {
+ case None => keyWithJs.get("props").map { js => (js \ column).as[JsValue] }.get
+ case Some(x) => x
+ }
+ }
+ }
+
+ private def buildReplaceJson(jsValue: JsValue)(mapper: JsValue => JsValue): JsValue = {
+ def traverse(js: JsValue): JsValue = js match {
+ case JsNull => mapper(JsNull)
+ case JsUndefined() => mapper(JsUndefined(""))
+ case JsNumber(v) => mapper(js)
+ case JsString(v) => mapper(js)
+ case JsBoolean(v) => mapper(js)
+ case JsArray(elements) => JsArray(elements.map { t => traverse(mapper(t)) })
+ case JsObject(values) => JsObject(values.map { case (k, v) => k -> traverse(mapper(v)) })
+ }
+
+ traverse(jsValue)
+ }
+
+ /** test query with filterOut is not working since it can not diffrentate filterOut */
+ private def buildNextQuery(jsonQuery: JsValue, _cursors: Seq[Seq[String]]): JsValue = {
+ val cursors = _cursors.flatten.iterator
+
+ buildReplaceJson(jsonQuery) {
+ case js@JsObject(fields) =>
+ val isStep = fields.find { case (k, _) => k == "label" } // find label group
+ if (isStep.isEmpty) js
+ else {
+ // TODO: Order not ensured
+ val withCursor = js.fieldSet | Set("cursor" -> JsString(cursors.next))
+ JsObject(withCursor.toSeq)
+ }
+ case js => js
+ }
+ }
+
+ private def buildRawEdges(queryOption: QueryOption,
+ queryRequestWithResultLs: Seq[QueryRequestWithResult],
+ excludeIds: Map[Int, Boolean],
+ scoreWeight: Double = 1.0): (ListBuffer[JsValue], ListBuffer[RAW_EDGE]) = {
+ val degrees = ListBuffer[JsValue]()
+ val rawEdges = ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]()
+ val metaPropNamesMap = scala.collection.mutable.Map[String, Int]()
+ for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ } yield {
+ val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ queryRequest.queryParam.label.metaPropNames.foreach { metaPropName =>
+ metaPropNamesMap.put(metaPropName, metaPropNamesMap.getOrElse(metaPropName, 0) + 1)
+ }
+ }
+ val propsExistInAll = metaPropNamesMap.filter(kv => kv._2 == queryRequestWithResultLs.length)
+ val orderByColumns = queryOption.orderByColumns.filter { case (column, _) =>
+ column match {
+ case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true
+ case _ =>
+ propsExistInAll.contains(column)
+// //TODO??
+// false
+// queryParam.label.metaPropNames.contains(column)
+ }
+ }
+
+ /** build result jsons */
+ for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ queryParam = queryRequest.queryParam
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ hashKey = toHashKey(edge, queryRequest.queryParam, queryOption.filterOutFields)
+ if !excludeIds.contains(hashKey)
+ } {
+
+ // edge to json
+ val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
+ val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
+ if (edge.isDegree && fromOpt.isDefined) {
+ if (queryOption.limitOpt.isEmpty) {
+ degrees += Json.obj(
+ "from" -> fromOpt.get,
+ "label" -> queryRequest.queryParam.label.label,
+ "direction" -> GraphUtil.fromDirection(edge.labelWithDir.dir),
+ LabelMeta.degree.name -> innerValToJsValue(edge.propsWithTs(LabelMeta.degreeSeq).innerVal, InnerVal.LONG)
+ )
+ }
+ } else {
+ val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam)
+ val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match {
+ case 0 =>
+ (None, None, None, None)
+ case 1 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, None, None, None)
+ case 2 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, None, None)
+ case 3 =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, v3, None)
+ case _ =>
+ val it = orderByColumns.iterator
+ val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1)
+ (v1, v2, v3, v4)
+ }
+
+ val currentEdge = (keyWithJs, score * scoreWeight, orderByValues)
+ rawEdges += currentEdge
+ }
+ }
+ (degrees, rawEdges)
+ }
+
+ private def buildResultJsValue(queryOption: QueryOption,
+ degrees: ListBuffer[JsValue],
+ rawEdges: ListBuffer[RAW_EDGE]): JsValue = {
+ if (queryOption.groupByColumns.isEmpty) {
+ // ordering
+ val filteredEdges = rawEdges.filter(t => t._2 >= queryOption.scoreThreshold)
+
+ val edges = queryOption.limitOpt match {
+ case None => orderBy(queryOption, filteredEdges).map(_._1)
+ case Some(limit) => orderBy(queryOption, filteredEdges).map(_._1).take(limit)
+ }
+ val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees
+
+ Json.obj(
+ "size" -> edges.size,
+ "degrees" -> resultDegrees,
+// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
+ "results" -> edges
+ )
+ } else {
+ val grouped = rawEdges.groupBy { case (keyWithJs, _, _) =>
+ val props = keyWithJs.get("props")
+
+ for {
+ column <- queryOption.groupByColumns
+ value <- keyWithJs.get(column) match {
+ case None => props.flatMap { js => (js \ column).asOpt[JsValue] }
+ case Some(x) => Some(x)
+ }
+ } yield column -> value
+ }
+
+ val groupedEdgesWithScoreSum =
+ for {
+ (groupByKeyVals, groupedRawEdges) <- grouped
+ scoreSum = groupedRawEdges.map(x => x._2).sum if scoreSum >= queryOption.scoreThreshold
+ } yield {
+ // ordering
+ val edges = orderBy(queryOption, groupedRawEdges).map(_._1)
+
+ //TODO: refactor this
+ val js = if (queryOption.returnAgg)
+ Json.obj(
+ "groupBy" -> Json.toJson(groupByKeyVals.toMap),
+ "scoreSum" -> scoreSum,
+ "agg" -> edges
+ )
+ else
+ Json.obj(
+ "groupBy" -> Json.toJson(groupByKeyVals.toMap),
+ "scoreSum" -> scoreSum,
+ "agg" -> Json.arr()
+ )
+ (js, scoreSum)
+ }
+
+ val groupedSortedJsons = queryOption.limitOpt match {
+ case None =>
+ groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1)
+ case Some(limit) =>
+ groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1).take(limit)
+ }
+ val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees
+ Json.obj(
+ "size" -> groupedSortedJsons.size,
+ "degrees" -> resultDegrees,
+// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
+ "results" -> groupedSortedJsons
+ )
+ }
+ }
+
+ def toSimpleVertexArrJsonMulti(queryOption: QueryOption,
+ resultWithExcludeLs: Seq[(Seq[QueryRequestWithResult], Seq[QueryRequestWithResult])],
+ excludes: Seq[QueryRequestWithResult]): JsValue = {
+ val excludeIds = (Seq((Seq.empty, excludes)) ++ resultWithExcludeLs).foldLeft(Map.empty[Int, Boolean]) { case (acc, (result, excludes)) =>
+ acc ++ resultInnerIds(excludes).map(hashKey => hashKey -> true).toMap
+ }
+
+ val (degrees, rawEdges) = (ListBuffer.empty[JsValue], ListBuffer.empty[RAW_EDGE])
+ for {
+ (result, localExclude) <- resultWithExcludeLs
+ } {
+ val newResult = result.map { queryRequestWithResult =>
+ val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ val newQuery = queryRequest.query.copy(queryOption = queryOption)
+ queryRequestWithResult.copy(queryRequest = queryRequest.copy(query = newQuery))
+ }
+ val (_degrees, _rawEdges) = buildRawEdges(queryOption, newResult, excludeIds)
+ degrees ++= _degrees
+ rawEdges ++= _rawEdges
+ }
+ buildResultJsValue(queryOption, degrees, rawEdges)
+ }
+
+ def toSimpleVertexArrJson(queryOption: QueryOption,
+ queryRequestWithResultLs: Seq[QueryRequestWithResult],
+ exclude: Seq[QueryRequestWithResult]): JsValue = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+ val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds)
+ buildResultJsValue(queryOption, degrees, rawEdges)
+ }
+
+
+ def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult],
+ exclude: Seq[QueryRequestWithResult]): JsValue = {
+
+ queryRequestWithResultLs.headOption.map { queryRequestWithResult =>
+ val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ val query = queryRequest.query
+ val queryOption = query.queryOption
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+ val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds)
+ buildResultJsValue(queryOption, degrees, rawEdges)
+ } getOrElse emptyResults
+ }
+
+ def verticesToJson(vertices: Iterable[Vertex]) = {
+ Json.toJson(vertices.flatMap { v => vertexToJson(v) })
+ }
+
+ def propsToJson(edge: Edge, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ for {
+ (seq, innerValWithTs) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
+ labelMeta <- queryParam.label.metaPropsMap.get(seq)
+ jsValue <- innerValToJsValue(innerValWithTs.innerVal, labelMeta.dataType)
+ } yield labelMeta.name -> jsValue
+ }
+
+ private def edgeParent(parentEdges: Seq[EdgeWithScore], q: Query, queryParam: QueryParam): JsValue = {
+ if (parentEdges.isEmpty) {
+ JsNull
+ } else {
+ val parents = for {
+ parent <- parentEdges
+ (parentEdge, parentScore) = EdgeWithScore.unapply(parent).get
+ parentQueryParam = QueryParam(parentEdge.labelWithDir)
+ parents = edgeParent(parentEdge.parentEdges, q, parentQueryParam) if parents != JsNull
+ } yield {
+ val originalEdge = parentEdge.originalEdgeOpt.getOrElse(parentEdge)
+ val edgeJson = edgeToJsonInner(originalEdge, parentScore, q, parentQueryParam) + ("parents" -> parents)
+ Json.toJson(edgeJson)
+ }
+
+ Json.toJson(parents)
+ }
+ }
+
+ /** TODO */
+ def edgeToJsonInner(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ val (srcColumn, tgtColumn) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
+
+ val kvMapOpt = for {
+ from <- innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
+ to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType)
+ } yield {
+ val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props"
+ val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam)
+ val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap
+
+ val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) =>
+ val jsValue = column match {
+ case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp))
+ case "from" => from
+ case "to" => to
+ case "label" => JsString(queryParam.label.label)
+ case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
+ case "_timestamp" | "timestamp" => JsNumber(edge.ts)
+ case "score" => JsNumber(score)
+ case "props" if propsMap.nonEmpty => Json.toJson(propsMap)
+ case _ => JsNull
+ }
+
+ if (jsValue == JsNull) map else map + (column -> jsValue)
+ }
+ kvMap
+ }
+
+ kvMapOpt.getOrElse(Map.empty)
+ }
+
+ def edgeToJson(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
+ val kvs = edgeToJsonInner(edge, score, q, queryParam)
+ if (kvs.nonEmpty && q.returnTree) kvs + ("parents" -> Json.toJson(edgeParent(edge.parentEdges, q, queryParam)))
+ else kvs
+ }
+
+ def vertexToJson(vertex: Vertex): Option[JsObject] = {
+ val serviceColumn = ServiceColumn.findById(vertex.id.colId)
+
+ for {
+ id <- innerValToJsValue(vertex.innerId, serviceColumn.columnType)
+ } yield {
+ Json.obj("serviceName" -> serviceColumn.service.serviceName,
+ "columnName" -> serviceColumn.columnName,
+ "id" -> id, "props" -> propsToJson(vertex),
+ "timestamp" -> vertex.ts,
+ // "belongsTo" -> vertex.belongLabelIds)
+ "belongsTo" -> vertex.belongLabelIds.flatMap(Label.findByIdOpt(_).map(_.label)))
+ }
+ }
+
+ private def keysToName(seqsToNames: Map[Int, String], props: Map[Int, InnerValLike]) = {
+ for {
+ (seq, value) <- props
+ name <- seqsToNames.get(seq)
+ } yield (name, value)
+ }
+
+ private def propsToJson(vertex: Vertex) = {
+ val serviceColumn = vertex.serviceColumn
+ val props = for {
+ (propKey, innerVal) <- vertex.props
+ columnMeta <- ColumnMeta.findByIdAndSeq(serviceColumn.id.get, propKey.toByte, useCache = true)
+ jsValue <- innerValToJsValue(innerVal, columnMeta.dataType)
+ } yield {
+ (columnMeta.name -> jsValue)
+ }
+ props.toMap
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def propsToJson(edge: Edge) = {
+ for {
+ (seq, v) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
+ metaProp <- edge.label.metaPropsMap.get(seq)
+ jsValue <- innerValToJsValue(v.innerVal, metaProp.dataType)
+ } yield {
+ (metaProp.name, jsValue)
+ }
+ }
+
+ @deprecated(message = "deprecated", since = "0.2")
+ def summarizeWithListExclude(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsObject = {
+ val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
+
+
+ val groupedEdgesWithRank = (for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
+ } yield {
+ (edge, score)
+ }).groupBy { case (edge, score) =>
+ (edge.label.tgtColumn, edge.label.srcColumn, edge.tgtVertex.innerId)
+ }
+
+ val jsons = for {
+ ((tgtColumn, srcColumn, target), edgesAndRanks) <- groupedEdgesWithRank
+ (edges, ranks) = edgesAndRanks.groupBy(x => x._1.srcVertex).map(_._2.head).unzip
+ tgtId <- innerValToJsValue(target, tgtColumn.columnType)
+ } yield {
+ Json.obj(tgtColumn.columnName -> tgtId,
+ s"${srcColumn.columnName}s" ->
+ edges.flatMap(edge => innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)), "scoreSum" -> ranks.sum)
+ }
+ val sortedJsons = jsons.toList.sortBy { jsObj => (jsObj \ "scoreSum").as[Double] }.reverse
+ if (queryRequestWithResultLs.isEmpty) {
+ Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
+ } else {
+ Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons,
+ "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId())
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
new file mode 100644
index 0000000..2febadd
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
@@ -0,0 +1,583 @@
+package org.apache.s2graph.core
+
+import com.google.common.hash.Hashing
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta}
+import org.apache.s2graph.core.parsers.{Where, WhereParser}
+import org.apache.s2graph.core.types.{HBaseSerializable, InnerVal, InnerValLike, LabelWithDirection}
+import org.hbase.async.ColumnRangeFilter
+import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
+
+import scala.util.hashing.MurmurHash3
+import scala.util.{Success, Try}
+
+object Query {
+ val initialScore = 1.0
+ lazy val empty = Query()
+
+ def toQuery(srcVertices: Seq[Vertex], queryParam: QueryParam) = Query(srcVertices, Vector(Step(List(queryParam))))
+
+ object DuplicatePolicy extends Enumeration {
+ type DuplicatePolicy = Value
+ val First, Sum, CountSum, Raw = Value
+
+ def apply(policy: String): Value = {
+ policy match {
+ case "sum" => Query.DuplicatePolicy.Sum
+ case "countSum" => Query.DuplicatePolicy.CountSum
+ case "raw" => Query.DuplicatePolicy.Raw
+ case _ => DuplicatePolicy.First
+ }
+ }
+ }
+}
+
+case class MultiQuery(queries: Seq[Query],
+ weights: Seq[Double],
+ queryOption: QueryOption,
+ jsonQuery: JsValue = JsNull)
+
+case class QueryOption(removeCycle: Boolean = false,
+ selectColumns: Seq[String] = Seq.empty,
+ groupByColumns: Seq[String] = Seq.empty,
+ orderByColumns: Seq[(String, Boolean)] = Seq.empty,
+ filterOutQuery: Option[Query] = None,
+ filterOutFields: Seq[String] = Seq(LabelMeta.to.name),
+ withScore: Boolean = true,
+ returnTree: Boolean = false,
+ limitOpt: Option[Int] = None,
+ returnAgg: Boolean = true,
+ scoreThreshold: Double = Double.MinValue,
+ returnDegree: Boolean = true)
+
+
+case class Query(vertices: Seq[Vertex] = Seq.empty[Vertex],
+ steps: IndexedSeq[Step] = Vector.empty[Step],
+ queryOption: QueryOption = QueryOption(),
+ jsonQuery: JsValue = JsNull) {
+
+ val removeCycle = queryOption.removeCycle
+ val selectColumns = queryOption.selectColumns
+// val groupBy = queryOption.groupBy
+ val groupByColumns = queryOption.groupByColumns
+ val orderByColumns = queryOption.orderByColumns
+ val filterOutQuery = queryOption.filterOutQuery
+ val filterOutFields = queryOption.filterOutFields
+ val withScore = queryOption.withScore
+ val returnTree = queryOption.returnTree
+ val limitOpt = queryOption.limitOpt
+ val returnAgg = queryOption.returnAgg
+ val returnDegree = queryOption.returnDegree
+
+ def cacheKeyBytes: Array[Byte] = {
+ val selectBytes = Bytes.toBytes(queryOption.selectColumns.toString)
+ val groupBytes = Bytes.toBytes(queryOption.groupByColumns.toString)
+ val orderByBytes = Bytes.toBytes(queryOption.orderByColumns.toString)
+ val filterOutBytes = queryOption.filterOutQuery.map(_.cacheKeyBytes).getOrElse(Array.empty[Byte])
+ val returnTreeBytes = Bytes.toBytes(queryOption.returnTree)
+
+ Seq(selectBytes, groupBytes, orderByBytes, filterOutBytes, returnTreeBytes).foldLeft(Array.empty[Byte])(Bytes.add)
+ }
+
+ lazy val selectColumnsSet = queryOption.selectColumns.map { c =>
+ if (c == "_from") "from"
+ else if (c == "_to") "to"
+ else c
+ }.toSet
+
+ /** return logical query id without considering parameter values */
+ def templateId(): JsValue = {
+ Json.toJson(for {
+ step <- steps
+ queryParam <- step.queryParams.sortBy(_.labelWithDir.labelId)
+ } yield {
+ Json.obj("label" -> queryParam.label.label, "direction" -> GraphUtil.fromDirection(queryParam.labelWithDir.dir))
+ })
+ }
+
+ def impressionId(): JsNumber = {
+ val hash = MurmurHash3.stringHash(templateId().toString())
+ JsNumber(hash)
+ }
+
+ def cursorStrings(): Seq[Seq[String]] = {
+ //Don`t know how to replace all cursor keys in json
+ steps.map { step =>
+ step.queryParams.map { queryParam =>
+ queryParam.cursorOpt.getOrElse("")
+ }
+ }
+ }
+}
+
+object EdgeTransformer {
+ val DefaultTransformField = Json.arr("_to")
+ val DefaultTransformFieldAsList = Json.arr("_to").as[List[String]]
+ val DefaultJson = Json.arr(DefaultTransformField)
+}
+
+/**
+ * TODO: step wise outputFields should be used with nextStepLimit, nextStepThreshold.
+ * @param jsValue
+ */
+case class EdgeTransformer(queryParam: QueryParam, jsValue: JsValue) {
+ val Delimiter = "\\$"
+ val targets = jsValue.asOpt[List[Vector[String]]].toList
+ val fieldsLs = for {
+ target <- targets
+ fields <- target
+ } yield fields
+ val isDefault = fieldsLs.size == 1 && fieldsLs.head.size == 1 && (fieldsLs.head.head == "_to" || fieldsLs.head.head == "to")
+
+ def toHashKeyBytes: Array[Byte] = if (isDefault) Array.empty[Byte] else Bytes.toBytes(jsValue.toString)
+
+ def replace(fmt: String,
+ values: Seq[InnerValLike],
+ nextStepOpt: Option[Step]): Seq[InnerValLike] = {
+
+ val tokens = fmt.split(Delimiter)
+ val _values = values.padTo(tokens.length, InnerVal.withStr("", queryParam.label.schemaVersion))
+ val mergedStr = tokens.zip(_values).map { case (prefix, innerVal) => prefix + innerVal.toString }.mkString
+ // logger.error(s"${tokens.toList}, ${values}, $mergedStr")
+ // println(s"${tokens.toList}, ${values}, $mergedStr")
+ nextStepOpt match {
+ case None =>
+ val columnType =
+ if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) queryParam.label.tgtColumnType
+ else queryParam.label.srcColumnType
+
+ if (columnType == InnerVal.STRING) Seq(InnerVal.withStr(mergedStr, queryParam.label.schemaVersion))
+ else Nil
+ case Some(nextStep) =>
+ val nextQueryParamsValid = nextStep.queryParams.filter { qParam =>
+ if (qParam.labelWithDir.dir == GraphUtil.directions("out")) qParam.label.srcColumnType == "string"
+ else qParam.label.tgtColumnType == "string"
+ }
+ for {
+ nextQueryParam <- nextQueryParamsValid
+ } yield {
+ InnerVal.withStr(mergedStr, nextQueryParam.label.schemaVersion)
+ }
+ }
+ }
+
+ def toInnerValOpt(edge: Edge, fieldName: String): Option[InnerValLike] = {
+ fieldName match {
+ case LabelMeta.to.name => Option(edge.tgtVertex.innerId)
+ case LabelMeta.from.name => Option(edge.srcVertex.innerId)
+ case _ =>
+ for {
+ labelMeta <- queryParam.label.metaPropsInvMap.get(fieldName)
+ value <- edge.propsWithTs.get(labelMeta.seq)
+ } yield value.innerVal
+ }
+ }
+
+ def transform(edge: Edge, nextStepOpt: Option[Step]): Seq[Edge] = {
+ if (isDefault) Seq(edge)
+ else {
+ val edges = for {
+ fields <- fieldsLs
+ innerVal <- {
+ if (fields.size == 1) {
+ val fieldName = fields.head
+ toInnerValOpt(edge, fieldName).toSeq
+ } else {
+ val fmt +: fieldNames = fields
+ replace(fmt, fieldNames.flatMap(fieldName => toInnerValOpt(edge, fieldName)), nextStepOpt)
+ }
+ }
+ } yield edge.updateTgtVertex(innerVal).copy(originalEdgeOpt = Option(edge))
+
+
+ edges
+ }
+ }
+}
+
+object Step {
+ val Delimiter = "|"
+}
+
+case class Step(queryParams: List[QueryParam],
+ labelWeights: Map[Int, Double] = Map.empty,
+ // scoreThreshold: Double = 0.0,
+ nextStepScoreThreshold: Double = 0.0,
+ nextStepLimit: Int = -1,
+ cacheTTL: Long = -1) {
+
+ lazy val excludes = queryParams.filter(_.exclude)
+ lazy val includes = queryParams.filterNot(_.exclude)
+ lazy val excludeIds = excludes.map(x => x.labelWithDir.labelId -> true).toMap
+
+ def toCacheKey(lss: Seq[Long]): Long = Hashing.murmur3_128().hashBytes(toCacheKeyRaw(lss)).asLong()
+// MurmurHash3.bytesHash(toCacheKeyRaw(lss))
+
+ def toCacheKeyRaw(lss: Seq[Long]): Array[Byte] = {
+ var bytes = Array.empty[Byte]
+ lss.sorted.foreach { h => bytes = Bytes.add(bytes, Bytes.toBytes(h)) }
+ bytes
+ }
+}
+
+case class VertexParam(vertices: Seq[Vertex]) {
+ var filters: Option[Map[Byte, InnerValLike]] = None
+
+ def has(what: Option[Map[Byte, InnerValLike]]): VertexParam = {
+ what match {
+ case None => this
+ case Some(w) => has(w)
+ }
+ }
+
+ def has(what: Map[Byte, InnerValLike]): VertexParam = {
+ this.filters = Some(what)
+ this
+ }
+
+}
+
+//object RankParam {
+// def apply(labelId: Int, keyAndWeights: Seq[(Byte, Double)]) = {
+// new RankParam(labelId, keyAndWeights)
+// }
+//}
+
+case class RankParam(labelId: Int, var keySeqAndWeights: Seq[(Byte, Double)] = Seq.empty[(Byte, Double)]) {
+ // empty => Count
+ lazy val rankKeysWeightsMap = keySeqAndWeights.toMap
+
+ def defaultKey() = {
+ this.keySeqAndWeights = List((LabelMeta.countSeq, 1.0))
+ this
+ }
+
+ def toHashKeyBytes(): Array[Byte] = {
+ var bytes = Array.empty[Byte]
+ keySeqAndWeights.map { case (key, weight) =>
+ bytes = Bytes.add(bytes, Array.fill(1)(key), Bytes.toBytes(weight))
+ }
+ bytes
+ }
+}
+
+object QueryParam {
+ lazy val Empty = QueryParam(LabelWithDirection(0, 0))
+ lazy val DefaultThreshold = Double.MinValue
+ val Delimiter = ","
+}
+
+case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System.currentTimeMillis()) {
+
+ import HBaseSerializable._
+ import Query.DuplicatePolicy
+ import Query.DuplicatePolicy._
+
+ lazy val label = Label.findById(labelWithDir.labelId)
+ val DefaultKey = LabelIndex.DefaultSeq
+ val fullKey = DefaultKey
+
+ var labelOrderSeq = fullKey
+
+ var sample = -1
+ var limit = 10
+ var offset = 0
+ var rank = new RankParam(labelWithDir.labelId, List(LabelMeta.countSeq -> 1))
+
+ var duration: Option[(Long, Long)] = None
+ var isInverted: Boolean = false
+
+ var columnRangeFilter: ColumnRangeFilter = null
+
+ var hasFilters: Map[Byte, InnerValLike] = Map.empty[Byte, InnerValLike]
+ var where: Try[Where] = Success(WhereParser.success)
+ var whereRawOpt: Option[String] = None
+ var duplicatePolicy = DuplicatePolicy.First
+ var rpcTimeoutInMillis = 1000
+ var maxAttempt = 2
+ var includeDegree = false
+ var tgtVertexInnerIdOpt: Option[InnerValLike] = None
+ var cacheTTLInMillis: Long = -1L
+ var threshold = QueryParam.DefaultThreshold
+ var timeDecay: Option[TimeDecay] = None
+ var transformer: EdgeTransformer = EdgeTransformer(this, EdgeTransformer.DefaultJson)
+ var scorePropagateOp: String = "multiply"
+ var exclude = false
+ var include = false
+ var shouldNormalize= false
+ var cursorOpt: Option[String] = None
+
+ var columnRangeFilterMinBytes = Array.empty[Byte]
+ var columnRangeFilterMaxBytes = Array.empty[Byte]
+
+ lazy val srcColumnWithDir = label.srcColumnWithDir(labelWithDir.dir)
+ lazy val tgtColumnWithDir = label.tgtColumnWithDir(labelWithDir.dir)
+
+ def toBytes(idxSeq: Byte, offset: Int, limit: Int, isInverted: Boolean): Array[Byte] = {
+ val front = Array[Byte](idxSeq, if (isInverted) 1.toByte else 0.toByte)
+ Bytes.add(front, Bytes.toBytes((offset.toLong << 32 | limit)))
+ }
+
+ /**
+ * consider only I/O specific parameters.
+ * properties that is used on Graph.filterEdges should not be considered.
+ * @param bytes
+ * @return
+ */
+ def toCacheKey(bytes: Array[Byte]): Long = {
+ val hashBytes = toCacheKeyRaw(bytes)
+ Hashing.murmur3_128().hashBytes(hashBytes).asLong()
+// MurmurHash3.bytesHash(hashBytes)
+ }
+
+ def toCacheKeyRaw(bytes: Array[Byte]): Array[Byte] = {
+ val transformBytes = transformer.toHashKeyBytes
+ //TODO: change this to binrary format.
+ val whereBytes = Bytes.toBytes(whereRawOpt.getOrElse(""))
+ val durationBytes = duration.map { case (min, max) =>
+ val minTs = min / cacheTTLInMillis
+ val maxTs = max / cacheTTLInMillis
+ Bytes.add(Bytes.toBytes(minTs), Bytes.toBytes(maxTs))
+ } getOrElse Array.empty[Byte]
+// Bytes.toBytes(duration.toString)
+ val conditionBytes = Bytes.add(transformBytes, whereBytes, durationBytes)
+ Bytes.add(Bytes.add(bytes, labelWithDir.bytes, toBytes(labelOrderSeq, offset, limit, isInverted)), rank.toHashKeyBytes(),
+ Bytes.add(columnRangeFilterMinBytes, columnRangeFilterMaxBytes, conditionBytes))
+ }
+
+ def isInverted(isInverted: Boolean): QueryParam = {
+ this.isInverted = isInverted
+ this
+ }
+
+ def labelOrderSeq(labelOrderSeq: Byte): QueryParam = {
+ this.labelOrderSeq = labelOrderSeq
+ this
+ }
+
+ def sample(n: Int): QueryParam = {
+ this.sample = n
+ this
+ }
+
+ def limit(offset: Int, limit: Int): QueryParam = {
+ /** since degree info is located on first always */
+ if (offset == 0 && this.columnRangeFilter == null) {
+ this.limit = limit + 1
+ this.offset = offset
+ } else {
+ this.limit = limit
+ this.offset = offset + 1
+ }
+ // this.columnPaginationFilter = new ColumnPaginationFilter(this.limit, this.offset)
+ this
+ }
+
+ def interval(fromTo: Option[(Seq[(Byte, InnerValLike)], Seq[(Byte, InnerValLike)])]): QueryParam = {
+ fromTo match {
+ case Some((from, to)) => interval(from, to)
+ case _ => this
+ }
+ }
+
+ def interval(from: Seq[(Byte, InnerValLike)], to: Seq[(Byte, InnerValLike)]): QueryParam = {
+ // val len = label.orderTypes.size.toByte
+ // val len = label.extraIndicesMap(labelOrderSeq).sortKeyTypes.size.toByte
+ // logger.error(s"indicesMap: ${label.indicesMap(labelOrderSeq)}")
+ val len = label.indicesMap(labelOrderSeq).sortKeyTypes.size.toByte
+
+ val minMetaByte = InnerVal.minMetaByte
+ // val maxMetaByte = InnerVal.maxMetaByte
+ val maxMetaByte = -1.toByte
+ val toVal = Bytes.add(propsToBytes(to), Array.fill(1)(minMetaByte))
+ //FIXME
+ val fromVal = Bytes.add(propsToBytes(from), Array.fill(10)(maxMetaByte))
+ toVal(0) = len
+ fromVal(0) = len
+ val maxBytes = fromVal
+ val minBytes = toVal
+ this.columnRangeFilterMaxBytes = maxBytes
+ this.columnRangeFilterMinBytes = minBytes
+ val rangeFilter = new ColumnRangeFilter(minBytes, true, maxBytes, true)
+ this.columnRangeFilter = rangeFilter
+ this
+ }
+
+ def duration(minMaxTs: Option[(Long, Long)]): QueryParam = {
+ minMaxTs match {
+ case Some((minTs, maxTs)) => duration(minTs, maxTs)
+ case _ => this
+ }
+ }
+
+ def duration(minTs: Long, maxTs: Long): QueryParam = {
+ this.duration = Some((minTs, maxTs))
+ this
+ }
+
+ def rank(r: RankParam): QueryParam = {
+ this.rank = r
+ this
+ }
+
+ def exclude(filterOut: Boolean): QueryParam = {
+ this.exclude = filterOut
+ this
+ }
+
+ def include(filterIn: Boolean): QueryParam = {
+ this.include = filterIn
+ this
+ }
+
+ def has(hasFilters: Map[Byte, InnerValLike]): QueryParam = {
+ this.hasFilters = hasFilters
+ this
+ }
+
+ def where(whereTry: Try[Where]): QueryParam = {
+ this.where = whereTry
+ this
+ }
+
+ def duplicatePolicy(policy: Option[DuplicatePolicy]): QueryParam = {
+ this.duplicatePolicy = policy.getOrElse(DuplicatePolicy.First)
+ this
+ }
+
+ def rpcTimeout(millis: Int): QueryParam = {
+ this.rpcTimeoutInMillis = millis
+ this
+ }
+
+ def maxAttempt(attempt: Int): QueryParam = {
+ this.maxAttempt = attempt
+ this
+ }
+
+ def includeDegree(includeDegree: Boolean): QueryParam = {
+ this.includeDegree = includeDegree
+ this
+ }
+
+ def tgtVertexInnerIdOpt(other: Option[InnerValLike]): QueryParam = {
+ this.tgtVertexInnerIdOpt = other
+ this
+ }
+
+ def cacheTTLInMillis(other: Long): QueryParam = {
+ this.cacheTTLInMillis = other
+ this
+ }
+
+ def timeDecay(other: Option[TimeDecay]): QueryParam = {
+ this.timeDecay = other
+ this
+ }
+
+ def threshold(other: Double): QueryParam = {
+ this.threshold = other
+ this
+ }
+
+ def transformer(other: Option[JsValue]): QueryParam = {
+ other match {
+ case Some(js) => this.transformer = EdgeTransformer(this, js)
+ case None =>
+ }
+ this
+ }
+
+ def scorePropagateOp(scorePropagateOp: String): QueryParam = {
+ this.scorePropagateOp = scorePropagateOp
+ this
+ }
+
+ def shouldNormalize(shouldNormalize: Boolean): QueryParam = {
+ this.shouldNormalize = shouldNormalize
+ this
+ }
+
+ def whereRawOpt(sqlOpt: Option[String]): QueryParam = {
+ this.whereRawOpt = sqlOpt
+ this
+ }
+
+ def cursorOpt(cursorOpt: Option[String]): QueryParam = {
+ this.cursorOpt = cursorOpt
+ this
+ }
+
+ def isSnapshotEdge = tgtVertexInnerIdOpt.isDefined
+
+ override def toString = {
+ List(label.label, labelOrderSeq, offset, limit, rank,
+ duration, isInverted, exclude, include, hasFilters).mkString("\t")
+ // duration, isInverted, exclude, include, hasFilters, outputFields).mkString("\t")
+ }
+
+ //
+ // def buildGetRequest(srcVertex: Vertex) = {
+ // val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
+ // val (srcInnerId, tgtInnerId) = tgtVertexInnerIdOpt match {
+ // case Some(tgtVertexInnerId) => // _to is given.
+ // /** we use toInvertedEdgeHashLike so dont need to swap src, tgt */
+ // val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ // val tgt = InnerVal.convertVersion(tgtVertexInnerId, tgtColumn.columnType, label.schemaVersion)
+ // (src, tgt)
+ // case None =>
+ // val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ // (src, src)
+ // }
+ //
+ // val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
+ // val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
+ // val edge = Edge(srcV, tgtV, labelWithDir)
+ //
+ // val get = if (tgtVertexInnerIdOpt.isDefined) {
+ // val snapshotEdge = edge.toInvertedEdgeHashLike
+ // val kv = snapshotEdge.kvs.head
+ // new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
+ // } else {
+ // val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == labelOrderSeq)
+ // assert(indexedEdgeOpt.isDefined)
+ // val indexedEdge = indexedEdgeOpt.get
+ // val kv = indexedEdge.kvs.head
+ // val table = label.hbaseTableName.getBytes
+ // //kv.table //
+ // val rowKey = kv.row // indexedEdge.rowKey.bytes
+ // val cf = edgeCf
+ // new GetRequest(table, rowKey, cf)
+ // }
+ //
+ // val (minTs, maxTs) = duration.getOrElse((0L, Long.MaxValue))
+ //
+ // get.maxVersions(1)
+ // get.setFailfast(true)
+ // get.setMaxResultsPerColumnFamily(limit)
+ // get.setRowOffsetPerColumnFamily(offset)
+ // get.setMinTimestamp(minTs)
+ // get.setMaxTimestamp(maxTs)
+ // get.setTimeout(rpcTimeoutInMillis)
+ // if (columnRangeFilter != null) get.setFilter(columnRangeFilter)
+ // // get.setMaxAttempt(maxAttempt.toByte)
+ // // get.setRpcTimeout(rpcTimeoutInMillis)
+ //
+ // // if (columnRangeFilter != null) get.filter(columnRangeFilter)
+ // // logger.debug(s"Get: $get, $offset, $limit")
+ //
+ // get
+ // }
+}
+
+case class TimeDecay(initial: Double = 1.0,
+ lambda: Double = 0.1,
+ timeUnit: Double = 60 * 60 * 24,
+ labelMetaSeq: Byte = LabelMeta.timeStampSeq) {
+ def decay(diff: Double): Double = {
+ //FIXME
+ val ret = initial * Math.pow(1.0 - lambda, diff / timeUnit)
+ // logger.debug(s"$initial, $lambda, $timeUnit, $diff, ${diff / timeUnit}, $ret")
+ ret
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
new file mode 100644
index 0000000..550c4c9
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -0,0 +1,42 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs}
+
+import scala.collection.Seq
+
+object QueryResult {
+ def fromVertices(query: Query): Seq[QueryRequestWithResult] = {
+ if (query.steps.isEmpty || query.steps.head.queryParams.isEmpty) {
+ Seq.empty
+ } else {
+ val queryParam = query.steps.head.queryParams.head
+ val label = queryParam.label
+ val currentTs = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq ->
+ InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs))
+ for {
+ vertex <- query.vertices
+ } yield {
+ val edge = Edge(vertex, vertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
+ val edgeWithScore = EdgeWithScore(edge, Graph.DefaultScore)
+ QueryRequestWithResult(QueryRequest(query, -1, vertex, queryParam),
+ QueryResult(edgeWithScoreLs = Seq(edgeWithScore)))
+ }
+ }
+ }
+}
+case class QueryRequestWithResult(queryRequest: QueryRequest, queryResult: QueryResult)
+
+case class QueryRequest(query: Query,
+ stepIdx: Int,
+ vertex: Vertex,
+ queryParam: QueryParam)
+
+
+case class QueryResult(edgeWithScoreLs: Seq[EdgeWithScore] = Nil,
+ tailCursor: Array[Byte] = Array.empty,
+ timestamp: Long = System.currentTimeMillis(),
+ isFailure: Boolean = false)
+
+case class EdgeWithScore(edge: Edge, score: Double)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Vertex.scala b/s2core/src/main/scala/org/apache/s2graph/core/Vertex.scala
new file mode 100644
index 0000000..f4f40b7
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Vertex.scala
@@ -0,0 +1,101 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.mysqls.{ColumnMeta, Service, ServiceColumn}
+import org.apache.s2graph.core.types.{InnerVal, InnerValLike, SourceVertexId, VertexId}
+import play.api.libs.json.Json
+
+case class Vertex(id: VertexId,
+ ts: Long = System.currentTimeMillis(),
+ props: Map[Int, InnerValLike] = Map.empty[Int, InnerValLike],
+ op: Byte = 0,
+ belongLabelIds: Seq[Int] = Seq.empty) extends GraphElement {
+
+ val innerId = id.innerId
+
+ def schemaVer = serviceColumn.schemaVersion
+
+ def serviceColumn = ServiceColumn.findById(id.colId)
+
+ def service = Service.findById(serviceColumn.serviceId)
+
+ lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
+
+ def defaultProps = Map(ColumnMeta.lastModifiedAtColumnSeq.toInt -> InnerVal.withLong(ts, schemaVer))
+
+ // lazy val kvs = Graph.client.vertexSerializer(this).toKeyValues
+
+ /** TODO: make this as configurable */
+ override def serviceName = service.serviceName
+
+ override def isAsync = false
+
+ override def queueKey = Seq(ts.toString, serviceName).mkString("|")
+
+ override def queuePartitionKey = id.innerId.toString
+
+ def propsWithName = for {
+ (seq, v) <- props
+ meta <- ColumnMeta.findByIdAndSeq(id.colId, seq.toByte)
+ } yield (meta.name -> v.toString)
+
+ // /** only used by bulk loader */
+ // def buildPuts(): List[Put] = {
+ // // logger.error(s"put: $this => $rowKey")
+ //// val put = new Put(rowKey.bytes)
+ //// for ((q, v) <- qualifiersWithValues) {
+ //// put.addColumn(vertexCf, q, ts, v)
+ //// }
+ //// List(put)
+ // val kv = kvs.head
+ // val put = new Put(kv.row)
+ // kvs.map { kv =>
+ // put.addColumn(kv.cf, kv.qualifier, kv.timestamp, kv.value)
+ // }
+ // List(put)
+ // }
+
+ def toEdgeVertex() = Vertex(SourceVertexId(id.colId, innerId), ts, props, op)
+
+
+ override def hashCode() = {
+ val hash = id.hashCode()
+ // logger.debug(s"Vertex.hashCode: $this -> $hash")
+ hash
+ }
+
+ override def equals(obj: Any) = {
+ obj match {
+ case otherVertex: Vertex =>
+ val ret = id == otherVertex.id
+ // logger.debug(s"Vertex.equals: $this, $obj => $ret")
+ ret
+ case _ => false
+ }
+ }
+
+ def withProps(newProps: Map[Int, InnerValLike]) = Vertex(id, ts, newProps, op)
+
+ def toLogString(): String = {
+ val (serviceName, columnName) =
+ if (!id.storeColId) ("", "")
+ else (serviceColumn.service.serviceName, serviceColumn.columnName)
+
+ if (propsWithName.nonEmpty)
+ Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName, Json.toJson(propsWithName)).mkString("\t")
+ else
+ Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
+ }
+}
+
+object Vertex {
+
+ def toPropKey(labelId: Int): Int = Byte.MaxValue + labelId
+
+ def toLabelId(propKey: Int): Int = propKey - Byte.MaxValue
+
+ def isLabelId(propKey: Int): Boolean = propKey > Byte.MaxValue
+
+ // val emptyVertex = Vertex(new CompositeId(CompositeId.defaultColId, CompositeId.defaultInnerId, false, true),
+ // System.currentTimeMillis())
+ def fromString(s: String): Option[Vertex] = Graph.toVertex(s)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Bucket.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Bucket.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Bucket.scala
new file mode 100644
index 0000000..c92c2a2
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Bucket.scala
@@ -0,0 +1,66 @@
+package org.apache.s2graph.core.mysqls
+
+/**
+ * Created by shon on 8/5/15.
+ */
+
+import scalikejdbc._
+
+object Bucket extends Model[Bucket] {
+
+ val rangeDelimiter = "~"
+ val INVALID_BUCKET_EXCEPTION = new RuntimeException("invalid bucket.")
+
+ def apply(rs: WrappedResultSet): Bucket = {
+ Bucket(rs.intOpt("id"),
+ rs.int("experiment_id"),
+ rs.string("modular"),
+ rs.string("http_verb"),
+ rs.string("api_path"),
+ rs.string("request_body"),
+ rs.int("timeout"),
+ rs.string("impression_id"),
+ rs.boolean("is_graph_query"),
+ rs.boolean("is_empty"))
+ }
+
+ def finds(experimentId: Int)(implicit session: DBSession = AutoSession): List[Bucket] = {
+ val cacheKey = "experimentId=" + experimentId
+ withCaches(cacheKey) {
+ sql"""select * from buckets where experiment_id = $experimentId"""
+ .map { rs => Bucket(rs) }.list().apply()
+ }
+ }
+
+ def toRange(str: String): Option[(Int, Int)] = {
+ val range = str.split(rangeDelimiter)
+ if (range.length == 2) Option(range.head.toInt, range.last.toInt)
+ else None
+ }
+
+ def findByImpressionId(impressionId: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Bucket] = {
+ val cacheKey = "impressionId=" + impressionId
+ val sql = sql"""select * from buckets where impression_id=$impressionId"""
+ .map { rs => Bucket(rs)}
+ if (useCache) {
+ withCache(cacheKey) {
+ sql.single().apply()
+ }
+ } else {
+ sql.single().apply()
+ }
+ }
+}
+
+case class Bucket(id: Option[Int],
+ experimentId: Int,
+ modular: String,
+ httpVerb: String, apiPath: String,
+ requestBody: String, timeout: Int, impressionId: String,
+ isGraphQuery: Boolean = true,
+ isEmpty: Boolean = false) {
+
+ import Bucket._
+
+ lazy val rangeOpt = toRange(modular)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ColumnMeta.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ColumnMeta.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ColumnMeta.scala
new file mode 100644
index 0000000..a726f40
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ColumnMeta.scala
@@ -0,0 +1,110 @@
+package org.apache.s2graph.core.mysqls
+
+import play.api.libs.json.Json
+import scalikejdbc._
+
+object ColumnMeta extends Model[ColumnMeta] {
+
+ val timeStampSeq = 0.toByte
+ val countSeq = -1.toByte
+ val lastModifiedAtColumnSeq = 0.toByte
+ val lastModifiedAtColumn = ColumnMeta(Some(0), 0, "lastModifiedAt", lastModifiedAtColumnSeq, "long")
+ val maxValue = Byte.MaxValue
+
+ def apply(rs: WrappedResultSet): ColumnMeta = {
+ ColumnMeta(Some(rs.int("id")), rs.int("column_id"), rs.string("name"), rs.byte("seq"), rs.string("data_type").toLowerCase())
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession) = {
+ // val cacheKey = s"id=$id"
+ val cacheKey = "id=" + id
+ withCache(cacheKey) {
+ sql"""select * from column_metas where id = ${id}""".map { rs => ColumnMeta(rs) }.single.apply
+ }.get
+ }
+
+ def findAllByColumn(columnId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
+ // val cacheKey = s"columnId=$columnId"
+ val cacheKey = "columnId=" + columnId
+ if (useCache) {
+ withCaches(cacheKey)( sql"""select *from column_metas where column_id = ${columnId} order by seq ASC"""
+ .map { rs => ColumnMeta(rs) }.list.apply())
+ } else {
+ sql"""select * from column_metas where column_id = ${columnId} order by seq ASC"""
+ .map { rs => ColumnMeta(rs) }.list.apply()
+ }
+ }
+
+ def findByName(columnId: Int, name: String)(implicit session: DBSession = AutoSession) = {
+ // val cacheKey = s"columnId=$columnId:name=$name"
+ val cacheKey = "columnId=" + columnId + ":name=" + name
+ withCache(cacheKey)( sql"""select * from column_metas where column_id = ${columnId} and name = ${name}"""
+ .map { rs => ColumnMeta(rs) }.single.apply())
+ }
+
+ def insert(columnId: Int, name: String, dataType: String)(implicit session: DBSession = AutoSession) = {
+ val ls = findAllByColumn(columnId, false)
+ val seq = ls.size + 1
+ if (seq <= maxValue) {
+ sql"""insert into column_metas(column_id, name, seq, data_type)
+ select ${columnId}, ${name}, ${seq}, ${dataType}"""
+ .updateAndReturnGeneratedKey.apply()
+ }
+ }
+
+ def findOrInsert(columnId: Int, name: String, dataType: String)(implicit session: DBSession = AutoSession): ColumnMeta = {
+ findByName(columnId, name) match {
+ case Some(c) => c
+ case None =>
+ insert(columnId, name, dataType)
+ expireCache(s"columnId=$columnId:name=$name")
+ findByName(columnId, name).get
+ }
+ }
+
+ def findByIdAndSeq(columnId: Int, seq: Byte, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
+ val cacheKey = "columnId=" + columnId + ":seq=" + seq
+ lazy val columnMetaOpt = sql"""
+ select * from column_metas where column_id = ${columnId} and seq = ${seq}
+ """.map { rs => ColumnMeta(rs) }.single.apply()
+
+ if (useCache) withCache(cacheKey)(columnMetaOpt)
+ else columnMetaOpt
+ }
+
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val columnMeta = findById(id)
+ val (columnId, name) = (columnMeta.columnId, columnMeta.name)
+ sql"""delete from column_metas where id = ${id}""".execute.apply()
+ val cacheKeys = List(s"id=$id", s"columnId=$columnId:name=$name", s"colunmId=$columnId")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ }
+
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from column_metas""".map { rs => ColumnMeta(rs) }.list().apply()
+
+ putsToCache(ls.map { x =>
+ val cacheKey = s"id=${x.id.get}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ val cacheKey = s"columnId=${x.columnId}:name=${x.name}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ val cacheKey = s"columnId=${x.columnId}:seq=${x.seq}"
+ (cacheKey -> x)
+ })
+ putsToCaches(ls.groupBy(x => x.columnId).map { case (columnId, ls) =>
+ val cacheKey = s"columnId=${columnId}"
+ (cacheKey -> ls)
+ }.toList)
+ }
+}
+
+case class ColumnMeta(id: Option[Int], columnId: Int, name: String, seq: Byte, dataType: String) {
+ lazy val toJson = Json.obj("name" -> name, "dataType" -> dataType)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Experiment.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Experiment.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Experiment.scala
new file mode 100644
index 0000000..3b5fefb
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Experiment.scala
@@ -0,0 +1,77 @@
+package org.apache.s2graph.core.mysqls
+
+import org.apache.s2graph.core.GraphUtil
+import scalikejdbc._
+
+import scala.util.Random
+
+object Experiment extends Model[Experiment] {
+ val impressionKey = "S2-Impression-Id"
+
+ def apply(rs: WrappedResultSet): Experiment = {
+ Experiment(rs.intOpt("id"),
+ rs.int("service_id"),
+ rs.string("name"),
+ rs.string("description"),
+ rs.string("experiment_type"),
+ rs.int("total_modular"))
+ }
+
+ def finds(serviceId: Int)(implicit session: DBSession = AutoSession): List[Experiment] = {
+ val cacheKey = "serviceId=" + serviceId
+ withCaches(cacheKey) {
+ sql"""select * from experiments where service_id = ${serviceId}"""
+ .map { rs => Experiment(rs) }.list().apply()
+ }
+ }
+
+ def findBy(serviceId: Int, name: String)(implicit session: DBSession = AutoSession): Option[Experiment] = {
+ val cacheKey = "serviceId=" + serviceId + ":name=" + name
+ withCache(cacheKey) {
+ sql"""select * from experiments where service_id = ${serviceId} and name = ${name}"""
+ .map { rs => Experiment(rs) }.single.apply
+ }
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession): Option[Experiment] = {
+ val cacheKey = "id=" + id
+ withCache(cacheKey)(
+ sql"""select * from experiments where id = ${id}"""
+ .map { rs => Experiment(rs) }.single.apply
+ )
+ }
+}
+
+case class Experiment(id: Option[Int],
+ serviceId: Int,
+ name: String,
+ description: String,
+ experimentType: String,
+ totalModular: Int) {
+
+ def buckets = Bucket.finds(id.get)
+
+ def rangeBuckets = for {
+ bucket <- buckets
+ range <- bucket.rangeOpt
+ } yield range -> bucket
+
+
+ def findBucket(uuid: String, impIdOpt: Option[String] = None): Option[Bucket] = {
+ impIdOpt match {
+ case Some(impId) => Bucket.findByImpressionId(impId)
+ case None =>
+ val seed = experimentType match {
+ case "u" => (GraphUtil.murmur3(uuid) % totalModular) + 1
+ case _ => Random.nextInt(totalModular) + 1
+ }
+ findBucket(seed)
+ }
+ }
+
+ def findBucket(uuidMod: Int): Option[Bucket] = {
+ rangeBuckets.find { case ((from, to), bucket) =>
+ from <= uuidMod && uuidMod <= to
+ }.map(_._2)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala
new file mode 100644
index 0000000..0958c2c
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Label.scala
@@ -0,0 +1,372 @@
+package org.apache.s2graph.core.mysqls
+
+import org.apache.s2graph.core.GraphExceptions.ModelNotFoundException
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{GraphExceptions, GraphUtil, JSONParser}
+import play.api.libs.json.Json
+import scalikejdbc._
+
+object Label extends Model[Label] {
+
+ val maxHBaseTableNames = 2
+
+ def apply(rs: WrappedResultSet): Label = {
+ Label(Option(rs.int("id")), rs.string("label"),
+ rs.int("src_service_id"), rs.string("src_column_name"), rs.string("src_column_type"),
+ rs.int("tgt_service_id"), rs.string("tgt_column_name"), rs.string("tgt_column_type"),
+ rs.boolean("is_directed"), rs.string("service_name"), rs.int("service_id"), rs.string("consistency_level"),
+ rs.string("hbase_table_name"), rs.intOpt("hbase_table_ttl"), rs.string("schema_version"), rs.boolean("is_async"), rs.string("compressionAlgorithm"))
+ }
+
+ def deleteAll(label: Label)(implicit session: DBSession) = {
+ val id = label.id
+ LabelMeta.findAllByLabelId(id.get, false).foreach { x => LabelMeta.delete(x.id.get) }
+ LabelIndex.findByLabelIdAll(id.get, false).foreach { x => LabelIndex.delete(x.id.get) }
+ Label.delete(id.get)
+ }
+
+ def findByName(labelName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Label] = {
+ val cacheKey = "label=" + labelName
+ lazy val labelOpt =
+ sql"""
+ select *
+ from labels
+ where label = ${labelName}""".map { rs => Label(rs) }.single.apply()
+
+ if (useCache) withCache(cacheKey)(labelOpt)
+ else labelOpt
+ }
+
+ def insert(label: String,
+ srcServiceId: Int,
+ srcColumnName: String,
+ srcColumnType: String,
+ tgtServiceId: Int,
+ tgtColumnName: String,
+ tgtColumnType: String,
+ isDirected: Boolean,
+ serviceName: String,
+ serviceId: Int,
+ consistencyLevel: String,
+ hTableName: String,
+ hTableTTL: Option[Int],
+ schemaVersion: String,
+ isAsync: Boolean,
+ compressionAlgorithm: String)(implicit session: DBSession = AutoSession) = {
+ sql"""
+ insert into labels(label,
+ src_service_id, src_column_name, src_column_type,
+ tgt_service_id, tgt_column_name, tgt_column_type,
+ is_directed, service_name, service_id, consistency_level, hbase_table_name, hbase_table_ttl, schema_version, is_async, compressionAlgorithm)
+ values (${label},
+ ${srcServiceId}, ${srcColumnName}, ${srcColumnType},
+ ${tgtServiceId}, ${tgtColumnName}, ${tgtColumnType},
+ ${isDirected}, ${serviceName}, ${serviceId}, ${consistencyLevel}, ${hTableName}, ${hTableTTL},
+ ${schemaVersion}, ${isAsync}, ${compressionAlgorithm})
+ """
+ .updateAndReturnGeneratedKey.apply()
+ }
+
+ def findByIdOpt(id: Int)(implicit session: DBSession = AutoSession): Option[Label] = {
+ val cacheKey = "id=" + id
+ withCache(cacheKey)(
+ sql"""
+ select *
+ from labels
+ where id = ${id}"""
+ .map { rs => Label(rs) }.single.apply())
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession): Label = {
+ val cacheKey = "id=" + id
+ withCache(cacheKey)(
+ sql"""
+ select *
+ from labels
+ where id = ${id}"""
+ .map { rs => Label(rs) }.single.apply()).get
+ }
+
+ def findByTgtColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ val cacheKey = "tgtColumnId=" + columnId
+ val col = ServiceColumn.findById(columnId)
+ withCaches(cacheKey)(
+ sql"""
+ select *
+ from labels
+ where tgt_column_name = ${col.columnName}
+ and service_id = ${col.serviceId}
+ """.map { rs => Label(rs) }.list().apply())
+ }
+
+ def findBySrcColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ val cacheKey = "srcColumnId=" + columnId
+ val col = ServiceColumn.findById(columnId)
+ withCaches(cacheKey)(
+ sql"""
+ select *
+ from labels
+ where src_column_name = ${col.columnName}
+ and service_id = ${col.serviceId}
+ """.map { rs => Label(rs) }.list().apply())
+ }
+
+ def findBySrcServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ val cacheKey = "srcServiceId=" + serviceId
+ withCaches(cacheKey)(
+ sql"""select * from labels where src_service_id = ${serviceId}""".map { rs => Label(rs) }.list().apply
+ )
+ }
+
+ def findByTgtServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ val cacheKey = "tgtServiceId=" + serviceId
+ withCaches(cacheKey)(
+ sql"""select * from labels where tgt_service_id = ${serviceId}""".map { rs => Label(rs) }.list().apply
+ )
+ }
+
+ def insertAll(labelName: String, srcServiceName: String, srcColumnName: String, srcColumnType: String,
+ tgtServiceName: String, tgtColumnName: String, tgtColumnType: String,
+ isDirected: Boolean = true,
+ serviceName: String,
+ indices: Seq[Index],
+ metaProps: Seq[Prop],
+ consistencyLevel: String,
+ hTableName: Option[String],
+ hTableTTL: Option[Int],
+ schemaVersion: String,
+ isAsync: Boolean,
+ compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Label = {
+
+ val srcServiceOpt = Service.findByName(srcServiceName, useCache = false)
+ val tgtServiceOpt = Service.findByName(tgtServiceName, useCache = false)
+ val serviceOpt = Service.findByName(serviceName, useCache = false)
+ if (srcServiceOpt.isEmpty) throw new RuntimeException(s"source service $srcServiceName is not created.")
+ if (tgtServiceOpt.isEmpty) throw new RuntimeException(s"target service $tgtServiceName is not created.")
+ if (serviceOpt.isEmpty) throw new RuntimeException(s"service $serviceName is not created.")
+
+ val newLabel = for {
+ srcService <- srcServiceOpt
+ tgtService <- tgtServiceOpt
+ service <- serviceOpt
+ } yield {
+ val srcServiceId = srcService.id.get
+ val tgtServiceId = tgtService.id.get
+ val serviceId = service.id.get
+
+ /** insert serviceColumn */
+ val srcCol = ServiceColumn.findOrInsert(srcServiceId, srcColumnName, Some(srcColumnType), schemaVersion)
+ val tgtCol = ServiceColumn.findOrInsert(tgtServiceId, tgtColumnName, Some(tgtColumnType), schemaVersion)
+
+ if (srcCol.columnType != srcColumnType) throw new RuntimeException(s"source service column type not matched ${srcCol.columnType} != ${srcColumnType}")
+ if (tgtCol.columnType != tgtColumnType) throw new RuntimeException(s"target service column type not matched ${tgtCol.columnType} != ${tgtColumnType}")
+
+ /** create label */
+ Label.findByName(labelName, useCache = false).getOrElse {
+
+ val createdId = insert(labelName, srcServiceId, srcColumnName, srcColumnType,
+ tgtServiceId, tgtColumnName, tgtColumnType, isDirected, serviceName, serviceId, consistencyLevel,
+ hTableName.getOrElse(service.hTableName), hTableTTL.orElse(service.hTableTTL), schemaVersion, isAsync, compressionAlgorithm).toInt
+
+ val labelMetaMap = metaProps.map { case Prop(propName, defaultValue, dataType) =>
+ val labelMeta = LabelMeta.findOrInsert(createdId, propName, defaultValue, dataType)
+ (propName -> labelMeta.seq)
+ }.toMap ++ LabelMeta.reservedMetas.map (labelMeta => labelMeta.name -> labelMeta.seq).toMap
+
+ if (indices.isEmpty) {
+ // make default index with _PK, _timestamp, 0
+ LabelIndex.findOrInsert(createdId, LabelIndex.DefaultName, LabelIndex.DefaultMetaSeqs.toList, "none")
+ } else {
+ indices.foreach { index =>
+ val metaSeq = index.propNames.map { name => labelMetaMap(name) }
+ LabelIndex.findOrInsert(createdId, index.name, metaSeq.toList, "none")
+ }
+ }
+
+ val cacheKeys = List(s"id=$createdId", s"label=$labelName")
+ val ret = findByName(labelName, useCache = false).get
+ putsToCache(cacheKeys.map(k => k -> ret))
+ ret
+ }
+ }
+
+ newLabel.getOrElse(throw new RuntimeException("failed to create label"))
+ }
+
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from labels""".map { rs => Label(rs) }.list().apply()
+ putsToCache(ls.map { x =>
+ val cacheKey = s"id=${x.id.get}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ val cacheKey = s"label=${x.label}"
+ (cacheKey -> x)
+ })
+ }
+
+ def updateName(oldName: String, newName: String)(implicit session: DBSession = AutoSession) = {
+ logger.info(s"rename label: $oldName -> $newName")
+ sql"""update labels set label = ${newName} where label = ${oldName}""".update.apply()
+ }
+
+ def updateHTableName(labelName: String, newHTableName: String)(implicit session: DBSession = AutoSession) = {
+ logger.info(s"update HTable of label $labelName to $newHTableName")
+ val cnt = sql"""update labels set hbase_table_name = $newHTableName where label = $labelName""".update().apply()
+ val label = Label.findByName(labelName, useCache = false).get
+
+ val cacheKeys = List(s"id=${label.id}", s"label=${label.label}")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ cnt
+ }
+
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val label = findById(id)
+ logger.info(s"delete label: $label")
+ val cnt = sql"""delete from labels where id = ${label.id.get}""".update().apply()
+ val cacheKeys = List(s"id=$id", s"label=${label.label}")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ cnt
+ }
+}
+
+case class Label(id: Option[Int], label: String,
+ srcServiceId: Int, srcColumnName: String, srcColumnType: String,
+ tgtServiceId: Int, tgtColumnName: String, tgtColumnType: String,
+ isDirected: Boolean = true, serviceName: String, serviceId: Int, consistencyLevel: String = "strong",
+ hTableName: String, hTableTTL: Option[Int],
+ schemaVersion: String, isAsync: Boolean = false,
+ compressionAlgorithm: String) extends JSONParser {
+ def metas = LabelMeta.findAllByLabelId(id.get)
+
+ def metaSeqsToNames = metas.map(x => (x.seq, x.name)) toMap
+
+ // lazy val firstHBaseTableName = hbaseTableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME)
+ lazy val srcService = Service.findById(srcServiceId)
+ lazy val tgtService = Service.findById(tgtServiceId)
+ lazy val service = Service.findById(serviceId)
+ /**
+ * TODO
+ * change this to apply hbase table from target serviceName
+ */
+ // lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.tableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME))
+ // lazy val (hbaseZkAddr, hbaseTableName) = (Config.HBASE_ZOOKEEPER_QUORUM, hTableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME))
+ // lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, hTableName.split(",").headOption.getOrElse(GraphConnection.getConfVal("hbase.table.name")))
+ lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, hTableName.split(",").head)
+
+ lazy val srcColumn = ServiceColumn.find(srcServiceId, srcColumnName).getOrElse(throw ModelNotFoundException("Source column not found"))
+ lazy val tgtColumn = ServiceColumn.find(tgtServiceId, tgtColumnName).getOrElse(throw ModelNotFoundException("Target column not found"))
+
+ lazy val direction = if (isDirected) "out" else "undirected"
+ lazy val defaultIndex = LabelIndex.findByLabelIdAndSeq(id.get, LabelIndex.DefaultSeq)
+
+ //TODO: Make sure this is correct
+ lazy val indices = LabelIndex.findByLabelIdAll(id.get, useCache = true)
+ lazy val indicesMap = indices.map(idx => (idx.seq, idx)) toMap
+ lazy val indexSeqsMap = indices.map(idx => (idx.metaSeqs, idx)) toMap
+ lazy val indexNameMap = indices.map(idx => (idx.name, idx)) toMap
+ lazy val extraIndices = indices.filter(idx => defaultIndex.isDefined && idx.id.get != defaultIndex.get.id.get)
+ // indices filterNot (_.id.get == defaultIndex.get.id.get)
+ lazy val extraIndicesMap = extraIndices.map(idx => (idx.seq, idx)) toMap
+
+ lazy val metaProps = LabelMeta.reservedMetas.map { m =>
+ if (m == LabelMeta.to) m.copy(dataType = tgtColumnType)
+ else if (m == LabelMeta.from) m.copy(dataType = srcColumnType)
+ else m
+ } ::: LabelMeta.findAllByLabelId(id.get, useCache = true)
+
+ lazy val metaPropsInner = LabelMeta.reservedMetasInner.map { m =>
+ if (m == LabelMeta.to) m.copy(dataType = tgtColumnType)
+ else if (m == LabelMeta.from) m.copy(dataType = srcColumnType)
+ else m
+ } ::: LabelMeta.findAllByLabelId(id.get, useCache = true)
+
+ lazy val metaPropsMap = metaProps.map(x => (x.seq, x)).toMap
+ lazy val metaPropsInvMap = metaProps.map(x => (x.name, x)).toMap
+ lazy val metaPropNames = metaProps.map(x => x.name)
+ lazy val metaPropNamesMap = metaProps.map(x => (x.seq, x.name)) toMap
+
+ /** this is used only by edgeToProps */
+ lazy val metaPropsDefaultMap = (for {
+ prop <- metaProps if LabelMeta.isValidSeq(prop.seq)
+ jsValue <- innerValToJsValue(toInnerVal(prop.defaultValue, prop.dataType, schemaVersion), prop.dataType)
+ } yield prop.name -> jsValue).toMap
+
+ lazy val metaPropsDefaultMapInner = (for {
+ prop <- metaPropsInner if LabelMeta.isValidSeq(prop.seq)
+ jsValue <- innerValToJsValue(toInnerVal(prop.defaultValue, prop.dataType, schemaVersion), prop.dataType)
+ } yield prop.name -> jsValue).toMap
+
+ def srcColumnWithDir(dir: Int) = {
+ if (dir == GraphUtil.directions("out")) srcColumn else tgtColumn
+ }
+
+ def tgtColumnWithDir(dir: Int) = {
+ if (dir == GraphUtil.directions("out")) tgtColumn else srcColumn
+ }
+
+ def srcTgtColumn(dir: Int) =
+ if (isDirected) {
+ (srcColumnWithDir(dir), tgtColumnWithDir(dir))
+ } else {
+ if (dir == GraphUtil.directions("in")) {
+ (tgtColumn, srcColumn)
+ } else {
+ (srcColumn, tgtColumn)
+ }
+ }
+
+ def init() = {
+ metas
+ metaSeqsToNames
+ service
+ srcColumn
+ tgtColumn
+ defaultIndex
+ indices
+ metaProps
+ }
+
+ // def srcColumnInnerVal(jsValue: JsValue) = {
+ // jsValueToInnerVal(jsValue, srcColumnType, version)
+ // }
+ // def tgtColumnInnerVal(jsValue: JsValue) = {
+ // jsValueToInnerVal(jsValue, tgtColumnType, version)
+ // }
+
+ override def toString(): String = {
+ val orderByKeys = LabelMeta.findAllByLabelId(id.get)
+ super.toString() + orderByKeys.toString()
+ }
+
+ // def findLabelIndexSeq(scoring: List[(Byte, Double)]): Byte = {
+ // if (scoring.isEmpty) LabelIndex.defaultSeq
+ // else {
+ // LabelIndex.findByLabelIdAndSeqs(id.get, scoring.map(_._1).sorted).map(_.seq).getOrElse(LabelIndex.defaultSeq)
+ //
+ //// LabelIndex.findByLabelIdAndSeqs(id.get, scoring.map(_._1).sorted).map(_.seq).getOrElse(LabelIndex.defaultSeq)
+ // }
+ // }
+ lazy val toJson = Json.obj("labelName" -> label,
+ "from" -> srcColumn.toJson, "to" -> tgtColumn.toJson,
+ "isDirected" -> isDirected,
+ "serviceName" -> serviceName,
+ "consistencyLevel" -> consistencyLevel,
+ "schemaVersion" -> schemaVersion,
+ "isAsync" -> isAsync,
+ "compressionAlgorithm" -> compressionAlgorithm,
+ "defaultIndex" -> defaultIndex.map(x => x.toJson),
+ "extraIndex" -> extraIndices.map(exIdx => exIdx.toJson),
+ "metaProps" -> metaProps.filter { labelMeta => LabelMeta.isValidSeqForAdmin(labelMeta.seq) }.map(_.toJson)
+ )
+
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala
new file mode 100644
index 0000000..31e63c3
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelIndex.scala
@@ -0,0 +1,143 @@
+package org.apache.s2graph.core.mysqls
+
+/**
+ * Created by shon on 6/3/15.
+ */
+
+import play.api.libs.json.Json
+import scalikejdbc._
+
+object LabelIndex extends Model[LabelIndex] {
+ val DefaultName = "_PK"
+ val DefaultMetaSeqs = Seq(LabelMeta.timeStampSeq)
+ val DefaultSeq = 1.toByte
+ val MaxOrderSeq = 7
+
+ def apply(rs: WrappedResultSet): LabelIndex = {
+ LabelIndex(rs.intOpt("id"), rs.int("label_id"), rs.string("name"), rs.byte("seq"),
+ rs.string("meta_seqs").split(",").filter(_ != "").map(s => s.toByte).toList match {
+ case metaSeqsList => metaSeqsList
+ },
+ rs.string("formulars"))
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession) = {
+ val cacheKey = "id=" + id
+ withCache(cacheKey) {
+ sql"""select * from label_indices where id = ${id}""".map { rs => LabelIndex(rs) }.single.apply
+ }.get
+ }
+
+ def findByLabelIdAll(labelId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
+ val cacheKey = "labelId=" + labelId
+ if (useCache) {
+ withCaches(cacheKey)( sql"""
+ select * from label_indices where label_id = ${labelId} and seq > 0 order by seq ASC
+ """.map { rs => LabelIndex(rs) }.list.apply)
+ } else {
+ sql"""
+ select * from label_indices where label_id = ${labelId} and seq > 0 order by seq ASC
+ """.map { rs => LabelIndex(rs) }.list.apply
+ }
+ }
+
+ def insert(labelId: Int, indexName: String, seq: Byte, metaSeqs: List[Byte], formulars: String)(implicit session: DBSession = AutoSession): Long = {
+ sql"""
+ insert into label_indices(label_id, name, seq, meta_seqs, formulars)
+ values (${labelId}, ${indexName}, ${seq}, ${metaSeqs.mkString(",")}, ${formulars})
+ """
+ .updateAndReturnGeneratedKey.apply()
+ }
+
+ def findOrInsert(labelId: Int, indexName: String, metaSeqs: List[Byte], formulars: String)(implicit session: DBSession = AutoSession): LabelIndex = {
+ findByLabelIdAndSeqs(labelId, metaSeqs) match {
+ case Some(s) => s
+ case None =>
+ val orders = findByLabelIdAll(labelId, false)
+ val seq = (orders.size + 1).toByte
+ assert(seq <= MaxOrderSeq)
+ val createdId = insert(labelId, indexName, seq, metaSeqs, formulars)
+ val cacheKeys = List(s"labelId=$labelId:seq=$seq",
+ s"labelId=$labelId:seqs=$metaSeqs", s"labelId=$labelId:seq=$seq", s"id=$createdId")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+
+ findByLabelIdAndSeq(labelId, seq).get
+ }
+ }
+
+ def findByLabelIdAndSeqs(labelId: Int, seqs: List[Byte])(implicit session: DBSession = AutoSession): Option[LabelIndex] = {
+ val cacheKey = "labelId=" + labelId + ":seqs=" + seqs.mkString(",")
+ withCache(cacheKey) {
+ sql"""
+ select * from label_indices where label_id = ${labelId} and meta_seqs = ${seqs.mkString(",")}
+ """.map { rs => LabelIndex(rs) }.single.apply
+ }
+ }
+
+ def findByLabelIdAndSeq(labelId: Int, seq: Byte, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[LabelIndex] = {
+ // val cacheKey = s"labelId=$labelId:seq=$seq"
+ val cacheKey = "labelId=" + labelId + ":seq=" + seq
+ if (useCache) {
+ withCache(cacheKey)( sql"""
+ select * from label_indices where label_id = ${labelId} and seq = ${seq}
+ """.map { rs => LabelIndex(rs) }.single.apply)
+ } else {
+ sql"""
+ select * from label_indices where label_id = ${labelId} and seq = ${seq}
+ """.map { rs => LabelIndex(rs) }.single.apply
+ }
+ }
+
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val labelIndex = findById(id)
+ val seqs = labelIndex.metaSeqs.mkString(",")
+ val (labelId, seq) = (labelIndex.labelId, labelIndex.seq)
+ sql"""delete from label_indices where id = ${id}""".execute.apply()
+
+ val cacheKeys = List(s"id=$id", s"labelId=$labelId", s"labelId=$labelId:seq=$seq", s"labelId=$labelId:seqs=$seqs")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ }
+
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from label_indices""".map { rs => LabelIndex(rs) }.list.apply
+ putsToCache(ls.map { x =>
+ var cacheKey = s"id=${x.id.get}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ var cacheKey = s"labelId=${x.labelId}:seq=${x.seq}}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ var cacheKey = s"labelId=${x.labelId}:seqs=${x.metaSeqs.mkString(",")}"
+ (cacheKey -> x)
+ })
+ putsToCaches(ls.groupBy(x => x.labelId).map { case (labelId, ls) =>
+ val cacheKey = s"labelId=${labelId}"
+ (cacheKey -> ls)
+ }.toList)
+ }
+}
+
+/**
+ * formular
+ * ex1): w1, w2, w3
+ * ex2): 1.5 * w1^2 + 3.4 * (w1 * w2), w2, w1
+ */
+
+case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, metaSeqs: Seq[Byte], formulars: String) {
+ lazy val label = Label.findById(labelId)
+ lazy val metas = label.metaPropsMap
+ lazy val sortKeyTypes = metaSeqs.flatMap(metaSeq => label.metaPropsMap.get(metaSeq))
+ lazy val propNames = sortKeyTypes.map { labelMeta => labelMeta.name }
+ lazy val toJson = Json.obj(
+ "name" -> name,
+ "propNames" -> sortKeyTypes.map(x => x.name)
+ )
+}
[15/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
new file mode 100644
index 0000000..34e9fcb
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -0,0 +1,1224 @@
+package org.apache.s2graph.core.storage
+
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.s2graph.core.ExceptionHandler.{KafkaMessage, Key, Val}
+import org.apache.s2graph.core.GraphExceptions.FetchTimeoutException
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.storage.serde.indexedge.wide.{IndexEdgeDeserializable, IndexEdgeSerializable}
+import org.apache.s2graph.core.storage.serde.snapshotedge.wide.SnapshotEdgeDeserializable
+import org.apache.s2graph.core.storage.serde.vertex.{VertexDeserializable, VertexSerializable}
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.utils.{Extensions, logger}
+
+import scala.annotation.tailrec
+import scala.collection.Seq
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Random, Try}
+
+abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) {
+ import HBaseType._
+
+ /** storage dependent configurations */
+ val MaxRetryNum = config.getInt("max.retry.number")
+ val MaxBackOff = config.getInt("max.back.off")
+ val DeleteAllFetchSize = config.getInt("delete.all.fetch.size")
+ val FailProb = config.getDouble("hbase.fail.prob")
+ val LockExpireDuration = Math.max(MaxRetryNum * MaxBackOff * 2, 10000)
+ val maxSize = config.getInt("future.cache.max.size")
+ val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+ /**
+ * Compatibility table
+ * | label schema version | snapshot edge | index edge | vertex | note |
+ * | v1 | serde.snapshotedge.wide | serde.indexedge.wide | serde.vertex | do not use this. this exist only for backward compatibility issue |
+ * | v2 | serde.snapshotedge.wide | serde.indexedge.wide | serde.vertex | do not use this. this exist only for backward compatibility issue |
+ * | v3 | serde.snapshotedge.tall | serde.indexedge.wide | serde.vertex | recommended with HBase. current stable schema |
+ * | v4 | serde.snapshotedge.tall | serde.indexedge.tall | serde.vertex | experimental schema. use scanner instead of get |
+ *
+ */
+
+ /**
+ * create serializer that knows how to convert given snapshotEdge into kvs: Seq[SKeyValue]
+ * so we can store this kvs.
+ * @param snapshotEdge: snapshotEdge to serialize
+ * @return serializer implementation for StorageSerializable which has toKeyValues return Seq[SKeyValue]
+ */
+ def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): Serializable[SnapshotEdge] = {
+ snapshotEdge.schemaVer match {
+ case VERSION1 | VERSION2 => new serde.snapshotedge.wide.SnapshotEdgeSerializable(snapshotEdge)
+ case VERSION3 | VERSION4 => new serde.snapshotedge.tall.SnapshotEdgeSerializable(snapshotEdge)
+ case _ => throw new RuntimeException(s"not supported version: ${snapshotEdge.schemaVer}")
+ }
+ }
+
+ /**
+ * create serializer that knows how to convert given indexEdge into kvs: Seq[SKeyValue]
+ * @param indexEdge: indexEdge to serialize
+ * @return serializer implementation
+ */
+ def indexEdgeSerializer(indexEdge: IndexEdge): Serializable[IndexEdge] = {
+ indexEdge.schemaVer match {
+ case VERSION1 | VERSION2 | VERSION3 => new IndexEdgeSerializable(indexEdge)
+ case VERSION4 => new serde.indexedge.tall.IndexEdgeSerializable(indexEdge)
+ case _ => throw new RuntimeException(s"not supported version: ${indexEdge.schemaVer}")
+
+ }
+ }
+
+ /**
+ * create serializer that knows how to convert given vertex into kvs: Seq[SKeyValue]
+ * @param vertex: vertex to serialize
+ * @return serializer implementation
+ */
+ def vertexSerializer(vertex: Vertex) = new VertexSerializable(vertex)
+
+ /**
+ * create deserializer that can parse stored CanSKeyValue into snapshotEdge.
+ * note that each storage implementation should implement implicit type class
+ * to convert storage dependent dataType into common SKeyValue type by implementing CanSKeyValue
+ *
+ * ex) Asynchbase use it's KeyValue class and CanSKeyValue object has implicit type conversion method.
+ * if any storaage use different class to represent stored byte array,
+ * then that storage implementation is responsible to provide implicit type conversion method on CanSKeyValue.
+ * */
+
+ val snapshotEdgeDeserializers: Map[String, Deserializable[SnapshotEdge]] = Map(
+ VERSION1 -> new SnapshotEdgeDeserializable,
+ VERSION2 -> new SnapshotEdgeDeserializable,
+ VERSION3 -> new serde.snapshotedge.tall.SnapshotEdgeDeserializable,
+ VERSION4 -> new serde.snapshotedge.tall.SnapshotEdgeDeserializable
+ )
+ def snapshotEdgeDeserializer(schemaVer: String) =
+ snapshotEdgeDeserializers.get(schemaVer).getOrElse(throw new RuntimeException(s"not supported version: ${schemaVer}"))
+
+ /** create deserializer that can parse stored CanSKeyValue into indexEdge. */
+ val indexEdgeDeserializers: Map[String, Deserializable[IndexEdge]] = Map(
+ VERSION1 -> new IndexEdgeDeserializable,
+ VERSION2 -> new IndexEdgeDeserializable,
+ VERSION3 -> new IndexEdgeDeserializable,
+ VERSION4 -> new serde.indexedge.tall.IndexEdgeDeserializable
+ )
+
+ def indexEdgeDeserializer(schemaVer: String) =
+ indexEdgeDeserializers.get(schemaVer).getOrElse(throw new RuntimeException(s"not supported version: ${schemaVer}"))
+
+ /** create deserializer that can parser stored CanSKeyValue into vertex. */
+ val vertexDeserializer = new VertexDeserializable
+
+
+ /**
+ * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
+ * note that this should be return true on all success.
+ * we assumes that each storage implementation has client as member variable.
+ *
+ *
+ * @param cluster: where this key values should be stored.
+ * @param kvs: sequence of SKeyValue that need to be stored in storage.
+ * @param withWait: flag to control wait ack from storage.
+ * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
+ * it never block thread, but rather submit work and notified by event loop when storage send ack back.
+ * @return ack message from storage.
+ */
+ def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean]
+
+// def writeToStorage(kv: SKeyValue, withWait: Boolean): Future[Boolean]
+
+ /**
+ * fetch SnapshotEdge for given request from storage.
+ * also storage datatype should be converted into SKeyValue.
+ * note that return type is Sequence rather than single SKeyValue for simplicity,
+ * even though there is assertions sequence.length == 1.
+ * @param request
+ * @return
+ */
+ def fetchSnapshotEdgeKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
+
+ /**
+ * write requestKeyValue into storage if the current value in storage that is stored matches.
+ * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
+ *
+ * Most important thing is this have to be 'atomic' operation.
+ * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
+ * either blocked or failed on write-write conflict case.
+ *
+ * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
+ * prevent wrong data for read.
+ *
+ * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
+ * compareAndSet to synchronize.
+ *
+ * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
+ * for storage that does not support concurrency control, then storage implementation
+ * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
+ * and write(writeLock).
+ * @param requestKeyValue
+ * @param expectedOpt
+ * @return
+ */
+ def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean]
+
+ /**
+ * build proper request which is specific into storage to call fetchIndexEdgeKeyValues or fetchSnapshotEdgeKeyValues.
+ * for example, Asynchbase use GetRequest, Scanner so this method is responsible to build
+ * client request(GetRequest, Scanner) based on user provided query.
+ * @param queryRequest
+ * @return
+ */
+ def buildRequest(queryRequest: QueryRequest): AnyRef
+
+ /**
+ * fetch IndexEdges for given queryParam in queryRequest.
+ * this expect previous step starting score to propagate score into next step.
+ * also parentEdges is necessary to return full bfs tree when query require it.
+ *
+ * note that return type is general type.
+ * for example, currently we wanted to use Asynchbase
+ * so single I/O return type should be Deferred[T].
+ *
+ * if we use native hbase client, then this return type can be Future[T] or just T.
+ * @param queryRequest
+ * @param prevStepScore
+ * @param isInnerCall
+ * @param parentEdges
+ * @return
+ */
+ def fetch(queryRequest: QueryRequest,
+ prevStepScore: Double,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): R
+
+ /**
+ * responsible to fire parallel fetch call into storage and create future that will return merged result.
+ * @param queryRequestWithScoreLs
+ * @param prevStepEdges
+ * @return
+ */
+ def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]]
+
+ /**
+ * fetch Vertex for given request from storage.
+ * @param request
+ * @return
+ */
+ def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
+
+ /**
+ * decide how to apply given edges(indexProps values + Map(_count -> countVal)) into storage.
+ * @param edges
+ * @param withWait
+ * @return
+ */
+ def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]]
+
+ /**
+ * this method need to be called when client shutdown. this is responsible to cleanUp the resources
+ * such as client into storage.
+ */
+ def flush(): Unit
+
+ /**
+ * create table on storage.
+ * if storage implementation does not support namespace or table, then there is nothing to be done
+ * @param zkAddr
+ * @param tableName
+ * @param cfs
+ * @param regionMultiplier
+ * @param ttl
+ * @param compressionAlgorithm
+ */
+ def createTable(zkAddr: String,
+ tableName: String,
+ cfs: List[String],
+ regionMultiplier: Int,
+ ttl: Option[Int],
+ compressionAlgorithm: String): Unit
+
+
+
+
+
+ /** Public Interface */
+
+ def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
+ def fromResult(queryParam: QueryParam,
+ kvs: Seq[SKeyValue],
+ version: String): Option[Vertex] = {
+ if (kvs.isEmpty) None
+ else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
+ }
+
+ val futures = vertices.map { vertex =>
+ val queryParam = QueryParam.Empty
+ val q = Query.toQuery(Seq(vertex), queryParam)
+ val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
+ fetchVertexKeyValues(buildRequest(queryRequest)).map { kvs =>
+ fromResult(queryParam, kvs, vertex.serviceColumn.schemaVersion)
+ } recoverWith { case ex: Throwable =>
+ Future.successful(None)
+ }
+ }
+
+ Future.sequence(futures).map { result => result.toList.flatten }
+ }
+
+ def mutateElements(elements: Seq[GraphElement],
+ withWait: Boolean = false): Future[Seq[Boolean]] = {
+
+ val edgeBuffer = ArrayBuffer[Edge]()
+ val vertexBuffer = ArrayBuffer[Vertex]()
+
+ elements.foreach {
+ case e: Edge => edgeBuffer += e
+ case v: Vertex => vertexBuffer += v
+ case any@_ => logger.error(s"Unknown type: ${any}")
+ }
+
+ val edgeFuture = mutateEdges(edgeBuffer, withWait)
+ val vertexFuture = mutateVertices(vertexBuffer, withWait)
+
+ val graphFuture = for {
+ edgesMutated <- edgeFuture
+ verticesMutated <- vertexFuture
+ } yield edgesMutated ++ verticesMutated
+
+ graphFuture
+ }
+
+ def mutateEdges(edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
+ val (strongEdges, weakEdges) =
+ (edges.partition(e => e.label.consistencyLevel == "strong" || e.op == GraphUtil.operations("insertBulk")))
+
+ val weakEdgesFutures = weakEdges.groupBy { e => e.label.hbaseZkAddr }.map { case (zkQuorum, edges) =>
+ val mutations = edges.flatMap { edge =>
+ val (_, edgeUpdate) =
+ if (edge.op == GraphUtil.operations("delete")) Edge.buildDeleteBulk(None, edge)
+ else Edge.buildOperation(None, Seq(edge))
+ buildVertexPutsAsync(edge) ++ indexedEdgeMutations(edgeUpdate) ++
+ snapshotEdgeMutations(edgeUpdate) ++ increments(edgeUpdate)
+ }
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ val strongEdgesFutures = mutateStrongEdges(strongEdges, withWait)
+ for {
+ weak <- Future.sequence(weakEdgesFutures)
+ strong <- strongEdgesFutures
+ } yield {
+ strong ++ weak
+ }
+ }
+ def mutateStrongEdges(_edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
+
+ val grouped = _edges.groupBy { edge => (edge.label, edge.srcVertex.innerId, edge.tgtVertex.innerId) } toSeq
+
+ val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
+ val (deleteAllEdges, edges) = edgeGroup.partition(_.op == GraphUtil.operations("deleteAll"))
+
+ // DeleteAll first
+ val deleteAllFutures = deleteAllEdges.map { edge =>
+ deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.label), edge.labelWithDir.dir, edge.ts)
+ }
+
+ // After deleteAll, process others
+ lazy val mutateEdgeFutures = edges.toList match {
+ case head :: tail =>
+ // val strongConsistency = edges.head.label.consistencyLevel == "strong"
+ // if (strongConsistency) {
+ val edgeFuture = mutateEdgesInner(edges, checkConsistency = true , withWait)(Edge.buildOperation)
+
+ //TODO: decide what we will do on failure on vertex put
+ val puts = buildVertexPutsAsync(head)
+ val vertexFuture = writeToStorage(head.label.hbaseZkAddr, puts, withWait)
+ Seq(edgeFuture, vertexFuture)
+ // } else {
+ // edges.map { edge => mutateEdge(edge, withWait = withWait) }
+ // }
+ case Nil => Nil
+ }
+
+ val composed = for {
+ deleteRet <- Future.sequence(deleteAllFutures)
+ mutateRet <- Future.sequence(mutateEdgeFutures)
+ } yield deleteRet ++ mutateRet
+
+ composed.map(_.forall(identity))
+ }
+
+ Future.sequence(mutateEdges)
+ }
+
+ def mutateVertex(vertex: Vertex, withWait: Boolean): Future[Boolean] = {
+ if (vertex.op == GraphUtil.operations("delete")) {
+ writeToStorage(vertex.hbaseZkAddr,
+ vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
+ } else if (vertex.op == GraphUtil.operations("deleteAll")) {
+ logger.info(s"deleteAll for vertex is truncated. $vertex")
+ Future.successful(true) // Ignore withWait parameter, because deleteAll operation may takes long time
+ } else {
+ writeToStorage(vertex.hbaseZkAddr, buildPutsAll(vertex), withWait)
+ }
+ }
+
+ def mutateVertices(vertices: Seq[Vertex],
+ withWait: Boolean = false): Future[Seq[Boolean]] = {
+ val futures = vertices.map { vertex => mutateVertex(vertex, withWait) }
+ Future.sequence(futures)
+ }
+
+
+ def mutateEdgesInner(edges: Seq[Edge],
+ checkConsistency: Boolean,
+ withWait: Boolean)(f: (Option[Edge], Seq[Edge]) => (Edge, EdgeMutate)): Future[Boolean] = {
+ if (!checkConsistency) {
+ val zkQuorum = edges.head.label.hbaseZkAddr
+ val futures = edges.map { edge =>
+ val (_, edgeUpdate) = f(None, Seq(edge))
+ val mutations =
+ indexedEdgeMutations(edgeUpdate) ++
+ snapshotEdgeMutations(edgeUpdate) ++
+ increments(edgeUpdate)
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ Future.sequence(futures).map { rets => rets.forall(identity) }
+ } else {
+ def commit(_edges: Seq[Edge], statusCode: Byte): Future[Boolean] = {
+
+ fetchSnapshotEdge(_edges.head) flatMap { case (queryParam, snapshotEdgeOpt, kvOpt) =>
+
+ val (newEdge, edgeUpdate) = f(snapshotEdgeOpt, _edges)
+ logger.debug(s"${snapshotEdgeOpt}\n${edgeUpdate.toLogString}")
+ //shouldReplace false.
+ if (edgeUpdate.newSnapshotEdge.isEmpty && statusCode <= 0) {
+ logger.debug(s"${newEdge.toLogString} drop.")
+ Future.successful(true)
+ } else {
+ commitUpdate(newEdge, statusCode)(snapshotEdgeOpt, kvOpt, edgeUpdate).map { ret =>
+ if (ret) {
+ logger.info(s"[Success] commit: \n${_edges.map(_.toLogString).mkString("\n")}")
+ } else {
+ throw new PartialFailureException(newEdge, 3, "commit failed.")
+ }
+ true
+ }
+ }
+ }
+ }
+ def retry(tryNum: Int)(edges: Seq[Edge], statusCode: Byte)(fn: (Seq[Edge], Byte) => Future[Boolean]): Future[Boolean] = {
+ if (tryNum >= MaxRetryNum) {
+ edges.foreach { edge =>
+ logger.error(s"commit failed after $MaxRetryNum\n${edge.toLogString}")
+ ExceptionHandler.enqueue(ExceptionHandler.toKafkaMessage(element = edge))
+ }
+ Future.successful(false)
+ } else {
+ val future = fn(edges, statusCode)
+ future.onSuccess {
+ case success =>
+ logger.debug(s"Finished. [$tryNum]\n${edges.head.toLogString}\n")
+ }
+ future recoverWith {
+ case FetchTimeoutException(retryEdge) =>
+ logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}")
+ retry(tryNum + 1)(edges, statusCode)(fn)
+
+ case PartialFailureException(retryEdge, failedStatusCode, faileReason) =>
+ val status = failedStatusCode match {
+ case 0 => "AcquireLock failed."
+ case 1 => "Mutation failed."
+ case 2 => "Increment failed."
+ case 3 => "ReleaseLock failed."
+ case 4 => "Unknown"
+ }
+
+ Thread.sleep(Random.nextInt(MaxBackOff))
+ logger.info(s"[Try: $tryNum], [Status: $status] partial fail.\n${retryEdge.toLogString}\nFailReason: ${faileReason}")
+ retry(tryNum + 1)(Seq(retryEdge), failedStatusCode)(fn)
+ case ex: Exception =>
+ logger.error("Unknown exception", ex)
+ Future.successful(false)
+ }
+ }
+ }
+ retry(1)(edges, 0)(commit)
+ }
+ }
+
+ def mutateLog(snapshotEdgeOpt: Option[Edge], edges: Seq[Edge],
+ newEdge: Edge, edgeMutate: EdgeMutate) =
+ Seq("----------------------------------------------",
+ s"SnapshotEdge: ${snapshotEdgeOpt.map(_.toLogString)}",
+ s"requestEdges: ${edges.map(_.toLogString).mkString("\n")}",
+ s"newEdge: ${newEdge.toLogString}",
+ s"mutation: \n${edgeMutate.toLogString}",
+ "----------------------------------------------").mkString("\n")
+
+
+ /** Delete All */
+ protected def deleteAllFetchedEdgesAsyncOld(queryRequest: QueryRequest,
+ queryResult: QueryResult,
+ requestTs: Long,
+ retryNum: Int): Future[Boolean] = {
+ val queryParam = queryRequest.queryParam
+ val zkQuorum = queryParam.label.hbaseZkAddr
+ val futures = for {
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ } yield {
+ /** reverted direction */
+ val reversedIndexedEdgesMutations = edge.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ buildIncrementsAsync(indexEdge, -1L)
+ }
+ val reversedSnapshotEdgeMutations = snapshotEdgeSerializer(edge.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ val forwardIndexedEdgeMutations = edge.edgesWithIndex.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ buildIncrementsAsync(indexEdge, -1L)
+ }
+ val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
+ writeToStorage(zkQuorum, mutations, withWait = true)
+ }
+
+ Future.sequence(futures).map { rets => rets.forall(identity) }
+ }
+
+ protected def buildEdgesToDelete(queryRequestWithResultLs: QueryRequestWithResult, requestTs: Long): QueryResult = {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs).get
+ val edgeWithScoreLs = queryResult.edgeWithScoreLs.filter { edgeWithScore =>
+ (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
+ }.map { edgeWithScore =>
+ val label = queryRequest.queryParam.label
+ val (newOp, newVersion, newPropsWithTs) = label.consistencyLevel match {
+ case "strong" =>
+ val _newPropsWithTs = edgeWithScore.edge.propsWithTs ++
+ Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(requestTs, requestTs, label.schemaVersion))
+ (GraphUtil.operations("delete"), requestTs, _newPropsWithTs)
+ case _ =>
+ val oldEdge = edgeWithScore.edge
+ (oldEdge.op, oldEdge.version, oldEdge.propsWithTs)
+ }
+
+ val copiedEdge =
+ edgeWithScore.edge.copy(op = newOp, version = newVersion, propsWithTs = newPropsWithTs)
+
+ val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
+// logger.debug(s"delete edge from deleteAll: ${edgeToDelete.edge.toLogString}")
+ edgeToDelete
+ }
+
+ queryResult.copy(edgeWithScoreLs = edgeWithScoreLs)
+ }
+
+ protected def deleteAllFetchedEdgesLs(queryRequestWithResultLs: Seq[QueryRequestWithResult], requestTs: Long): Future[(Boolean, Boolean)] = {
+ val queryResultLs = queryRequestWithResultLs.map(_.queryResult)
+ queryResultLs.foreach { queryResult =>
+ if (queryResult.isFailure) throw new RuntimeException("fetched result is fallback.")
+ }
+ val futures = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ deleteQueryResult = buildEdgesToDelete(queryRequestWithResult, requestTs)
+ if deleteQueryResult.edgeWithScoreLs.nonEmpty
+ } yield {
+ val label = queryRequest.queryParam.label
+ label.schemaVersion match {
+ case HBaseType.VERSION3 | HBaseType.VERSION4 =>
+ if (label.consistencyLevel == "strong") {
+ /**
+ * read: snapshotEdge on queryResult = O(N)
+ * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
+ */
+ mutateEdges(deleteQueryResult.edgeWithScoreLs.map(_.edge), withWait = true).map(_.forall(identity))
+ } else {
+ deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
+ }
+ case _ =>
+
+ /**
+ * read: x
+ * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
+ */
+ deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
+ }
+ }
+
+ if (futures.isEmpty) {
+ // all deleted.
+ Future.successful(true -> true)
+ } else {
+ Future.sequence(futures).map { rets => false -> rets.forall(identity) }
+ }
+ }
+
+ protected def fetchAndDeleteAll(query: Query, requestTs: Long): Future[(Boolean, Boolean)] = {
+ val future = for {
+ queryRequestWithResultLs <- getEdges(query)
+ (allDeleted, ret) <- deleteAllFetchedEdgesLs(queryRequestWithResultLs, requestTs)
+ } yield {
+// logger.debug(s"fetchAndDeleteAll: ${allDeleted}, ${ret}")
+ (allDeleted, ret)
+ }
+
+ Extensions.retryOnFailure(MaxRetryNum) {
+ future
+ } {
+ logger.error(s"fetch and deleteAll failed.")
+ (true, false)
+ }
+
+ }
+
+ def deleteAllAdjacentEdges(srcVertices: Seq[Vertex],
+ labels: Seq[Label],
+ dir: Int,
+ ts: Long): Future[Boolean] = {
+
+ def enqueueLogMessage() = {
+ val kafkaMessages = for {
+ vertice <- srcVertices
+ id = vertice.innerId.toIdString()
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", GraphUtil.fromOp(dir.toByte)).mkString("\t")
+ val topic = ExceptionHandler.failTopic
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+
+ ExceptionHandler.enqueues(kafkaMessages)
+ }
+
+ val requestTs = ts
+ val queryParams = for {
+ label <- labels
+ } yield {
+ val labelWithDir = LabelWithDirection(label.id.get, dir)
+ QueryParam(labelWithDir).limit(0, DeleteAllFetchSize).duplicatePolicy(Option(Query.DuplicatePolicy.Raw))
+ }
+
+ val step = Step(queryParams.toList)
+ val q = Query(srcVertices, Vector(step))
+
+ // Extensions.retryOnSuccessWithBackoff(MaxRetryNum, Random.nextInt(MaxBackOff) + 1) {
+ val retryFuture = Extensions.retryOnSuccess(MaxRetryNum) {
+ fetchAndDeleteAll(q, requestTs)
+ } { case (allDeleted, deleteSuccess) =>
+ allDeleted && deleteSuccess
+ }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
+
+ retryFuture onFailure {
+ case ex =>
+ logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
+ enqueueLogMessage()
+ }
+
+ retryFuture
+ }
+
+ /** End Of Delete All */
+
+
+
+
+ /** Parsing Logic: parse from kv from Storage into Edge */
+ def toEdge[K: CanSKeyValue](kv: K,
+ queryParam: QueryParam,
+ cacheElementOpt: Option[IndexEdge],
+ parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
+// logger.debug(s"toEdge: $kv")
+ try {
+ val schemaVer = queryParam.label.schemaVersion
+ val indexEdgeOpt = indexEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
+ indexEdgeOpt.map(indexEdge => indexEdge.toEdge.copy(parentEdges = parentEdges))
+ } catch {
+ case ex: Exception =>
+ logger.error(s"Fail on toEdge: ${kv.toString}, ${queryParam}", ex)
+ None
+ }
+ }
+
+ def toSnapshotEdge[K: CanSKeyValue](kv: K,
+ queryParam: QueryParam,
+ cacheElementOpt: Option[SnapshotEdge] = None,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
+// logger.debug(s"SnapshottoEdge: $kv")
+ val schemaVer = queryParam.label.schemaVersion
+ val snapshotEdgeOpt = snapshotEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
+
+ if (isInnerCall) {
+ snapshotEdgeOpt.flatMap { snapshotEdge =>
+ val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
+ if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
+ else None
+ }
+ } else {
+ snapshotEdgeOpt.flatMap { snapshotEdge =>
+ if (Edge.allPropsDeleted(snapshotEdge.props)) None
+ else {
+ val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
+ if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
+ else None
+ }
+ }
+ }
+ }
+
+ def toEdges[K: CanSKeyValue](kvs: Seq[K],
+ queryParam: QueryParam,
+ prevScore: Double = 1.0,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
+ if (kvs.isEmpty) Seq.empty
+ else {
+ val first = kvs.head
+ val kv = first
+ val schemaVer = queryParam.label.schemaVersion
+ val cacheElementOpt =
+ if (queryParam.isSnapshotEdge) None
+ else indexEdgeDeserializer(schemaVer).fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, None)
+
+ for {
+ kv <- kvs
+ edge <-
+ if (queryParam.isSnapshotEdge) toSnapshotEdge(kv, queryParam, None, isInnerCall, parentEdges)
+ else toEdge(kv, queryParam, cacheElementOpt, parentEdges)
+ } yield {
+ //TODO: Refactor this.
+ val currentScore =
+ queryParam.scorePropagateOp match {
+ case "plus" => edge.rank(queryParam.rank) + prevScore
+ case _ => edge.rank(queryParam.rank) * prevScore
+ }
+ EdgeWithScore(edge, currentScore)
+ }
+ }
+ }
+
+ /** End Of Parse Logic */
+
+// /** methods for consistency */
+// protected def writeAsyncSimple(zkQuorum: String, elementRpcs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
+// if (elementRpcs.isEmpty) {
+// Future.successful(true)
+// } else {
+// val futures = elementRpcs.map { rpc => writeToStorage(rpc, withWait) }
+// Future.sequence(futures).map(_.forall(identity))
+// }
+// }
+
+ case class PartialFailureException(edge: Edge, statusCode: Byte, failReason: String) extends Exception
+
+ protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge) = {
+ val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}").mkString("\n")
+ logger.debug(msg)
+ }
+
+ protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge, edgeMutate: EdgeMutate) = {
+ val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}",
+ s"${edgeMutate.toLogString}").mkString("\n")
+ logger.debug(msg)
+ }
+
+ protected def buildLockEdge(snapshotEdgeOpt: Option[Edge], edge: Edge, kvOpt: Option[SKeyValue]) = {
+ val currentTs = System.currentTimeMillis()
+ val lockTs = snapshotEdgeOpt match {
+ case None => Option(currentTs)
+ case Some(snapshotEdge) =>
+ snapshotEdge.pendingEdgeOpt match {
+ case None => Option(currentTs)
+ case Some(pendingEdge) => pendingEdge.lockTs
+ }
+ }
+ val newVersion = kvOpt.map(_.timestamp).getOrElse(edge.ts) + 1
+ // snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
+ val pendingEdge = edge.copy(version = newVersion, statusCode = 1, lockTs = lockTs)
+ val base = snapshotEdgeOpt match {
+ case None =>
+ // no one ever mutated on this snapshotEdge.
+ edge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
+ case Some(snapshotEdge) =>
+ // there is at least one mutation have been succeed.
+ snapshotEdgeOpt.get.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
+ }
+ base.copy(version = newVersion, statusCode = 1, lockTs = None)
+ }
+
+ protected def buildReleaseLockEdge(snapshotEdgeOpt: Option[Edge], lockEdge: SnapshotEdge,
+ edgeMutate: EdgeMutate) = {
+ val newVersion = lockEdge.version + 1
+ val base = edgeMutate.newSnapshotEdge match {
+ case None =>
+ // shouldReplace false
+ assert(snapshotEdgeOpt.isDefined)
+ snapshotEdgeOpt.get.toSnapshotEdge
+ case Some(newSnapshotEdge) => newSnapshotEdge
+ }
+ base.copy(version = newVersion, statusCode = 0, pendingEdgeOpt = None)
+ }
+
+ protected def acquireLock(statusCode: Byte,
+ edge: Edge,
+ oldSnapshotEdgeOpt: Option[Edge],
+ lockEdge: SnapshotEdge,
+ oldBytes: Array[Byte]): Future[Boolean] = {
+ if (statusCode >= 1) {
+ logger.debug(s"skip acquireLock: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 0, s"$p")
+ else {
+ val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
+ val oldPut = oldSnapshotEdgeOpt.map(e => snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head)
+// val lockEdgePut = buildPutAsync(lockEdge).head
+// val oldPut = oldSnapshotEdgeOpt.map(e => buildPutAsync(e.toSnapshotEdge).head)
+ writeLock(lockEdgePut, oldPut).recoverWith { case ex: Exception =>
+ logger.error(s"AcquireLock RPC Failed.")
+ throw new PartialFailureException(edge, 0, "AcquireLock RPC Failed")
+ }.map { ret =>
+ if (ret) {
+ val log = Seq(
+ "\n",
+ "=" * 50,
+ s"[Success]: acquireLock",
+ s"[RequestEdge]: ${edge.toLogString}",
+ s"[LockEdge]: ${lockEdge.toLogString()}",
+ s"[PendingEdge]: ${lockEdge.pendingEdgeOpt.map(_.toLogString).getOrElse("")}",
+ "=" * 50, "\n").mkString("\n")
+
+ logger.debug(log)
+ // debug(ret, "acquireLock", edge.toSnapshotEdge)
+ } else {
+ throw new PartialFailureException(edge, 0, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+ }
+
+
+
+ protected def releaseLock(predicate: Boolean,
+ edge: Edge,
+ lockEdge: SnapshotEdge,
+ releaseLockEdge: SnapshotEdge,
+ _edgeMutate: EdgeMutate,
+ oldBytes: Array[Byte]): Future[Boolean] = {
+ if (!predicate) {
+ throw new PartialFailureException(edge, 3, "predicate failed.")
+ }
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 3, s"$p")
+ else {
+ val releaseLockEdgePut = snapshotEdgeSerializer(releaseLockEdge).toKeyValues.head
+ val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
+ writeLock(releaseLockEdgePut, Option(lockEdgePut)).recoverWith {
+ case ex: Exception =>
+ logger.error(s"ReleaseLock RPC Failed.")
+ throw new PartialFailureException(edge, 3, "ReleaseLock RPC Failed")
+ }.map { ret =>
+ if (ret) {
+ debug(ret, "releaseLock", edge.toSnapshotEdge)
+ } else {
+ val msg = Seq("\nFATAL ERROR\n",
+ "=" * 50,
+ oldBytes.toList,
+ lockEdgePut,
+ releaseLockEdgePut,
+ // lockEdgePut.value.toList,
+ // releaseLockEdgePut.value().toList,
+ "=" * 50,
+ "\n"
+ )
+ logger.error(msg.mkString("\n"))
+ // error(ret, "releaseLock", edge.toSnapshotEdge)
+ throw new PartialFailureException(edge, 3, "hbase fail.")
+ }
+ true
+ }
+ }
+ Future.successful(true)
+ }
+
+
+ protected def mutate(predicate: Boolean,
+ edge: Edge,
+ statusCode: Byte,
+ _edgeMutate: EdgeMutate): Future[Boolean] = {
+ if (!predicate) throw new PartialFailureException(edge, 1, "predicate failed.")
+
+ if (statusCode >= 2) {
+ logger.debug(s"skip mutate: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 1, s"$p")
+ else
+ writeToStorage(edge.label.hbaseZkAddr, indexedEdgeMutations(_edgeMutate), withWait = true).map { ret =>
+ if (ret) {
+ debug(ret, "mutate", edge.toSnapshotEdge, _edgeMutate)
+ } else {
+ throw new PartialFailureException(edge, 1, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+
+ protected def increment(predicate: Boolean,
+ edge: Edge,
+ statusCode: Byte, _edgeMutate: EdgeMutate): Future[Boolean] = {
+ if (!predicate) throw new PartialFailureException(edge, 2, "predicate failed.")
+ if (statusCode >= 3) {
+ logger.debug(s"skip increment: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 2, s"$p")
+ else
+ writeToStorage(edge.label.hbaseZkAddr, increments(_edgeMutate), withWait = true).map { ret =>
+ if (ret) {
+ debug(ret, "increment", edge.toSnapshotEdge, _edgeMutate)
+ } else {
+ throw new PartialFailureException(edge, 2, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+
+
+ /** this may be overrided by specific storage implementation */
+ protected def commitProcess(edge: Edge, statusCode: Byte)
+ (snapshotEdgeOpt: Option[Edge], kvOpt: Option[SKeyValue])
+ (lockEdge: SnapshotEdge, releaseLockEdge: SnapshotEdge, _edgeMutate: EdgeMutate): Future[Boolean] = {
+ val oldBytes = kvOpt.map(kv => kv.value).getOrElse(Array.empty[Byte])
+ for {
+ locked <- acquireLock(statusCode, edge, snapshotEdgeOpt, lockEdge, oldBytes)
+ mutated <- mutate(locked, edge, statusCode, _edgeMutate)
+ incremented <- increment(mutated, edge, statusCode, _edgeMutate)
+ released <- releaseLock(incremented, edge, lockEdge, releaseLockEdge, _edgeMutate, oldBytes)
+ } yield {
+ released
+ }
+ }
+
+ protected def commitUpdate(edge: Edge,
+ statusCode: Byte)(snapshotEdgeOpt: Option[Edge],
+ kvOpt: Option[SKeyValue],
+ edgeUpdate: EdgeMutate): Future[Boolean] = {
+ val label = edge.label
+ def oldBytes = kvOpt.map(_.value).getOrElse(Array.empty)
+
+ val lockEdge = buildLockEdge(snapshotEdgeOpt, edge, kvOpt)
+ val releaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, edgeUpdate)
+ val _process = commitProcess(edge, statusCode)(snapshotEdgeOpt, kvOpt)_
+ snapshotEdgeOpt match {
+ case None =>
+ // no one ever did success on acquire lock.
+ _process(lockEdge, releaseLockEdge, edgeUpdate)
+ // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
+ case Some(snapshotEdge) =>
+ // someone did success on acquire lock at least one.
+ snapshotEdge.pendingEdgeOpt match {
+ case None =>
+ // not locked
+ _process(lockEdge, releaseLockEdge, edgeUpdate)
+ // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
+ case Some(pendingEdge) =>
+ def isLockExpired = pendingEdge.lockTs.get + LockExpireDuration < System.currentTimeMillis()
+ if (isLockExpired) {
+ val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
+ val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(pendingEdge))
+ val newLockEdge = buildLockEdge(snapshotEdgeOpt, pendingEdge, kvOpt)
+ val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, newLockEdge, newEdgeUpdate)
+ commitProcess(edge, statusCode = 0)(snapshotEdgeOpt, kvOpt)(newLockEdge, newReleaseLockEdge, newEdgeUpdate).flatMap { ret =>
+ // process(newLockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode = 0).flatMap { ret =>
+ val log = s"[Success]: Resolving expired pending edge.\n${pendingEdge.toLogString}"
+ throw new PartialFailureException(edge, 0, log)
+ }
+ } else {
+ // locked
+ if (pendingEdge.ts == edge.ts && statusCode > 0) {
+ // self locked
+ val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
+ val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(edge))
+ val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, newEdgeUpdate)
+
+ /** lockEdge will be ignored */
+ _process(lockEdge, newReleaseLockEdge, newEdgeUpdate)
+ // process(lockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode)
+ } else {
+ throw new PartialFailureException(edge, statusCode, s"others[${pendingEdge.ts}] is mutating. me[${edge.ts}]")
+ }
+ }
+ }
+ }
+ }
+
+ /** end of methods for consistency */
+
+
+ // def futureCache[T] = Cache[Long, (Long, T)]
+
+ protected def toRequestEdge(queryRequest: QueryRequest): Edge = {
+ val srcVertex = queryRequest.vertex
+ // val tgtVertexOpt = queryRequest.tgtVertexOpt
+ val edgeCf = Serializable.edgeCf
+ val queryParam = queryRequest.queryParam
+ val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
+ val label = queryParam.label
+ val labelWithDir = queryParam.labelWithDir
+ val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
+ val (srcInnerId, tgtInnerId) = tgtVertexIdOpt match {
+ case Some(tgtVertexId) => // _to is given.
+ /** we use toSnapshotEdge so dont need to swap src, tgt */
+ val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ val tgt = InnerVal.convertVersion(tgtVertexId, tgtColumn.columnType, label.schemaVersion)
+ (src, tgt)
+ case None =>
+ val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ (src, src)
+ }
+
+ val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
+ val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
+ val currentTs = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs)).toMap
+ Edge(srcV, tgtV, labelWithDir, propsWithTs = propsWithTs)
+ }
+
+
+
+ protected def fetchSnapshotEdge(edge: Edge): Future[(QueryParam, Option[Edge], Option[SKeyValue])] = {
+ val labelWithDir = edge.labelWithDir
+ val queryParam = QueryParam(labelWithDir)
+ val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
+ val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
+ val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
+
+ fetchSnapshotEdgeKeyValues(buildRequest(queryRequest)).map { kvs =>
+ val (edgeOpt, kvOpt) =
+ if (kvs.isEmpty) (None, None)
+ else {
+ val _edgeOpt = toEdges(kvs, queryParam, 1.0, isInnerCall = true, parentEdges = Nil).headOption.map(_.edge)
+ val _kvOpt = kvs.headOption
+ (_edgeOpt, _kvOpt)
+ }
+ (queryParam, edgeOpt, kvOpt)
+ } recoverWith { case ex: Throwable =>
+ logger.error(s"fetchQueryParam failed. fallback return.", ex)
+ throw new FetchTimeoutException(s"${edge.toLogString}")
+ }
+ }
+
+ protected def fetchStep(orgQuery: Query, queryRequestWithResultsLs: Seq[QueryRequestWithResult]): Future[Seq[QueryRequestWithResult]] = {
+ if (queryRequestWithResultsLs.isEmpty) Future.successful(Nil)
+ else {
+ val queryRequest = queryRequestWithResultsLs.head.queryRequest
+ val q = orgQuery
+ val queryResultsLs = queryRequestWithResultsLs.map(_.queryResult)
+
+ val stepIdx = queryRequest.stepIdx + 1
+
+ val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
+ val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
+ val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
+ val step = q.steps(stepIdx)
+ val alreadyVisited =
+ if (stepIdx == 0) Map.empty[(LabelWithDirection, Vertex), Boolean]
+ else Graph.alreadyVisitedVertices(queryResultsLs)
+
+ val groupedBy = queryResultsLs.flatMap { queryResult =>
+ queryResult.edgeWithScoreLs.map { case edgeWithScore =>
+ edgeWithScore.edge.tgtVertex -> edgeWithScore
+ }
+ }.groupBy { case (vertex, edgeWithScore) => vertex }
+
+ val groupedByFiltered = for {
+ (vertex, edgesWithScore) <- groupedBy
+ aggregatedScore = edgesWithScore.map(_._2.score).sum if aggregatedScore >= prevStepThreshold
+ } yield vertex -> aggregatedScore
+
+ val prevStepTgtVertexIdEdges = for {
+ (vertex, edgesWithScore) <- groupedBy
+ } yield vertex.id -> edgesWithScore.map { case (vertex, edgeWithScore) => edgeWithScore }
+
+ val nextStepSrcVertices = if (prevStepLimit >= 0) {
+ groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
+ } else {
+ groupedByFiltered.toSeq
+ }
+
+ val queryRequests = for {
+ (vertex, prevStepScore) <- nextStepSrcVertices
+ queryParam <- step.queryParams
+ } yield (QueryRequest(q, stepIdx, vertex, queryParam), prevStepScore)
+
+ Graph.filterEdges(fetches(queryRequests, prevStepTgtVertexIdEdges), alreadyVisited)(ec)
+ }
+ }
+
+ protected def fetchStepFuture(orgQuery: Query, queryRequestWithResultLsFuture: Future[Seq[QueryRequestWithResult]]): Future[Seq[QueryRequestWithResult]] = {
+ for {
+ queryRequestWithResultLs <- queryRequestWithResultLsFuture
+ ret <- fetchStep(orgQuery, queryRequestWithResultLs)
+ } yield ret
+ }
+
+ def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = {
+ val fallback = {
+ val queryRequest = QueryRequest(query = q, stepIdx = 0, q.vertices.head, queryParam = QueryParam.Empty)
+ Future.successful(q.vertices.map(v => QueryRequestWithResult(queryRequest, QueryResult())))
+ }
+ Try {
+
+ if (q.steps.isEmpty) {
+ // TODO: this should be get vertex query.
+ fallback
+ } else {
+ // current stepIdx = -1
+ val startQueryResultLs = QueryResult.fromVertices(q)
+ q.steps.foldLeft(Future.successful(startQueryResultLs)) { case (acc, step) =>
+ fetchStepFuture(q, acc)
+// fetchStepFuture(q, acc).map { stepResults =>
+// step.queryParams.zip(stepResults).foreach { case (qParam, queryRequestWithResult) =>
+// val cursor = Base64.getEncoder.encodeToString(queryRequestWithResult.queryResult.tailCursor)
+// qParam.cursorOpt = Option(cursor)
+// }
+// stepResults
+// }
+ }
+ }
+ } recover {
+ case e: Exception =>
+ logger.error(s"getEdgesAsync: $e", e)
+ fallback
+ } get
+ }
+
+ def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = {
+ val ts = System.currentTimeMillis()
+ val futures = for {
+ (srcVertex, tgtVertex, queryParam) <- params
+ propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, queryParam.label.schemaVersion))
+ edge = Edge(srcVertex, tgtVertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
+ } yield {
+ fetchSnapshotEdge(edge).map { case (queryParam, edgeOpt, kvOpt) =>
+ val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
+ val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
+ val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
+ val queryResult = QueryResult(edgeOpt.toSeq.map(e => EdgeWithScore(e, 1.0)))
+ QueryRequestWithResult(queryRequest, queryResult)
+ }
+ }
+
+ Future.sequence(futures)
+ }
+
+
+
+ @tailrec
+ final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (range < sampleNumber || set.size == sampleNumber) set
+ else randomInt(sampleNumber, range, set + Random.nextInt(range))
+ }
+
+ protected def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
+ if (edges.size <= n){
+ edges
+ }else{
+ val plainEdges = if (queryRequest.queryParam.offset == 0) {
+ edges.tail
+ } else edges
+
+ val randoms = randomInt(n, plainEdges.size)
+ var samples = List.empty[EdgeWithScore]
+ var idx = 0
+ plainEdges.foreach { e =>
+ if (randoms.contains(idx)) samples = e :: samples
+ idx += 1
+ }
+ samples.toSeq
+ }
+
+ }
+ /** end of query */
+
+ /** Mutation Builder */
+
+
+ /** EdgeMutate */
+ def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] = {
+ val deleteMutations = edgeMutate.edgesToDelete.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
+ }
+ val insertMutations = edgeMutate.edgesToInsert.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ }
+
+ deleteMutations ++ insertMutations
+ }
+
+ def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+ edgeMutate.newSnapshotEdge.map(e => snapshotEdgeSerializer(e).toKeyValues).getOrElse(Nil)
+
+ def increments(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+ (edgeMutate.edgesToDelete.isEmpty, edgeMutate.edgesToInsert.isEmpty) match {
+ case (true, true) =>
+
+ /** when there is no need to update. shouldUpdate == false */
+ List.empty
+ case (true, false) =>
+
+ /** no edges to delete but there is new edges to insert so increase degree by 1 */
+ edgeMutate.edgesToInsert.flatMap { e => buildIncrementsAsync(e) }
+ case (false, true) =>
+
+ /** no edges to insert but there is old edges to delete so decrease degree by 1 */
+ edgeMutate.edgesToDelete.flatMap { e => buildIncrementsAsync(e, -1L) }
+ case (false, false) =>
+
+ /** update on existing edges so no change on degree */
+ List.empty
+ }
+
+ /** IndexEdge */
+ def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
+ val newProps = indexedEdge.props ++ Map(LabelMeta.degreeSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
+ val _indexedEdge = indexedEdge.copy(props = newProps)
+ indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
+ }
+
+ def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
+ val newProps = indexedEdge.props ++ Map(LabelMeta.countSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
+ val _indexedEdge = indexedEdge.copy(props = newProps)
+ indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
+ }
+ def buildDeleteBelongsToId(vertex: Vertex): Seq[SKeyValue] = {
+ val kvs = vertexSerializer(vertex).toKeyValues
+ val kv = kvs.head
+ vertex.belongLabelIds.map { id =>
+ kv.copy(qualifier = Bytes.toBytes(Vertex.toPropKey(id)), operation = SKeyValue.Delete)
+ }
+ }
+
+ def buildVertexPutsAsync(edge: Edge): Seq[SKeyValue] =
+ if (edge.op == GraphUtil.operations("delete"))
+ buildDeleteBelongsToId(edge.srcForVertex) ++ buildDeleteBelongsToId(edge.tgtForVertex)
+ else
+ vertexSerializer(edge.srcForVertex).toKeyValues ++ vertexSerializer(edge.tgtForVertex).toKeyValues
+
+ def buildPutsAll(vertex: Vertex): Seq[SKeyValue] = {
+ vertex.op match {
+ case d: Byte if d == GraphUtil.operations("delete") => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
+ case _ => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala
new file mode 100644
index 0000000..74cd308
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageDeserializable.scala
@@ -0,0 +1,95 @@
+package org.apache.s2graph.core.storage
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.QueryParam
+import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, InnerValLikeWithTs}
+import org.apache.s2graph.core.utils.logger
+
+object StorageDeserializable {
+ /** Deserializer */
+ def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
+ val byte = bytes(offset)
+ val isInverted = if ((byte & 1) != 0) true else false
+ val labelOrderSeq = byte >> 1
+ (labelOrderSeq.toByte, isInverted)
+ }
+
+ def bytesToKeyValues(bytes: Array[Byte],
+ offset: Int,
+ length: Int,
+ version: String): (Array[(Byte, InnerValLike)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLike)](len)
+ var i = 0
+ while (i < len) {
+ val k = bytes(pos)
+ pos += 1
+ val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ val ret = (kvs, pos)
+ // logger.debug(s"bytesToProps: $ret")
+ ret
+ }
+
+ def bytesToKeyValuesWithTs(bytes: Array[Byte],
+ offset: Int,
+ version: String): (Array[(Byte, InnerValLikeWithTs)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLikeWithTs)](len)
+ var i = 0
+ while (i < len) {
+ val k = bytes(pos)
+ pos += 1
+ val (v, numOfBytesUsed) = InnerValLikeWithTs.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ val ret = (kvs, pos)
+ // logger.debug(s"bytesToProps: $ret")
+ ret
+ }
+
+ def bytesToProps(bytes: Array[Byte],
+ offset: Int,
+ version: String): (Array[(Byte, InnerValLike)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLike)](len)
+ var i = 0
+ while (i < len) {
+ val k = HBaseType.EMPTY_SEQ_BYTE
+ val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ // logger.error(s"bytesToProps: $kvs")
+ val ret = (kvs, pos)
+
+ ret
+ }
+
+ def bytesToLong(bytes: Array[Byte], offset: Int): Long = Bytes.toLong(bytes, offset)
+}
+
+trait StorageDeserializable[E] {
+ def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): Option[E] = {
+ try {
+ Option(fromKeyValuesInner(queryParam, kvs, version, cacheElementOpt))
+ } catch {
+ case e: Exception =>
+ logger.error(s"${this.getClass.getName} fromKeyValues failed.", e)
+ None
+ }
+ }
+ def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): E
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala
new file mode 100644
index 0000000..46ce539
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerializable.scala
@@ -0,0 +1,41 @@
+package org.apache.s2graph.core.storage
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.types.{InnerValLike, InnerValLikeWithTs}
+
+object StorageSerializable {
+ /** serializer */
+ def propsToBytes(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, v.bytes)
+ bytes
+ }
+
+ def propsToKeyValues(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
+ bytes
+ }
+
+ def propsToKeyValuesWithTs(props: Seq[(Byte, InnerValLikeWithTs)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
+ bytes
+ }
+
+ def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
+ assert(labelOrderSeq < (1 << 6))
+ val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
+ Array.fill(1)(byte.toByte)
+ }
+}
+
+trait StorageSerializable[E] {
+ def toKeyValues: Seq[SKeyValue]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
new file mode 100644
index 0000000..2560c9d
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -0,0 +1,534 @@
+package org.apache.s2graph.core.storage.hbase
+
+import java.util
+import java.util.Base64
+
+import com.stumbleupon.async.Deferred
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.client.{ConnectionFactory, Durability}
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.regionserver.BloomType
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.types.{HBaseType, VertexId}
+import org.apache.s2graph.core.utils.{DeferCache, Extensions, FutureCache, logger}
+import org.hbase.async._
+
+import scala.collection.JavaConversions._
+import scala.collection.{Map, Seq}
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext, Future, duration}
+import scala.util.hashing.MurmurHash3
+
+
+object AsynchbaseStorage {
+ val vertexCf = Serializable.vertexCf
+ val edgeCf = Serializable.edgeCf
+ val emptyKVs = new util.ArrayList[KeyValue]()
+
+
+ def makeClient(config: Config, overrideKv: (String, String)*) = {
+ val asyncConfig: org.hbase.async.Config =
+ if (config.hasPath("hbase.security.auth.enable") && config.getBoolean("hbase.security.auth.enable")) {
+ val krb5Conf = config.getString("java.security.krb5.conf")
+ val jaas = config.getString("java.security.auth.login.config")
+
+ System.setProperty("java.security.krb5.conf", krb5Conf)
+ System.setProperty("java.security.auth.login.config", jaas)
+ new org.hbase.async.Config()
+ } else {
+ new org.hbase.async.Config()
+ }
+
+ for (entry <- config.entrySet() if entry.getKey.contains("hbase")) {
+ asyncConfig.overrideConfig(entry.getKey, entry.getValue.unwrapped().toString)
+ }
+
+ for ((k, v) <- overrideKv) {
+ asyncConfig.overrideConfig(k, v)
+ }
+
+ val client = new HBaseClient(asyncConfig)
+ logger.info(s"Asynchbase: ${client.getConfig.dumpConfiguration()}")
+ client
+ }
+}
+
+
+class AsynchbaseStorage(override val config: Config)(implicit ec: ExecutionContext)
+ extends Storage[Deferred[QueryRequestWithResult]](config) {
+
+ import Extensions.DeferOps
+
+ /**
+ * Asynchbase client setup.
+ * note that we need two client, one for bulk(withWait=false) and another for withWait=true
+ */
+ val configWithFlush = config.withFallback(ConfigFactory.parseMap(Map("hbase.rpcs.buffered_flush_interval" -> "0")))
+ val client = AsynchbaseStorage.makeClient(config)
+
+ private val clientWithFlush = AsynchbaseStorage.makeClient(config, "hbase.rpcs.buffered_flush_interval" -> "0")
+ private val clients = Seq(client, clientWithFlush)
+ private val clientFlushInterval = config.getInt("hbase.rpcs.buffered_flush_interval").toString().toShort
+ private val emptyKeyValues = new util.ArrayList[KeyValue]()
+ private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
+
+ /** Future Cache to squash request */
+ private val futureCache = new DeferCache[QueryResult](config)(ec)
+
+ /** Simple Vertex Cache */
+ private val vertexCache = new FutureCache[Seq[SKeyValue]](config)(ec)
+
+
+ /**
+ * fire rpcs into proper hbase cluster using client and
+ * return true on all mutation success. otherwise return false.
+ */
+ override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
+ if (kvs.isEmpty) Future.successful(true)
+ else {
+ val _client = client(withWait)
+ val futures = kvs.map { kv =>
+ val _defer = kv.operation match {
+ case SKeyValue.Put => _client.put(new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp))
+ case SKeyValue.Delete =>
+ if (kv.qualifier == null) _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.timestamp))
+ else _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.timestamp))
+ case SKeyValue.Increment =>
+ _client.atomicIncrement(new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, Bytes.toLong(kv.value)))
+ }
+ val future = _defer.withCallback { ret => true }.recoverWith { ex =>
+ logger.error(s"mutation failed. $kv", ex)
+ false
+ }.toFuture
+
+ if (withWait) future else Future.successful(true)
+ }
+
+ Future.sequence(futures).map(_.forall(identity))
+ }
+ }
+
+
+ override def fetchSnapshotEdgeKeyValues(hbaseRpc: AnyRef): Future[Seq[SKeyValue]] = {
+ val defer = fetchKeyValuesInner(hbaseRpc)
+ defer.toFuture.map { kvsArr =>
+ kvsArr.map { kv =>
+ implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
+ } toSeq
+ }
+ }
+
+ /**
+ * since HBase natively provide CheckAndSet on storage level, implementation becomes simple.
+ * @param rpc: key value that is need to be stored on storage.
+ * @param expectedOpt: last valid value for rpc's KeyValue.value from fetching.
+ * @return return true if expected value matches and our rpc is successfully applied, otherwise false.
+ * note that when some other thread modified same cell and have different value on this KeyValue,
+ * then HBase atomically return false.
+ */
+ override def writeLock(rpc: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean] = {
+ val put = new PutRequest(rpc.table, rpc.row, rpc.cf, rpc.qualifier, rpc.value, rpc.timestamp)
+ val expected = expectedOpt.map(_.value).getOrElse(Array.empty)
+ client(withWait = true).compareAndSet(put, expected).withCallback(ret => ret.booleanValue()).toFuture
+ }
+
+
+ /**
+ * given queryRequest, build storage specific RPC Request.
+ * In HBase case, we either build Scanner or GetRequest.
+ *
+ * IndexEdge layer:
+ * Tall schema(v4): use scanner.
+ * Wide schema(label's schema version in v1, v2, v3): use GetRequest with columnRangeFilter
+ * when query is given with itnerval option.
+ * SnapshotEdge layer:
+ * Tall schema(v3, v4): use GetRequest without column filter.
+ * Wide schema(label's schema version in v1, v2): use GetRequest with columnRangeFilter.
+ * Vertex layer:
+ * all version: use GetRequest without column filter.
+ * @param queryRequest
+ * @return Scanner or GetRequest with proper setup with StartKey, EndKey, RangeFilter.
+ */
+ override def buildRequest(queryRequest: QueryRequest): AnyRef = {
+ import Serializable._
+ val queryParam = queryRequest.queryParam
+ val label = queryParam.label
+ val edge = toRequestEdge(queryRequest)
+
+ val kv = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
+ val snapshotEdge = edge.toSnapshotEdge
+ snapshotEdgeSerializer(snapshotEdge).toKeyValues.head
+ // new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
+ } else {
+ val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == queryParam.labelOrderSeq)
+ assert(indexedEdgeOpt.isDefined)
+
+ val indexedEdge = indexedEdgeOpt.get
+ indexEdgeSerializer(indexedEdge).toKeyValues.head
+ }
+
+ val (minTs, maxTs) = queryParam.duration.getOrElse((0L, Long.MaxValue))
+
+ label.schemaVersion match {
+ case HBaseType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty =>
+ val scanner = client.newScanner(label.hbaseTableName.getBytes)
+ scanner.setFamily(edgeCf)
+
+ /**
+ * TODO: remove this part.
+ */
+ val indexEdgeOpt = edge.edgesWithIndex.filter(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq).headOption
+ val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
+
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+ // val labelIndexSeqWithIsInvertedStopBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = true)
+ val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, Bytes.add(labelIndexSeqWithIsInvertedBytes, Array.fill(1)(edge.op)))
+ val (startKey, stopKey) =
+ if (queryParam.columnRangeFilter != null) {
+ // interval is set.
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Bytes.add(Base64.getDecoder.decode(cursor), Array.fill(1)(0))
+ case None => Bytes.add(baseKey, queryParam.columnRangeFilterMinBytes)
+ }
+ (_startKey, Bytes.add(baseKey, queryParam.columnRangeFilterMaxBytes))
+ } else {
+ /**
+ * note: since propsToBytes encode size of property map at first byte, we are sure about max value here
+ */
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Bytes.add(Base64.getDecoder.decode(cursor), Array.fill(1)(0))
+ case None => baseKey
+ }
+ (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
+ }
+// logger.debug(s"[StartKey]: ${startKey.toList}")
+// logger.debug(s"[StopKey]: ${stopKey.toList}")
+
+ scanner.setStartKey(startKey)
+ scanner.setStopKey(stopKey)
+
+ if (queryParam.limit == Int.MinValue) logger.debug(s"MinValue: $queryParam")
+
+ scanner.setMaxVersions(1)
+ scanner.setMaxNumRows(queryParam.limit)
+ scanner.setMaxTimestamp(maxTs)
+ scanner.setMinTimestamp(minTs)
+ scanner.setRpcTimeout(queryParam.rpcTimeoutInMillis)
+ // SET option for this rpc properly.
+ scanner
+ case _ =>
+ val get =
+ if (queryParam.tgtVertexInnerIdOpt.isDefined) new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
+ else new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf)
+
+ get.maxVersions(1)
+ get.setFailfast(true)
+ get.setMaxResultsPerColumnFamily(queryParam.limit)
+ get.setRowOffsetPerColumnFamily(queryParam.offset)
+ get.setMinTimestamp(minTs)
+ get.setMaxTimestamp(maxTs)
+ get.setTimeout(queryParam.rpcTimeoutInMillis)
+
+ if (queryParam.columnRangeFilter != null) get.setFilter(queryParam.columnRangeFilter)
+
+ get
+ }
+ }
+
+ /**
+ * we are using future cache to squash requests into same key on storage.
+ *
+ * @param queryRequest
+ * @param prevStepScore
+ * @param isInnerCall
+ * @param parentEdges
+ * @return we use Deferred here since it has much better performrance compared to scala.concurrent.Future.
+ * seems like map, flatMap on scala.concurrent.Future is slower than Deferred's addCallback
+ */
+ override def fetch(queryRequest: QueryRequest,
+ prevStepScore: Double,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): Deferred[QueryRequestWithResult] = {
+
+ def fetchInner(hbaseRpc: AnyRef): Deferred[QueryResult] = {
+ fetchKeyValuesInner(hbaseRpc).withCallback { kvs =>
+ val edgeWithScores = toEdges(kvs, queryRequest.queryParam, prevStepScore, isInnerCall, parentEdges)
+ val resultEdgesWithScores = if (queryRequest.queryParam.sample >= 0) {
+ sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
+ } else edgeWithScores
+ QueryResult(resultEdgesWithScores, tailCursor = kvs.lastOption.map(_.key).getOrElse(Array.empty[Byte]))
+// QueryRequestWithResult(queryRequest, QueryResult(resultEdgesWithScores, tailCursor = kvs.lastOption.map(_.key).getOrElse(Array.empty)))
+
+ } recoverWith { ex =>
+ logger.error(s"fetchInner failed. fallback return. $hbaseRpc}", ex)
+ QueryResult(isFailure = true)
+// QueryRequestWithResult(queryRequest, QueryResult(isFailure = true))
+ }
+ }
+
+ val queryParam = queryRequest.queryParam
+ val cacheTTL = queryParam.cacheTTLInMillis
+ val request = buildRequest(queryRequest)
+
+ val defer =
+ if (cacheTTL <= 0) fetchInner(request)
+ else {
+ val cacheKeyBytes = Bytes.add(queryRequest.query.cacheKeyBytes, toCacheKeyBytes(request))
+ val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
+ futureCache.getOrElseUpdate(cacheKey, cacheTTL)(fetchInner(request))
+ }
+ defer withCallback { queryResult => QueryRequestWithResult(queryRequest, queryResult)}
+ }
+
+
+ override def fetches(queryRequestWithScoreLs: scala.Seq[(QueryRequest, Double)],
+ prevStepEdges: Predef.Map[VertexId, scala.Seq[EdgeWithScore]]): Future[scala.Seq[QueryRequestWithResult]] = {
+ val defers: Seq[Deferred[QueryRequestWithResult]] = for {
+ (queryRequest, prevStepScore) <- queryRequestWithScoreLs
+ parentEdges <- prevStepEdges.get(queryRequest.vertex.id)
+ } yield fetch(queryRequest, prevStepScore, isInnerCall = false, parentEdges)
+
+ val grouped: Deferred[util.ArrayList[QueryRequestWithResult]] = Deferred.group(defers)
+ grouped withCallback {
+ queryResults: util.ArrayList[QueryRequestWithResult] =>
+ queryResults.toIndexedSeq
+ } toFuture
+ }
+
+
+ def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]] = fetchSnapshotEdgeKeyValues(request)
+
+
+ /**
+ * when withWait is given, we use client with flushInterval set to 0.
+ * if we are not using this, then we are adding extra wait time as much as flushInterval in worst case.
+ *
+ * @param edges
+ * @param withWait
+ * @return
+ */
+ override def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = {
+ val _client = client(withWait)
+ val defers: Seq[Deferred[(Boolean, Long)]] = for {
+ edge <- edges
+ } yield {
+ val edgeWithIndex = edge.edgesWithIndex.head
+ val countWithTs = edge.propsWithTs(LabelMeta.countSeq)
+ val countVal = countWithTs.innerVal.toString().toLong
+ val incr = buildIncrementsCountAsync(edgeWithIndex, countVal).head
+ val request = incr.asInstanceOf[AtomicIncrementRequest]
+ _client.bufferAtomicIncrement(request) withCallback { resultCount: java.lang.Long =>
+ (true, resultCount.longValue())
+ } recoverWith { ex =>
+ logger.error(s"mutation failed. $request", ex)
+ (false, -1L)
+ }
+ }
+
+ val grouped: Deferred[util.ArrayList[(Boolean, Long)]] = Deferred.groupInOrder(defers)
+ grouped.toFuture.map(_.toSeq)
+ }
+
+
+ override def flush(): Unit = clients.foreach { client =>
+ val timeout = Duration((clientFlushInterval + 10) * 20, duration.MILLISECONDS)
+ Await.result(client.flush().toFuture, timeout)
+ }
+
+
+ override def createTable(zkAddr: String,
+ tableName: String,
+ cfs: List[String],
+ regionMultiplier: Int,
+ ttl: Option[Int],
+ compressionAlgorithm: String): Unit = {
+ logger.info(s"create table: $tableName on $zkAddr, $cfs, $regionMultiplier, $compressionAlgorithm")
+ val admin = getAdmin(zkAddr)
+ val regionCount = admin.getClusterStatus.getServersSize * regionMultiplier
+ if (!admin.tableExists(TableName.valueOf(tableName))) {
+ try {
+ val desc = new HTableDescriptor(TableName.valueOf(tableName))
+ desc.setDurability(Durability.ASYNC_WAL)
+ for (cf <- cfs) {
+ val columnDesc = new HColumnDescriptor(cf)
+ .setCompressionType(Algorithm.valueOf(compressionAlgorithm.toUpperCase))
+ .setBloomFilterType(BloomType.ROW)
+ .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)
+ .setMaxVersions(1)
+ .setTimeToLive(2147483647)
+ .setMinVersions(0)
+ .setBlocksize(32768)
+ .setBlockCacheEnabled(true)
+ if (ttl.isDefined) columnDesc.setTimeToLive(ttl.get)
+ desc.addFamily(columnDesc)
+ }
+
+ if (regionCount <= 1) admin.createTable(desc)
+ else admin.createTable(desc, getStartKey(regionCount), getEndKey(regionCount), regionCount)
+ } catch {
+ case e: Throwable =>
+ logger.error(s"$zkAddr, $tableName failed with $e", e)
+ throw e
+ }
+ } else {
+ logger.info(s"$zkAddr, $tableName, $cfs already exist.")
+ }
+ }
+
+
+ /** Asynchbase implementation override default getVertices to use future Cache */
+ override def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
+ def fromResult(queryParam: QueryParam,
+ kvs: Seq[SKeyValue],
+ version: String): Option[Vertex] = {
+
+ if (kvs.isEmpty) None
+ else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
+ }
+
+ val futures = vertices.map { vertex =>
+ val kvs = vertexSerializer(vertex).toKeyValues
+ val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
+ // get.setTimeout(this.singleGetTimeout.toShort)
+ get.setFailfast(true)
+ get.maxVersions(1)
+
+ val cacheKey = MurmurHash3.stringHash(get.toString)
+ vertexCache.getOrElseUpdate(cacheKey, cacheTTL = 10000)(fetchVertexKeyValues(get)).map { kvs =>
+ fromResult(QueryParam.Empty, kvs, vertex.serviceColumn.schemaVersion)
+ }
+ }
+
+ Future.sequence(futures).map { result => result.toList.flatten }
+ }
+
+
+
+
+
+ /**
+ * Private Methods which is specific to Asynchbase implementation.
+ */
+ private def fetchKeyValuesInner(rpc: AnyRef): Deferred[util.ArrayList[KeyValue]] = {
+ rpc match {
+ case getRequest: GetRequest => client.get(getRequest)
+ case scanner: Scanner =>
+ scanner.nextRows().withCallback { kvsLs =>
+ val ls = new util.ArrayList[KeyValue]
+ if (kvsLs == null) {
+
+ } else {
+ kvsLs.foreach { kvs =>
+ if (kvs != null) kvs.foreach { kv => ls.add(kv) }
+ else {
+
+ }
+ }
+ }
+ scanner.close()
+ ls
+ }.recoverWith { ex =>
+ logger.error(s"fetchKeyValuesInner failed.", ex)
+ scanner.close()
+ emptyKeyValues
+ }
+ case _ => Deferred.fromError(new RuntimeException(s"fetchKeyValues failed. $rpc"))
+ }
+ }
+
+ private def toCacheKeyBytes(hbaseRpc: AnyRef): Array[Byte] = {
+ hbaseRpc match {
+ case getRequest: GetRequest => getRequest.key()
+ case scanner: Scanner => scanner.getCurrentKey()
+ case _ =>
+ logger.error(s"toCacheKeyBytes failed. not supported class type. $hbaseRpc")
+ Array.empty[Byte]
+ }
+ }
+
+ private def getSecureClusterAdmin(zkAddr: String) = {
+ val jaas = config.getString("java.security.auth.login.config")
+ val krb5Conf = config.getString("java.security.krb5.conf")
+ val realm = config.getString("realm")
+ val principal = config.getString("principal")
+ val keytab = config.getString("keytab")
+
+
+
+ System.setProperty("java.security.auth.login.config", jaas)
+ System.setProperty("java.security.krb5.conf", krb5Conf)
+ // System.setProperty("sun.security.krb5.debug", "true")
+ // System.setProperty("sun.security.spnego.debug", "true")
+ val conf = new Configuration(true)
+ val hConf = HBaseConfiguration.create(conf)
+
+ hConf.set("hbase.zookeeper.quorum", zkAddr)
+
+ hConf.set("hadoop.security.authentication", "Kerberos")
+ hConf.set("hbase.security.authentication", "Kerberos")
+ hConf.set("hbase.master.kerberos.principal", "hbase/_HOST@" + realm)
+ hConf.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@" + realm)
+
+ System.out.println("Connecting secure cluster, using keytab\n")
+ UserGroupInformation.setConfiguration(hConf)
+ UserGroupInformation.loginUserFromKeytab(principal, keytab)
+ val currentUser = UserGroupInformation.getCurrentUser()
+ System.out.println("current user : " + currentUser + "\n")
+
+ // get table list
+ val conn = ConnectionFactory.createConnection(hConf)
+ conn.getAdmin
+ }
+
+ /**
+ * following configuration need to come together to use secured hbase cluster.
+ * 1. set hbase.security.auth.enable = true
+ * 2. set file path to jaas file java.security.auth.login.config
+ * 3. set file path to kerberos file java.security.krb5.conf
+ * 4. set realm
+ * 5. set principal
+ * 6. set file path to keytab
+ * @param zkAddr
+ * @return
+ */
+ private def getAdmin(zkAddr: String) = {
+ if (config.hasPath("hbase.security.auth.enable") && config.getBoolean("hbase.security.auth.enable")) {
+ getSecureClusterAdmin(zkAddr)
+ } else {
+ val conf = HBaseConfiguration.create()
+ conf.set("hbase.zookeeper.quorum", zkAddr)
+ val conn = ConnectionFactory.createConnection(conf)
+ conn.getAdmin
+ }
+ }
+
+ private def enableTable(zkAddr: String, tableName: String) = {
+ getAdmin(zkAddr).enableTable(TableName.valueOf(tableName))
+ }
+
+ private def disableTable(zkAddr: String, tableName: String) = {
+ getAdmin(zkAddr).disableTable(TableName.valueOf(tableName))
+ }
+
+ private def dropTable(zkAddr: String, tableName: String) = {
+ getAdmin(zkAddr).disableTable(TableName.valueOf(tableName))
+ getAdmin(zkAddr).deleteTable(TableName.valueOf(tableName))
+ }
+
+ private def getStartKey(regionCount: Int): Array[Byte] = {
+ Bytes.toBytes((Int.MaxValue / regionCount))
+ }
+
+ private def getEndKey(regionCount: Int): Array[Byte] = {
+ Bytes.toBytes((Int.MaxValue / regionCount * (regionCount - 1)))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
new file mode 100644
index 0000000..143f02d
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
@@ -0,0 +1,132 @@
+package org.apache.s2graph.core.storage.serde.indexedge.tall
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.storage.StorageDeserializable._
+import org.apache.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue, StorageDeserializable}
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.{GraphUtil, IndexEdge, QueryParam, Vertex}
+
+class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] {
+ import StorageDeserializable._
+
+ type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
+ type ValueRaw = (Array[(Byte, InnerValLike)], Int)
+
+ private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+ // val degree = Bytes.toLong(kv.value)
+ val degree = bytesToLongFunc(kv.value, 0)
+ val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
+ val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
+ (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
+ }
+
+ private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+ var qualifierLen = 0
+ var pos = 0
+ val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
+ val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
+ pos = endAt
+ qualifierLen += endAt
+ val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
+ (HBaseType.defaultTgtVertexId, 0)
+ } else {
+ TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
+ }
+ qualifierLen += tgtVertexIdLen
+ (props, endAt, tgtVertexId, tgtVertexIdLen)
+ }
+ val (op, opLen) =
+ if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
+ else (kv.qualifier(qualifierLen), 1)
+
+ qualifierLen += opLen
+
+ (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
+ }
+
+ private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
+ val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
+ (props, endAt)
+ }
+
+ private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
+ (Array.empty[(Byte, InnerValLike)], 0)
+ }
+
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[IndexEdge]): IndexEdge = {
+
+ assert(_kvs.size == 1)
+
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+
+ val kv = kvs.head
+
+ // logger.debug(s"[Des]: ${kv.row.toList}, ${kv.qualifier.toList}, ${kv.value.toList}")
+ var pos = 0
+ val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
+ pos += srcIdLen
+ val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
+ pos += 4
+ val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
+ pos += 1
+
+ val op = kv.row(pos)
+ pos += 1
+
+ if (pos == kv.row.length) {
+ // degree
+ // val degreeVal = Bytes.toLong(kv.value)
+ val degreeVal = bytesToLongFunc(kv.value, 0)
+ val ts = kv.timestamp
+ val props = Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, version),
+ LabelMeta.degreeSeq -> InnerVal.withLong(degreeVal, version))
+ val tgtVertexId = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
+ IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, props)
+ } else {
+ // not degree edge
+ val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${queryParam.label.id.get}, ${labelIdxSeq}"))
+
+ val (idxPropsRaw, endAt) = bytesToProps(kv.row, pos, version)
+ pos = endAt
+ val (tgtVertexIdRaw, tgtVertexIdLen) = if (endAt == kv.row.length) {
+ (HBaseType.defaultTgtVertexId, 0)
+ } else {
+ TargetVertexId.fromBytes(kv.row, endAt, kv.row.length, version)
+ }
+
+ val idxProps = for {
+ (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
+ } yield if (k == LabelMeta.degreeSeq) k -> v else seq -> v
+
+ val idxPropsMap = idxProps.toMap
+
+ val tgtVertexId =
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => tgtVertexIdRaw
+ case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
+ }
+
+ val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
+ // val countVal = Bytes.toLong(kv.value)
+ val countVal = bytesToLongFunc(kv.value, 0)
+ val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
+ (dummyProps, 8)
+ } else {
+ bytesToKeyValues(kv.value, 0, kv.value.length, version)
+ }
+
+ val _mergedProps = (idxProps ++ props).toMap
+ val mergedProps =
+ if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
+ else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
+
+ val ts = kv.timestamp
+ IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
+
+ }
+ }
+ }
[23/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/test/scala/subscriber/TransferToHFileTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/subscriber/TransferToHFileTest.scala
deleted file mode 100644
index 7b3f72f..0000000
--- a/loader/src/test/scala/subscriber/TransferToHFileTest.scala
+++ /dev/null
@@ -1,169 +0,0 @@
-package subscriber
-
-import com.kakao.s2graph.core.Management
-import com.kakao.s2graph.core.types.HBaseType
-import org.apache.spark.{SparkConf, SparkContext}
-import org.scalatest._
-import subscriber.TransferToHFile._
-
-/**
- * Created by Eric on 2015. 12. 2..
- */
-class TransferToHFileTest extends FlatSpec with BeforeAndAfterAll with Matchers {
-
- private val master = "local[2]"
- private val appName = "example-spark"
-
- private var sc: SparkContext = _
-
- val dataWithoutDir =
- """
- |1447686000000 insertBulk e a b friends_rel {}
- |1447686000000 insertBulk e a c friends_rel {}
- |1447686000000 insertBulk e a d friends_rel {}
- |1447686000000 insertBulk e b d friends_rel {}
- |1447686000000 insertBulk e b e friends_rel {}
- """.stripMargin.trim
-
- val dataWithDir =
- """
- |1447686000000 insertBulk e a b friends_rel {} out
- |1447686000000 insertBulk e b a friends_rel {} in
- |1447686000000 insertBulk e a c friends_rel {} out
- |1447686000000 insertBulk e c a friends_rel {} in
- |1447686000000 insertBulk e a d friends_rel {} out
- |1447686000000 insertBulk e d a friends_rel {} in
- |1447686000000 insertBulk e b d friends_rel {} out
- |1447686000000 insertBulk e d b friends_rel {} in
- |1447686000000 insertBulk e b e friends_rel {} out
- |1447686000000 insertBulk e e b friends_rel {} in
- """.stripMargin.trim
-
- override def beforeAll(): Unit = {
- println("### beforeAll")
-
- GraphSubscriberHelper.apply("dev", "none", "none", "none")
- // 1. create service
- if(Management.findService("loader-test").isEmpty) {
- println(">>> create service...")
- Management.createService("loader-test", "localhost", "loader-test-dev", 1, None, "gz")
- }
-
- // 2. create label
- if(Management.findLabel("friends_rel").isEmpty) {
- println(">>> create label...")
- Management.createLabel(
- "friends_rel",
- "loader-test", "user_id", "string",
- "loader-test", "user_id", "string",
- true,
- "loader-test",
- Seq(),
- Seq(),
- "weak",
- None, None,
- HBaseType.DEFAULT_VERSION,
- false,
- Management.defaultCompressionAlgorithm
- )
- }
-
- // create spark context
- val conf = new SparkConf()
- .setMaster(master)
- .setAppName(appName)
-
- sc = new SparkContext(conf)
- }
-
- override def afterAll(): Unit = {
- println("### afterALL")
- if (sc != null) {
- sc.stop()
- }
-
- Management.deleteLabel("friends_rel")
- }
-
- "buildDegreePutRequest" should "transform degree to PutRequest" in {
- val putReqs = buildDegreePutRequests("a", "friends_rel", "out", 3L)
- putReqs.size should equal(1)
- }
-
- "toKeyValues" should "transform edges to KeyValues on edge format data without direction" in {
- val rdd = sc.parallelize(dataWithoutDir.split("\n"))
-
- val kvs = rdd.mapPartitions { iter =>
- GraphSubscriberHelper.apply("dev", "none", "none", "none")
- TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
- }
- kvs.foreach(println)
- // edges * 2 (snapshot edges + indexed edges)
- kvs.count() should equal(10)
-
-
- val kvsAutoCreated = rdd.mapPartitions { iter =>
- GraphSubscriberHelper.apply("dev", "none", "none", "none")
- TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], true)
- }
-
- // edges * 3 (snapshot edges + indexed edges + reverse edges)
- kvsAutoCreated.count() should equal(15)
- }
-
- "toKeyValues" should "transform edges to KeyValues on edge format data with direction" in {
- val rdd = sc.parallelize(dataWithDir.split("\n"))
-
- val kvs = rdd.mapPartitions { iter =>
- GraphSubscriberHelper.apply("dev", "none", "none", "none")
- TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
- }
-
- // edges * 2 (snapshot edges + indexed edges)
- kvs.count() should equal(20)
- }
-
- "buildDegrees" should "build degrees on edge format data without direction" in {
- val rdd = sc.parallelize(dataWithoutDir.split("\n"))
-
- // autoCreate = false
- val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
- agg + current
- }.collectAsMap()
- degrees.size should equal(2)
-
- degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
- degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
-
-
- // autoCreate = true
- val degreesAutoCreated = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], true).reduceByKey { (agg, current) =>
- agg + current
- }.collectAsMap()
- degreesAutoCreated.size should equal(6)
-
- degreesAutoCreated should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
- degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
- degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
- degreesAutoCreated should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
- degreesAutoCreated should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
- degreesAutoCreated should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
- }
-
- "buildDegrees" should "build degrees on edge format data with direction" in {
- val rdd = sc.parallelize(dataWithDir.split("\n"))
-
- val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
- agg + current
- }.collectAsMap()
-
- degrees.size should equal(6)
-
- degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
- degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
- degrees should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
- degrees should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
- degrees should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
- degrees should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
deleted file mode 100644
index 8e6ad7d..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
+++ /dev/null
@@ -1,568 +0,0 @@
-package com.kakao.s2graph.core
-
-
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json.{JsNumber, Json}
-
-import scala.collection.JavaConversions._
-import scala.util.hashing.MurmurHash3
-
-
-case class SnapshotEdge(srcVertex: Vertex,
- tgtVertex: Vertex,
- labelWithDir: LabelWithDirection,
- op: Byte,
- version: Long,
- props: Map[Byte, InnerValLikeWithTs],
- pendingEdgeOpt: Option[Edge],
- statusCode: Byte = 0,
- lockTs: Option[Long]) extends JSONParser {
-
- if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
-
- val label = Label.findById(labelWithDir.labelId)
- val schemaVer = label.schemaVersion
- lazy val propsWithoutTs = props.mapValues(_.innerVal)
- val ts = props(LabelMeta.timeStampSeq).innerVal.toString().toLong
-
- def toEdge: Edge = {
- val ts = props.get(LabelMeta.timeStampSeq).map(v => v.ts).getOrElse(version)
- Edge(srcVertex, tgtVertex, labelWithDir, op,
- version, props, pendingEdgeOpt = pendingEdgeOpt,
- statusCode = statusCode, lockTs = lockTs)
- }
-
- def propsWithName = (for {
- (seq, v) <- props
- meta <- label.metaPropsMap.get(seq)
- jsValue <- innerValToJsValue(v.innerVal, meta.dataType)
- } yield meta.name -> jsValue) ++ Map("version" -> JsNumber(version))
-
- // only for debug
- def toLogString() = {
- List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, propsWithName).mkString("\t")
- }
-}
-
-case class IndexEdge(srcVertex: Vertex,
- tgtVertex: Vertex,
- labelWithDir: LabelWithDirection,
- op: Byte,
- version: Long,
- labelIndexSeq: Byte,
- props: Map[Byte, InnerValLike]) extends JSONParser {
- if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
- // assert(props.containsKey(LabelMeta.timeStampSeq))
-
- val ts = props(LabelMeta.timeStampSeq).toString.toLong
- val degreeEdge = props.contains(LabelMeta.degreeSeq)
- lazy val label = Label.findById(labelWithDir.labelId)
- val schemaVer = label.schemaVersion
- lazy val labelIndex = LabelIndex.findByLabelIdAndSeq(labelWithDir.labelId, labelIndexSeq).get
- lazy val defaultIndexMetas = labelIndex.sortKeyTypes.map { meta =>
- val innerVal = toInnerVal(meta.defaultValue, meta.dataType, schemaVer)
- meta.seq -> innerVal
- }.toMap
-
- lazy val labelIndexMetaSeqs = labelIndex.metaSeqs
-
- /** TODO: make sure call of this class fill props as this assumes */
- lazy val orders = for (k <- labelIndexMetaSeqs) yield {
- props.get(k) match {
- case None =>
-
- /**
- * TODO: agly hack
- * now we double store target vertex.innerId/srcVertex.innerId for easy development. later fix this to only store id once
- */
- val v = k match {
- case LabelMeta.timeStampSeq => InnerVal.withLong(version, schemaVer)
- case LabelMeta.toSeq => tgtVertex.innerId
- case LabelMeta.fromSeq => //srcVertex.innerId
- // for now, it does not make sense to build index on srcVertex.innerId since all edges have same data.
- throw new RuntimeException("_from on indexProps is not supported")
- case _ => defaultIndexMetas(k)
- }
-
- k -> v
- case Some(v) => k -> v
- }
- }
-
- lazy val ordersKeyMap = orders.map { case (byte, _) => byte }.toSet
- lazy val metas = for ((k, v) <- props if !ordersKeyMap.contains(k)) yield k -> v
-
- lazy val propsWithTs = props.map { case (k, v) => k -> InnerValLikeWithTs(v, version) }
-
- //TODO:
- // lazy val kvs = Graph.client.indexedEdgeSerializer(this).toKeyValues.toList
-
- lazy val hasAllPropsForIndex = orders.length == labelIndexMetaSeqs.length
-
- def propsWithName = for {
- (seq, v) <- props
- meta <- label.metaPropsMap.get(seq) if seq >= 0
- jsValue <- innerValToJsValue(v, meta.dataType)
- } yield meta.name -> jsValue
-
-
- def toEdge: Edge = Edge(srcVertex, tgtVertex, labelWithDir, op, version, propsWithTs)
-
- // only for debug
- def toLogString() = {
- List(version, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, Json.toJson(propsWithName)).mkString("\t")
- }
-}
-
-case class Edge(srcVertex: Vertex,
- tgtVertex: Vertex,
- labelWithDir: LabelWithDirection,
- op: Byte = GraphUtil.defaultOpByte,
- version: Long = System.currentTimeMillis(),
- propsWithTs: Map[Byte, InnerValLikeWithTs],
- parentEdges: Seq[EdgeWithScore] = Nil,
- originalEdgeOpt: Option[Edge] = None,
- pendingEdgeOpt: Option[Edge] = None,
- statusCode: Byte = 0,
- lockTs: Option[Long] = None) extends GraphElement with JSONParser {
-
- if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
- // assert(propsWithTs.containsKey(LabelMeta.timeStampSeq))
- val schemaVer = label.schemaVersion
- val ts = propsWithTs(LabelMeta.timeStampSeq).innerVal.toString.toLong
-
- def props = propsWithTs.mapValues(_.innerVal)
-
- def relatedEdges = {
- if (labelWithDir.isDirected) List(this, duplicateEdge)
- else {
- val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
- val base = copy(labelWithDir = outDir)
- List(base, base.reverseSrcTgtEdge)
- }
- }
-
- // def relatedEdges = List(this)
-
- def srcForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- Vertex(VertexId(label.tgtColumn.id.get, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- } else {
- Vertex(VertexId(label.srcColumn.id.get, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- }
- }
-
- def tgtForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- Vertex(VertexId(label.srcColumn.id.get, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- } else {
- Vertex(VertexId(label.tgtColumn.id.get, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- }
- }
-
- def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
-
- def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
-
- def reverseSrcTgtEdge = copy(srcVertex = tgtVertex, tgtVertex = srcVertex)
-
- def label = Label.findById(labelWithDir.labelId)
-
- def labelOrders = LabelIndex.findByLabelIdAll(labelWithDir.labelId)
-
- override def serviceName = label.serviceName
-
- override def queueKey = Seq(ts.toString, tgtVertex.serviceName).mkString("|")
-
- override def queuePartitionKey = Seq(srcVertex.innerId, tgtVertex.innerId).mkString("|")
-
- override def isAsync = label.isAsync
-
- def isDegree = propsWithTs.contains(LabelMeta.degreeSeq)
-
- def propsPlusTs = propsWithTs.get(LabelMeta.timeStampSeq) match {
- case Some(_) => props
- case None => props ++ Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, schemaVer))
- }
-
- def propsPlusTsValid = propsPlusTs.filter(kv => kv._1 >= 0)
-
- def edgesWithIndex = for (labelOrder <- labelOrders) yield {
- IndexEdge(srcVertex, tgtVertex, labelWithDir, op, version, labelOrder.seq, propsPlusTs)
- }
-
- def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
- IndexEdge(srcVertex, tgtVertex, labelWithDir, op, version, labelOrder.seq, propsPlusTsValid)
- }
-
- /** force direction as out on invertedEdge */
- def toSnapshotEdge: SnapshotEdge = {
- val (smaller, larger) = (srcForVertex, tgtForVertex)
-
- val newLabelWithDir = LabelWithDirection(labelWithDir.labelId, GraphUtil.directions("out"))
-
- val ret = SnapshotEdge(smaller, larger, newLabelWithDir, op, version,
- Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(ts, schemaVer), ts)) ++ propsWithTs,
- pendingEdgeOpt = pendingEdgeOpt, statusCode = statusCode, lockTs = lockTs)
- ret
- }
-
- override def hashCode(): Int = {
- MurmurHash3.stringHash(srcVertex.innerId + "," + labelWithDir + "," + tgtVertex.innerId)
- }
-
- override def equals(other: Any): Boolean = other match {
- case e: Edge =>
- srcVertex.innerId == e.srcVertex.innerId &&
- tgtVertex.innerId == e.tgtVertex.innerId &&
- labelWithDir == e.labelWithDir
- case _ => false
- }
-
- def propsWithName = for {
- (seq, v) <- props
- meta <- label.metaPropsMap.get(seq) if seq > 0
- jsValue <- innerValToJsValue(v, meta.dataType)
- } yield meta.name -> jsValue
-
- def updateTgtVertex(id: InnerValLike) = {
- val newId = TargetVertexId(tgtVertex.id.colId, id)
- val newTgtVertex = Vertex(newId, tgtVertex.ts, tgtVertex.props)
- Edge(srcVertex, newTgtVertex, labelWithDir, op, version, propsWithTs)
- }
-
- def rank(r: RankParam): Double =
- if (r.keySeqAndWeights.size <= 0) 1.0f
- else {
- var sum: Double = 0
-
- for ((seq, w) <- r.keySeqAndWeights) {
- seq match {
- case LabelMeta.countSeq => sum += 1
- case _ => {
- propsWithTs.get(seq) match {
- case None => // do nothing
- case Some(innerValWithTs) => {
- val cost = try innerValWithTs.innerVal.toString.toDouble catch {
- case e: Exception =>
- logger.error("toInnerval failed in rank", e)
- 1.0
- }
- sum += w * cost
- }
- }
- }
- }
- }
- sum
- }
-
- def toLogString: String = {
- val ret =
- if (propsWithName.nonEmpty)
- List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, Json.toJson(propsWithName))
- else
- List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label)
-
- ret.mkString("\t")
- }
-}
-
-case class EdgeMutate(edgesToDelete: List[IndexEdge] = List.empty[IndexEdge],
- edgesToInsert: List[IndexEdge] = List.empty[IndexEdge],
- newSnapshotEdge: Option[SnapshotEdge] = None) {
-
- def toLogString: String = {
- val l = (0 until 50).map(_ => "-").mkString("")
- val deletes = s"deletes: ${edgesToDelete.map(e => e.toLogString).mkString("\n")}"
- val inserts = s"inserts: ${edgesToInsert.map(e => e.toLogString).mkString("\n")}"
- val updates = s"snapshot: ${newSnapshotEdge.map(e => e.toLogString).mkString("\n")}"
-
- List("\n", l, deletes, inserts, updates, l, "\n").mkString("\n")
- }
-}
-
-object Edge extends JSONParser {
- val incrementVersion = 1L
- val minTsVal = 0L
-
- /** now version information is required also **/
- type State = Map[Byte, InnerValLikeWithTs]
- type PropsPairWithTs = (State, State, Long, String)
- type MergeState = PropsPairWithTs => (State, Boolean)
- type UpdateFunc = (Option[Edge], Edge, MergeState)
-
- def allPropsDeleted(props: Map[Byte, InnerValLikeWithTs]): Boolean =
- if (!props.containsKey(LabelMeta.lastDeletedAt)) false
- else {
- val lastDeletedAt = props.get(LabelMeta.lastDeletedAt).get.ts
- val propsWithoutLastDeletedAt = props - LabelMeta.lastDeletedAt
-
- propsWithoutLastDeletedAt.forall { case (_, v) => v.ts <= lastDeletedAt }
- }
-
- def buildDeleteBulk(invertedEdge: Option[Edge], requestEdge: Edge): (Edge, EdgeMutate) = {
- // assert(invertedEdge.isEmpty)
- // assert(requestEdge.op == GraphUtil.operations("delete"))
-
- val edgesToDelete = requestEdge.relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
- val edgeInverted = Option(requestEdge.toSnapshotEdge)
-
- (requestEdge, EdgeMutate(edgesToDelete, edgesToInsert = Nil, edgeInverted))
- }
-
- def buildOperation(invertedEdge: Option[Edge], requestEdges: Seq[Edge]): (Edge, EdgeMutate) = {
- // logger.debug(s"oldEdge: ${invertedEdge.map(_.toStringRaw)}")
- // logger.debug(s"requestEdge: ${requestEdge.toStringRaw}")
- val oldPropsWithTs =
- if (invertedEdge.isEmpty) Map.empty[Byte, InnerValLikeWithTs] else invertedEdge.get.propsWithTs
-
- val funcs = requestEdges.map { edge =>
- if (edge.op == GraphUtil.operations("insert")) {
- edge.label.consistencyLevel match {
- case "strong" => Edge.mergeUpsert _
- case _ => Edge.mergeInsertBulk _
- }
- } else if (edge.op == GraphUtil.operations("insertBulk")) {
- Edge.mergeInsertBulk _
- } else if (edge.op == GraphUtil.operations("delete")) {
- edge.label.consistencyLevel match {
- case "strong" => Edge.mergeDelete _
- case _ => throw new RuntimeException("not supported")
- }
- }
- else if (edge.op == GraphUtil.operations("update")) Edge.mergeUpdate _
- else if (edge.op == GraphUtil.operations("increment")) Edge.mergeIncrement _
- else throw new RuntimeException(s"not supported operation on edge: $edge")
- }
-
- val oldTs = invertedEdge.map(_.ts).getOrElse(minTsVal)
- val requestWithFuncs = requestEdges.zip(funcs).filter(oldTs != _._1.ts).sortBy(_._1.ts)
-
- if (requestWithFuncs.isEmpty) {
- (requestEdges.head, EdgeMutate())
- } else {
- val requestEdge = requestWithFuncs.last._1
- var prevPropsWithTs = oldPropsWithTs
-
- for {
- (requestEdge, func) <- requestWithFuncs
- } {
- val (_newPropsWithTs, _) = func(prevPropsWithTs, requestEdge.propsWithTs, requestEdge.ts, requestEdge.schemaVer)
- prevPropsWithTs = _newPropsWithTs
- // logger.debug(s"${requestEdge.toLogString}\n$oldPropsWithTs\n$prevPropsWithTs\n")
- }
- val requestTs = requestEdge.ts
- /** version should be monotoniously increasing so our RPC mutation should be applied safely */
- val newVersion = invertedEdge.map(e => e.version + incrementVersion).getOrElse(requestTs)
- val maxTs = prevPropsWithTs.map(_._2.ts).max
- val newTs = if (maxTs > requestTs) maxTs else requestTs
- val propsWithTs = prevPropsWithTs ++
- Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(newTs, requestEdge.label.schemaVersion), newTs))
- val edgeMutate = buildMutation(invertedEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
-
- // logger.debug(s"${edgeMutate.toLogString}\n${propsWithTs}")
- // logger.error(s"$propsWithTs")
- (requestEdge, edgeMutate)
- }
- }
-
- def buildMutation(snapshotEdgeOpt: Option[Edge],
- requestEdge: Edge,
- newVersion: Long,
- oldPropsWithTs: Map[Byte, InnerValLikeWithTs],
- newPropsWithTs: Map[Byte, InnerValLikeWithTs]): EdgeMutate = {
- if (oldPropsWithTs == newPropsWithTs) {
- // all requests should be dropped. so empty mutation.
- // logger.error(s"Case 1")
- EdgeMutate(edgesToDelete = Nil, edgesToInsert = Nil, newSnapshotEdge = None)
- } else {
- val withOutDeletedAt = newPropsWithTs.filter(kv => kv._1 != LabelMeta.lastDeletedAt)
- val newOp = snapshotEdgeOpt match {
- case None => requestEdge.op
- case Some(old) =>
- val oldMaxTs = old.propsWithTs.map(_._2.ts).max
- if (oldMaxTs > requestEdge.ts) old.op
- else requestEdge.op
- }
-
- val newSnapshotEdgeOpt =
- Option(requestEdge.copy(op = newOp, propsWithTs = newPropsWithTs, version = newVersion).toSnapshotEdge)
- // delete request must always update snapshot.
- if (withOutDeletedAt == oldPropsWithTs && newPropsWithTs.containsKey(LabelMeta.lastDeletedAt)) {
- // no mutation on indexEdges. only snapshotEdge should be updated to record lastDeletedAt.
- // logger.error(s"Case 2")
- EdgeMutate(edgesToDelete = Nil, edgesToInsert = Nil, newSnapshotEdge = newSnapshotEdgeOpt)
- } else {
- // logger.error(s"Case 3")
- val edgesToDelete = snapshotEdgeOpt match {
- case Some(snapshotEdge) if snapshotEdge.op != GraphUtil.operations("delete") =>
- snapshotEdge.copy(op = GraphUtil.defaultOpByte).
- relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
- case _ => Nil
- }
-
- val edgesToInsert =
- if (newPropsWithTs.isEmpty || allPropsDeleted(newPropsWithTs)) Nil
- else
- requestEdge.copy(version = newVersion, propsWithTs = newPropsWithTs, op = GraphUtil.defaultOpByte).
- relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
-
- EdgeMutate(edgesToDelete = edgesToDelete, edgesToInsert = edgesToInsert, newSnapshotEdge = newSnapshotEdgeOpt)
- }
- }
- }
-
- def mergeUpsert(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
- var shouldReplace = false
- val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
- val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
- val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
- propsWithTs.get(k) match {
- case Some(newValWithTs) =>
- assert(oldValWithTs.ts >= lastDeletedAt)
- val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
- else {
- shouldReplace = true
- newValWithTs
- }
- Some(k -> v)
-
- case None =>
- assert(oldValWithTs.ts >= lastDeletedAt)
- if (oldValWithTs.ts >= requestTs || k < 0) Some(k -> oldValWithTs)
- else {
- shouldReplace = true
- None
- }
- }
- }
- val existInNew =
- for {
- (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
- } yield {
- shouldReplace = true
- Some(k -> newValWithTs)
- }
-
- ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
- }
-
- def mergeUpdate(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
- var shouldReplace = false
- val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
- val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
- val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
- propsWithTs.get(k) match {
- case Some(newValWithTs) =>
- assert(oldValWithTs.ts >= lastDeletedAt)
- val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
- else {
- shouldReplace = true
- newValWithTs
- }
- Some(k -> v)
- case None =>
- // important: update need to merge previous valid values.
- assert(oldValWithTs.ts >= lastDeletedAt)
- Some(k -> oldValWithTs)
- }
- }
- val existInNew = for {
- (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
- } yield {
- shouldReplace = true
- Some(k -> newValWithTs)
- }
-
- ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
- }
-
- def mergeIncrement(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
- var shouldReplace = false
- val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
- val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
- val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
- propsWithTs.get(k) match {
- case Some(newValWithTs) =>
- if (k == LabelMeta.timeStampSeq) {
- val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
- else {
- shouldReplace = true
- newValWithTs
- }
- Some(k -> v)
- } else {
- if (oldValWithTs.ts >= newValWithTs.ts) {
- Some(k -> oldValWithTs)
- } else {
- assert(oldValWithTs.ts < newValWithTs.ts && oldValWithTs.ts >= lastDeletedAt)
- shouldReplace = true
- // incr(t0), incr(t2), d(t1) => deleted
- Some(k -> InnerValLikeWithTs(oldValWithTs.innerVal + newValWithTs.innerVal, oldValWithTs.ts))
- }
- }
-
- case None =>
- assert(oldValWithTs.ts >= lastDeletedAt)
- Some(k -> oldValWithTs)
- // if (oldValWithTs.ts >= lastDeletedAt) Some(k -> oldValWithTs) else None
- }
- }
- val existInNew = for {
- (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
- } yield {
- shouldReplace = true
- Some(k -> newValWithTs)
- }
-
- ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
- }
-
- def mergeDelete(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
- var shouldReplace = false
- val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
- val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt) match {
- case Some(prevDeletedAt) =>
- if (prevDeletedAt.ts >= requestTs) prevDeletedAt.ts
- else {
- shouldReplace = true
- requestTs
- }
- case None => {
- shouldReplace = true
- requestTs
- }
- }
- val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
- if (k == LabelMeta.timeStampSeq) {
- if (oldValWithTs.ts >= requestTs) Some(k -> oldValWithTs)
- else {
- shouldReplace = true
- Some(k -> InnerValLikeWithTs.withLong(requestTs, requestTs, version))
- }
- } else {
- if (oldValWithTs.ts >= lastDeletedAt) Some(k -> oldValWithTs)
- else {
- shouldReplace = true
- None
- }
- }
- }
- val mustExistInNew = Map(LabelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(lastDeletedAt, lastDeletedAt, version))
- ((existInOld.flatten ++ mustExistInNew).toMap, shouldReplace)
- }
-
- def mergeInsertBulk(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
- val (_, propsWithTs, _, _) = propsPairWithTs
- (propsWithTs, true)
- }
-
- def fromString(s: String): Option[Edge] = Graph.toEdge(s)
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/ExceptionHandler.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/ExceptionHandler.scala b/s2core/src/main/scala/com/kakao/s2graph/core/ExceptionHandler.scala
deleted file mode 100644
index a965e90..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/ExceptionHandler.scala
+++ /dev/null
@@ -1,162 +0,0 @@
-package com.kakao.s2graph.core
-
-import java.util.Properties
-import java.util.concurrent.atomic.AtomicLong
-
-import akka.actor._
-import akka.routing.{Broadcast, RoundRobinPool}
-import com.typesafe.config.Config
-import org.apache.kafka.clients.producer._
-
-import scala.concurrent.duration._
-
-/**
- * Created by shon on 7/16/15.
- */
-object ExceptionHandler {
-
- var producer: Option[Producer[Key, Val]] = None
- var properties: Option[Properties] = None
- val numOfRoutees = 1
- val actorSystem = ActorSystem("ExceptionHandler")
- var routees: Option[ActorRef] = None
- var shutdownTime = 1000 millis
- var phase = "dev"
- lazy val failTopic = s"mutateFailed_${phase}"
-
- def apply(config: Config) = {
- properties =
- if (config.hasPath("kafka.metadata.broker.list")) Option(kafkaConfig(config))
- else None
- phase = if (config.hasPath("phase")) config.getString("phase") else "dev"
- producer = for {
- props <- properties
- p <- try {
- Option(new KafkaProducer[Key, Val](props))
- } catch {
- case e: Throwable => None
- }
- } yield {
- p
- }
- init()
- }
-
- def props(producer: Producer[Key, Val]) = Props(classOf[KafkaAggregatorActor], producer)
-
- def init() = {
- for {
- p <- producer
- } {
- routees = Option(actorSystem.actorOf(RoundRobinPool(numOfRoutees).props(props(p))))
- }
- }
-
- def shutdown() = {
- routees.map(_ ! Broadcast(PoisonPill))
- Thread.sleep(shutdownTime.length)
- }
-
- def enqueues(msgs: Seq[KafkaMessage]) = {
- msgs.foreach(enqueue)
- }
-
- def enqueue(msg: KafkaMessage) = {
- routees.map(_ ! msg)
- }
-
-
- def kafkaConfig(config: Config) = {
- val props = new Properties();
-
- /** all default configuration for new producer */
- val brokers =
- if (config.hasPath("kafka.metadata.broker.list")) config.getString("kafka.metadata.broker.list")
- else "localhost"
- props.put("bootstrap.servers", brokers)
- props.put("acks", "1")
- props.put("buffer.memory", "33554432")
- props.put("compression.type", "snappy")
- props.put("retries", "0")
- props.put("batch.size", "16384")
- props.put("linger.ms", "0")
- props.put("max.request.size", "1048576")
- props.put("receive.buffer.bytes", "32768")
- props.put("send.buffer.bytes", "131072")
- props.put("timeout.ms", "30000")
- props.put("block.on.buffer.full", "false")
- props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
- props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
- props
- }
-
- type Key = String
- type Val = String
-
- def toKafkaMessage(topic: String = failTopic, element: GraphElement, originalString: Option[String] = None) = {
- KafkaMessage(new ProducerRecord[Key, Val](topic, element.queuePartitionKey,
- originalString.getOrElse(element.toLogString())))
- }
-
- case class KafkaMessage(msg: ProducerRecord[Key, Val])
-
- case class Message(topic: String, msg: String)
-
- case class BufferedKafkaMessage(msgs: Seq[ProducerRecord[Key, Val]], bufferSize: Int)
-
- case class BufferedMessage(topic: String, bufferedMsgs: String, bufferSize: Int)
-
- case object FlushBuffer
-
- case class UpdateHealth(isHealty: Boolean)
-
- case object ShowMetrics
-
-}
-
-class KafkaAggregatorActor(kafkaProducer: Producer[String, String]) extends Stash with ActorLogging {
-
- import ExceptionHandler._
-
- val failedCount = new AtomicLong(0L)
- val successCount = new AtomicLong(0L)
- val stashCount = new AtomicLong(0L)
-
- implicit val ex = context.system.dispatcher
-
- context.system.scheduler.schedule(0 millis, 10 seconds) {
- self ! ShowMetrics
- }
-
- override def receive = {
- case ShowMetrics =>
- log.info(s"[Stats]: failed[${failedCount.get}], stashed[${stashCount.get}], success[${successCount.get}]")
-
- case m: KafkaMessage =>
- val replayTo = self
- try {
- kafkaProducer.send(m.msg, new Callback() {
- override def onCompletion(meta: RecordMetadata, e: Exception) = {
- if (e == null) {
- // success
- successCount.incrementAndGet()
- unstashAll()
- stashCount.set(0L)
- } else {
- // failure
- log.error(s"onCompletion: $e", e)
- failedCount.incrementAndGet()
- replayTo ! m
- }
- }
- })
- } catch {
- case e@(_: org.apache.kafka.clients.producer.BufferExhaustedException | _: Throwable) =>
- log.error(s"$e", e)
- log.info(s"stash")
- stash()
- stashCount.incrementAndGet()
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
deleted file mode 100644
index fdc8553..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
+++ /dev/null
@@ -1,381 +0,0 @@
-package com.kakao.s2graph.core
-
-import java.util
-import java.util.concurrent.ConcurrentHashMap
-
-import com.google.common.cache.CacheBuilder
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.parsers.WhereParser
-import com.kakao.s2graph.core.storage.Storage
-import com.kakao.s2graph.core.storage.hbase._
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.logger
-import com.typesafe.config.{Config, ConfigFactory}
-
-import scala.collection.JavaConversions._
-import scala.collection._
-import scala.collection.mutable.ListBuffer
-import scala.concurrent._
-import scala.util.Try
-
-object Graph {
- val DefaultScore = 1.0
-
- private val DefaultConfigs: Map[String, AnyRef] = Map(
- "hbase.zookeeper.quorum" -> "localhost",
- "hbase.table.name" -> "s2graph",
- "hbase.table.compression.algorithm" -> "gz",
- "phase" -> "dev",
- "db.default.driver" -> "com.mysql.jdbc.Driver",
- "db.default.url" -> "jdbc:mysql://localhost:3306/graph_dev",
- "db.default.password" -> "graph",
- "db.default.user" -> "graph",
- "cache.max.size" -> java.lang.Integer.valueOf(10000),
- "cache.ttl.seconds" -> java.lang.Integer.valueOf(60),
- "hbase.client.retries.number" -> java.lang.Integer.valueOf(20),
- "hbase.rpcs.buffered_flush_interval" -> java.lang.Short.valueOf(100.toShort),
- "hbase.rpc.timeout" -> java.lang.Integer.valueOf(1000),
- "max.retry.number" -> java.lang.Integer.valueOf(100),
- "lock.expire.time" -> java.lang.Integer.valueOf(1000 * 60 * 10),
- "max.back.off" -> java.lang.Integer.valueOf(100),
- "hbase.fail.prob" -> java.lang.Double.valueOf(-0.1),
- "delete.all.fetch.size" -> java.lang.Integer.valueOf(1000),
- "future.cache.max.size" -> java.lang.Integer.valueOf(100000),
- "future.cache.expire.after.write" -> java.lang.Integer.valueOf(10000),
- "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000),
- "s2graph.storage.backend" -> "hbase"
- )
-
- var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs)
-
- /** helpers for filterEdges */
- type HashKey = (Int, Int, Int, Int, Boolean)
- type FilterHashKey = (Int, Int)
- type Result = (ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]],
- ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)],
- ListBuffer[(HashKey, FilterHashKey, Edge, Double)])
-
- def toHashKey(queryParam: QueryParam, edge: Edge, isDegree: Boolean): (HashKey, FilterHashKey) = {
- val src = edge.srcVertex.innerId.hashCode()
- val tgt = edge.tgtVertex.innerId.hashCode()
- val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
- val filterHashKey = (src, tgt)
-
- (hashKey, filterHashKey)
- }
-
- def alreadyVisitedVertices(queryResultLs: Seq[QueryResult]): Map[(LabelWithDirection, Vertex), Boolean] = {
- val vertices = for {
- queryResult <- queryResultLs
- edgeWithScore <- queryResult.edgeWithScoreLs
- edge = edgeWithScore.edge
- vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
- } yield (edge.labelWithDir, vertex) -> true
-
- vertices.toMap
- }
-
- /** common methods for filter out, transform, aggregate queryResult */
- def convertEdges(queryParam: QueryParam, edge: Edge, nextStepOpt: Option[Step]): Seq[Edge] = {
- for {
- convertedEdge <- queryParam.transformer.transform(edge, nextStepOpt) if !edge.isDegree
- } yield convertedEdge
- }
-
- def processTimeDecay(queryParam: QueryParam, edge: Edge) = {
- /** process time decay */
- val tsVal = queryParam.timeDecay match {
- case None => 1.0
- case Some(timeDecay) =>
- val tsVal = try {
- val labelMeta = edge.label.metaPropsMap(timeDecay.labelMetaSeq)
- val innerValWithTsOpt = edge.propsWithTs.get(timeDecay.labelMetaSeq)
- innerValWithTsOpt.map { innerValWithTs =>
- val innerVal = innerValWithTs.innerVal
- labelMeta.dataType match {
- case InnerVal.LONG => innerVal.value match {
- case n: BigDecimal => n.bigDecimal.longValue()
- case _ => innerVal.toString().toLong
- }
- case _ => innerVal.toString().toLong
- }
- } getOrElse(edge.ts)
- } catch {
- case e: Exception =>
- logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
- edge.ts
- }
- val timeDiff = queryParam.timestamp - tsVal
- timeDecay.decay(timeDiff)
- }
-
- tsVal
- }
-
- def aggregateScore(newScore: Double,
- resultEdges: ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)],
- duplicateEdges: ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]],
- edgeWithScoreSorted: ListBuffer[(HashKey, FilterHashKey, Edge, Double)],
- hashKey: HashKey,
- filterHashKey: FilterHashKey,
- queryParam: QueryParam,
- convertedEdge: Edge) = {
-
- /** skip duplicate policy check if consistencyLevel is strong */
- if (queryParam.label.consistencyLevel != "strong" && resultEdges.containsKey(hashKey)) {
- val (oldFilterHashKey, oldEdge, oldScore) = resultEdges.get(hashKey)
- //TODO:
- queryParam.duplicatePolicy match {
- case Query.DuplicatePolicy.First => // do nothing
- case Query.DuplicatePolicy.Raw =>
- if (duplicateEdges.containsKey(hashKey)) {
- duplicateEdges.get(hashKey).append(convertedEdge -> newScore)
- } else {
- val newBuffer = new ListBuffer[(Edge, Double)]
- newBuffer.append(convertedEdge -> newScore)
- duplicateEdges.put(hashKey, newBuffer)
- }
- case Query.DuplicatePolicy.CountSum =>
- resultEdges.put(hashKey, (filterHashKey, oldEdge, oldScore + 1))
- case _ =>
- resultEdges.put(hashKey, (filterHashKey, oldEdge, oldScore + newScore))
- }
- } else {
- resultEdges.put(hashKey, (filterHashKey, convertedEdge, newScore))
- edgeWithScoreSorted.append((hashKey, filterHashKey, convertedEdge, newScore))
- }
- }
-
- def aggregateResults(queryRequestWithResult: QueryRequestWithResult,
- queryParamResult: Result,
- edgesToInclude: util.HashSet[FilterHashKey],
- edgesToExclude: util.HashSet[FilterHashKey]): QueryRequestWithResult = {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- val (query, stepIdx, _, queryParam) = QueryRequest.unapply(queryRequest).get
-
- val (duplicateEdges, resultEdges, edgeWithScoreSorted) = queryParamResult
- val edgesWithScores = for {
- (hashKey, filterHashKey, edge, _) <- edgeWithScoreSorted if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
- score = resultEdges.get(hashKey)._3
- (duplicateEdge, aggregatedScore) <- fetchDuplicatedEdges(edge, score, hashKey, duplicateEdges) if aggregatedScore >= queryParam.threshold
- } yield EdgeWithScore(duplicateEdge, aggregatedScore)
-
- QueryRequestWithResult(queryRequest, QueryResult(edgesWithScores))
- }
-
- def fetchDuplicatedEdges(edge: Edge,
- score: Double,
- hashKey: HashKey,
- duplicateEdges: ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]]) = {
- (edge -> score) +: (if (duplicateEdges.containsKey(hashKey)) duplicateEdges.get(hashKey) else Seq.empty)
- }
-
- def queryResultWithFilter(queryRequestWithResult: QueryRequestWithResult) = {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- val (_, _, _, queryParam) = QueryRequest.unapply(queryRequest).get
- val whereFilter = queryParam.where.get
- if (whereFilter == WhereParser.success) queryResult.edgeWithScoreLs
- else queryResult.edgeWithScoreLs.withFilter(edgeWithScore => whereFilter.filter(edgeWithScore.edge))
- }
-
- def filterEdges(queryResultLsFuture: Future[Seq[QueryRequestWithResult]],
- alreadyVisited: Map[(LabelWithDirection, Vertex), Boolean] = Map.empty[(LabelWithDirection, Vertex), Boolean])
- (implicit ec: scala.concurrent.ExecutionContext): Future[Seq[QueryRequestWithResult]] = {
-
- queryResultLsFuture.map { queryRequestWithResultLs =>
- if (queryRequestWithResultLs.isEmpty) Nil
- else {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs.head).get
- val (q, stepIdx, srcVertex, queryParam) = QueryRequest.unapply(queryRequest).get
- val step = q.steps(stepIdx)
-
- val nextStepOpt = if (stepIdx < q.steps.size - 1) Option(q.steps(stepIdx + 1)) else None
-
- val excludeLabelWithDirSet = new util.HashSet[(Int, Int)]
- val includeLabelWithDirSet = new util.HashSet[(Int, Int)]
- step.queryParams.filter(_.exclude).foreach(l => excludeLabelWithDirSet.add(l.labelWithDir.labelId -> l.labelWithDir.dir))
- step.queryParams.filter(_.include).foreach(l => includeLabelWithDirSet.add(l.labelWithDir.labelId -> l.labelWithDir.dir))
-
- val edgesToExclude = new util.HashSet[FilterHashKey]()
- val edgesToInclude = new util.HashSet[FilterHashKey]()
-
- val queryParamResultLs = new ListBuffer[Result]
- queryRequestWithResultLs.foreach { queryRequestWithResult =>
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- val queryParam = queryRequest.queryParam
- val duplicateEdges = new util.concurrent.ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]]()
- val resultEdges = new util.concurrent.ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)]()
- val edgeWithScoreSorted = new ListBuffer[(HashKey, FilterHashKey, Edge, Double)]
- val labelWeight = step.labelWeights.getOrElse(queryParam.labelWithDir.labelId, 1.0)
-
- // store degree value with Array.empty so if degree edge exist, it comes at very first.
- def checkDegree() = queryResult.edgeWithScoreLs.headOption.exists { edgeWithScore =>
- edgeWithScore.edge.isDegree
- }
- var isDegree = checkDegree()
-
- val includeExcludeKey = queryParam.labelWithDir.labelId -> queryParam.labelWithDir.dir
- val shouldBeExcluded = excludeLabelWithDirSet.contains(includeExcludeKey)
- val shouldBeIncluded = includeLabelWithDirSet.contains(includeExcludeKey)
-
- queryResultWithFilter(queryRequestWithResult).foreach { edgeWithScore =>
- val (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if (queryParam.transformer.isDefault) {
- val convertedEdge = edge
-
- val (hashKey, filterHashKey) = toHashKey(queryParam, convertedEdge, isDegree)
-
- /** check if this edge should be exlcuded. */
- if (shouldBeExcluded && !isDegree) {
- edgesToExclude.add(filterHashKey)
- } else {
- if (shouldBeIncluded && !isDegree) {
- edgesToInclude.add(filterHashKey)
- }
- val tsVal = processTimeDecay(queryParam, convertedEdge)
- val newScore = labelWeight * score * tsVal
- aggregateScore(newScore, resultEdges, duplicateEdges, edgeWithScoreSorted, hashKey, filterHashKey, queryParam, convertedEdge)
- }
- } else {
- convertEdges(queryParam, edge, nextStepOpt).foreach { convertedEdge =>
- val (hashKey, filterHashKey) = toHashKey(queryParam, convertedEdge, isDegree)
-
- /** check if this edge should be exlcuded. */
- if (shouldBeExcluded && !isDegree) {
- edgesToExclude.add(filterHashKey)
- } else {
- if (shouldBeIncluded && !isDegree) {
- edgesToInclude.add(filterHashKey)
- }
- val tsVal = processTimeDecay(queryParam, convertedEdge)
- val newScore = labelWeight * score * tsVal
- aggregateScore(newScore, resultEdges, duplicateEdges, edgeWithScoreSorted, hashKey, filterHashKey, queryParam, convertedEdge)
- }
- }
- }
- isDegree = false
- }
- val ret = (duplicateEdges, resultEdges, edgeWithScoreSorted)
- queryParamResultLs.append(ret)
- }
-
- val aggregatedResults = for {
- (queryRequestWithResult, queryParamResult) <- queryRequestWithResultLs.zip(queryParamResultLs)
- } yield {
- aggregateResults(queryRequestWithResult, queryParamResult, edgesToInclude, edgesToExclude)
- }
-
- aggregatedResults
- }
- }
- }
-
- def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] = Try {
- val parts = GraphUtil.split(s)
- val logType = parts(2)
- val element = if (logType == "edge" | logType == "e") {
- /** current only edge is considered to be bulk loaded */
- labelMapping.get(parts(5)) match {
- case None =>
- case Some(toReplace) =>
- parts(5) = toReplace
- }
- toEdge(parts)
- } else if (logType == "vertex" | logType == "v") {
- toVertex(parts)
- } else {
- throw new GraphExceptions.JsonParseException("log type is not exist in log.")
- }
-
- element
- } recover {
- case e: Exception =>
- logger.error(s"$e", e)
- None
- } get
-
-
- def toVertex(s: String): Option[Vertex] = {
- toVertex(GraphUtil.split(s))
- }
-
- def toEdge(s: String): Option[Edge] = {
- toEdge(GraphUtil.split(s))
- }
-
- //"1418342849000\tu\te\t3286249\t71770\ttalk_friend\t{\"is_hidden\":false}"
- //{"from":1,"to":101,"label":"graph_test","props":{"time":-1, "weight":10},"timestamp":1417616431},
- def toEdge(parts: Array[String]): Option[Edge] = Try {
- val (ts, operation, logType, srcId, tgtId, label) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
- val props = if (parts.length >= 7) parts(6) else "{}"
- val tempDirection = if (parts.length >= 8) parts(7) else "out"
- val direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
-
- val edge = Management.toEdge(ts.toLong, operation, srcId, tgtId, label, direction, props)
- // logger.debug(s"toEdge: $edge")
- Some(edge)
- } recover {
- case e: Exception =>
- logger.error(s"toEdge: $e", e)
- throw e
- } get
-
- //"1418342850000\ti\tv\t168756793\ttalk_user_id\t{\"country_iso\":\"KR\"}"
- def toVertex(parts: Array[String]): Option[Vertex] = Try {
- val (ts, operation, logType, srcId, serviceName, colName) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
- val props = if (parts.length >= 7) parts(6) else "{}"
- Some(Management.toVertex(ts.toLong, operation, srcId, serviceName, colName, props))
- } recover {
- case e: Throwable =>
- logger.error(s"toVertex: $e", e)
- throw e
- } get
-
- def initStorage(config: Config)(ec: ExecutionContext) = {
- config.getString("s2graph.storage.backend") match {
- case "hbase" => new AsynchbaseStorage(config)(ec)
- case _ => throw new RuntimeException("not supported storage.")
- }
- }
-}
-
-class Graph(_config: Config)(implicit val ec: ExecutionContext) {
- val config = _config.withFallback(Graph.DefaultConfig)
-
- Model.apply(config)
- Model.loadCache()
-
- // TODO: Make storage client by config param
- val storage = Graph.initStorage(config)(ec)
-
-
- for {
- entry <- config.entrySet() if Graph.DefaultConfigs.contains(entry.getKey)
- (k, v) = (entry.getKey, entry.getValue)
- } logger.info(s"[Initialized]: $k, ${this.config.getAnyRef(k)}")
-
- /** select */
- def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = storage.checkEdges(params)
-
- def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = storage.getEdges(q)
-
- def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = storage.getVertices(vertices)
-
- /** write */
- def deleteAllAdjacentEdges(srcVertices: List[Vertex], labels: Seq[Label], dir: Int, ts: Long): Future[Boolean] =
- storage.deleteAllAdjacentEdges(srcVertices, labels, dir, ts)
-
- def mutateElements(elements: Seq[GraphElement], withWait: Boolean = false): Future[Seq[Boolean]] =
- storage.mutateElements(elements, withWait)
-
- def mutateEdges(edges: Seq[Edge], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateEdges(edges, withWait)
-
- def mutateVertices(vertices: Seq[Vertex], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateVertices(vertices, withWait)
-
- def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = storage.incrementCounts(edges, withWait)
-
- def shutdown(): Unit = {
- storage.flush()
- Model.shutdown()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/GraphElement.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/GraphElement.scala b/s2core/src/main/scala/com/kakao/s2graph/core/GraphElement.scala
deleted file mode 100644
index e6c043d..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/GraphElement.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package com.kakao.s2graph.core
-
-import org.hbase.async.{HBaseRpc}
-
-trait GraphElement {
- def serviceName: String
- def ts: Long
- def isAsync: Boolean
- def queueKey: String
- def queuePartitionKey: String
- def toLogString(): String
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/GraphExceptions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/GraphExceptions.scala b/s2core/src/main/scala/com/kakao/s2graph/core/GraphExceptions.scala
deleted file mode 100644
index fa186c2..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/GraphExceptions.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-package com.kakao.s2graph.core
-
-
-object GraphExceptions {
-
- case class JsonParseException(msg: String) extends Exception(msg)
-
- case class LabelNotExistException(msg: String) extends Exception(msg)
-
- case class ModelNotFoundException(msg: String) extends Exception(msg)
-
- case class MaxPropSizeReachedException(msg: String) extends Exception(msg)
-
- case class LabelAlreadyExistException(msg: String) extends Exception(msg)
-
- case class InternalException(msg: String) extends Exception(msg)
-
- case class IllegalDataTypeException(msg: String) extends Exception(msg)
-
- case class WhereParserException(msg: String, ex: Exception = null) extends Exception(msg, ex)
-
- case class BadQueryException(msg: String, ex: Throwable = null) extends Exception(msg, ex)
-
- case class InvalidHTableException(msg: String) extends Exception(msg)
-
- case class FetchTimeoutException(msg: String) extends Exception(msg)
-
- case class DropRequestException(msg: String) extends Exception(msg)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/GraphUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/GraphUtil.scala b/s2core/src/main/scala/com/kakao/s2graph/core/GraphUtil.scala
deleted file mode 100644
index 0359963..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/GraphUtil.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-package com.kakao.s2graph.core
-
-import scala.util.Random
-import scala.util.hashing.MurmurHash3
-import java.util.regex.Pattern
-import play.api.libs.json.Json
-
-object GraphUtil {
- private val TOKEN_DELIMITER = Pattern.compile("[\t]")
- val operations = Map("i" -> 0, "insert" -> 0, "u" -> 1, "update" -> 1,
- "increment" -> 2, "d" -> 3, "delete" -> 3,
- "deleteAll" -> 4, "insertBulk" -> 5, "incrementCount" -> 6).map {
- case (k, v) =>
- k -> v.toByte
- }
- val BitsForMurMurHash = 16
- val bytesForMurMurHash = 2
- val defaultOpByte = operations("insert")
- val directions = Map("out" -> 0, "in" -> 1, "undirected" -> 2, "u" -> 2, "directed" -> 0, "d" -> 0)
- val consistencyLevel = Map("weak" -> 0, "strong" -> 1)
-
- def toType(t: String) = {
- t.trim().toLowerCase match {
- case "e" | "edge" => "edge"
- case "v" | "vertex" => "vertex"
- }
- }
-
- def toDir(direction: String): Option[Byte] = {
- val d = direction.trim().toLowerCase match {
- case "directed" | "d" => Some(0)
- case "undirected" | "u" => Some(2)
- case "out" => Some(0)
- case "in" => Some(1)
- case _ => None
- }
- d.map(x => x.toByte)
- }
-
- def toDirection(direction: String): Int = {
- direction.trim().toLowerCase match {
- case "directed" | "d" => 0
- case "undirected" | "u" => 2
- case "out" => 0
- case "in" => 1
- case _ => 2
- }
- }
-
- def fromDirection(direction: Int) = {
- direction match {
- case 0 => "out"
- case 1 => "in"
- case 2 => "undirected"
- }
- }
-
- def toggleDir(dir: Int) = {
- dir match {
- case 0 => 1
- case 1 => 0
- case 2 => 2
- case _ => throw new UnsupportedOperationException(s"toggleDirection: $dir")
- }
- }
-
- def toOp(op: String): Option[Byte] = {
- op.trim() match {
- case "i" | "insert" => Some(0)
- case "d" | "delete" => Some(3)
- case "u" | "update" => Some(1)
- case "increment" => Some(2)
- case "deleteAll" => Some(4)
- case "insertBulk" => Some(5)
- case "incrementCount" => Option(6)
- case _ => None
- }
- }
-
- def fromOp(op: Byte): String = {
- op match {
- case 0 => "insert"
- case 3 => "delete"
- case 1 => "update"
- case 2 => "increment"
- case 4 => "deleteAll"
- case 5 => "insertBulk"
- case 6 => "incrementCount"
- case _ =>
- throw new UnsupportedOperationException(s"op : $op (only support 0(insert),1(delete),2(updaet),3(increment))")
- }
- }
-
- // def toggleOp(op: Byte): Byte = {
- // val ret = op match {
- // case 0 => 1
- // case 1 => 0
- // case x: Byte => x
- // }
- // ret.toByte
- // }
- // 2^31 - 1
-
- def transformHash(h: Int): Int = {
- // h / 2 + (Int.MaxValue / 2 - 1)
- if (h < 0) -1 * (h + 1) else h
- }
- def murmur3Int(s: String): Int = {
- val hash = MurmurHash3.stringHash(s)
- transformHash(hash)
- }
- def murmur3(s: String): Short = {
- val hash = MurmurHash3.stringHash(s)
- val positiveHash = transformHash(hash) >> BitsForMurMurHash
- positiveHash.toShort
-// Random.nextInt(Short.MaxValue).toShort
- }
-
- def smartSplit(s: String, delemiter: String) = {
- val trimed_string = s.trim()
- if (trimed_string.equals("")) {
- Seq[String]()
- } else {
- trimed_string.split(delemiter).toList
- }
- }
-
- def split(line: String) = TOKEN_DELIMITER.split(line)
-
- def parseString(s: String): List[String] = {
- if (!s.startsWith("[")) {
- s.split("\n").toList
- } else {
- Json.parse(s).asOpt[List[String]].getOrElse(List.empty[String])
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/JSONParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/JSONParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/JSONParser.scala
deleted file mode 100644
index 9663f69..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/JSONParser.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLike}
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json._
-
-
-trait JSONParser {
-
- //TODO: check result notation on bigDecimal.
- def innerValToJsValue(innerVal: InnerValLike, dataType: String): Option[JsValue] = {
- try {
- val dType = InnerVal.toInnerDataType(dataType)
- val jsValue = dType match {
- case InnerVal.STRING => JsString(innerVal.value.asInstanceOf[String])
- case InnerVal.BOOLEAN => JsBoolean(innerVal.value.asInstanceOf[Boolean])
- case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
- // case t if InnerVal.NUMERICS.contains(t) =>
- innerVal.value match {
- case l: Long => JsNumber(l)
- case i: Int => JsNumber(i)
- case s: Short => JsNumber(s.toLong)
- case b: Byte => JsNumber(b.toLong)
- case f: Float => JsNumber(f.toDouble)
- case d: Double =>
- // JsNumber(d)
- dType match {
- case InnerVal.BYTE => JsNumber(d.toInt)
- case InnerVal.SHORT => JsNumber(d.toInt)
- case InnerVal.INT => JsNumber(d.toInt)
- case InnerVal.LONG => JsNumber(d.toLong)
- case InnerVal.FLOAT => JsNumber(d.toDouble)
- case InnerVal.DOUBLE => JsNumber(d.toDouble)
- case _ => throw new RuntimeException(s"$innerVal, $dType => $dataType")
- }
- case num: BigDecimal =>
- // JsNumber(num)
- // JsNumber(InnerVal.scaleNumber(num.asInstanceOf[BigDecimal], dType))
- dType match {
- case InnerVal.BYTE => JsNumber(num.toInt)
- case InnerVal.SHORT => JsNumber(num.toInt)
- case InnerVal.INT => JsNumber(num.toInt)
- case InnerVal.LONG => JsNumber(num.toLong)
- case InnerVal.FLOAT => JsNumber(num.toDouble)
- case InnerVal.DOUBLE => JsNumber(num.toDouble)
- case _ => throw new RuntimeException(s"$innerVal, $dType => $dataType")
- }
- // JsNumber(num.toLong)
- case _ => throw new RuntimeException(s"$innerVal, Numeric Unknown => $dataType")
- }
- // JsNumber(InnerVal.scaleNumber(innerVal.asInstanceOf[BigDecimal], dType))
- case _ => throw new RuntimeException(s"$innerVal, Unknown => $dataType")
- }
- Some(jsValue)
- } catch {
- case e: Exception =>
- logger.info(s"JSONParser.innerValToJsValue: $e")
- None
- }
- }
-
- // def innerValToString(innerVal: InnerValLike, dataType: String): String = {
- // val dType = InnerVal.toInnerDataType(dataType)
- // InnerVal.toInnerDataType(dType) match {
- // case InnerVal.STRING => innerVal.toString
- // case InnerVal.BOOLEAN => innerVal.toString
- // // case t if InnerVal.NUMERICS.contains(t) =>
- // case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
- // BigDecimal(innerVal.toString).bigDecimal.toPlainString
- // case _ => innerVal.toString
- // // throw new RuntimeException("innerVal to jsValue failed.")
- // }
- // }
-
- def toInnerVal(str: String, dataType: String, version: String): InnerValLike = {
- //TODO:
- // logger.error(s"toInnerVal: $str, $dataType, $version")
- val s =
- if (str.startsWith("\"") && str.endsWith("\"")) str.substring(1, str.length - 1)
- else str
- val dType = InnerVal.toInnerDataType(dataType)
-
- dType match {
- case InnerVal.STRING => InnerVal.withStr(s, version)
- // case t if InnerVal.NUMERICS.contains(t) => InnerVal.withNumber(BigDecimal(s), version)
- case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
- InnerVal.withNumber(BigDecimal(s), version)
- case InnerVal.BOOLEAN => InnerVal.withBoolean(s.toBoolean, version)
- case InnerVal.BLOB => InnerVal.withBlob(s.getBytes, version)
- case _ =>
- // InnerVal.withStr("")
- throw new RuntimeException(s"illegal datatype for string: dataType is $dataType for $s")
- }
- }
-
- def jsValueToInnerVal(jsValue: JsValue, dataType: String, version: String): Option[InnerValLike] = {
- val ret = try {
- val dType = InnerVal.toInnerDataType(dataType.toLowerCase())
- jsValue match {
- case n: JsNumber =>
- dType match {
- case InnerVal.STRING => Some(InnerVal.withStr(jsValue.toString, version))
- // case t if InnerVal.NUMERICS.contains(t) =>
- case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
- Some(InnerVal.withNumber(n.value, version))
- case _ => None
- }
- case s: JsString =>
- dType match {
- case InnerVal.STRING => Some(InnerVal.withStr(s.value, version))
- case InnerVal.BOOLEAN => Some(InnerVal.withBoolean(s.as[String].toBoolean, version))
- // case t if InnerVal.NUMERICS.contains(t) =>
- case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
- Some(InnerVal.withNumber(BigDecimal(s.value), version))
- case _ => None
- }
- case b: JsBoolean =>
- dType match {
- case InnerVal.STRING => Some(InnerVal.withStr(b.toString, version))
- case InnerVal.BOOLEAN => Some(InnerVal.withBoolean(b.value, version))
- case _ => None
- }
- case _ =>
- None
- }
- } catch {
- case e: Exception =>
- logger.error(e.getMessage)
- None
- }
-
- ret
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
deleted file mode 100644
index ccf9d1f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
+++ /dev/null
@@ -1,368 +0,0 @@
-package com.kakao.s2graph.core
-
-
-import com.kakao.s2graph.core.GraphExceptions.{InvalidHTableException, LabelAlreadyExistException, LabelNotExistException}
-import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.types.HBaseType._
-import com.kakao.s2graph.core.types._
-import play.api.libs.json.Reads._
-import play.api.libs.json._
-import scala.util.Try
-
-/**
- * This is designed to be bridge between rest to s2core.
- * s2core never use this for finding models.
- */
-object Management extends JSONParser {
-
- object JsonModel {
-
- case class Prop(name: String, defaultValue: String, datatType: String)
-
- object Prop extends ((String, String, String) => Prop)
-
- case class Index(name: String, propNames: Seq[String])
-
- }
-
- import HBaseType._
-
- val DefaultCompressionAlgorithm = "gz"
-
-
- def findService(serviceName: String) = {
- Service.findByName(serviceName, useCache = false)
- }
-
- def deleteService(serviceName: String) = {
- Service.findByName(serviceName).foreach { service =>
- // service.deleteAll()
- }
- }
-
- def updateHTable(labelName: String, newHTableName: String): Try[Int] = Try {
- val targetLabel = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(s"Target label $labelName does not exist."))
- if (targetLabel.hTableName == newHTableName) throw new InvalidHTableException(s"New HTable name is already in use for target label.")
-
- Label.updateHTableName(targetLabel.label, newHTableName)
- }
-
-
-
- def createServiceColumn(serviceName: String,
- columnName: String,
- columnType: String,
- props: Seq[Prop],
- schemaVersion: String = DEFAULT_VERSION) = {
-
- Model withTx { implicit session =>
- val serviceOpt = Service.findByName(serviceName)
- serviceOpt match {
- case None => throw new RuntimeException(s"create service $serviceName has not been created.")
- case Some(service) =>
- val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion)
- for {
- Prop(propName, defaultValue, dataType) <- props
- } yield {
- ColumnMeta.findOrInsert(serviceColumn.id.get, propName, dataType)
- }
- }
- }
- }
-
- def deleteColumn(serviceName: String, columnName: String, schemaVersion: String = DEFAULT_VERSION) = {
- Model withTx { implicit session =>
- val service = Service.findByName(serviceName, useCache = false).getOrElse(throw new RuntimeException("Service not Found"))
- val serviceColumns = ServiceColumn.find(service.id.get, columnName, useCache = false)
- val columnNames = serviceColumns.map { serviceColumn =>
- ServiceColumn.delete(serviceColumn.id.get)
- serviceColumn.columnName
- }
-
- columnNames.getOrElse(throw new RuntimeException("column not found"))
- }
- }
-
- def findLabel(labelName: String): Option[Label] = {
- Label.findByName(labelName, useCache = false)
- }
-
- def deleteLabel(labelName: String) = {
- Model withTx { implicit session =>
- Label.findByName(labelName, useCache = false).foreach { label =>
- Label.deleteAll(label)
- }
- labelName
- }
- }
-
- def addIndex(labelStr: String, indices: Seq[Index]): Try[Label] = {
- Model withTx { implicit session =>
- val label = Label.findByName(labelStr).getOrElse(throw LabelNotExistException(s"$labelStr not found"))
- val labelMetaMap = label.metaPropsInvMap
-
- indices.foreach { index =>
- val metaSeq = index.propNames.map { name => labelMetaMap(name).seq }
- LabelIndex.findOrInsert(label.id.get, index.name, metaSeq.toList, "none")
- }
-
- label
- }
- }
-
- def addProp(labelStr: String, prop: Prop) = {
- Model withTx { implicit session =>
- val labelOpt = Label.findByName(labelStr)
- val label = labelOpt.getOrElse(throw LabelNotExistException(s"$labelStr not found"))
-
- LabelMeta.findOrInsert(label.id.get, prop.name, prop.defaultValue, prop.datatType)
- }
- }
-
- def addProps(labelStr: String, props: Seq[Prop]) = {
- Model withTx { implicit session =>
- val labelOpt = Label.findByName(labelStr)
- val label = labelOpt.getOrElse(throw LabelNotExistException(s"$labelStr not found"))
-
- props.map {
- case Prop(propName, defaultValue, dataType) =>
- LabelMeta.findOrInsert(label.id.get, propName, defaultValue, dataType)
- }
- }
- }
-
- def addVertexProp(serviceName: String,
- columnName: String,
- propsName: String,
- propsType: String,
- schemaVersion: String = DEFAULT_VERSION): ColumnMeta = {
- val result = for {
- service <- Service.findByName(serviceName, useCache = false)
- serviceColumn <- ServiceColumn.find(service.id.get, columnName)
- } yield {
- ColumnMeta.findOrInsert(serviceColumn.id.get, propsName, propsType)
- }
- result.getOrElse({
- throw new RuntimeException(s"add property on vertex failed")
- })
- }
-
- def getServiceLable(label: String): Option[Label] = {
- Label.findByName(label, useCache = true)
- }
-
- /**
- *
- */
-
- def toLabelWithDirectionAndOp(label: Label, direction: String): Option[LabelWithDirection] = {
- for {
- labelId <- label.id
- dir = GraphUtil.toDirection(direction)
- } yield LabelWithDirection(labelId, dir)
- }
-
- def tryOption[A, R](key: A, f: A => Option[R]) = {
- f(key) match {
- case None => throw new GraphExceptions.InternalException(s"$key is not found in DB. create $key first.")
- case Some(r) => r
- }
- }
-
- def toEdge(ts: Long, operation: String, srcId: String, tgtId: String,
- labelStr: String, direction: String = "", props: String): Edge = {
-
- val label = tryOption(labelStr, getServiceLable)
- val dir =
- if (direction == "")
-// GraphUtil.toDirection(label.direction)
- GraphUtil.directions("out")
- else
- GraphUtil.toDirection(direction)
-
- // logger.debug(s"$srcId, ${label.srcColumnWithDir(dir)}")
- // logger.debug(s"$tgtId, ${label.tgtColumnWithDir(dir)}")
-
- val srcVertexId = toInnerVal(srcId, label.srcColumn.columnType, label.schemaVersion)
- val tgtVertexId = toInnerVal(tgtId, label.tgtColumn.columnType, label.schemaVersion)
-
- val srcColId = label.srcColumn.id.get
- val tgtColId = label.tgtColumn.id.get
- val (srcVertex, tgtVertex) = if (dir == GraphUtil.directions("out")) {
- (Vertex(SourceVertexId(srcColId, srcVertexId), System.currentTimeMillis()),
- Vertex(TargetVertexId(tgtColId, tgtVertexId), System.currentTimeMillis()))
- } else {
- (Vertex(SourceVertexId(tgtColId, tgtVertexId), System.currentTimeMillis()),
- Vertex(TargetVertexId(srcColId, srcVertexId), System.currentTimeMillis()))
- }
-
- // val dir = if (direction == "") GraphUtil.toDirection(label.direction) else GraphUtil.toDirection(direction)
- val labelWithDir = LabelWithDirection(label.id.get, dir)
- val op = tryOption(operation, GraphUtil.toOp)
-
- val jsObject = Json.parse(props).asOpt[JsObject].getOrElse(Json.obj())
- val parsedProps = toProps(label, jsObject.fields).toMap
- val propsWithTs = parsedProps.map(kv => (kv._1 -> InnerValLikeWithTs(kv._2, ts))) ++
- Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(ts, label.schemaVersion), ts))
-
- Edge(srcVertex, tgtVertex, labelWithDir, op, version = ts, propsWithTs = propsWithTs)
-
- }
-
- def toVertex(ts: Long, operation: String, id: String, serviceName: String, columnName: String, props: String): Vertex = {
- Service.findByName(serviceName) match {
- case None => throw new RuntimeException(s"$serviceName does not exist. create service first.")
- case Some(service) =>
- ServiceColumn.find(service.id.get, columnName) match {
- case None => throw new RuntimeException(s"$columnName is not exist. create service column first.")
- case Some(col) =>
- val idVal = toInnerVal(id, col.columnType, col.schemaVersion)
- val op = tryOption(operation, GraphUtil.toOp)
- val jsObject = Json.parse(props).asOpt[JsObject].getOrElse(Json.obj())
- val parsedProps = toProps(col, jsObject).toMap
- Vertex(VertexId(col.id.get, idVal), ts, parsedProps, op = op)
- }
- }
- }
-
- def toProps(column: ServiceColumn, js: JsObject): Seq[(Int, InnerValLike)] = {
-
- val props = for {
- (k, v) <- js.fields
- meta <- column.metasInvMap.get(k)
- } yield {
- val innerVal = jsValueToInnerVal(v, meta.dataType, column.schemaVersion).getOrElse(
- throw new RuntimeException(s"$k is not defined. create schema for vertex."))
-
- (meta.seq.toInt, innerVal)
- }
- props
-
- }
-
- def toProps(label: Label, js: Seq[(String, JsValue)]): Seq[(Byte, InnerValLike)] = {
- val props = for {
- (k, v) <- js
- meta <- label.metaPropsInvMap.get(k)
- innerVal <- jsValueToInnerVal(v, meta.dataType, label.schemaVersion)
- } yield (meta.seq, innerVal)
-
- props
- }
-
-
- /**
- * update label name.
- */
- def updateLabelName(oldLabelName: String, newLabelName: String) = {
- for {
- old <- Label.findByName(oldLabelName)
- } {
- Label.findByName(newLabelName) match {
- case None =>
- Label.updateName(oldLabelName, newLabelName)
- case Some(_) =>
- // throw new RuntimeException(s"$newLabelName already exist")
- }
- }
- }
-}
-
-class Management(graph: Graph) {
- import Management._
- val storage = graph.storage
-
- def createTable(zkAddr: String,
- tableName: String,
- cfs: List[String],
- regionMultiplier: Int,
- ttl: Option[Int],
- compressionAlgorithm: String = DefaultCompressionAlgorithm): Unit =
- storage.createTable(zkAddr, tableName, cfs, regionMultiplier, ttl, compressionAlgorithm)
-
- /** HBase specific code */
- def createService(serviceName: String,
- cluster: String, hTableName: String,
- preSplitSize: Int, hTableTTL: Option[Int],
- compressionAlgorithm: String = DefaultCompressionAlgorithm): Try[Service] = {
-
- Model withTx { implicit session =>
- val service = Service.findOrInsert(serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
- /** create hbase table for service */
- storage.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
- service
- }
- }
-
- /** HBase specific code */
- def createLabel(label: String,
- srcServiceName: String,
- srcColumnName: String,
- srcColumnType: String,
- tgtServiceName: String,
- tgtColumnName: String,
- tgtColumnType: String,
- isDirected: Boolean = true,
- serviceName: String,
- indices: Seq[Index],
- props: Seq[Prop],
- consistencyLevel: String,
- hTableName: Option[String],
- hTableTTL: Option[Int],
- schemaVersion: String = DEFAULT_VERSION,
- isAsync: Boolean,
- compressionAlgorithm: String = "gz"): Try[Label] = {
-
- val labelOpt = Label.findByName(label, useCache = false)
-
- Model withTx { implicit session =>
- labelOpt match {
- case Some(l) =>
- throw new GraphExceptions.LabelAlreadyExistException(s"Label name ${l.label} already exist.")
- case None =>
- /** create all models */
- val newLabel = Label.insertAll(label,
- srcServiceName, srcColumnName, srcColumnType,
- tgtServiceName, tgtColumnName, tgtColumnType,
- isDirected, serviceName, indices, props, consistencyLevel,
- hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
-
- /** create hbase table */
- val service = newLabel.service
- (hTableName, hTableTTL) match {
- case (None, None) => // do nothing
- case (None, Some(hbaseTableTTL)) => throw new RuntimeException("if want to specify ttl, give hbaseTableName also")
- case (Some(hbaseTableName), None) =>
- // create own hbase table with default ttl on service level.
- storage.createTable(service.cluster, hbaseTableName, List("e", "v"), service.preSplitSize, service.hTableTTL, compressionAlgorithm)
- case (Some(hbaseTableName), Some(hbaseTableTTL)) =>
- // create own hbase table with own ttl.
- storage.createTable(service.cluster, hbaseTableName, List("e", "v"), service.preSplitSize, hTableTTL, compressionAlgorithm)
- }
- newLabel
- }
- }
- }
-
- /**
- * label
- */
- /**
- * copy label when if oldLabel exist and newLabel do not exist.
- * copy label: only used by bulk load job. not sure if we need to parameterize hbase cluster.
- */
- def copyLabel(oldLabelName: String, newLabelName: String, hTableName: Option[String]) = {
- val old = Label.findByName(oldLabelName).getOrElse(throw new LabelAlreadyExistException(s"Old label $oldLabelName not exists."))
- if (Label.findByName(newLabelName).isDefined) throw new LabelAlreadyExistException(s"New label $newLabelName already exists.")
-
- val allProps = old.metas.map { labelMeta => Prop(labelMeta.name, labelMeta.defaultValue, labelMeta.dataType) }
- val allIndices = old.indices.map { index => Index(index.name, index.propNames) }
-
- createLabel(newLabelName, old.srcService.serviceName, old.srcColumnName, old.srcColumnType,
- old.tgtService.serviceName, old.tgtColumnName, old.tgtColumnType,
- old.isDirected, old.serviceName,
- allIndices, allProps,
- old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala b/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
deleted file mode 100644
index 33ec7d9..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/OrderingUtil.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.types.InnerValLike
-import play.api.libs.json.{JsNumber, JsString, JsValue}
-
-object OrderingUtil {
-
- implicit object JsValueOrdering extends Ordering[JsValue] {
- override def compare(x: JsValue, y: JsValue): Int = {
- (x, y) match {
- case (JsNumber(xv), JsNumber(yv)) =>
- Ordering.BigDecimal.compare(xv, yv)
- case (JsString(xv), JsString(yv)) =>
- Ordering.String.compare(xv, yv)
- case _ => throw new Exception(s"unsupported type")
- }
- }
- }
-
- implicit object InnerValLikeOrdering extends Ordering[InnerValLike] {
- override def compare(x: InnerValLike, y: InnerValLike): Int = {
- x.compare(y)
- }
- }
-
- implicit object MultiValueOrdering extends Ordering[Any] {
- override def compare(x: Any, y: Any): Int = {
- (x, y) match {
- case (xv: Int, yv: Int) => implicitly[Ordering[Int]].compare(xv, yv)
- case (xv: Long, yv: Long) => implicitly[Ordering[Long]].compare(xv, yv)
- case (xv: Double, yv: Double) => implicitly[Ordering[Double]].compare(xv, yv)
- case (xv: String, yv: String) => implicitly[Ordering[String]].compare(xv, yv)
- case (xv: JsValue, yv: JsValue) => implicitly[Ordering[JsValue]].compare(xv, yv)
- case (xv: InnerValLike, yv: InnerValLike) => implicitly[Ordering[InnerValLike]].compare(xv, yv)
- }
- }
- }
-
- def TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]): Ordering[(T, T, T, T)] = {
- new Ordering[(T, T, T, T)] {
- override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
- val len = ascendingLs.length
- val it = ascendingLs.iterator
- if (len >= 1) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare1 = ord.compare(x._1, y._1)
- if (compare1 != 0) return compare1
- }
-
- if (len >= 2) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare2 = ord.compare(x._2, y._2)
- if (compare2 != 0) return compare2
- }
-
- if (len >= 3) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare3 = ord.compare(x._3, y._3)
- if (compare3 != 0) return compare3
- }
-
- if (len >= 4) {
- val (x, y) = it.next() match {
- case true => tx -> ty
- case false => ty -> tx
- }
- val compare4 = ord.compare(x._4, y._4)
- if (compare4 != 0) return compare4
- }
- 0
- }
- }
- }
-}
-
-class SeqMultiOrdering[T](ascendingLs: Seq[Boolean], defaultAscending: Boolean = true)(implicit ord: Ordering[T]) extends Ordering[Seq[T]] {
- override def compare(x: Seq[T], y: Seq[T]): Int = {
- val xe = x.iterator
- val ye = y.iterator
- val oe = ascendingLs.iterator
-
- while (xe.hasNext && ye.hasNext) {
- val ascending = if (oe.hasNext) oe.next() else defaultAscending
- val (xev, yev) = ascending match {
- case true => xe.next() -> ye.next()
- case false => ye.next() -> xe.next()
- }
- val res = ord.compare(xev, yev)
- if (res != 0) return res
- }
-
- Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
- }
-}
-
-//class TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]) extends Ordering[(T, T, T, T)] {
-// override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
-// val len = ascendingLs.length
-// val it = ascendingLs.iterator
-// if (len >= 1) {
-// val (x, y) = it.next() match {
-// case true => tx -> ty
-// case false => ty -> tx
-// }
-// val compare1 = ord.compare(x._1, y._1)
-// if (compare1 != 0) return compare1
-// }
-//
-// if (len >= 2) {
-// val (x, y) = it.next() match {
-// case true => tx -> ty
-// case false => ty -> tx
-// }
-// val compare2 = ord.compare(x._2, y._2)
-// if (compare2 != 0) return compare2
-// }
-//
-// if (len >= 3) {
-// val (x, y) = it.next() match {
-// case true => tx -> ty
-// case false => ty -> tx
-// }
-// val compare3 = ord.compare(x._3, y._3)
-// if (compare3 != 0) return compare3
-// }
-//
-// if (len >= 4) {
-// val (x, y) = it.next() match {
-// case true => tx -> ty
-// case false => ty -> tx
-// }
-// val compare4 = ord.compare(x._4, y._4)
-// if (compare4 != 0) return compare4
-// }
-// 0
-// }
-//}
[18/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/Edge.scala
new file mode 100644
index 0000000..c1a5738
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Edge.scala
@@ -0,0 +1,567 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta}
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.json.{JsNumber, Json}
+
+import scala.collection.JavaConversions._
+import scala.util.hashing.MurmurHash3
+
+
+case class SnapshotEdge(srcVertex: Vertex,
+ tgtVertex: Vertex,
+ labelWithDir: LabelWithDirection,
+ op: Byte,
+ version: Long,
+ props: Map[Byte, InnerValLikeWithTs],
+ pendingEdgeOpt: Option[Edge],
+ statusCode: Byte = 0,
+ lockTs: Option[Long]) extends JSONParser {
+
+ if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
+
+ val label = Label.findById(labelWithDir.labelId)
+ val schemaVer = label.schemaVersion
+ lazy val propsWithoutTs = props.mapValues(_.innerVal)
+ val ts = props(LabelMeta.timeStampSeq).innerVal.toString().toLong
+
+ def toEdge: Edge = {
+ val ts = props.get(LabelMeta.timeStampSeq).map(v => v.ts).getOrElse(version)
+ Edge(srcVertex, tgtVertex, labelWithDir, op,
+ version, props, pendingEdgeOpt = pendingEdgeOpt,
+ statusCode = statusCode, lockTs = lockTs)
+ }
+
+ def propsWithName = (for {
+ (seq, v) <- props
+ meta <- label.metaPropsMap.get(seq)
+ jsValue <- innerValToJsValue(v.innerVal, meta.dataType)
+ } yield meta.name -> jsValue) ++ Map("version" -> JsNumber(version))
+
+ // only for debug
+ def toLogString() = {
+ List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, propsWithName).mkString("\t")
+ }
+}
+
+case class IndexEdge(srcVertex: Vertex,
+ tgtVertex: Vertex,
+ labelWithDir: LabelWithDirection,
+ op: Byte,
+ version: Long,
+ labelIndexSeq: Byte,
+ props: Map[Byte, InnerValLike]) extends JSONParser {
+ if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
+ // assert(props.containsKey(LabelMeta.timeStampSeq))
+
+ val ts = props(LabelMeta.timeStampSeq).toString.toLong
+ val degreeEdge = props.contains(LabelMeta.degreeSeq)
+ lazy val label = Label.findById(labelWithDir.labelId)
+ val schemaVer = label.schemaVersion
+ lazy val labelIndex = LabelIndex.findByLabelIdAndSeq(labelWithDir.labelId, labelIndexSeq).get
+ lazy val defaultIndexMetas = labelIndex.sortKeyTypes.map { meta =>
+ val innerVal = toInnerVal(meta.defaultValue, meta.dataType, schemaVer)
+ meta.seq -> innerVal
+ }.toMap
+
+ lazy val labelIndexMetaSeqs = labelIndex.metaSeqs
+
+ /** TODO: make sure call of this class fill props as this assumes */
+ lazy val orders = for (k <- labelIndexMetaSeqs) yield {
+ props.get(k) match {
+ case None =>
+
+ /**
+ * TODO: agly hack
+ * now we double store target vertex.innerId/srcVertex.innerId for easy development. later fix this to only store id once
+ */
+ val v = k match {
+ case LabelMeta.timeStampSeq => InnerVal.withLong(version, schemaVer)
+ case LabelMeta.toSeq => tgtVertex.innerId
+ case LabelMeta.fromSeq => //srcVertex.innerId
+ // for now, it does not make sense to build index on srcVertex.innerId since all edges have same data.
+ throw new RuntimeException("_from on indexProps is not supported")
+ case _ => defaultIndexMetas(k)
+ }
+
+ k -> v
+ case Some(v) => k -> v
+ }
+ }
+
+ lazy val ordersKeyMap = orders.map { case (byte, _) => byte }.toSet
+ lazy val metas = for ((k, v) <- props if !ordersKeyMap.contains(k)) yield k -> v
+
+ lazy val propsWithTs = props.map { case (k, v) => k -> InnerValLikeWithTs(v, version) }
+
+ //TODO:
+ // lazy val kvs = Graph.client.indexedEdgeSerializer(this).toKeyValues.toList
+
+ lazy val hasAllPropsForIndex = orders.length == labelIndexMetaSeqs.length
+
+ def propsWithName = for {
+ (seq, v) <- props
+ meta <- label.metaPropsMap.get(seq) if seq >= 0
+ jsValue <- innerValToJsValue(v, meta.dataType)
+ } yield meta.name -> jsValue
+
+
+ def toEdge: Edge = Edge(srcVertex, tgtVertex, labelWithDir, op, version, propsWithTs)
+
+ // only for debug
+ def toLogString() = {
+ List(version, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, Json.toJson(propsWithName)).mkString("\t")
+ }
+}
+
+case class Edge(srcVertex: Vertex,
+ tgtVertex: Vertex,
+ labelWithDir: LabelWithDirection,
+ op: Byte = GraphUtil.defaultOpByte,
+ version: Long = System.currentTimeMillis(),
+ propsWithTs: Map[Byte, InnerValLikeWithTs],
+ parentEdges: Seq[EdgeWithScore] = Nil,
+ originalEdgeOpt: Option[Edge] = None,
+ pendingEdgeOpt: Option[Edge] = None,
+ statusCode: Byte = 0,
+ lockTs: Option[Long] = None) extends GraphElement with JSONParser {
+
+ if (!props.containsKey(LabelMeta.timeStampSeq)) throw new Exception("Timestamp is required.")
+ // assert(propsWithTs.containsKey(LabelMeta.timeStampSeq))
+ val schemaVer = label.schemaVersion
+ val ts = propsWithTs(LabelMeta.timeStampSeq).innerVal.toString.toLong
+
+ def props = propsWithTs.mapValues(_.innerVal)
+
+ def relatedEdges = {
+ if (labelWithDir.isDirected) List(this, duplicateEdge)
+ else {
+ val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
+ val base = copy(labelWithDir = outDir)
+ List(base, base.reverseSrcTgtEdge)
+ }
+ }
+
+ // def relatedEdges = List(this)
+
+ def srcForVertex = {
+ val belongLabelIds = Seq(labelWithDir.labelId)
+ if (labelWithDir.dir == GraphUtil.directions("in")) {
+ Vertex(VertexId(label.tgtColumn.id.get, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ Vertex(VertexId(label.srcColumn.id.get, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+
+ def tgtForVertex = {
+ val belongLabelIds = Seq(labelWithDir.labelId)
+ if (labelWithDir.dir == GraphUtil.directions("in")) {
+ Vertex(VertexId(label.srcColumn.id.get, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ Vertex(VertexId(label.tgtColumn.id.get, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+
+ def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
+
+ def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
+
+ def reverseSrcTgtEdge = copy(srcVertex = tgtVertex, tgtVertex = srcVertex)
+
+ def label = Label.findById(labelWithDir.labelId)
+
+ def labelOrders = LabelIndex.findByLabelIdAll(labelWithDir.labelId)
+
+ override def serviceName = label.serviceName
+
+ override def queueKey = Seq(ts.toString, tgtVertex.serviceName).mkString("|")
+
+ override def queuePartitionKey = Seq(srcVertex.innerId, tgtVertex.innerId).mkString("|")
+
+ override def isAsync = label.isAsync
+
+ def isDegree = propsWithTs.contains(LabelMeta.degreeSeq)
+
+ def propsPlusTs = propsWithTs.get(LabelMeta.timeStampSeq) match {
+ case Some(_) => props
+ case None => props ++ Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, schemaVer))
+ }
+
+ def propsPlusTsValid = propsPlusTs.filter(kv => kv._1 >= 0)
+
+ def edgesWithIndex = for (labelOrder <- labelOrders) yield {
+ IndexEdge(srcVertex, tgtVertex, labelWithDir, op, version, labelOrder.seq, propsPlusTs)
+ }
+
+ def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
+ IndexEdge(srcVertex, tgtVertex, labelWithDir, op, version, labelOrder.seq, propsPlusTsValid)
+ }
+
+ /** force direction as out on invertedEdge */
+ def toSnapshotEdge: SnapshotEdge = {
+ val (smaller, larger) = (srcForVertex, tgtForVertex)
+
+ val newLabelWithDir = LabelWithDirection(labelWithDir.labelId, GraphUtil.directions("out"))
+
+ val ret = SnapshotEdge(smaller, larger, newLabelWithDir, op, version,
+ Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(ts, schemaVer), ts)) ++ propsWithTs,
+ pendingEdgeOpt = pendingEdgeOpt, statusCode = statusCode, lockTs = lockTs)
+ ret
+ }
+
+ override def hashCode(): Int = {
+ MurmurHash3.stringHash(srcVertex.innerId + "," + labelWithDir + "," + tgtVertex.innerId)
+ }
+
+ override def equals(other: Any): Boolean = other match {
+ case e: Edge =>
+ srcVertex.innerId == e.srcVertex.innerId &&
+ tgtVertex.innerId == e.tgtVertex.innerId &&
+ labelWithDir == e.labelWithDir
+ case _ => false
+ }
+
+ def propsWithName = for {
+ (seq, v) <- props
+ meta <- label.metaPropsMap.get(seq) if seq > 0
+ jsValue <- innerValToJsValue(v, meta.dataType)
+ } yield meta.name -> jsValue
+
+ def updateTgtVertex(id: InnerValLike) = {
+ val newId = TargetVertexId(tgtVertex.id.colId, id)
+ val newTgtVertex = Vertex(newId, tgtVertex.ts, tgtVertex.props)
+ Edge(srcVertex, newTgtVertex, labelWithDir, op, version, propsWithTs)
+ }
+
+ def rank(r: RankParam): Double =
+ if (r.keySeqAndWeights.size <= 0) 1.0f
+ else {
+ var sum: Double = 0
+
+ for ((seq, w) <- r.keySeqAndWeights) {
+ seq match {
+ case LabelMeta.countSeq => sum += 1
+ case _ => {
+ propsWithTs.get(seq) match {
+ case None => // do nothing
+ case Some(innerValWithTs) => {
+ val cost = try innerValWithTs.innerVal.toString.toDouble catch {
+ case e: Exception =>
+ logger.error("toInnerval failed in rank", e)
+ 1.0
+ }
+ sum += w * cost
+ }
+ }
+ }
+ }
+ }
+ sum
+ }
+
+ def toLogString: String = {
+ val ret =
+ if (propsWithName.nonEmpty)
+ List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label, Json.toJson(propsWithName))
+ else
+ List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, label.label)
+
+ ret.mkString("\t")
+ }
+}
+
+case class EdgeMutate(edgesToDelete: List[IndexEdge] = List.empty[IndexEdge],
+ edgesToInsert: List[IndexEdge] = List.empty[IndexEdge],
+ newSnapshotEdge: Option[SnapshotEdge] = None) {
+
+ def toLogString: String = {
+ val l = (0 until 50).map(_ => "-").mkString("")
+ val deletes = s"deletes: ${edgesToDelete.map(e => e.toLogString).mkString("\n")}"
+ val inserts = s"inserts: ${edgesToInsert.map(e => e.toLogString).mkString("\n")}"
+ val updates = s"snapshot: ${newSnapshotEdge.map(e => e.toLogString).mkString("\n")}"
+
+ List("\n", l, deletes, inserts, updates, l, "\n").mkString("\n")
+ }
+}
+
+object Edge extends JSONParser {
+ val incrementVersion = 1L
+ val minTsVal = 0L
+
+ /** now version information is required also **/
+ type State = Map[Byte, InnerValLikeWithTs]
+ type PropsPairWithTs = (State, State, Long, String)
+ type MergeState = PropsPairWithTs => (State, Boolean)
+ type UpdateFunc = (Option[Edge], Edge, MergeState)
+
+ def allPropsDeleted(props: Map[Byte, InnerValLikeWithTs]): Boolean =
+ if (!props.containsKey(LabelMeta.lastDeletedAt)) false
+ else {
+ val lastDeletedAt = props.get(LabelMeta.lastDeletedAt).get.ts
+ val propsWithoutLastDeletedAt = props - LabelMeta.lastDeletedAt
+
+ propsWithoutLastDeletedAt.forall { case (_, v) => v.ts <= lastDeletedAt }
+ }
+
+ def buildDeleteBulk(invertedEdge: Option[Edge], requestEdge: Edge): (Edge, EdgeMutate) = {
+ // assert(invertedEdge.isEmpty)
+ // assert(requestEdge.op == GraphUtil.operations("delete"))
+
+ val edgesToDelete = requestEdge.relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
+ val edgeInverted = Option(requestEdge.toSnapshotEdge)
+
+ (requestEdge, EdgeMutate(edgesToDelete, edgesToInsert = Nil, edgeInverted))
+ }
+
+ def buildOperation(invertedEdge: Option[Edge], requestEdges: Seq[Edge]): (Edge, EdgeMutate) = {
+ // logger.debug(s"oldEdge: ${invertedEdge.map(_.toStringRaw)}")
+ // logger.debug(s"requestEdge: ${requestEdge.toStringRaw}")
+ val oldPropsWithTs =
+ if (invertedEdge.isEmpty) Map.empty[Byte, InnerValLikeWithTs] else invertedEdge.get.propsWithTs
+
+ val funcs = requestEdges.map { edge =>
+ if (edge.op == GraphUtil.operations("insert")) {
+ edge.label.consistencyLevel match {
+ case "strong" => Edge.mergeUpsert _
+ case _ => Edge.mergeInsertBulk _
+ }
+ } else if (edge.op == GraphUtil.operations("insertBulk")) {
+ Edge.mergeInsertBulk _
+ } else if (edge.op == GraphUtil.operations("delete")) {
+ edge.label.consistencyLevel match {
+ case "strong" => Edge.mergeDelete _
+ case _ => throw new RuntimeException("not supported")
+ }
+ }
+ else if (edge.op == GraphUtil.operations("update")) Edge.mergeUpdate _
+ else if (edge.op == GraphUtil.operations("increment")) Edge.mergeIncrement _
+ else throw new RuntimeException(s"not supported operation on edge: $edge")
+ }
+
+ val oldTs = invertedEdge.map(_.ts).getOrElse(minTsVal)
+ val requestWithFuncs = requestEdges.zip(funcs).filter(oldTs != _._1.ts).sortBy(_._1.ts)
+
+ if (requestWithFuncs.isEmpty) {
+ (requestEdges.head, EdgeMutate())
+ } else {
+ val requestEdge = requestWithFuncs.last._1
+ var prevPropsWithTs = oldPropsWithTs
+
+ for {
+ (requestEdge, func) <- requestWithFuncs
+ } {
+ val (_newPropsWithTs, _) = func(prevPropsWithTs, requestEdge.propsWithTs, requestEdge.ts, requestEdge.schemaVer)
+ prevPropsWithTs = _newPropsWithTs
+ // logger.debug(s"${requestEdge.toLogString}\n$oldPropsWithTs\n$prevPropsWithTs\n")
+ }
+ val requestTs = requestEdge.ts
+ /** version should be monotoniously increasing so our RPC mutation should be applied safely */
+ val newVersion = invertedEdge.map(e => e.version + incrementVersion).getOrElse(requestTs)
+ val maxTs = prevPropsWithTs.map(_._2.ts).max
+ val newTs = if (maxTs > requestTs) maxTs else requestTs
+ val propsWithTs = prevPropsWithTs ++
+ Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(newTs, requestEdge.label.schemaVersion), newTs))
+ val edgeMutate = buildMutation(invertedEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
+
+ // logger.debug(s"${edgeMutate.toLogString}\n${propsWithTs}")
+ // logger.error(s"$propsWithTs")
+ (requestEdge, edgeMutate)
+ }
+ }
+
+ def buildMutation(snapshotEdgeOpt: Option[Edge],
+ requestEdge: Edge,
+ newVersion: Long,
+ oldPropsWithTs: Map[Byte, InnerValLikeWithTs],
+ newPropsWithTs: Map[Byte, InnerValLikeWithTs]): EdgeMutate = {
+ if (oldPropsWithTs == newPropsWithTs) {
+ // all requests should be dropped. so empty mutation.
+ // logger.error(s"Case 1")
+ EdgeMutate(edgesToDelete = Nil, edgesToInsert = Nil, newSnapshotEdge = None)
+ } else {
+ val withOutDeletedAt = newPropsWithTs.filter(kv => kv._1 != LabelMeta.lastDeletedAt)
+ val newOp = snapshotEdgeOpt match {
+ case None => requestEdge.op
+ case Some(old) =>
+ val oldMaxTs = old.propsWithTs.map(_._2.ts).max
+ if (oldMaxTs > requestEdge.ts) old.op
+ else requestEdge.op
+ }
+
+ val newSnapshotEdgeOpt =
+ Option(requestEdge.copy(op = newOp, propsWithTs = newPropsWithTs, version = newVersion).toSnapshotEdge)
+ // delete request must always update snapshot.
+ if (withOutDeletedAt == oldPropsWithTs && newPropsWithTs.containsKey(LabelMeta.lastDeletedAt)) {
+ // no mutation on indexEdges. only snapshotEdge should be updated to record lastDeletedAt.
+ // logger.error(s"Case 2")
+ EdgeMutate(edgesToDelete = Nil, edgesToInsert = Nil, newSnapshotEdge = newSnapshotEdgeOpt)
+ } else {
+ // logger.error(s"Case 3")
+ val edgesToDelete = snapshotEdgeOpt match {
+ case Some(snapshotEdge) if snapshotEdge.op != GraphUtil.operations("delete") =>
+ snapshotEdge.copy(op = GraphUtil.defaultOpByte).
+ relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
+ case _ => Nil
+ }
+
+ val edgesToInsert =
+ if (newPropsWithTs.isEmpty || allPropsDeleted(newPropsWithTs)) Nil
+ else
+ requestEdge.copy(version = newVersion, propsWithTs = newPropsWithTs, op = GraphUtil.defaultOpByte).
+ relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
+
+ EdgeMutate(edgesToDelete = edgesToDelete, edgesToInsert = edgesToInsert, newSnapshotEdge = newSnapshotEdgeOpt)
+ }
+ }
+ }
+
+ def mergeUpsert(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
+ var shouldReplace = false
+ val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
+ val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
+ val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
+ propsWithTs.get(k) match {
+ case Some(newValWithTs) =>
+ assert(oldValWithTs.ts >= lastDeletedAt)
+ val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
+ else {
+ shouldReplace = true
+ newValWithTs
+ }
+ Some(k -> v)
+
+ case None =>
+ assert(oldValWithTs.ts >= lastDeletedAt)
+ if (oldValWithTs.ts >= requestTs || k < 0) Some(k -> oldValWithTs)
+ else {
+ shouldReplace = true
+ None
+ }
+ }
+ }
+ val existInNew =
+ for {
+ (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
+ } yield {
+ shouldReplace = true
+ Some(k -> newValWithTs)
+ }
+
+ ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
+ }
+
+ def mergeUpdate(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
+ var shouldReplace = false
+ val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
+ val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
+ val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
+ propsWithTs.get(k) match {
+ case Some(newValWithTs) =>
+ assert(oldValWithTs.ts >= lastDeletedAt)
+ val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
+ else {
+ shouldReplace = true
+ newValWithTs
+ }
+ Some(k -> v)
+ case None =>
+ // important: update need to merge previous valid values.
+ assert(oldValWithTs.ts >= lastDeletedAt)
+ Some(k -> oldValWithTs)
+ }
+ }
+ val existInNew = for {
+ (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
+ } yield {
+ shouldReplace = true
+ Some(k -> newValWithTs)
+ }
+
+ ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
+ }
+
+ def mergeIncrement(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
+ var shouldReplace = false
+ val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
+ val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt).map(v => v.ts).getOrElse(minTsVal)
+ val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
+ propsWithTs.get(k) match {
+ case Some(newValWithTs) =>
+ if (k == LabelMeta.timeStampSeq) {
+ val v = if (oldValWithTs.ts >= newValWithTs.ts) oldValWithTs
+ else {
+ shouldReplace = true
+ newValWithTs
+ }
+ Some(k -> v)
+ } else {
+ if (oldValWithTs.ts >= newValWithTs.ts) {
+ Some(k -> oldValWithTs)
+ } else {
+ assert(oldValWithTs.ts < newValWithTs.ts && oldValWithTs.ts >= lastDeletedAt)
+ shouldReplace = true
+ // incr(t0), incr(t2), d(t1) => deleted
+ Some(k -> InnerValLikeWithTs(oldValWithTs.innerVal + newValWithTs.innerVal, oldValWithTs.ts))
+ }
+ }
+
+ case None =>
+ assert(oldValWithTs.ts >= lastDeletedAt)
+ Some(k -> oldValWithTs)
+ // if (oldValWithTs.ts >= lastDeletedAt) Some(k -> oldValWithTs) else None
+ }
+ }
+ val existInNew = for {
+ (k, newValWithTs) <- propsWithTs if !oldPropsWithTs.contains(k) && newValWithTs.ts > lastDeletedAt
+ } yield {
+ shouldReplace = true
+ Some(k -> newValWithTs)
+ }
+
+ ((existInOld.flatten ++ existInNew.flatten).toMap, shouldReplace)
+ }
+
+ def mergeDelete(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
+ var shouldReplace = false
+ val (oldPropsWithTs, propsWithTs, requestTs, version) = propsPairWithTs
+ val lastDeletedAt = oldPropsWithTs.get(LabelMeta.lastDeletedAt) match {
+ case Some(prevDeletedAt) =>
+ if (prevDeletedAt.ts >= requestTs) prevDeletedAt.ts
+ else {
+ shouldReplace = true
+ requestTs
+ }
+ case None => {
+ shouldReplace = true
+ requestTs
+ }
+ }
+ val existInOld = for ((k, oldValWithTs) <- oldPropsWithTs) yield {
+ if (k == LabelMeta.timeStampSeq) {
+ if (oldValWithTs.ts >= requestTs) Some(k -> oldValWithTs)
+ else {
+ shouldReplace = true
+ Some(k -> InnerValLikeWithTs.withLong(requestTs, requestTs, version))
+ }
+ } else {
+ if (oldValWithTs.ts >= lastDeletedAt) Some(k -> oldValWithTs)
+ else {
+ shouldReplace = true
+ None
+ }
+ }
+ }
+ val mustExistInNew = Map(LabelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(lastDeletedAt, lastDeletedAt, version))
+ ((existInOld.flatten ++ mustExistInNew).toMap, shouldReplace)
+ }
+
+ def mergeInsertBulk(propsPairWithTs: PropsPairWithTs): (State, Boolean) = {
+ val (_, propsWithTs, _, _) = propsPairWithTs
+ (propsWithTs, true)
+ }
+
+ def fromString(s: String): Option[Edge] = Graph.toEdge(s)
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/ExceptionHandler.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/ExceptionHandler.scala b/s2core/src/main/scala/org/apache/s2graph/core/ExceptionHandler.scala
new file mode 100644
index 0000000..d3177b8
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/ExceptionHandler.scala
@@ -0,0 +1,159 @@
+package org.apache.s2graph.core
+
+import java.util.Properties
+import java.util.concurrent.atomic.AtomicLong
+
+import akka.actor._
+import akka.routing.{Broadcast, RoundRobinPool}
+import com.typesafe.config.Config
+import org.apache.kafka.clients.producer._
+
+import scala.concurrent.duration._
+
+object ExceptionHandler {
+
+ var producer: Option[Producer[Key, Val]] = None
+ var properties: Option[Properties] = None
+ val numOfRoutees = 1
+ val actorSystem = ActorSystem("ExceptionHandler")
+ var routees: Option[ActorRef] = None
+ var shutdownTime = 1000 millis
+ var phase = "dev"
+ lazy val failTopic = s"mutateFailed_${phase}"
+
+ def apply(config: Config) = {
+ properties =
+ if (config.hasPath("kafka.metadata.broker.list")) Option(kafkaConfig(config))
+ else None
+ phase = if (config.hasPath("phase")) config.getString("phase") else "dev"
+ producer = for {
+ props <- properties
+ p <- try {
+ Option(new KafkaProducer[Key, Val](props))
+ } catch {
+ case e: Throwable => None
+ }
+ } yield {
+ p
+ }
+ init()
+ }
+
+ def props(producer: Producer[Key, Val]) = Props(classOf[KafkaAggregatorActor], producer)
+
+ def init() = {
+ for {
+ p <- producer
+ } {
+ routees = Option(actorSystem.actorOf(RoundRobinPool(numOfRoutees).props(props(p))))
+ }
+ }
+
+ def shutdown() = {
+ routees.map(_ ! Broadcast(PoisonPill))
+ Thread.sleep(shutdownTime.length)
+ }
+
+ def enqueues(msgs: Seq[KafkaMessage]) = {
+ msgs.foreach(enqueue)
+ }
+
+ def enqueue(msg: KafkaMessage) = {
+ routees.map(_ ! msg)
+ }
+
+
+ def kafkaConfig(config: Config) = {
+ val props = new Properties();
+
+ /** all default configuration for new producer */
+ val brokers =
+ if (config.hasPath("kafka.metadata.broker.list")) config.getString("kafka.metadata.broker.list")
+ else "localhost"
+ props.put("bootstrap.servers", brokers)
+ props.put("acks", "1")
+ props.put("buffer.memory", "33554432")
+ props.put("compression.type", "snappy")
+ props.put("retries", "0")
+ props.put("batch.size", "16384")
+ props.put("linger.ms", "0")
+ props.put("max.request.size", "1048576")
+ props.put("receive.buffer.bytes", "32768")
+ props.put("send.buffer.bytes", "131072")
+ props.put("timeout.ms", "30000")
+ props.put("block.on.buffer.full", "false")
+ props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")
+ props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")
+ props
+ }
+
+ type Key = String
+ type Val = String
+
+ def toKafkaMessage(topic: String = failTopic, element: GraphElement, originalString: Option[String] = None) = {
+ KafkaMessage(new ProducerRecord[Key, Val](topic, element.queuePartitionKey,
+ originalString.getOrElse(element.toLogString())))
+ }
+
+ case class KafkaMessage(msg: ProducerRecord[Key, Val])
+
+ case class Message(topic: String, msg: String)
+
+ case class BufferedKafkaMessage(msgs: Seq[ProducerRecord[Key, Val]], bufferSize: Int)
+
+ case class BufferedMessage(topic: String, bufferedMsgs: String, bufferSize: Int)
+
+ case object FlushBuffer
+
+ case class UpdateHealth(isHealty: Boolean)
+
+ case object ShowMetrics
+
+}
+
+class KafkaAggregatorActor(kafkaProducer: Producer[String, String]) extends Stash with ActorLogging {
+
+ import ExceptionHandler._
+
+ val failedCount = new AtomicLong(0L)
+ val successCount = new AtomicLong(0L)
+ val stashCount = new AtomicLong(0L)
+
+ implicit val ex = context.system.dispatcher
+
+ context.system.scheduler.schedule(0 millis, 10 seconds) {
+ self ! ShowMetrics
+ }
+
+ override def receive = {
+ case ShowMetrics =>
+ log.info(s"[Stats]: failed[${failedCount.get}], stashed[${stashCount.get}], success[${successCount.get}]")
+
+ case m: KafkaMessage =>
+ val replayTo = self
+ try {
+ kafkaProducer.send(m.msg, new Callback() {
+ override def onCompletion(meta: RecordMetadata, e: Exception) = {
+ if (e == null) {
+ // success
+ successCount.incrementAndGet()
+ unstashAll()
+ stashCount.set(0L)
+ } else {
+ // failure
+ log.error(s"onCompletion: $e", e)
+ failedCount.incrementAndGet()
+ replayTo ! m
+ }
+ }
+ })
+ } catch {
+ case e@(_: org.apache.kafka.clients.producer.BufferExhaustedException | _: Throwable) =>
+ log.error(s"$e", e)
+ log.info(s"stash")
+ stash()
+ stashCount.incrementAndGet()
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala
new file mode 100644
index 0000000..b2cbd19
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala
@@ -0,0 +1,379 @@
+package org.apache.s2graph.core
+
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.s2graph.core.mysqls.{Label, Model}
+import org.apache.s2graph.core.parsers.WhereParser
+import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
+import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection}
+import org.apache.s2graph.core.utils.logger
+
+import scala.collection.JavaConversions._
+import scala.collection._
+import scala.collection.mutable.ListBuffer
+import scala.concurrent._
+import scala.util.Try
+
+object Graph {
+ val DefaultScore = 1.0
+
+ private val DefaultConfigs: Map[String, AnyRef] = Map(
+ "hbase.zookeeper.quorum" -> "localhost",
+ "hbase.table.name" -> "s2graph",
+ "hbase.table.compression.algorithm" -> "gz",
+ "phase" -> "dev",
+ "db.default.driver" -> "com.mysql.jdbc.Driver",
+ "db.default.url" -> "jdbc:mysql://localhost:3306/graph_dev",
+ "db.default.password" -> "graph",
+ "db.default.user" -> "graph",
+ "cache.max.size" -> java.lang.Integer.valueOf(10000),
+ "cache.ttl.seconds" -> java.lang.Integer.valueOf(60),
+ "hbase.client.retries.number" -> java.lang.Integer.valueOf(20),
+ "hbase.rpcs.buffered_flush_interval" -> java.lang.Short.valueOf(100.toShort),
+ "hbase.rpc.timeout" -> java.lang.Integer.valueOf(1000),
+ "max.retry.number" -> java.lang.Integer.valueOf(100),
+ "lock.expire.time" -> java.lang.Integer.valueOf(1000 * 60 * 10),
+ "max.back.off" -> java.lang.Integer.valueOf(100),
+ "hbase.fail.prob" -> java.lang.Double.valueOf(-0.1),
+ "delete.all.fetch.size" -> java.lang.Integer.valueOf(1000),
+ "future.cache.max.size" -> java.lang.Integer.valueOf(100000),
+ "future.cache.expire.after.write" -> java.lang.Integer.valueOf(10000),
+ "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000),
+ "s2graph.storage.backend" -> "hbase"
+ )
+
+ var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs)
+
+ /** helpers for filterEdges */
+ type HashKey = (Int, Int, Int, Int, Boolean)
+ type FilterHashKey = (Int, Int)
+ type Result = (ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]],
+ ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)],
+ ListBuffer[(HashKey, FilterHashKey, Edge, Double)])
+
+ def toHashKey(queryParam: QueryParam, edge: Edge, isDegree: Boolean): (HashKey, FilterHashKey) = {
+ val src = edge.srcVertex.innerId.hashCode()
+ val tgt = edge.tgtVertex.innerId.hashCode()
+ val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
+ val filterHashKey = (src, tgt)
+
+ (hashKey, filterHashKey)
+ }
+
+ def alreadyVisitedVertices(queryResultLs: Seq[QueryResult]): Map[(LabelWithDirection, Vertex), Boolean] = {
+ val vertices = for {
+ queryResult <- queryResultLs
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ edge = edgeWithScore.edge
+ vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
+ } yield (edge.labelWithDir, vertex) -> true
+
+ vertices.toMap
+ }
+
+ /** common methods for filter out, transform, aggregate queryResult */
+ def convertEdges(queryParam: QueryParam, edge: Edge, nextStepOpt: Option[Step]): Seq[Edge] = {
+ for {
+ convertedEdge <- queryParam.transformer.transform(edge, nextStepOpt) if !edge.isDegree
+ } yield convertedEdge
+ }
+
+ def processTimeDecay(queryParam: QueryParam, edge: Edge) = {
+ /** process time decay */
+ val tsVal = queryParam.timeDecay match {
+ case None => 1.0
+ case Some(timeDecay) =>
+ val tsVal = try {
+ val labelMeta = edge.label.metaPropsMap(timeDecay.labelMetaSeq)
+ val innerValWithTsOpt = edge.propsWithTs.get(timeDecay.labelMetaSeq)
+ innerValWithTsOpt.map { innerValWithTs =>
+ val innerVal = innerValWithTs.innerVal
+ labelMeta.dataType match {
+ case InnerVal.LONG => innerVal.value match {
+ case n: BigDecimal => n.bigDecimal.longValue()
+ case _ => innerVal.toString().toLong
+ }
+ case _ => innerVal.toString().toLong
+ }
+ } getOrElse(edge.ts)
+ } catch {
+ case e: Exception =>
+ logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
+ edge.ts
+ }
+ val timeDiff = queryParam.timestamp - tsVal
+ timeDecay.decay(timeDiff)
+ }
+
+ tsVal
+ }
+
+ def aggregateScore(newScore: Double,
+ resultEdges: ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)],
+ duplicateEdges: ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]],
+ edgeWithScoreSorted: ListBuffer[(HashKey, FilterHashKey, Edge, Double)],
+ hashKey: HashKey,
+ filterHashKey: FilterHashKey,
+ queryParam: QueryParam,
+ convertedEdge: Edge) = {
+
+ /** skip duplicate policy check if consistencyLevel is strong */
+ if (queryParam.label.consistencyLevel != "strong" && resultEdges.containsKey(hashKey)) {
+ val (oldFilterHashKey, oldEdge, oldScore) = resultEdges.get(hashKey)
+ //TODO:
+ queryParam.duplicatePolicy match {
+ case Query.DuplicatePolicy.First => // do nothing
+ case Query.DuplicatePolicy.Raw =>
+ if (duplicateEdges.containsKey(hashKey)) {
+ duplicateEdges.get(hashKey).append(convertedEdge -> newScore)
+ } else {
+ val newBuffer = new ListBuffer[(Edge, Double)]
+ newBuffer.append(convertedEdge -> newScore)
+ duplicateEdges.put(hashKey, newBuffer)
+ }
+ case Query.DuplicatePolicy.CountSum =>
+ resultEdges.put(hashKey, (filterHashKey, oldEdge, oldScore + 1))
+ case _ =>
+ resultEdges.put(hashKey, (filterHashKey, oldEdge, oldScore + newScore))
+ }
+ } else {
+ resultEdges.put(hashKey, (filterHashKey, convertedEdge, newScore))
+ edgeWithScoreSorted.append((hashKey, filterHashKey, convertedEdge, newScore))
+ }
+ }
+
+ def aggregateResults(queryRequestWithResult: QueryRequestWithResult,
+ queryParamResult: Result,
+ edgesToInclude: util.HashSet[FilterHashKey],
+ edgesToExclude: util.HashSet[FilterHashKey]): QueryRequestWithResult = {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ val (query, stepIdx, _, queryParam) = QueryRequest.unapply(queryRequest).get
+
+ val (duplicateEdges, resultEdges, edgeWithScoreSorted) = queryParamResult
+ val edgesWithScores = for {
+ (hashKey, filterHashKey, edge, _) <- edgeWithScoreSorted if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
+ score = resultEdges.get(hashKey)._3
+ (duplicateEdge, aggregatedScore) <- fetchDuplicatedEdges(edge, score, hashKey, duplicateEdges) if aggregatedScore >= queryParam.threshold
+ } yield EdgeWithScore(duplicateEdge, aggregatedScore)
+
+ QueryRequestWithResult(queryRequest, QueryResult(edgesWithScores))
+ }
+
+ def fetchDuplicatedEdges(edge: Edge,
+ score: Double,
+ hashKey: HashKey,
+ duplicateEdges: ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]]) = {
+ (edge -> score) +: (if (duplicateEdges.containsKey(hashKey)) duplicateEdges.get(hashKey) else Seq.empty)
+ }
+
+ def queryResultWithFilter(queryRequestWithResult: QueryRequestWithResult) = {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ val (_, _, _, queryParam) = QueryRequest.unapply(queryRequest).get
+ val whereFilter = queryParam.where.get
+ if (whereFilter == WhereParser.success) queryResult.edgeWithScoreLs
+ else queryResult.edgeWithScoreLs.withFilter(edgeWithScore => whereFilter.filter(edgeWithScore.edge))
+ }
+
+ def filterEdges(queryResultLsFuture: Future[Seq[QueryRequestWithResult]],
+ alreadyVisited: Map[(LabelWithDirection, Vertex), Boolean] = Map.empty[(LabelWithDirection, Vertex), Boolean])
+ (implicit ec: scala.concurrent.ExecutionContext): Future[Seq[QueryRequestWithResult]] = {
+
+ queryResultLsFuture.map { queryRequestWithResultLs =>
+ if (queryRequestWithResultLs.isEmpty) Nil
+ else {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs.head).get
+ val (q, stepIdx, srcVertex, queryParam) = QueryRequest.unapply(queryRequest).get
+ val step = q.steps(stepIdx)
+
+ val nextStepOpt = if (stepIdx < q.steps.size - 1) Option(q.steps(stepIdx + 1)) else None
+
+ val excludeLabelWithDirSet = new util.HashSet[(Int, Int)]
+ val includeLabelWithDirSet = new util.HashSet[(Int, Int)]
+ step.queryParams.filter(_.exclude).foreach(l => excludeLabelWithDirSet.add(l.labelWithDir.labelId -> l.labelWithDir.dir))
+ step.queryParams.filter(_.include).foreach(l => includeLabelWithDirSet.add(l.labelWithDir.labelId -> l.labelWithDir.dir))
+
+ val edgesToExclude = new util.HashSet[FilterHashKey]()
+ val edgesToInclude = new util.HashSet[FilterHashKey]()
+
+ val queryParamResultLs = new ListBuffer[Result]
+ queryRequestWithResultLs.foreach { queryRequestWithResult =>
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ val queryParam = queryRequest.queryParam
+ val duplicateEdges = new util.concurrent.ConcurrentHashMap[HashKey, ListBuffer[(Edge, Double)]]()
+ val resultEdges = new util.concurrent.ConcurrentHashMap[HashKey, (FilterHashKey, Edge, Double)]()
+ val edgeWithScoreSorted = new ListBuffer[(HashKey, FilterHashKey, Edge, Double)]
+ val labelWeight = step.labelWeights.getOrElse(queryParam.labelWithDir.labelId, 1.0)
+
+ // store degree value with Array.empty so if degree edge exist, it comes at very first.
+ def checkDegree() = queryResult.edgeWithScoreLs.headOption.exists { edgeWithScore =>
+ edgeWithScore.edge.isDegree
+ }
+ var isDegree = checkDegree()
+
+ val includeExcludeKey = queryParam.labelWithDir.labelId -> queryParam.labelWithDir.dir
+ val shouldBeExcluded = excludeLabelWithDirSet.contains(includeExcludeKey)
+ val shouldBeIncluded = includeLabelWithDirSet.contains(includeExcludeKey)
+
+ queryResultWithFilter(queryRequestWithResult).foreach { edgeWithScore =>
+ val (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ if (queryParam.transformer.isDefault) {
+ val convertedEdge = edge
+
+ val (hashKey, filterHashKey) = toHashKey(queryParam, convertedEdge, isDegree)
+
+ /** check if this edge should be exlcuded. */
+ if (shouldBeExcluded && !isDegree) {
+ edgesToExclude.add(filterHashKey)
+ } else {
+ if (shouldBeIncluded && !isDegree) {
+ edgesToInclude.add(filterHashKey)
+ }
+ val tsVal = processTimeDecay(queryParam, convertedEdge)
+ val newScore = labelWeight * score * tsVal
+ aggregateScore(newScore, resultEdges, duplicateEdges, edgeWithScoreSorted, hashKey, filterHashKey, queryParam, convertedEdge)
+ }
+ } else {
+ convertEdges(queryParam, edge, nextStepOpt).foreach { convertedEdge =>
+ val (hashKey, filterHashKey) = toHashKey(queryParam, convertedEdge, isDegree)
+
+ /** check if this edge should be exlcuded. */
+ if (shouldBeExcluded && !isDegree) {
+ edgesToExclude.add(filterHashKey)
+ } else {
+ if (shouldBeIncluded && !isDegree) {
+ edgesToInclude.add(filterHashKey)
+ }
+ val tsVal = processTimeDecay(queryParam, convertedEdge)
+ val newScore = labelWeight * score * tsVal
+ aggregateScore(newScore, resultEdges, duplicateEdges, edgeWithScoreSorted, hashKey, filterHashKey, queryParam, convertedEdge)
+ }
+ }
+ }
+ isDegree = false
+ }
+ val ret = (duplicateEdges, resultEdges, edgeWithScoreSorted)
+ queryParamResultLs.append(ret)
+ }
+
+ val aggregatedResults = for {
+ (queryRequestWithResult, queryParamResult) <- queryRequestWithResultLs.zip(queryParamResultLs)
+ } yield {
+ aggregateResults(queryRequestWithResult, queryParamResult, edgesToInclude, edgesToExclude)
+ }
+
+ aggregatedResults
+ }
+ }
+ }
+
+ def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] = Try {
+ val parts = GraphUtil.split(s)
+ val logType = parts(2)
+ val element = if (logType == "edge" | logType == "e") {
+ /** current only edge is considered to be bulk loaded */
+ labelMapping.get(parts(5)) match {
+ case None =>
+ case Some(toReplace) =>
+ parts(5) = toReplace
+ }
+ toEdge(parts)
+ } else if (logType == "vertex" | logType == "v") {
+ toVertex(parts)
+ } else {
+ throw new GraphExceptions.JsonParseException("log type is not exist in log.")
+ }
+
+ element
+ } recover {
+ case e: Exception =>
+ logger.error(s"$e", e)
+ None
+ } get
+
+
+ def toVertex(s: String): Option[Vertex] = {
+ toVertex(GraphUtil.split(s))
+ }
+
+ def toEdge(s: String): Option[Edge] = {
+ toEdge(GraphUtil.split(s))
+ }
+
+ //"1418342849000\tu\te\t3286249\t71770\ttalk_friend\t{\"is_hidden\":false}"
+ //{"from":1,"to":101,"label":"graph_test","props":{"time":-1, "weight":10},"timestamp":1417616431},
+ def toEdge(parts: Array[String]): Option[Edge] = Try {
+ val (ts, operation, logType, srcId, tgtId, label) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
+ val props = if (parts.length >= 7) parts(6) else "{}"
+ val tempDirection = if (parts.length >= 8) parts(7) else "out"
+ val direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
+
+ val edge = Management.toEdge(ts.toLong, operation, srcId, tgtId, label, direction, props)
+ // logger.debug(s"toEdge: $edge")
+ Some(edge)
+ } recover {
+ case e: Exception =>
+ logger.error(s"toEdge: $e", e)
+ throw e
+ } get
+
+ //"1418342850000\ti\tv\t168756793\ttalk_user_id\t{\"country_iso\":\"KR\"}"
+ def toVertex(parts: Array[String]): Option[Vertex] = Try {
+ val (ts, operation, logType, srcId, serviceName, colName) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
+ val props = if (parts.length >= 7) parts(6) else "{}"
+ Some(Management.toVertex(ts.toLong, operation, srcId, serviceName, colName, props))
+ } recover {
+ case e: Throwable =>
+ logger.error(s"toVertex: $e", e)
+ throw e
+ } get
+
+ def initStorage(config: Config)(ec: ExecutionContext) = {
+ config.getString("s2graph.storage.backend") match {
+ case "hbase" => new AsynchbaseStorage(config)(ec)
+ case _ => throw new RuntimeException("not supported storage.")
+ }
+ }
+}
+
+class Graph(_config: Config)(implicit val ec: ExecutionContext) {
+ val config = _config.withFallback(Graph.DefaultConfig)
+
+ Model.apply(config)
+ Model.loadCache()
+
+ // TODO: Make storage client by config param
+ val storage = Graph.initStorage(config)(ec)
+
+
+ for {
+ entry <- config.entrySet() if Graph.DefaultConfigs.contains(entry.getKey)
+ (k, v) = (entry.getKey, entry.getValue)
+ } logger.info(s"[Initialized]: $k, ${this.config.getAnyRef(k)}")
+
+ /** select */
+ def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = storage.checkEdges(params)
+
+ def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = storage.getEdges(q)
+
+ def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = storage.getVertices(vertices)
+
+ /** write */
+ def deleteAllAdjacentEdges(srcVertices: List[Vertex], labels: Seq[Label], dir: Int, ts: Long): Future[Boolean] =
+ storage.deleteAllAdjacentEdges(srcVertices, labels, dir, ts)
+
+ def mutateElements(elements: Seq[GraphElement], withWait: Boolean = false): Future[Seq[Boolean]] =
+ storage.mutateElements(elements, withWait)
+
+ def mutateEdges(edges: Seq[Edge], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateEdges(edges, withWait)
+
+ def mutateVertices(vertices: Seq[Vertex], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateVertices(vertices, withWait)
+
+ def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = storage.incrementCounts(edges, withWait)
+
+ def shutdown(): Unit = {
+ storage.flush()
+ Model.shutdown()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/GraphElement.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElement.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElement.scala
new file mode 100644
index 0000000..12bb941
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElement.scala
@@ -0,0 +1,10 @@
+package org.apache.s2graph.core
+
+trait GraphElement {
+ def serviceName: String
+ def ts: Long
+ def isAsync: Boolean
+ def queueKey: String
+ def queuePartitionKey: String
+ def toLogString(): String
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala
new file mode 100644
index 0000000..e7d2d76
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphExceptions.scala
@@ -0,0 +1,28 @@
+package org.apache.s2graph.core
+
+object GraphExceptions {
+
+ case class JsonParseException(msg: String) extends Exception(msg)
+
+ case class LabelNotExistException(msg: String) extends Exception(msg)
+
+ case class ModelNotFoundException(msg: String) extends Exception(msg)
+
+ case class MaxPropSizeReachedException(msg: String) extends Exception(msg)
+
+ case class LabelAlreadyExistException(msg: String) extends Exception(msg)
+
+ case class InternalException(msg: String) extends Exception(msg)
+
+ case class IllegalDataTypeException(msg: String) extends Exception(msg)
+
+ case class WhereParserException(msg: String, ex: Exception = null) extends Exception(msg, ex)
+
+ case class BadQueryException(msg: String, ex: Throwable = null) extends Exception(msg, ex)
+
+ case class InvalidHTableException(msg: String) extends Exception(msg)
+
+ case class FetchTimeoutException(msg: String) extends Exception(msg)
+
+ case class DropRequestException(msg: String) extends Exception(msg)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
new file mode 100644
index 0000000..1a2b916
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
@@ -0,0 +1,139 @@
+package org.apache.s2graph.core
+
+import java.util.regex.Pattern
+
+import play.api.libs.json.Json
+
+import scala.util.hashing.MurmurHash3
+
+object GraphUtil {
+ private val TOKEN_DELIMITER = Pattern.compile("[\t]")
+ val operations = Map("i" -> 0, "insert" -> 0, "u" -> 1, "update" -> 1,
+ "increment" -> 2, "d" -> 3, "delete" -> 3,
+ "deleteAll" -> 4, "insertBulk" -> 5, "incrementCount" -> 6).map {
+ case (k, v) =>
+ k -> v.toByte
+ }
+ val BitsForMurMurHash = 16
+ val bytesForMurMurHash = 2
+ val defaultOpByte = operations("insert")
+ val directions = Map("out" -> 0, "in" -> 1, "undirected" -> 2, "u" -> 2, "directed" -> 0, "d" -> 0)
+ val consistencyLevel = Map("weak" -> 0, "strong" -> 1)
+
+ def toType(t: String) = {
+ t.trim().toLowerCase match {
+ case "e" | "edge" => "edge"
+ case "v" | "vertex" => "vertex"
+ }
+ }
+
+ def toDir(direction: String): Option[Byte] = {
+ val d = direction.trim().toLowerCase match {
+ case "directed" | "d" => Some(0)
+ case "undirected" | "u" => Some(2)
+ case "out" => Some(0)
+ case "in" => Some(1)
+ case _ => None
+ }
+ d.map(x => x.toByte)
+ }
+
+ def toDirection(direction: String): Int = {
+ direction.trim().toLowerCase match {
+ case "directed" | "d" => 0
+ case "undirected" | "u" => 2
+ case "out" => 0
+ case "in" => 1
+ case _ => 2
+ }
+ }
+
+ def fromDirection(direction: Int) = {
+ direction match {
+ case 0 => "out"
+ case 1 => "in"
+ case 2 => "undirected"
+ }
+ }
+
+ def toggleDir(dir: Int) = {
+ dir match {
+ case 0 => 1
+ case 1 => 0
+ case 2 => 2
+ case _ => throw new UnsupportedOperationException(s"toggleDirection: $dir")
+ }
+ }
+
+ def toOp(op: String): Option[Byte] = {
+ op.trim() match {
+ case "i" | "insert" => Some(0)
+ case "d" | "delete" => Some(3)
+ case "u" | "update" => Some(1)
+ case "increment" => Some(2)
+ case "deleteAll" => Some(4)
+ case "insertBulk" => Some(5)
+ case "incrementCount" => Option(6)
+ case _ => None
+ }
+ }
+
+ def fromOp(op: Byte): String = {
+ op match {
+ case 0 => "insert"
+ case 3 => "delete"
+ case 1 => "update"
+ case 2 => "increment"
+ case 4 => "deleteAll"
+ case 5 => "insertBulk"
+ case 6 => "incrementCount"
+ case _ =>
+ throw new UnsupportedOperationException(s"op : $op (only support 0(insert),1(delete),2(updaet),3(increment))")
+ }
+ }
+
+ // def toggleOp(op: Byte): Byte = {
+ // val ret = op match {
+ // case 0 => 1
+ // case 1 => 0
+ // case x: Byte => x
+ // }
+ // ret.toByte
+ // }
+ // 2^31 - 1
+
+ def transformHash(h: Int): Int = {
+ // h / 2 + (Int.MaxValue / 2 - 1)
+ if (h < 0) -1 * (h + 1) else h
+ }
+ def murmur3Int(s: String): Int = {
+ val hash = MurmurHash3.stringHash(s)
+ transformHash(hash)
+ }
+ def murmur3(s: String): Short = {
+ val hash = MurmurHash3.stringHash(s)
+ val positiveHash = transformHash(hash) >> BitsForMurMurHash
+ positiveHash.toShort
+// Random.nextInt(Short.MaxValue).toShort
+ }
+
+ def smartSplit(s: String, delemiter: String) = {
+ val trimed_string = s.trim()
+ if (trimed_string.equals("")) {
+ Seq[String]()
+ } else {
+ trimed_string.split(delemiter).toList
+ }
+ }
+
+ def split(line: String) = TOKEN_DELIMITER.split(line)
+
+ def parseString(s: String): List[String] = {
+ if (!s.startsWith("[")) {
+ s.split("\n").toList
+ } else {
+ Json.parse(s).asOpt[List[String]].getOrElse(List.empty[String])
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
new file mode 100644
index 0000000..8b9a228
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
@@ -0,0 +1,134 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.types.{InnerVal, InnerValLike}
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.json._
+
+
+trait JSONParser {
+
+ //TODO: check result notation on bigDecimal.
+ def innerValToJsValue(innerVal: InnerValLike, dataType: String): Option[JsValue] = {
+ try {
+ val dType = InnerVal.toInnerDataType(dataType)
+ val jsValue = dType match {
+ case InnerVal.STRING => JsString(innerVal.value.asInstanceOf[String])
+ case InnerVal.BOOLEAN => JsBoolean(innerVal.value.asInstanceOf[Boolean])
+ case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
+ // case t if InnerVal.NUMERICS.contains(t) =>
+ innerVal.value match {
+ case l: Long => JsNumber(l)
+ case i: Int => JsNumber(i)
+ case s: Short => JsNumber(s.toLong)
+ case b: Byte => JsNumber(b.toLong)
+ case f: Float => JsNumber(f.toDouble)
+ case d: Double =>
+ // JsNumber(d)
+ dType match {
+ case InnerVal.BYTE => JsNumber(d.toInt)
+ case InnerVal.SHORT => JsNumber(d.toInt)
+ case InnerVal.INT => JsNumber(d.toInt)
+ case InnerVal.LONG => JsNumber(d.toLong)
+ case InnerVal.FLOAT => JsNumber(d.toDouble)
+ case InnerVal.DOUBLE => JsNumber(d.toDouble)
+ case _ => throw new RuntimeException(s"$innerVal, $dType => $dataType")
+ }
+ case num: BigDecimal =>
+ // JsNumber(num)
+ // JsNumber(InnerVal.scaleNumber(num.asInstanceOf[BigDecimal], dType))
+ dType match {
+ case InnerVal.BYTE => JsNumber(num.toInt)
+ case InnerVal.SHORT => JsNumber(num.toInt)
+ case InnerVal.INT => JsNumber(num.toInt)
+ case InnerVal.LONG => JsNumber(num.toLong)
+ case InnerVal.FLOAT => JsNumber(num.toDouble)
+ case InnerVal.DOUBLE => JsNumber(num.toDouble)
+ case _ => throw new RuntimeException(s"$innerVal, $dType => $dataType")
+ }
+ // JsNumber(num.toLong)
+ case _ => throw new RuntimeException(s"$innerVal, Numeric Unknown => $dataType")
+ }
+ // JsNumber(InnerVal.scaleNumber(innerVal.asInstanceOf[BigDecimal], dType))
+ case _ => throw new RuntimeException(s"$innerVal, Unknown => $dataType")
+ }
+ Some(jsValue)
+ } catch {
+ case e: Exception =>
+ logger.info(s"JSONParser.innerValToJsValue: $e")
+ None
+ }
+ }
+
+ // def innerValToString(innerVal: InnerValLike, dataType: String): String = {
+ // val dType = InnerVal.toInnerDataType(dataType)
+ // InnerVal.toInnerDataType(dType) match {
+ // case InnerVal.STRING => innerVal.toString
+ // case InnerVal.BOOLEAN => innerVal.toString
+ // // case t if InnerVal.NUMERICS.contains(t) =>
+ // case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
+ // BigDecimal(innerVal.toString).bigDecimal.toPlainString
+ // case _ => innerVal.toString
+ // // throw new RuntimeException("innerVal to jsValue failed.")
+ // }
+ // }
+
+ def toInnerVal(str: String, dataType: String, version: String): InnerValLike = {
+ //TODO:
+ // logger.error(s"toInnerVal: $str, $dataType, $version")
+ val s =
+ if (str.startsWith("\"") && str.endsWith("\"")) str.substring(1, str.length - 1)
+ else str
+ val dType = InnerVal.toInnerDataType(dataType)
+
+ dType match {
+ case InnerVal.STRING => InnerVal.withStr(s, version)
+ // case t if InnerVal.NUMERICS.contains(t) => InnerVal.withNumber(BigDecimal(s), version)
+ case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
+ InnerVal.withNumber(BigDecimal(s), version)
+ case InnerVal.BOOLEAN => InnerVal.withBoolean(s.toBoolean, version)
+ case InnerVal.BLOB => InnerVal.withBlob(s.getBytes, version)
+ case _ =>
+ // InnerVal.withStr("")
+ throw new RuntimeException(s"illegal datatype for string: dataType is $dataType for $s")
+ }
+ }
+
+ def jsValueToInnerVal(jsValue: JsValue, dataType: String, version: String): Option[InnerValLike] = {
+ val ret = try {
+ val dType = InnerVal.toInnerDataType(dataType.toLowerCase())
+ jsValue match {
+ case n: JsNumber =>
+ dType match {
+ case InnerVal.STRING => Some(InnerVal.withStr(jsValue.toString, version))
+ // case t if InnerVal.NUMERICS.contains(t) =>
+ case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
+ Some(InnerVal.withNumber(n.value, version))
+ case _ => None
+ }
+ case s: JsString =>
+ dType match {
+ case InnerVal.STRING => Some(InnerVal.withStr(s.value, version))
+ case InnerVal.BOOLEAN => Some(InnerVal.withBoolean(s.as[String].toBoolean, version))
+ // case t if InnerVal.NUMERICS.contains(t) =>
+ case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE =>
+ Some(InnerVal.withNumber(BigDecimal(s.value), version))
+ case _ => None
+ }
+ case b: JsBoolean =>
+ dType match {
+ case InnerVal.STRING => Some(InnerVal.withStr(b.toString, version))
+ case InnerVal.BOOLEAN => Some(InnerVal.withBoolean(b.value, version))
+ case _ => None
+ }
+ case _ =>
+ None
+ }
+ } catch {
+ case e: Exception =>
+ logger.error(e.getMessage)
+ None
+ }
+
+ ret
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
new file mode 100644
index 0000000..b355757
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -0,0 +1,368 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.GraphExceptions.{InvalidHTableException, LabelAlreadyExistException, LabelNotExistException}
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.mysqls._
+import org.apache.s2graph.core.types.HBaseType._
+import org.apache.s2graph.core.types._
+import play.api.libs.json.Reads._
+import play.api.libs.json._
+
+import scala.util.Try
+
+/**
+ * This is designed to be bridge between rest to s2core.
+ * s2core never use this for finding models.
+ */
+object Management extends JSONParser {
+
+ object JsonModel {
+
+ case class Prop(name: String, defaultValue: String, datatType: String)
+
+ object Prop extends ((String, String, String) => Prop)
+
+ case class Index(name: String, propNames: Seq[String])
+
+ }
+
+ import HBaseType._
+
+ val DefaultCompressionAlgorithm = "gz"
+
+
+ def findService(serviceName: String) = {
+ Service.findByName(serviceName, useCache = false)
+ }
+
+ def deleteService(serviceName: String) = {
+ Service.findByName(serviceName).foreach { service =>
+ // service.deleteAll()
+ }
+ }
+
+ def updateHTable(labelName: String, newHTableName: String): Try[Int] = Try {
+ val targetLabel = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(s"Target label $labelName does not exist."))
+ if (targetLabel.hTableName == newHTableName) throw new InvalidHTableException(s"New HTable name is already in use for target label.")
+
+ Label.updateHTableName(targetLabel.label, newHTableName)
+ }
+
+
+
+ def createServiceColumn(serviceName: String,
+ columnName: String,
+ columnType: String,
+ props: Seq[Prop],
+ schemaVersion: String = DEFAULT_VERSION) = {
+
+ Model withTx { implicit session =>
+ val serviceOpt = Service.findByName(serviceName)
+ serviceOpt match {
+ case None => throw new RuntimeException(s"create service $serviceName has not been created.")
+ case Some(service) =>
+ val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion)
+ for {
+ Prop(propName, defaultValue, dataType) <- props
+ } yield {
+ ColumnMeta.findOrInsert(serviceColumn.id.get, propName, dataType)
+ }
+ }
+ }
+ }
+
+ def deleteColumn(serviceName: String, columnName: String, schemaVersion: String = DEFAULT_VERSION) = {
+ Model withTx { implicit session =>
+ val service = Service.findByName(serviceName, useCache = false).getOrElse(throw new RuntimeException("Service not Found"))
+ val serviceColumns = ServiceColumn.find(service.id.get, columnName, useCache = false)
+ val columnNames = serviceColumns.map { serviceColumn =>
+ ServiceColumn.delete(serviceColumn.id.get)
+ serviceColumn.columnName
+ }
+
+ columnNames.getOrElse(throw new RuntimeException("column not found"))
+ }
+ }
+
+ def findLabel(labelName: String): Option[Label] = {
+ Label.findByName(labelName, useCache = false)
+ }
+
+ def deleteLabel(labelName: String) = {
+ Model withTx { implicit session =>
+ Label.findByName(labelName, useCache = false).foreach { label =>
+ Label.deleteAll(label)
+ }
+ labelName
+ }
+ }
+
+ def addIndex(labelStr: String, indices: Seq[Index]): Try[Label] = {
+ Model withTx { implicit session =>
+ val label = Label.findByName(labelStr).getOrElse(throw LabelNotExistException(s"$labelStr not found"))
+ val labelMetaMap = label.metaPropsInvMap
+
+ indices.foreach { index =>
+ val metaSeq = index.propNames.map { name => labelMetaMap(name).seq }
+ LabelIndex.findOrInsert(label.id.get, index.name, metaSeq.toList, "none")
+ }
+
+ label
+ }
+ }
+
+ def addProp(labelStr: String, prop: Prop) = {
+ Model withTx { implicit session =>
+ val labelOpt = Label.findByName(labelStr)
+ val label = labelOpt.getOrElse(throw LabelNotExistException(s"$labelStr not found"))
+
+ LabelMeta.findOrInsert(label.id.get, prop.name, prop.defaultValue, prop.datatType)
+ }
+ }
+
+ def addProps(labelStr: String, props: Seq[Prop]) = {
+ Model withTx { implicit session =>
+ val labelOpt = Label.findByName(labelStr)
+ val label = labelOpt.getOrElse(throw LabelNotExistException(s"$labelStr not found"))
+
+ props.map {
+ case Prop(propName, defaultValue, dataType) =>
+ LabelMeta.findOrInsert(label.id.get, propName, defaultValue, dataType)
+ }
+ }
+ }
+
+ def addVertexProp(serviceName: String,
+ columnName: String,
+ propsName: String,
+ propsType: String,
+ schemaVersion: String = DEFAULT_VERSION): ColumnMeta = {
+ val result = for {
+ service <- Service.findByName(serviceName, useCache = false)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName)
+ } yield {
+ ColumnMeta.findOrInsert(serviceColumn.id.get, propsName, propsType)
+ }
+ result.getOrElse({
+ throw new RuntimeException(s"add property on vertex failed")
+ })
+ }
+
+ def getServiceLable(label: String): Option[Label] = {
+ Label.findByName(label, useCache = true)
+ }
+
+ /**
+ *
+ */
+
+ def toLabelWithDirectionAndOp(label: Label, direction: String): Option[LabelWithDirection] = {
+ for {
+ labelId <- label.id
+ dir = GraphUtil.toDirection(direction)
+ } yield LabelWithDirection(labelId, dir)
+ }
+
+ def tryOption[A, R](key: A, f: A => Option[R]) = {
+ f(key) match {
+ case None => throw new GraphExceptions.InternalException(s"$key is not found in DB. create $key first.")
+ case Some(r) => r
+ }
+ }
+
+ def toEdge(ts: Long, operation: String, srcId: String, tgtId: String,
+ labelStr: String, direction: String = "", props: String): Edge = {
+
+ val label = tryOption(labelStr, getServiceLable)
+ val dir =
+ if (direction == "")
+// GraphUtil.toDirection(label.direction)
+ GraphUtil.directions("out")
+ else
+ GraphUtil.toDirection(direction)
+
+ // logger.debug(s"$srcId, ${label.srcColumnWithDir(dir)}")
+ // logger.debug(s"$tgtId, ${label.tgtColumnWithDir(dir)}")
+
+ val srcVertexId = toInnerVal(srcId, label.srcColumn.columnType, label.schemaVersion)
+ val tgtVertexId = toInnerVal(tgtId, label.tgtColumn.columnType, label.schemaVersion)
+
+ val srcColId = label.srcColumn.id.get
+ val tgtColId = label.tgtColumn.id.get
+ val (srcVertex, tgtVertex) = if (dir == GraphUtil.directions("out")) {
+ (Vertex(SourceVertexId(srcColId, srcVertexId), System.currentTimeMillis()),
+ Vertex(TargetVertexId(tgtColId, tgtVertexId), System.currentTimeMillis()))
+ } else {
+ (Vertex(SourceVertexId(tgtColId, tgtVertexId), System.currentTimeMillis()),
+ Vertex(TargetVertexId(srcColId, srcVertexId), System.currentTimeMillis()))
+ }
+
+ // val dir = if (direction == "") GraphUtil.toDirection(label.direction) else GraphUtil.toDirection(direction)
+ val labelWithDir = LabelWithDirection(label.id.get, dir)
+ val op = tryOption(operation, GraphUtil.toOp)
+
+ val jsObject = Json.parse(props).asOpt[JsObject].getOrElse(Json.obj())
+ val parsedProps = toProps(label, jsObject.fields).toMap
+ val propsWithTs = parsedProps.map(kv => (kv._1 -> InnerValLikeWithTs(kv._2, ts))) ++
+ Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(ts, label.schemaVersion), ts))
+
+ Edge(srcVertex, tgtVertex, labelWithDir, op, version = ts, propsWithTs = propsWithTs)
+
+ }
+
+ def toVertex(ts: Long, operation: String, id: String, serviceName: String, columnName: String, props: String): Vertex = {
+ Service.findByName(serviceName) match {
+ case None => throw new RuntimeException(s"$serviceName does not exist. create service first.")
+ case Some(service) =>
+ ServiceColumn.find(service.id.get, columnName) match {
+ case None => throw new RuntimeException(s"$columnName is not exist. create service column first.")
+ case Some(col) =>
+ val idVal = toInnerVal(id, col.columnType, col.schemaVersion)
+ val op = tryOption(operation, GraphUtil.toOp)
+ val jsObject = Json.parse(props).asOpt[JsObject].getOrElse(Json.obj())
+ val parsedProps = toProps(col, jsObject).toMap
+ Vertex(VertexId(col.id.get, idVal), ts, parsedProps, op = op)
+ }
+ }
+ }
+
+ def toProps(column: ServiceColumn, js: JsObject): Seq[(Int, InnerValLike)] = {
+
+ val props = for {
+ (k, v) <- js.fields
+ meta <- column.metasInvMap.get(k)
+ } yield {
+ val innerVal = jsValueToInnerVal(v, meta.dataType, column.schemaVersion).getOrElse(
+ throw new RuntimeException(s"$k is not defined. create schema for vertex."))
+
+ (meta.seq.toInt, innerVal)
+ }
+ props
+
+ }
+
+ def toProps(label: Label, js: Seq[(String, JsValue)]): Seq[(Byte, InnerValLike)] = {
+ val props = for {
+ (k, v) <- js
+ meta <- label.metaPropsInvMap.get(k)
+ innerVal <- jsValueToInnerVal(v, meta.dataType, label.schemaVersion)
+ } yield (meta.seq, innerVal)
+
+ props
+ }
+
+
+ /**
+ * update label name.
+ */
+ def updateLabelName(oldLabelName: String, newLabelName: String) = {
+ for {
+ old <- Label.findByName(oldLabelName)
+ } {
+ Label.findByName(newLabelName) match {
+ case None =>
+ Label.updateName(oldLabelName, newLabelName)
+ case Some(_) =>
+ // throw new RuntimeException(s"$newLabelName already exist")
+ }
+ }
+ }
+}
+
+class Management(graph: Graph) {
+ import Management._
+ val storage = graph.storage
+
+ def createTable(zkAddr: String,
+ tableName: String,
+ cfs: List[String],
+ regionMultiplier: Int,
+ ttl: Option[Int],
+ compressionAlgorithm: String = DefaultCompressionAlgorithm): Unit =
+ storage.createTable(zkAddr, tableName, cfs, regionMultiplier, ttl, compressionAlgorithm)
+
+ /** HBase specific code */
+ def createService(serviceName: String,
+ cluster: String, hTableName: String,
+ preSplitSize: Int, hTableTTL: Option[Int],
+ compressionAlgorithm: String = DefaultCompressionAlgorithm): Try[Service] = {
+
+ Model withTx { implicit session =>
+ val service = Service.findOrInsert(serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
+ /** create hbase table for service */
+ storage.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
+ service
+ }
+ }
+
+ /** HBase specific code */
+ def createLabel(label: String,
+ srcServiceName: String,
+ srcColumnName: String,
+ srcColumnType: String,
+ tgtServiceName: String,
+ tgtColumnName: String,
+ tgtColumnType: String,
+ isDirected: Boolean = true,
+ serviceName: String,
+ indices: Seq[Index],
+ props: Seq[Prop],
+ consistencyLevel: String,
+ hTableName: Option[String],
+ hTableTTL: Option[Int],
+ schemaVersion: String = DEFAULT_VERSION,
+ isAsync: Boolean,
+ compressionAlgorithm: String = "gz"): Try[Label] = {
+
+ val labelOpt = Label.findByName(label, useCache = false)
+
+ Model withTx { implicit session =>
+ labelOpt match {
+ case Some(l) =>
+ throw new GraphExceptions.LabelAlreadyExistException(s"Label name ${l.label} already exist.")
+ case None =>
+ /** create all models */
+ val newLabel = Label.insertAll(label,
+ srcServiceName, srcColumnName, srcColumnType,
+ tgtServiceName, tgtColumnName, tgtColumnType,
+ isDirected, serviceName, indices, props, consistencyLevel,
+ hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
+
+ /** create hbase table */
+ val service = newLabel.service
+ (hTableName, hTableTTL) match {
+ case (None, None) => // do nothing
+ case (None, Some(hbaseTableTTL)) => throw new RuntimeException("if want to specify ttl, give hbaseTableName also")
+ case (Some(hbaseTableName), None) =>
+ // create own hbase table with default ttl on service level.
+ storage.createTable(service.cluster, hbaseTableName, List("e", "v"), service.preSplitSize, service.hTableTTL, compressionAlgorithm)
+ case (Some(hbaseTableName), Some(hbaseTableTTL)) =>
+ // create own hbase table with own ttl.
+ storage.createTable(service.cluster, hbaseTableName, List("e", "v"), service.preSplitSize, hTableTTL, compressionAlgorithm)
+ }
+ newLabel
+ }
+ }
+ }
+
+ /**
+ * label
+ */
+ /**
+ * copy label when if oldLabel exist and newLabel do not exist.
+ * copy label: only used by bulk load job. not sure if we need to parameterize hbase cluster.
+ */
+ def copyLabel(oldLabelName: String, newLabelName: String, hTableName: Option[String]) = {
+ val old = Label.findByName(oldLabelName).getOrElse(throw new LabelAlreadyExistException(s"Old label $oldLabelName not exists."))
+ if (Label.findByName(newLabelName).isDefined) throw new LabelAlreadyExistException(s"New label $newLabelName already exists.")
+
+ val allProps = old.metas.map { labelMeta => Prop(labelMeta.name, labelMeta.defaultValue, labelMeta.dataType) }
+ val allIndices = old.indices.map { index => Index(index.name, index.propNames) }
+
+ createLabel(newLabelName, old.srcService.serviceName, old.srcColumnName, old.srcColumnType,
+ old.tgtService.serviceName, old.tgtColumnName, old.tgtColumnType,
+ old.isDirected, old.serviceName,
+ allIndices, allProps,
+ old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/OrderingUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/OrderingUtil.scala b/s2core/src/main/scala/org/apache/s2graph/core/OrderingUtil.scala
new file mode 100644
index 0000000..c31aa79
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/OrderingUtil.scala
@@ -0,0 +1,146 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.types.InnerValLike
+import play.api.libs.json.{JsNumber, JsString, JsValue}
+
+object OrderingUtil {
+
+ implicit object JsValueOrdering extends Ordering[JsValue] {
+ override def compare(x: JsValue, y: JsValue): Int = {
+ (x, y) match {
+ case (JsNumber(xv), JsNumber(yv)) =>
+ Ordering.BigDecimal.compare(xv, yv)
+ case (JsString(xv), JsString(yv)) =>
+ Ordering.String.compare(xv, yv)
+ case _ => throw new Exception(s"unsupported type")
+ }
+ }
+ }
+
+ implicit object InnerValLikeOrdering extends Ordering[InnerValLike] {
+ override def compare(x: InnerValLike, y: InnerValLike): Int = {
+ x.compare(y)
+ }
+ }
+
+ implicit object MultiValueOrdering extends Ordering[Any] {
+ override def compare(x: Any, y: Any): Int = {
+ (x, y) match {
+ case (xv: Int, yv: Int) => implicitly[Ordering[Int]].compare(xv, yv)
+ case (xv: Long, yv: Long) => implicitly[Ordering[Long]].compare(xv, yv)
+ case (xv: Double, yv: Double) => implicitly[Ordering[Double]].compare(xv, yv)
+ case (xv: String, yv: String) => implicitly[Ordering[String]].compare(xv, yv)
+ case (xv: JsValue, yv: JsValue) => implicitly[Ordering[JsValue]].compare(xv, yv)
+ case (xv: InnerValLike, yv: InnerValLike) => implicitly[Ordering[InnerValLike]].compare(xv, yv)
+ }
+ }
+ }
+
+ def TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]): Ordering[(T, T, T, T)] = {
+ new Ordering[(T, T, T, T)] {
+ override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
+ val len = ascendingLs.length
+ val it = ascendingLs.iterator
+ if (len >= 1) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare1 = ord.compare(x._1, y._1)
+ if (compare1 != 0) return compare1
+ }
+
+ if (len >= 2) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare2 = ord.compare(x._2, y._2)
+ if (compare2 != 0) return compare2
+ }
+
+ if (len >= 3) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare3 = ord.compare(x._3, y._3)
+ if (compare3 != 0) return compare3
+ }
+
+ if (len >= 4) {
+ val (x, y) = it.next() match {
+ case true => tx -> ty
+ case false => ty -> tx
+ }
+ val compare4 = ord.compare(x._4, y._4)
+ if (compare4 != 0) return compare4
+ }
+ 0
+ }
+ }
+ }
+}
+
+class SeqMultiOrdering[T](ascendingLs: Seq[Boolean], defaultAscending: Boolean = true)(implicit ord: Ordering[T]) extends Ordering[Seq[T]] {
+ override def compare(x: Seq[T], y: Seq[T]): Int = {
+ val xe = x.iterator
+ val ye = y.iterator
+ val oe = ascendingLs.iterator
+
+ while (xe.hasNext && ye.hasNext) {
+ val ascending = if (oe.hasNext) oe.next() else defaultAscending
+ val (xev, yev) = ascending match {
+ case true => xe.next() -> ye.next()
+ case false => ye.next() -> xe.next()
+ }
+ val res = ord.compare(xev, yev)
+ if (res != 0) return res
+ }
+
+ Ordering.Boolean.compare(xe.hasNext, ye.hasNext)
+ }
+}
+
+//class TupleMultiOrdering[T](ascendingLs: Seq[Boolean])(implicit ord: Ordering[T]) extends Ordering[(T, T, T, T)] {
+// override def compare(tx: (T, T, T, T), ty: (T, T, T, T)): Int = {
+// val len = ascendingLs.length
+// val it = ascendingLs.iterator
+// if (len >= 1) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare1 = ord.compare(x._1, y._1)
+// if (compare1 != 0) return compare1
+// }
+//
+// if (len >= 2) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare2 = ord.compare(x._2, y._2)
+// if (compare2 != 0) return compare2
+// }
+//
+// if (len >= 3) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare3 = ord.compare(x._3, y._3)
+// if (compare3 != 0) return compare3
+// }
+//
+// if (len >= 4) {
+// val (x, y) = it.next() match {
+// case true => tx -> ty
+// case false => ty -> tx
+// }
+// val compare4 = ord.compare(x._4, y._4)
+// if (compare4 != 0) return compare4
+// }
+// 0
+// }
+//}
[12/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
deleted file mode 100644
index 127f2fe..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.types.{InnerValLike, InnerVal}
-import org.scalatest.{Matchers, FunSuite}
-
-class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser {
-
- import types.HBaseType._
- import InnerVal._
-
- val innerValsPerVersion = for {
- version <- List(VERSION2, VERSION1)
- } yield {
- val innerVals = List(
- (InnerVal.withStr("ABC123", version), STRING),
- (InnerVal.withNumber(23, version), BYTE),
- (InnerVal.withNumber(23, version), INT),
- (InnerVal.withNumber(Int.MaxValue, version), INT),
- (InnerVal.withNumber(Int.MinValue, version), INT),
- (InnerVal.withNumber(Long.MaxValue, version), LONG),
- (InnerVal.withNumber(Long.MinValue, version), LONG),
- (InnerVal.withBoolean(true, version), BOOLEAN)
- )
- val doubleVals = if (version == VERSION2) {
- List(
- (InnerVal.withDouble(Double.MaxValue, version), DOUBLE),
- (InnerVal.withDouble(Double.MinValue, version), DOUBLE),
- (InnerVal.withDouble(0.1, version), DOUBLE),
- (InnerVal.withFloat(Float.MinValue, version), FLOAT),
- (InnerVal.withFloat(Float.MaxValue, version), FLOAT),
- (InnerVal.withFloat(0.9f, version), FLOAT)
- )
- } else {
- List.empty[(InnerValLike, String)]
- }
- (innerVals ++ doubleVals, version)
- }
-
- def testInnerValToJsValue(innerValWithDataTypes: Seq[(InnerValLike, String)],
- version: String) = {
- for {
- (innerVal, dataType) <- innerValWithDataTypes
- } {
- val jsValueOpt = innerValToJsValue(innerVal, dataType)
- val decodedOpt = jsValueOpt.flatMap { jsValue =>
- jsValueToInnerVal(jsValue, dataType, version)
- }
- println(s"jsValue: $jsValueOpt")
- println(s"innerVal: $decodedOpt")
- (decodedOpt.isDefined && innerVal == decodedOpt.get) shouldBe true
- }
- }
-
- test("aa") {
- val innerVal = InnerVal.withStr("abc", VERSION2)
- val tmp = innerValToJsValue(innerVal, "string")
- println(tmp)
- }
- test("JsValue <-> InnerVal with dataType") {
- for {
- (innerValWithDataTypes, version) <- innerValsPerVersion
- } {
- testInnerValToJsValue(innerValWithDataTypes, version)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
deleted file mode 100644
index 61818fc..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/OrderingUtilTest.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.OrderingUtil._
-import org.scalatest.{FunSuite, Matchers}
-import play.api.libs.json.JsString
-
-class OrderingUtilTest extends FunSuite with Matchers {
- test("test SeqMultiOrdering") {
- val jsLs: Seq[Seq[Any]] = Seq(
- Seq(0, "a"),
- Seq(0, "b"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(2, "c")
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[Seq[Any]] = Seq(
- Seq(2, "c"),
- Seq(1, "a"),
- Seq(1, "b"),
- Seq(0, "a"),
- Seq(0, "b")
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() should equal(sortedJsLs.toString())
- }
-
- test("test tuple 1 TupleMultiOrdering") {
- val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, None, None, None),
- (0, None, None, None),
- (1, None, None, None),
- (1, None, None, None),
- (2, None, None, None)
- )
-
- val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (2, None, None, None),
- (1, None, None, None),
- (1, None, None, None),
- (0, None, None, None),
- (0, None, None, None)
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false)
- val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() should equal(sortedJsLs.toString())
- }
-
- test("test tuple 2 TupleMultiOrdering") {
- val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, "a", None, None),
- (0, "b", None, None),
- (1, "a", None, None),
- (1, "b", None, None),
- (2, "c", None, None)
- )
-
- // number descending, string ascending
- val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (2, "c", None, None),
- (1, "a", None, None),
- (1, "b", None, None),
- (0, "a", None, None),
- (0, "b", None, None)
- )
-
- val ascendingLs: Seq[Boolean] = Seq(false, true)
- val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() should equal(sortedJsLs.toString())
- }
-
- test("test tuple 3 TupleMultiOrdering") {
- val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, "a", 0l, None),
- (0, "a", 1l, None),
- (0, "b", 0l, None),
- (1, "a", 0l, None),
- (1, "b", 0l, None),
- (2, "c", 0l, None)
- )
-
- val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, "a", 1l, None),
- (0, "a", 0l, None),
- (0, "b", 0l, None),
- (1, "a", 0l, None),
- (1, "b", 0l, None),
- (2, "c", 0l, None)
- )
-
- val ascendingLs: Seq[Boolean] = Seq(true, true, false)
- val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() should equal(sortedJsLs.toString())
- }
-
- test("test tuple 4 TupleMultiOrdering") {
- val jsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, "a", 0l, JsString("a")),
- (0, "a", 0l, JsString("b")),
- (0, "a", 1l, JsString("a")),
- (0, "b", 0l, JsString("b")),
- (1, "a", 0l, JsString("b")),
- (1, "b", 0l, JsString("b")),
- (2, "c", 0l, JsString("b"))
- )
-
- val sortedJsLs: Seq[(Any, Any, Any, Any)] = Seq(
- (0, "a", 0l, JsString("b")),
- (0, "a", 0l, JsString("a")),
- (0, "a", 1l, JsString("a")),
- (0, "b", 0l, JsString("b")),
- (1, "a", 0l, JsString("b")),
- (1, "b", 0l, JsString("b")),
- (2, "c", 0l, JsString("b"))
- )
-
- val ascendingLs: Seq[Boolean] = Seq(true, true, true, false)
- val resultJsLs = jsLs.sorted(TupleMultiOrdering[Any](ascendingLs))
-
- resultJsLs.toString() should equal(sortedJsLs.toString())
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
deleted file mode 100644
index 544c17c..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.types.LabelWithDirection
-import org.apache.hadoop.hbase.util.Bytes
-import org.scalatest.{FunSuite, Matchers}
-
-class QueryParamTest extends FunSuite with Matchers with TestCommon {
-// val version = HBaseType.VERSION2
-// val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString)
-// test("EdgeTransformer toInnerValOpt") {
-//
-// /** only labelNameV2 has string type output */
-// val jsVal = Json.arr(Json.arr("_to"), Json.arr("phone_number.$", "phone_number"), Json.arr("age.$", "age"))
-// val transformer = EdgeTransformer(queryParamV2, jsVal)
-// val convertedLs = transformer.transform(testEdge, None)
-//
-// convertedLs(0).tgtVertex.innerId.toString == "10" shouldBe true
-// convertedLs(1).tgtVertex.innerId.toString == "phone_number.xxxx" shouldBe true
-// convertedLs(2).tgtVertex.innerId.toString == "age.20" shouldBe true
-// true
-// }
-
- val dummyRequests = {
- for {
- id <- 0 until 1000
- } yield {
- Bytes.toBytes(id)
- }
- }
-
- test("QueryParam toCacheKey bytes") {
- val startedAt = System.nanoTime()
- val queryParam = QueryParam(LabelWithDirection(1, 0))
-
- for {
- i <- dummyRequests.indices
- x = queryParam.toCacheKey(dummyRequests(i))
- } {
- for {
- j <- dummyRequests.indices if i != j
- y = queryParam.toCacheKey(dummyRequests(j))
- } {
- x should not equal y
- }
- }
-
- dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
- val xHash = queryParam.toCacheKey(x)
- val yHash = queryParam.toCacheKey(y)
-// println(xHash, yHash)
- xHash should be(yHash)
- }
- val duration = System.nanoTime() - startedAt
-
- println(s">> bytes: $duration")
- }
-
- test("QueryParam toCacheKey with variable params") {
- val startedAt = System.nanoTime()
- val queryParam = QueryParam(LabelWithDirection(1, 0))
-
- dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
- x shouldBe y
- queryParam.limit(0, 10)
- var xHash = queryParam.toCacheKey(x)
- xHash shouldBe queryParam.toCacheKey(y)
- queryParam.limit(1, 10)
- var yHash = queryParam.toCacheKey(y)
- queryParam.toCacheKey(x) shouldBe yHash
-// println(xHash, yHash)
- xHash should not be yHash
-
- queryParam.limit(0, 10)
- xHash = queryParam.toCacheKey(x)
- queryParam.limit(0, 11)
- yHash = queryParam.toCacheKey(y)
-
- xHash should not be yHash
- }
-
- val duration = System.nanoTime() - startedAt
-
- println(s">> diff: $duration")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
deleted file mode 100644
index ed9aaa5..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
+++ /dev/null
@@ -1,192 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.mysqls._
-//import com.kakao.s2graph.core.models._
-
-
-import com.kakao.s2graph.core.types._
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.{PutRequest, KeyValue}
-
-
-trait TestCommon {
- val ts = System.currentTimeMillis()
- val testServiceId = 1
- val testColumnId = 1
- val testLabelId = 1
- val testDir = GraphUtil.directions("out")
- val testOp = GraphUtil.operations("insert")
- val testLabelOrderSeq = LabelIndex.DefaultSeq
- val testLabelWithDir = LabelWithDirection(testLabelId, testDir)
- val labelMeta = LabelMeta
-
- def equalsTo(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) == 0
-
- def largerThan(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) > 0
-
- def lessThan(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) < 0
-
- def lessThanEqual(x: Array[Byte], y: Array[Byte]) = Bytes.compareTo(x, y) <= 0
-
- /** */
- import HBaseType.{VERSION2, VERSION1}
- private val tsValSmall = InnerVal.withLong(ts, VERSION1)
- private val tsValLarge = InnerVal.withLong(ts + 1, VERSION1)
- private val boolValSmall = InnerVal.withBoolean(false, VERSION1)
- private val boolValLarge = InnerVal.withBoolean(true, VERSION1)
- private val doubleValSmall = InnerVal.withDouble(-0.1, VERSION1)
- private val doubleValLarge = InnerVal.withDouble(0.1, VERSION1)
- private val toSeq = LabelMeta.toSeq.toInt
- private val toVal = InnerVal.withLong(Long.MinValue, VERSION1)
-
-
- private val tsValSmallV2 = InnerVal.withLong(ts, VERSION2)
- private val tsValLargeV2 = InnerVal.withLong(ts + 1, VERSION2)
- private val boolValSmallV2 = InnerVal.withBoolean(false, VERSION2)
- private val boolValLargeV2 = InnerVal.withBoolean(true, VERSION2)
- private val doubleValSmallV2 = InnerVal.withDouble(-0.1, VERSION2)
- private val doubleValLargeV2 = InnerVal.withDouble(0.1, VERSION2)
- private val toValV2 = InnerVal.withLong(Long.MinValue, VERSION2)
-
- val intVals = (Int.MinValue until Int.MinValue + 10) ++ (-129 to -126) ++ (-1 to 1) ++ (126 to 129) ++
- (Int.MaxValue - 10 until Int.MaxValue)
- val intInnerVals = intVals.map { v => InnerVal.withNumber(BigDecimal(v), VERSION1) }
-
- val intInnerValsV2 = intVals.map { v => InnerVal.withNumber(BigDecimal(v), VERSION2) }
-
- val stringVals = List("abc", "abd", "ac", "aca", "b")
- val stringInnerVals = stringVals.map { s => InnerVal.withStr(s, VERSION1)}
- val stringInnerValsV2 = stringVals.map { s => InnerVal.withStr(s, VERSION2)}
-
- val numVals = (Long.MinValue until Long.MinValue + 10).map(BigDecimal(_)) ++
- (Int.MinValue until Int.MinValue + 10).map(BigDecimal(_)) ++
- (Int.MaxValue - 10 until Int.MaxValue).map(BigDecimal(_)) ++
- (Long.MaxValue - 10 until Long.MaxValue).map(BigDecimal(_))
- val numInnerVals = numVals.map { n => InnerVal.withLong(n.toLong, VERSION1)}
- val numInnerValsV2 = numVals.map { n => InnerVal.withNumber(n, VERSION2)}
-
- val doubleStep = Double.MaxValue / 5
- val doubleVals = (Double.MinValue until 0 by doubleStep).map(BigDecimal(_)) ++
- (-9999.9 until -9994.1 by 1.1).map(BigDecimal(_)) ++
- (-128.0 until 128.0 by 1.2).map(BigDecimal(_)) ++
- (129.0 until 142.0 by 1.1).map(BigDecimal(_)) ++
- (doubleStep until Double.MaxValue by doubleStep).map(BigDecimal(_))
- val doubleInnerVals = doubleVals.map { d => InnerVal.withDouble(d.toDouble, VERSION1)}
- val doubleInnerValsV2 = doubleVals.map { d => InnerVal.withDouble(d.toDouble, VERSION2)}
-
- /** version 1 string order is broken */
- val idxPropsLs = Seq(
- Seq((0 -> tsValSmall), (1 -> boolValSmall), (2 -> InnerVal.withStr("ac", VERSION1)),(toSeq -> toVal)),
- Seq((0 -> tsValSmall), (1 -> boolValSmall), (2 -> InnerVal.withStr("ab", VERSION1)), (toSeq -> toVal)),
- Seq((0 -> tsValSmall), (1 -> boolValSmall), (2-> InnerVal.withStr("b", VERSION1)), (toSeq -> toVal)),
- Seq((0 -> tsValSmall), (1 -> boolValLarge), (2 -> InnerVal.withStr("b", VERSION1)), (toSeq -> toVal)),
- Seq((0 -> tsValLarge), (1 -> boolValSmall), (2 -> InnerVal.withStr("a", VERSION1)), (toSeq -> toVal))
- ).map(seq => seq.map(t => t._1.toByte -> t._2 ))
-
- val idxPropsLsV2 = Seq(
- Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 -> doubleValSmallV2), (toSeq -> toValV2)),
- Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 -> doubleValLargeV2), (toSeq -> toValV2)),
- Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("ab", VERSION2)), (3 -> doubleValLargeV2), (toSeq -> toValV2)),
- Seq((0 -> tsValSmallV2), (1 -> boolValSmallV2), (2-> InnerVal.withStr("b", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2)),
- Seq((0 -> tsValSmallV2), (1 -> boolValLargeV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2)),
- Seq((0 -> tsValLargeV2), (1 -> boolValSmallV2), (2 -> InnerVal.withStr("a", VERSION2)), (3 ->doubleValLargeV2), (toSeq -> toValV2))
- ).map(seq => seq.map(t => t._1.toByte -> t._2 ) )
-
- val idxPropsWithTsLs = idxPropsLs.map { idxProps =>
- idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }
- }
- val idxPropsWithTsLsV2 = idxPropsLsV2.map { idxProps =>
- idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }
- }
- //
- // def testOrder(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]],
- // innerVals: Iterable[InnerValLike], skipHashBytes: Boolean = false)
- // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
- // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
- // /** check if increasing target vertex id is ordered properly with same indexProps */
- // val rets = for {
- // idxProps <- idxPropsLs
- // } yield {
- // val head = createFunc(idxProps, innerVals.head)
- // val start = head
- // var prev = head
- // val rets = for {
- // innerVal <- innerVals.tail
- // } yield {
- // val current = createFunc(idxProps, innerVal)
- // val bytes = current.bytes
- // val decoded = fromBytesFunc(bytes)
- // println(s"current: $current")
- // println(s"decoded: $decoded")
- //
- // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
- // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
- // val (isSame, orderPreserved) = (current, decoded) match {
- // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
- // /** _to is used in indexProps */
- // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
- // case _ =>
- // (current == decoded, lessThan(currentBytes, prevBytes))
- // }
- //
- // println(s"$current ${bytes.toList}")
- // println(s"$prev ${prev.bytes.toList}")
- // println(s"SerDe[$isSame], Order[$orderPreserved]")
- // prev = current
- // isSame && orderPreserved
- // }
- // rets.forall(x => x)
- // }
- // rets.forall(x => x)
- // }
- // def testOrderReverse(idxPropsLs: Seq[Seq[(Byte, InnerValLike)]], innerVals: Iterable[InnerValLike],
- // skipHashBytes: Boolean = false)
- // (createFunc: (Seq[(Byte, InnerValLike)], InnerValLike) => HBaseSerializable,
- // fromBytesFunc: Array[Byte] => HBaseSerializable) = {
- // /** check if increasing target vertex id is ordered properly with same indexProps */
- // val rets = for {
- // innerVal <- innerVals
- // } yield {
- // val head = createFunc(idxPropsLs.head, innerVal)
- // val start = head
- // var prev = head
- // val rets = for {
- // idxProps <- idxPropsLs.tail
- // } yield {
- // val current = createFunc(idxProps, innerVal)
- // val bytes = current.bytes
- // val decoded = fromBytesFunc(bytes)
- // println(s"current: $current")
- // println(s"decoded: $decoded")
- //
- // val prevBytes = if (skipHashBytes) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
- // val currentBytes = if (skipHashBytes) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
- // val (isSame, orderPreserved) = (current, decoded) match {
- // case (c: v2.EdgeQualifier, d: v2.EdgeQualifier) if (idxProps.map(_._1).contains(toSeq)) =>
- // /** _to is used in indexProps */
- // (c.props.map(_._2) == d.props.map(_._2) && c.op == d.op, Bytes.compareTo(currentBytes, prevBytes) <= 0)
- // case _ =>
- // (current == decoded, lessThan(currentBytes, prevBytes))
- // }
- //
- // println(s"$current ${bytes.toList}")
- // println(s"$prev ${prev.bytes.toList}")
- // println(s"SerDe[$isSame], Order[$orderPreserved]")
- // prev = current
- // isSame && orderPreserved
- // }
- //
- // rets.forall(x => x)
- // }
- //
- // rets.forall(x => x)
- // }
- //
- //
- // def putToKeyValues(put: PutRequest) = {
- // val ts = put.timestamp()
- // for ((q, v) <- put.qualifiers().zip(put.values)) yield {
- // new KeyValue(put.key(), put.family(), q, ts, v)
- // }
- // }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
deleted file mode 100644
index f7c4bc1..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
-import com.kakao.s2graph.core.mysqls._
-import scalikejdbc.AutoSession
-
-//import com.kakao.s2graph.core.models._
-
-import com.kakao.s2graph.core.types.{InnerVal, LabelWithDirection}
-import com.typesafe.config.{Config, ConfigFactory}
-
-import scala.concurrent.ExecutionContext
-
-trait TestCommonWithModels {
-
- import InnerVal._
- import types.HBaseType._
-
- var graph: Graph = _
- var config: Config = _
- var management: Management = _
-
- def initTests() = {
- config = ConfigFactory.load()
- graph = new Graph(config)(ExecutionContext.Implicits.global)
- management = new Management(graph)
-
- implicit val session = AutoSession
-
- deleteTestLabel()
- deleteTestService()
-
- createTestService()
- createTestLabel()
- }
-
- def zkQuorum = config.getString("hbase.zookeeper.quorum")
-
- def cluster = config.getString("hbase.zookeeper.quorum")
-
- implicit val session = AutoSession
-
- val serviceName = "_test_service"
- val serviceNameV2 = "_test_service_v2"
- val serviceNameV3 = "_test_service_v3"
- val serviceNameV4 = "_test_service_v4"
-
- val columnName = "user_id"
- val columnNameV2 = "user_id_v2"
- val columnNameV3 = "user_id_v3"
- val columnNameV4 = "user_id_v4"
-
- val columnType = "long"
- val columnTypeV2 = "long"
- val columnTypeV3 = "long"
- val columnTypeV4 = "long"
-
- val tgtColumnName = "itme_id"
- val tgtColumnNameV2 = "item_id_v2"
- val tgtColumnNameV3 = "item_id_v3"
- val tgtColumnNameV4 = "item_id_v4"
-
- val tgtColumnType = "string"
- val tgtColumnTypeV2 = "string"
- val tgtColumnTypeV3 = "string"
- val tgtColumnTypeV4 = "string"
-
- val hTableName = "_test_cases"
- val preSplitSize = 0
-
- val labelName = "_test_label"
- val labelNameV2 = "_test_label_v2"
- val labelNameV3 = "_test_label_v3"
- val labelNameV4 = "_test_label_v4"
-
- val undirectedLabelName = "_test_label_undirected"
- val undirectedLabelNameV2 = "_test_label_undirected_v2"
-
- val testProps = Seq(
- Prop("affinity_score", "0.0", DOUBLE),
- Prop("is_blocked", "false", BOOLEAN),
- Prop("time", "0", INT),
- Prop("weight", "0", INT),
- Prop("is_hidden", "true", BOOLEAN),
- Prop("phone_number", "xxx-xxx-xxxx", STRING),
- Prop("score", "0.1", FLOAT),
- Prop("age", "10", INT)
- )
- val testIdxProps = Seq(Index("_PK", Seq("_timestamp", "affinity_score")))
- val consistencyLevel = "strong"
- val hTableTTL = None
-
-
- def createTestService() = {
- implicit val session = AutoSession
- management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
- management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
- management.createService(serviceNameV3, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
- management.createService(serviceNameV4, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
- }
-
- def deleteTestService() = {
- implicit val session = AutoSession
- Management.deleteService(serviceName)
- Management.deleteService(serviceNameV2)
- Management.deleteService(serviceNameV3)
- Management.deleteService(serviceNameV4)
- }
-
- def deleteTestLabel() = {
- implicit val session = AutoSession
- Management.deleteLabel(labelName)
- Management.deleteLabel(labelNameV2)
- Management.deleteLabel(undirectedLabelName)
- Management.deleteLabel(undirectedLabelNameV2)
- }
-
- def createTestLabel() = {
- implicit val session = AutoSession
- management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
- isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4")
-
- management.createLabel(labelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
- isDirected = true, serviceNameV2, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
-
- management.createLabel(labelNameV3, serviceNameV3, columnNameV3, columnTypeV3, serviceNameV3, tgtColumnNameV3, tgtColumnTypeV3,
- isDirected = true, serviceNameV3, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4")
-
- management.createLabel(labelNameV4, serviceNameV4, columnNameV4, columnTypeV4, serviceNameV4, tgtColumnNameV4, tgtColumnTypeV4,
- isDirected = true, serviceNameV4, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4")
-
- management.createLabel(undirectedLabelName, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType,
- isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4")
-
- management.createLabel(undirectedLabelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
- isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
- }
-
- def service = Service.findByName(serviceName, useCache = false).get
-
- def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
-
- def serviceV3 = Service.findByName(serviceNameV3, useCache = false).get
-
- def serviceV4 = Service.findByName(serviceNameV4, useCache = false).get
-
- def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
-
- def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
-
- def columnV3 = ServiceColumn.find(serviceV3.id.get, columnNameV3, useCache = false).get
-
- def columnV4 = ServiceColumn.find(serviceV4.id.get, columnNameV4, useCache = false).get
-
- def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
-
- def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
-
- def tgtColumnV3 = ServiceColumn.find(serviceV3.id.get, tgtColumnNameV3, useCache = false).get
-
- def tgtColumnV4 = ServiceColumn.find(serviceV4.id.get, tgtColumnNameV4, useCache = false).get
-
- def label = Label.findByName(labelName, useCache = false).get
-
- def labelV2 = Label.findByName(labelNameV2, useCache = false).get
-
- def labelV3 = Label.findByName(labelNameV3, useCache = false).get
-
- def labelV4 = Label.findByName(labelNameV4, useCache = false).get
-
- def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
-
- def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
-
- def dir = GraphUtil.directions("out")
-
- def op = GraphUtil.operations("insert")
-
- def labelOrderSeq = LabelIndex.DefaultSeq
-
- def labelWithDir = LabelWithDirection(label.id.get, dir)
-
- def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
-
- def labelWithDirV3 = LabelWithDirection(labelV3.id.get, dir)
-
- def labelWithDirV4 = LabelWithDirection(labelV4.id.get, dir)
-
- def queryParam = QueryParam(labelWithDir)
-
- def queryParamV2 = QueryParam(labelWithDirV2)
-
- def queryParamV3 = QueryParam(labelWithDirV3)
-
- def queryParamV4 = QueryParam(labelWithDirV4)
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/VertexTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/VertexTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/VertexTest.scala
deleted file mode 100644
index 62b98b5..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/VertexTest.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-//package com.kakao.s2graph.core
-//
-//import com.kakao.s2graph.core.types.{VertexId, InnerVal, InnerValLike}
-//import org.scalatest.{Matchers, FunSuite}
-//
-//
-///**
-// * Created by shon on 5/29/15.
-// */
-//class VertexTest extends FunSuite with Matchers with TestCommonWithModels with TestCommon {
-//
-// import types.HBaseType.{VERSION1, VERSION2}
-// val idxPropsList = idxPropsLs.map { seq => seq.map { kv => kv._1.toInt -> kv._2 }}
-// val idxPropsListV2 = idxPropsLsV2.map { seq => seq.map { kv => kv._1.toInt -> kv._2 }}
-// def equalsExact(left: Vertex, right: Vertex) = {
-// left.id == right.id && left.ts == right.ts &&
-// left.props == right.props && left.op == right.op
-// }
-// def vertexId(innerVal: InnerValLike)(version: String) = {
-// val colId = if (version == VERSION2) columnV2.id.get else column.id.get
-// VertexId(colId, innerVal)
-// }
-//
-// /** assumes innerVal is sorted */
-// def testVertexEncodeDecode(innerVals: Seq[InnerValLike],
-// propsLs: Seq[Seq[(Int, InnerValLike)]], version: String) = {
-// for {
-// props <- propsLs
-// } {
-// val currentTs = BigDecimal(props.toMap.get(0.toByte).get.toString).toLong
-// val head = Vertex(vertexId(innerVals.head)(version), currentTs, props.toMap, op)
-// val start = head
-// var prev = head
-// for {
-// innerVal <- innerVals.tail
-// } {
-// var current = Vertex(vertexId(innerVal)(version), currentTs, props.toMap, op)
-// val puts = current.buildPutsAsync()
-// val kvs = for {put <- puts; kv <- putToKeyValues(put)} yield kv
-// val decodedOpt = Vertex(kvs, version)
-// val prevBytes = prev.rowKey.bytes.drop(GraphUtil.bytesForMurMurHash)
-// val currentBytes = current.rowKey.bytes.drop(GraphUtil.bytesForMurMurHash)
-// decodedOpt.isDefined shouldBe true
-// val isSame = equalsExact(decodedOpt.get, current)
-// val comp = lessThan(currentBytes, prevBytes)
-//
-// println(s"current: $current")
-// println(s"decoded: ${decodedOpt.get}")
-// println(s"$isSame, $comp")
-// prev = current
-// isSame && comp shouldBe true
-// }
-// }
-// }
-//
-// test("test with int innerVals as id version 1") {
-// testVertexEncodeDecode(intInnerVals, idxPropsList, VERSION1)
-// }
-// test("test with int innerVals as id version 2") {
-// testVertexEncodeDecode(intInnerValsV2, idxPropsListV2, VERSION2)
-// }
-// test("test with string stringVals as id versoin 2") {
-// testVertexEncodeDecode(stringInnerValsV2, idxPropsListV2, VERSION2)
-// }
-// // test("test vertex encoding/decoding") {
-// // val innerVal1 = new InnerVal(BigDecimal(10))
-// // val innerVal2 = new InnerValV1(Some(10L), None, None)
-// // println(s"${innerVal1.bytes.toList}")
-// // println(s"${innerVal2.bytes.toList}")
-// // val id1 = new CompositeId(0, innerVal1, isEdge = false, useHash = true)
-// // val id2 = new CompositeIdV1(0, innerVal2, isEdge = false, useHash = true)
-// // val ts = System.currentTimeMillis()
-// // val v1 = Vertex(id1, ts)
-// // val v2 = Vertex(id2, ts)
-// //
-// // println(s"${v1.rowKey.bytes.toList}")
-// // println(s"${v2.rowKey.bytes.toList}")
-// // }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
deleted file mode 100644
index f66eabf..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-package com.kakao.s2graph.core.models
-
-import java.util.concurrent.ExecutorService
-
-import com.kakao.s2graph.core.mysqls.{Label, Model}
-import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon, Graph}
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{BeforeAndAfterAll, Sequential, FunSuite, Matchers}
-
-import scala.concurrent.ExecutionContext
-
-class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll {
- override def beforeAll(): Unit = {
- initTests()
- }
-
- override def afterAll(): Unit = {
- graph.shutdown()
- }
-
- // val serviceName = "testService"
- // val newServiceName = "newTestService"
- // val cluster = "localhost"
- // val hbaseTableName = "s2graph-dev"
- // val columnName = "user_id"
- // val columnType = "long"
- // val labelName = "model_test_label"
- // val newLabelName = "new_model_test_label"
- // val columnMetaName = "is_valid_user"
- // val labelMetaName = "is_hidden"
- // val hbaseTableTTL = -1
- // val id = 1
- //
- // val service = HService(Map("id" -> id, "serviceName" -> serviceName, "cluster" -> cluster,
- // "hbaseTableName" -> hbaseTableName, "preSplitSize" -> 0, "hbaseTableTTL" -> -1))
- // val serviceColumn = HServiceColumn(Map("id" -> id, "serviceId" -> service.id.get,
- // "columnName" -> columnName, "columnType" -> columnType))
- // val columnMeta = HColumnMeta(Map("id" -> id, "columnId" -> serviceColumn.id.get, "name" -> columnMetaName, "seq" -> 1.toByte))
- // val label = HLabel(Map("id" -> id, "label" -> labelName,
- // "srcServiceId" -> service.id.get, "srcColumnName" -> columnName, "srcColumnType" -> columnType,
- // "tgtServiceId" -> service.id.get, "tgtColumnName" -> columnName, "tgtColumnType" -> columnType,
- // "isDirected" -> true, "serviceName" -> service.serviceName, "serviceId" -> service.id.get,
- // "consistencyLevel" -> "weak", "hTableName" -> hbaseTableName, "hTableTTL" -> -1
- // ))
- // val labelMeta = HLabelMeta(Map("id" -> id, "labelId" -> label.id.get, "name" -> labelMetaName, "seq" -> 1.toByte,
- // "defaultValue" -> false, "dataType" -> "boolean", "usedInIndex" -> false))
- // val labelIndex = HLabelIndex(Map("id" -> id, "labelId" -> label.id.get, "seq" -> 1.toByte,
- // "metaSeqs" -> "0", "formular" -> "none"))
- test("test Label.findByName") {
- val labelOpt = Label.findByName(labelName, useCache = false)
- println(labelOpt)
- labelOpt.isDefined shouldBe true
- val indices = labelOpt.get.indices
- indices.size > 0 shouldBe true
- println(indices)
- val defaultIndexOpt = labelOpt.get.defaultIndex
- println(defaultIndexOpt)
- defaultIndexOpt.isDefined shouldBe true
- val metas = labelOpt.get.metaProps
- println(metas)
- metas.size > 0 shouldBe true
- val srcService = labelOpt.get.srcService
- println(srcService)
- val tgtService = labelOpt.get.tgtService
- println(tgtService)
- val service = labelOpt.get.service
- println(service)
- val srcColumn = labelOpt.get.srcService
- println(srcColumn)
- val tgtColumn = labelOpt.get.tgtService
- println(tgtColumn)
- }
- // test("test create") {
- // service.create()
- // HService.findByName(serviceName, useCache = false) == Some(service)
- //
- // serviceColumn.create()
- // HServiceColumn.findsByServiceId(service.id.get, useCache = false).headOption == Some(serviceColumn)
- //
- // columnMeta.create()
- // HColumnMeta.findByName(serviceColumn.id.get, columnMetaName, useCache = false) == Some(columnMeta)
- //
- // label.create()
- // HLabel.findByName(labelName, useCache = false) == Some(label)
- //
- // labelMeta.create()
- // HLabelMeta.findByName(label.id.get, labelMetaName, useCache = false) == Some(labelMeta)
- //
- // labelIndex.create()
- // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).headOption == Some(labelIndex)
- // }
- //
- // test("test update") {
- // service.update("cluster", "...")
- // HService.findById(service.id.get, useCache = false).cluster == "..."
- //
- // service.update("serviceName", newServiceName)
- // assert(HService.findByName(serviceName, useCache = false) == None)
- // HService.findByName(newServiceName, useCache = false).map { service => service.id.get == service.id.get}
- //
- // label.update("label", newLabelName)
- // HLabel.findById(label.id.get, useCache = false).label == "newLabelName"
- //
- // label.update("consistencyLevel", "strong")
- // HLabel.findById(label.id.get, useCache = false).consistencyLevel == "strong" &&
- // HLabel.findByName(newLabelName).isDefined &&
- // HLabel.findByName(labelName) == None
- //
- // }
- // test("test read by index") {
- // val labels = HLabel.findBySrcServiceId(service.id.get, useCache = false)
- // val idxs = HLabelIndex.findByLabelIdAll(label.id.get, useCache = false)
- // labels.length == 1 &&
- // labels.head == label
- // idxs.length == 1 &&
- // idxs.head == labelIndex
- // }
- // test("test delete") {
- //// HLabel.findByName(labelName).foreach { label =>
- //// label.deleteAll()
- //// }
- // HLabel.findByName(newLabelName).foreach { label =>
- // label.deleteAll()
- // }
- // HLabelMeta.findAllByLabelId(label.id.get, useCache = false).isEmpty &&
- // HLabelIndex.findByLabelIdAll(label.id.get, useCache = false).isEmpty
- //
- // service.deleteAll()
- // }
-
- // test("test labelIndex") {
- // println(HLabelIndex.findByLabelIdAll(1))
- // }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
deleted file mode 100644
index 078b5a9..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-import java.util.Properties
-
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-import scalikejdbc._
-
-class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
- val Ttl = 2
- override def beforeAll(): Unit = {
- /*
- maxSize = config.getInt("cache.max.size")
- ttl = config.getInt("cache.ttl.seconds")
- */
- val props = new Properties()
- props.setProperty("cache.ttl.seconds", Ttl.toString)
- Model.apply(ConfigFactory.load(ConfigFactory.parseProperties(props)))
-
- implicit val session = AutoSession
- sql"""DELETE FROM buckets""".update().apply()
- sql"""DELETE FROM experiments""".update().apply()
-
- val expId = sql"""INSERT INTO experiments(service_id, service_name, `name`, description) VALUES(1, "s1", "exp1", "")""".updateAndReturnGeneratedKey().apply()
- sql"""INSERT INTO
- buckets(experiment_id, modular, http_verb, api_path, request_body, impression_id)
- VALUES($expId, "1~100", "POST", "/a/b/c", "None", "imp1")""".update().apply()
-
- }
-
- "Experiment" should "find bucket list" in {
- Experiment.findBy(1, "exp1") should not be empty
-
- Experiment.findBy(1, "exp1").foreach { exp =>
- val bucket = exp.buckets.head
- bucket.impressionId should equal("imp1")
- }
- }
-
- it should "update bucket list after cache ttl time" in {
- Experiment.findBy(1, "exp1").foreach { exp =>
- val bucket = exp.buckets.head
- bucket.impressionId should equal("imp1")
-
- implicit val session = AutoSession
-
- sql"""UPDATE buckets SET impression_id = "imp2" WHERE id = ${bucket.id}""".update().apply()
- }
-
- // sleep ttl time
- Thread.sleep((Ttl + 1) * 1000)
-
- // update experiment and bucket
- Experiment.findBy(1, "exp1").foreach(exp => exp.buckets)
-
- // wait for cache updating
- Thread.sleep(1 * 1000)
-
- // should be updated
- Experiment.findBy(1, "exp1").foreach { exp =>
- exp.buckets.head.impressionId should equal("imp2")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
deleted file mode 100644
index febad3d..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
+++ /dev/null
@@ -1,234 +0,0 @@
-package com.kakao.s2graph.core.parsers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{Experiment, Label, LabelMeta}
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.logger
-import org.scalatest.{FunSuite, Matchers}
-import play.api.libs.json.Json
-
-class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
- initTests()
-
- // dummy data for dummy edge
- initTests()
-
- import HBaseType.{VERSION1, VERSION2}
-
- val ts = System.currentTimeMillis()
- val dummyTs = (LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
-
- def ids(version: String) = {
- val colId = if (version == VERSION2) columnV2.id.get else column.id.get
- val srcId = SourceVertexId(colId, InnerVal.withLong(1, version))
- val tgtId = TargetVertexId(colId, InnerVal.withLong(2, version))
-
- val srcIdStr = SourceVertexId(colId, InnerVal.withStr("abc", version))
- val tgtIdStr = TargetVertexId(colId, InnerVal.withStr("def", version))
-
- val srcVertex = Vertex(srcId, ts)
- val tgtVertex = Vertex(tgtId, ts)
- val srcVertexStr = Vertex(srcIdStr, ts)
- val tgtVertexStr = Vertex(tgtIdStr, ts)
- (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, version)
- }
-
-
- def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = {
- val whereOpt = WhereParser(label).parse(sql)
- whereOpt.isSuccess shouldBe true
-
- println("=================================================================")
- println(sql)
- println(whereOpt.get)
-
- val ret = whereOpt.get.filter(edge)
- if (ret != expected) {
- println("==================")
- println(s"$whereOpt")
- println(s"$edge")
- println("==================")
- }
- ret shouldBe expected
- }
-
- test("check where clause not nested") {
- for {
- (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
- } {
- /** test for each version */
- val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
- val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
- val f = validate(label)(edge) _
-
- /** labelName label is long-long relation */
- f(s"_to=${tgtVertex.innerId.toString}")(true)
-
- // currently this throw exception since label`s _to is long type.
- f(s"_to=19230495")(false)
- f(s"_to!=19230495")(true)
- }
- }
-
- test("check where clause nested") {
- for {
- (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
- } {
- /** test for each version */
- val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
- val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
-
- val f = validate(label)(edge) _
-
- // time == 3
- f("time >= 3")(true)
- f("time > 2")(true)
- f("time <= 3")(true)
- f("time < 2")(false)
-
- f("(time in (1, 2, 3) and is_blocked = true) or is_hidden = false")(false)
- f("(time in (1, 2, 3) or is_blocked = true) or is_hidden = false")(true)
- f("(time in (1, 2, 3) and is_blocked = true) or is_hidden = true")(true)
- f("(time in (1, 2, 3) or is_blocked = true) and is_hidden = true")(true)
-
- f("((time in ( 1, 2, 3) and weight between 1 and 10) or is_hidden=false)")(true)
- f("(time in (1, 2, 4 ) or weight between 1 and 9) or (is_hidden = false)")(false)
- f("(time in ( 1,2,4 ) or weight between 1 and 9) or is_hidden= true")(true)
- f("(time in (1,2,3) or weight between 1 and 10) and is_hidden =false")(false)
- }
- }
-
- test("check where clause with from/to long") {
- for {
- (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
- } {
- /** test for each version */
- val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
- val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir
- val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner)
- val lname = if (schemaVer == VERSION2) labelNameV2 else labelName
- val f = validate(label)(edge) _
-
- f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true)
- f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true)
- f(s"_from = ${tgtVertex.innerId.value} and _to = 102934")(false)
- f(s"_from = -1")(false)
- f(s"_from in (-1, -0.1)")(false)
- }
- }
-
-
- test("check where clause with parent") {
- for {
- (srcId, tgtId, srcIdStr, tgtIdStr, srcVertex, tgtVertex, srcVertexStr, tgtVertexStr, schemaVer) <- List(ids(VERSION1), ids(VERSION2))
- } {
- /** test for each version */
- val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 1, "name" -> "abc")
- val parentJs = Json.obj("is_hidden" -> false, "is_blocked" -> false, "weight" -> 20, "time" -> 3, "name" -> "a")
-
- val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val parentPropsInner = Management.toProps(label, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
-
- val grandParentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner)
- val parentEdge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, parentPropsInner,
- parentEdges = Seq(EdgeWithScore(grandParentEdge, 1.0)))
- val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner,
- parentEdges = Seq(EdgeWithScore(parentEdge, 1.0)))
-
- println(edge.toString)
- println(parentEdge.toString)
- println(grandParentEdge.toString)
-
- val f = validate(label)(edge) _
-
- // Compare edge's prop(`_from`) with edge's prop(`name`)
- f("_from = 1")(true)
- f("_to = 2")(true)
- f("_from = 123")(false)
- f("_from = time")(true)
-
- // Compare edge's prop(`weight`) with edge's prop(`time`)
- f("weight = time")(false)
- f("weight = is_blocked")(false)
-
- // Compare edge's prop(`weight`) with parent edge's prop(`weight`)
- f("_parent.is_blocked = is_blocked")(true)
- f("is_hidden = _parent.is_hidden")(false)
- f("_parent.weight = weight")(false)
-
- // Compare edge's prop(`is_hidden`) with parent of parent edge's prop(`is_hidden`)
- f("_parent._parent.is_hidden = is_hidden")(false)
- f("_parent._parent.is_blocked = is_blocked")(true)
- f("_parent._parent.weight = weight")(false)
- f("_parent._parent.weight = _parent.weight")(true)
- }
- }
-
- test("replace reserved") {
- val ts = 0
- import com.kakao.s2graph.core.rest.TemplateHelper._
-
- calculate(ts, 1, "hour") should be(hour + ts)
- calculate(ts, 1, "day") should be(day + ts)
-
- calculate(ts + 10, 1, "HOUR") should be(hour + ts + 10)
- calculate(ts + 10, 1, "DAY") should be(day + ts + 10)
-
- val body = """{
- "day": ${1day},
- "hour": ${1hour},
- "-day": "${-10 day}",
- "-hour": ${-10 hour},
- "now": "${now}"
- }
- """
-
- val parsed = replaceVariable(ts, body)
- val json = Json.parse(parsed)
-
- (json \ "day").as[Long] should be (1 * day + ts)
- (json \ "hour").as[Long] should be (1 * hour + ts)
-
- (json \ "-day").as[Long] should be (-10 * day + ts)
- (json \ "-hour").as[Long] should be (-10 * hour + ts)
-
- (json \ "now").as[Long] should be (ts)
-
- val otherBody = """{
- "nextday": "${next_day}",
- "3dayago": "${next_day - 3 day}",
- "nexthour": "${next_hour}"
- }"""
-
- val currentTs = System.currentTimeMillis()
- val expectedDayTs = currentTs / day * day + day
- val expectedHourTs = currentTs / hour * hour + hour
- val threeDayAgo = expectedDayTs - 3 * day
- val currentTsLs = (1 until 1000).map(currentTs + _)
-
- currentTsLs.foreach { ts =>
- val parsed = replaceVariable(ts, otherBody)
- val json = Json.parse(parsed)
-
- (json \ "nextday").as[Long] should be(expectedDayTs)
- (json \ "nexthour").as[Long] should be(expectedHourTs)
- (json \ "3dayago").as[Long] should be(threeDayAgo)
- }
- }
-
- // test("time decay") {
- // val ts = System.currentTimeMillis()
- //
- // for {
- // i <- (0 until 10)
- // } {
- // val timeUnit = 60 * 60
- // val diff = i * timeUnit
- // val x = TimeDecay(1.0, 0.05, timeUnit)
- // println(x.decay(diff))
- // }
- // }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
deleted file mode 100644
index 128f2d7..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-//package com.kakao.s2graph.core.storage.hbase
-//
-//import com.kakao.s2graph.core.Graph
-//import com.typesafe.config.ConfigFactory
-//
-//import org.apache.hadoop.hbase.util.Bytes
-//import org.hbase.async.GetRequest
-//import org.scalatest.{FunSuite, Matchers}
-//
-//import scala.concurrent.ExecutionContext
-//
-//class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
-//
-// val dummyRequests = {
-// for {
-// id <- 0 until 1000
-// } yield {
-// new GetRequest("a", Bytes.toBytes(id))
-// }
-// }
-//
-// implicit val ec = ExecutionContext.Implicits.global
-// val config = ConfigFactory.load()
-// val graph = new Graph(config)
-//
-// val qb = new AsynchbaseQueryBuilder(graph.storage.asInstanceOf[AsynchbaseStorage])
-//
-// test("test toCacheKeyBytes") {
-// val startedAt = System.nanoTime()
-//
-// for {
-// i <- dummyRequests.indices
-// x = qb.toCacheKeyBytes(dummyRequests(i))
-// } {
-// for {
-// j <- dummyRequests.indices if i != j
-// y = qb.toCacheKeyBytes(dummyRequests(j))
-// } {
-// x should not equal y
-// }
-// }
-//
-// dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
-// val xHash = qb.toCacheKeyBytes(x)
-// val yHash = qb.toCacheKeyBytes(y)
-// // println(xHash, yHash)
-// xHash should be(yHash)
-// }
-// val duration = System.nanoTime() - startedAt
-//
-// println(s">> bytes: $duration")
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
deleted file mode 100644
index 87a0dcb..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorageTest.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.typesafe.config.ConfigFactory
-import org.hbase.async.{GetRequest, PutRequest}
-import org.scalatest.{Matchers, FunSuite}
-import scala.collection.JavaConversions._
-
-class AsynchbaseStorageTest extends FunSuite with Matchers {
-
- /** need secured cluster */
-// test("test secure cluster connection") {
-// val config = ConfigFactory.parseMap(
-// Map(
-// "hbase.zookeeper.quorum" -> "localhost",
-// "hbase.security.auth.enable" -> "true",
-// "hbase.security.authentication" -> "kerberos",
-// "hbase.kerberos.regionserver.principal" -> "hbase/_HOST@LOCAL.HBASE",
-// "hbase.sasl.clientconfig" -> "Client",
-// "java.security.krb5.conf" -> "krb5.conf",
-// "java.security.auth.login.config" -> "async-client.jaas.conf")
-// )
-//
-// val client = AsynchbaseStorage.makeClient(config)
-// val table = "test".getBytes()
-//
-// val putRequest = new PutRequest(table, "a".getBytes(), "e".getBytes, "a".getBytes, "a".getBytes)
-// val getRequest = new GetRequest(table, "a".getBytes(), "e".getBytes)
-// val ret = client.put(putRequest).join()
-// val kvs = client.get(getRequest).join()
-// for {
-// kv <- kvs
-// } {
-// println(kv.toString)
-// }
-// }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeTest.scala
deleted file mode 100644
index e68fc20..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeTest.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.mysqls.{Label, LabelMeta, LabelIndex}
-import com.kakao.s2graph.core.{IndexEdge, Vertex, TestCommonWithModels}
-import com.kakao.s2graph.core.types._
-import org.scalatest.{FunSuite, Matchers}
-
-
-class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels {
- initTests()
-
- /**
- * check if storage serializer/deserializer can translate from/to bytes array.
- * @param l: label for edge.
- * @param ts: timestamp for edge.
- * @param to: to VertexId for edge.
- * @param props: expected props of edge.
- */
- def check(l: Label, ts: Long, to: InnerValLike, props: Map[Byte, InnerValLike]): Unit = {
- val from = InnerVal.withLong(1, l.schemaVersion)
- val vertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, from)
- val tgtVertexId = TargetVertexId(HBaseType.DEFAULT_COL_ID, to)
- val vertex = Vertex(vertexId, ts)
- val tgtVertex = Vertex(tgtVertexId, ts)
- val labelWithDir = LabelWithDirection(l.id.get, 0)
-
- val indexEdge = IndexEdge(vertex, tgtVertex, labelWithDir, 0, ts, LabelIndex.DefaultSeq, props)
- val _indexEdgeOpt = graph.storage.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(queryParam,
- graph.storage.indexEdgeSerializer(indexEdge).toKeyValues, l.schemaVersion, None)
-
- _indexEdgeOpt should not be empty
- indexEdge should be(_indexEdgeOpt.get)
- }
-
-
- /** note that props have to be properly set up for equals */
- test("test serializer/deserializer for index edge.") {
- val ts = System.currentTimeMillis()
- for {
- l <- Seq(label, labelV2, labelV3, labelV4)
- } {
- val to = InnerVal.withLong(101, l.schemaVersion)
- val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
- val props = Map(LabelMeta.timeStampSeq -> tsInnerVal,
- 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion))
-
- check(l, ts, to, props)
- }
- }
-
- test("test serializer/deserializer for degree edge.") {
- val ts = System.currentTimeMillis()
- for {
- l <- Seq(label, labelV2, labelV3, labelV4)
- } {
- val to = InnerVal.withStr("0", l.schemaVersion)
- val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
- val props = Map(
- LabelMeta.degreeSeq -> InnerVal.withLong(10, l.schemaVersion),
- LabelMeta.timeStampSeq -> tsInnerVal)
-
- check(l, ts, to, props)
- }
- }
-
- test("test serializer/deserializer for incrementCount index edge.") {
- val ts = System.currentTimeMillis()
- for {
- l <- Seq(label, labelV2, labelV3, labelV4)
- } {
- val to = InnerVal.withLong(101, l.schemaVersion)
-
- val tsInnerVal = InnerVal.withLong(ts, l.schemaVersion)
- val props = Map(LabelMeta.timeStampSeq -> tsInnerVal,
- 1.toByte -> InnerVal.withDouble(2.1, l.schemaVersion),
- LabelMeta.countSeq -> InnerVal.withLong(10, l.schemaVersion))
-
- check(l, ts, to, props)
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/CompositeIdTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/CompositeIdTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/CompositeIdTest.scala
deleted file mode 100644
index d014208..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/CompositeIdTest.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.{TestCommonWithModels, GraphUtil, TestCommon}
-//import com.kakao.s2graph.core.types2._
-//import org.apache.hadoop.hbase.util.Bytes
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 5/29/15.
-// */
-//class CompositeIdTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels {
-// /** these constants need to be sorted asc order for test to run */
-// import InnerVal.{VERSION1, VERSION2}
-//
-//
-// val functions = for {
-// isEdge <- List(true, false)
-// useHash <- List(false)
-// } yield {
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// CompositeId(testColumnId, innerVal, isEdge, useHash)
-// val deserializer = (bytes: Array[Byte]) => CompositeId.fromBytes(bytes, 0, isEdge, useHash)
-// (serializer, deserializer)
-// }
-//
-// def testOrder(innerVals: Iterable[InnerValLike], isEdge: Boolean, useHash: Boolean, version: String) = {
-// /** check if increasing target vertex id is ordered properly with same indexProps */
-// import InnerVal.{VERSION1, VERSION2}
-// val colId = version match {
-// case VERSION2 => columnV2.id.get
-// case VERSION1 => column.id.get
-// case _ => throw new RuntimeException("!")
-// }
-// val head = CompositeId(colId, innerVals.head, isEdge = isEdge, useHash = useHash)
-// var prev = head
-//
-// for {
-// innerVal <- innerVals.tail
-// } {
-// val current = CompositeId(colId, innerVal, isEdge = isEdge, useHash = useHash)
-// val bytes = current.bytes
-// val decoded = CompositeId.fromBytes(bytes, 0, isEdge, useHash, version)
-//
-// println(s"current: $current")
-// println(s"decoded: $decoded")
-//
-// val prevBytes = if (useHash) prev.bytes.drop(GraphUtil.bytesForMurMurHash) else prev.bytes
-// val currentBytes = if (useHash) bytes.drop(GraphUtil.bytesForMurMurHash) else bytes
-// println(s"prev: $prev, ${Bytes.compareTo(currentBytes, prevBytes)}")
-// val comp = lessThan(currentBytes, prevBytes) && current == decoded
-// prev = current
-// comp shouldBe true
-// }
-// }
-// /** version 1 */
-// test("order of compositeId numeric v1") {
-// for {
-// isEdge <- List(true, false)
-// useHash <- List(true, false)
-// } {
-// testOrder(numInnerVals, isEdge, useHash, VERSION1)
-// }
-// }
-// /** string order in v1 is not actually descending. it depends on string length */
-//// test("order of compositeId string v1") {
-//// for {
-//// isEdge <- List(true, false)
-//// useHash <- List(true, false)
-//// } {
-//// testOrder(stringInnerVals, isEdge, useHash, VERSION1)
-//// }
-//// }
-//// test("order of compositeId double v1") {
-//// for {
-//// isEdge <- List(true, false)
-//// useHash <- List(true, false)
-//// } {
-//// testOrder(doubleInnerVals, isEdge, useHash, VERSION1)
-//// }
-//// }
-// /** version 2 */
-// test("order of compositeId numeric v2") {
-// for {
-// isEdge <- List(true, false)
-// useHash <- List(true, false)
-// } {
-// testOrder(numInnerValsV2, isEdge, useHash, VERSION2)
-// }
-// }
-// test("order of compositeId string v2") {
-// for {
-// isEdge <- List(true, false)
-// useHash <- List(true, false)
-// } {
-// testOrder(stringInnerValsV2, isEdge, useHash, VERSION2)
-// }
-// }
-// test("order of compositeId double v2") {
-// for {
-// isEdge <- List(true, false)
-// useHash <- List(true, false)
-// } {
-// testOrder(doubleInnerValsV2, isEdge, useHash, VERSION2)
-// }
-// }
-//
-//
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/EdgeTypeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/EdgeTypeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/EdgeTypeTest.scala
deleted file mode 100644
index cfcedd1..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/EdgeTypeTest.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.types2._
-//import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon}
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 5/29/15.
-// */
-//class EdgeTypeTest extends FunSuite with Matchers with TestCommon {
-//
-// import HBaseType.{VERSION1, VERSION2}
-// def vertexId(innerVal: InnerValLike) = VertexId(testColumnId, innerVal)
-// def sourceVertexId(innerVal: InnerValLike) = SourceVertexId(testColumnId, innerVal)
-// def targetVertexId(innerVal: InnerValLike) = TargetVertexId(testColumnId, innerVal)
-// val skipHashBytes = true
-//
-// test("test edge row key order with int source vertex id version 1") {
-// val version = VERSION1
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeRowKey(sourceVertexId(innerVal), testLabelWithDir, testLabelOrderSeq, isInverted = false)(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeRowKey.fromBytes(bytes, 0, bytes.length, version)
-// testOrder(idxPropsLs, intInnerVals, skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-// test("test edge row key order with int source vertex id version 2") {
-// val version = VERSION2
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeRowKey(sourceVertexId(innerVal), testLabelWithDir, testLabelOrderSeq, isInverted = false)(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeRowKey.fromBytes(bytes, 0, bytes.length, version)
-// testOrder(idxPropsLsV2, intInnerValsV2, skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-//
-// test("test edge row qualifier with int target vertex id version 1") {
-// val version = VERSION1
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeQualifier(idxProps, targetVertexId(innerVal), testOp)(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeQualifier.fromBytes(bytes, 0, bytes.length, version)
-//
-// testOrder(idxPropsLs, intInnerVals, !skipHashBytes)(serializer, deserializer) shouldBe true
-// testOrderReverse(idxPropsLs, intInnerVals, !skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-// test("test edge row qualifier with int target vertex id version 2") {
-// val version = VERSION2
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeQualifier(idxProps, targetVertexId(innerVal), testOp)(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeQualifier.fromBytes(bytes, 0, bytes.length, version)
-//
-// testOrder(idxPropsLsV2, intInnerValsV2, !skipHashBytes)(serializer, deserializer) shouldBe true
-// testOrderReverse(idxPropsLsV2, intInnerValsV2, !skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-//
-// test("test edge row qualifier inverted with int target vertex id version 1") {
-// val version = VERSION1
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeQualifierInverted(targetVertexId(innerVal))(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeQualifierInverted.fromBytes(bytes, 0, bytes.length, version)
-//
-// testOrder(idxPropsLs, intInnerVals, !skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-// test("test edge row qualifier inverted with int target vertex id version 2") {
-// val version = VERSION2
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// EdgeQualifierInverted(targetVertexId(innerVal))(version)
-// val deserializer = (bytes: Array[Byte]) => EdgeQualifierInverted.fromBytes(bytes, 0, bytes.length, version)
-//
-// testOrder(idxPropsLsV2, intInnerValsV2, !skipHashBytes)(serializer, deserializer) shouldBe true
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
deleted file mode 100644
index 69a299d..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-package com.kakao.s2graph.core.types
-
-import com.kakao.s2graph.core.TestCommonWithModels
-import com.kakao.s2graph.core.types._
-import org.apache.hadoop.hbase.util.{Bytes, OrderedBytes, SimplePositionedByteRange}
-import org.scalatest.{Matchers, FunSuite}
-import play.api.libs.json.Json
-
-class InnerValTest extends FunSuite with Matchers with TestCommonWithModels {
- initTests()
-
- import HBaseType.{VERSION2, VERSION1}
- val decimals = List(
- BigDecimal(Long.MinValue),
- BigDecimal(Int.MinValue),
- BigDecimal(Double.MinValue),
- BigDecimal(Float.MinValue),
- BigDecimal(Short.MinValue),
- BigDecimal(Byte.MinValue),
-
- BigDecimal(-1),
- BigDecimal(0),
- BigDecimal(1),
- BigDecimal(Long.MaxValue),
- BigDecimal(Int.MaxValue),
- BigDecimal(Double.MaxValue),
- BigDecimal(Float.MaxValue),
- BigDecimal(Short.MaxValue),
- BigDecimal(Byte.MaxValue)
- )
- val booleans = List(
- false, true
- )
- val strings = List(
- "abc", "abd", "ac", "aca"
- )
- val texts = List(
- (0 until 1000).map(x => "a").mkString
- )
- val blobs = List(
- (0 until 1000).map(x => Byte.MaxValue).toArray
- )
- def testEncodeDecode(ranges: List[InnerValLike], version: String) = {
- for {
- innerVal <- ranges
- } {
- val bytes = innerVal.bytes
- val (decoded, numOfBytesUsed) = InnerVal.fromBytes(bytes, 0, bytes.length, version)
- innerVal == decoded shouldBe true
- bytes.length == numOfBytesUsed shouldBe true
- }
- }
- // test("big decimal") {
- // for {
- // version <- List(VERSION2, VERSION1)
- // } {
- // val innerVals = decimals.map { num => InnerVal.withNumber(num, version)}
- // testEncodeDecode(innerVals, version)
- // }
- // }
- // test("text") {
- // for {
- // version <- List(VERSION2)
- // } {
- // val innerVals = texts.map { t => InnerVal.withStr(t, version) }
- // testEncodeDecode(innerVals, version)
- // }
- // }
- // test("string") {
- // for {
- // version <- List(VERSION2, VERSION1)
- // } {
- // val innerVals = strings.map { t => InnerVal.withStr(t, version) }
- // testEncodeDecode(innerVals, version)
- // }
- // }
- // test("blob") {
- // for {
- // version <- List(VERSION2)
- // } {
- // val innerVals = blobs.map { t => InnerVal.withBlob(t, version) }
- // testEncodeDecode(innerVals, version)
- // }
- // }
- // test("boolean") {
- // for {
- // version <- List(VERSION2, VERSION1)
- // } {
- // val innerVals = booleans.map { t => InnerVal.withBoolean(t, version) }
- // testEncodeDecode(innerVals, version)
- // }
- // }
- test("korean") {
- val small = InnerVal.withStr("가", VERSION2)
- val large = InnerVal.withStr("나", VERSION2)
- val smallBytes = small.bytes
- val largeBytes = large.bytes
-
- println (Bytes.compareTo(smallBytes, largeBytes))
- true
- }
- // test("innerVal") {
- // val srcVal = InnerVal.withLong(44391298, VERSION2)
- // val srcValV1 = InnerVal.withLong(44391298, VERSION1)
- // val tgtVal = InnerVal.withLong(7295564, VERSION2)
- //
- // val a = VertexId(0, srcVal)
- // val b = SourceVertexId(0, srcVal)
- // val c = TargetVertexId(0, srcVal)
- // val aa = VertexId(0, srcValV1)
- // val bb = SourceVertexId(0, srcValV1)
- // val cc = TargetVertexId(0, srcValV1)
- // println(a.bytes.toList)
- // println(b.bytes.toList)
- // println(c.bytes.toList)
- //
- // println(aa.bytes.toList)
- // println(bb.bytes.toList)
- // println(cc.bytes.toList)
- // }
- // test("aa") {
- // val bytes = InnerVal.withLong(Int.MaxValue, VERSION2).bytes
- // val pbr = new SimplePositionedByteRange(bytes)
- // pbr.setOffset(1)
- // println(pbr.getPosition)
- // val num = OrderedBytes.decodeNumericAsBigDecimal(pbr)
- // println(pbr.getPosition)
- // true
- // }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/SourceVertexIdTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/SourceVertexIdTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/SourceVertexIdTest.scala
deleted file mode 100644
index b77e27b..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/SourceVertexIdTest.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.types2._
-//import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon}
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 6/10/15.
-// */
-//class SourceVertexIdTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels{
-// import HBaseType.{VERSION1, VERSION2}
-//
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) => SourceVertexId(testColumnId, innerVal)
-// def deserializer(version: String) = (bytes: Array[Byte]) => SourceVertexId.fromBytes(bytes, 0, bytes.length, version)
-// def skipHashBytes = true
-// val emptyIndexPropsLs = Seq(Seq.empty[(Byte, InnerValLike)])
-// /** version 1 */
-// test("order of compositeId numeric v1") {
-// val version = VERSION1
-// testOrder(emptyIndexPropsLs, numInnerVals, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// /** string order in v1 is not actually descending. it depends on string length */
-// // test("order of compositeId string v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(stringInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// // test("order of compositeId double v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(doubleInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// /** version 2 */
-// test("order of compositeId numeric v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, numInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId string v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, stringInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId double v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, doubleInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/TargetVertexIdTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/TargetVertexIdTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/TargetVertexIdTest.scala
deleted file mode 100644
index 845de95..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/TargetVertexIdTest.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.types2._
-//import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon}
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 6/10/15.
-// */
-//class TargetVertexIdTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels{
-// import HBaseType.{VERSION1, VERSION2}
-//
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) => TargetVertexId(testColumnId, innerVal)
-// def deserializer(version: String) = (bytes: Array[Byte]) => TargetVertexId.fromBytes(bytes, 0, bytes.length, version)
-// def skipHashBytes = false
-//
-// val emptyIndexPropsLs = Seq(Seq.empty[(Byte, InnerValLike)])
-// /** version 1 */
-// test("order of compositeId numeric v1") {
-// val version = VERSION1
-// testOrder(emptyIndexPropsLs, numInnerVals, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// /** string order in v1 is not actually descending. it depends on string length */
-// // test("order of compositeId string v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(stringInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// // test("order of compositeId double v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(doubleInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// /** version 2 */
-// test("order of compositeId numeric v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, numInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId string v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, stringInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId double v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, doubleInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexIdTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexIdTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexIdTest.scala
deleted file mode 100644
index c0d1e59..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexIdTest.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.types2.{HBaseType, VertexId, InnerValLike, InnerVal}
-//import com.kakao.s2graph.core.{GraphUtil, TestCommonWithModels, TestCommon}
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 6/10/15.
-// */
-//class VertexIdTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels {
-// import HBaseType.{VERSION1, VERSION2}
-//
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) => VertexId(testColumnId, innerVal)
-// def deserializer(version: String) = (bytes: Array[Byte]) => VertexId.fromBytes(bytes, 0, bytes.length, version)
-// val skipHashBytes = true
-// val emptyIndexPropsLs = Seq(Seq.empty[(Byte, InnerValLike)])
-// /** version 1 */
-// test("order of compositeId numeric v1") {
-// val version = VERSION1
-// testOrder(emptyIndexPropsLs, numInnerVals, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// /** string order in v1 is not actually descending. it depends on string length */
-// // test("order of compositeId string v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(stringInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// // test("order of compositeId double v1") {
-// // for {
-// // isEdge <- List(true, false)
-// // useHash <- List(true, false)
-// // } {
-// // testOrder(doubleInnerVals, isEdge, useHash, VERSION1)
-// // }
-// // }
-// /** version 2 */
-// test("order of compositeId numeric v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, numInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId string v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, stringInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-// test("order of compositeId double v2") {
-// val version = VERSION2
-// testOrder(emptyIndexPropsLs, doubleInnerValsV2, skipHashBytes)(serializer, deserializer(version)) shouldBe true
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexTypeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexTypeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexTypeTest.scala
deleted file mode 100644
index 284601b..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/types/VertexTypeTest.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-//package com.kakao.s2graph.core.types
-//
-//import com.kakao.s2graph.core.TestCommon
-//import com.kakao.s2graph.core.types2._
-//import org.apache.hadoop.hbase.util.Bytes
-//import org.scalatest.{Matchers, FunSuite}
-//
-///**
-// * Created by shon on 5/29/15.
-// */
-//class VertexTypeTest extends FunSuite with Matchers with TestCommon {
-//
-//
-// import HBaseType.{VERSION2, VERSION1}
-// val skipHashBytes = true
-////
-//// def functions = {
-//// for {
-//// version <- List(VERSION1, VERSION2)
-//// } yield {
-//// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-//// VertexRowKey(VertexId(testColumnId, innerVal))(version)
-//// val deserializer = (bytes: Array[Byte]) => VertexRowKey.fromBytes(bytes, 0, bytes.length, version)
-//// (serializer, deserializer, version)
-//// }
-//// }
-//
-// test("test vertex row key order with int id type version 1") {
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// VertexRowKey(VertexId(testColumnId, innerVal))(VERSION1)
-// val deserializer = (bytes: Array[Byte]) => VertexRowKey.fromBytes(bytes, 0, bytes.length, VERSION1)
-// testOrder(idxPropsLs, intInnerVals, skipHashBytes)(serializer, deserializer)
-// }
-// test("test vertex row key order with int id type version 2") {
-// val serializer = (idxProps: Seq[(Byte, InnerValLike)], innerVal: InnerValLike) =>
-// VertexRowKey(VertexId(testColumnId, innerVal))(VERSION2)
-// val deserializer = (bytes: Array[Byte]) => VertexRowKey.fromBytes(bytes, 0, bytes.length, VERSION2)
-// testOrder(idxPropsLsV2, intInnerValsV2, skipHashBytes)(serializer, deserializer)
-// }
-//
-//
-//}
[13/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
deleted file mode 100644
index 2aad32f..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
+++ /dev/null
@@ -1,536 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLikeWithTs, VertexId}
-import com.kakao.s2graph.core.utils.logger
-import org.scalatest.FunSuite
-import org.scalatest.matchers.Matcher
-
-class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
- initTests()
-
- test("toLogString") {
- val testLabelName = labelNameV2
- val bulkQueries = List(
- ("1445240543366", "update", "{\"is_blocked\":true}"),
- ("1445240543362", "insert", "{\"is_hidden\":false}"),
- ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
- ("1445240543363", "delete", "{}"),
- ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
-
- val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
-
- val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
- Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
- }).mkString("\n")
-
- val expected = Seq(
- Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"),
- Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"),
- Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"),
- Seq("1445240543363", "delete", "e", "1", "2", testLabelName),
- Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}")
- ).map(_.mkString("\t")).mkString("\n")
-
- assert(bulkEdge === expected)
- }
-
- test("buildOperation") {
- val schemaVersion = "v2"
- val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = Vertex(vertexId)
- val tgtVertex = srcVertex
-
- val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
-
- val snapshotEdge = None
- val propsWithTs = Map(timestampProp)
- val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
- val newVersion = 0L
-
- val newPropsWithTs = Map(
- timestampProp,
- 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1)
- )
-
- val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
- logger.info(edgeMutate.toLogString)
-
- assert(edgeMutate.newSnapshotEdge.isDefined)
- assert(edgeMutate.edgesToInsert.nonEmpty)
- assert(edgeMutate.edgesToDelete.isEmpty)
- }
-
- test("buildMutation: snapshotEdge: None with newProps") {
- val schemaVersion = "v2"
- val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = Vertex(vertexId)
- val tgtVertex = srcVertex
-
- val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
-
- val snapshotEdge = None
- val propsWithTs = Map(timestampProp)
- val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
- val newVersion = 0L
-
- val newPropsWithTs = Map(
- timestampProp,
- 1.toByte -> InnerValLikeWithTs(InnerVal.withBoolean(false, schemaVersion), 1)
- )
-
- val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
- logger.info(edgeMutate.toLogString)
-
- assert(edgeMutate.newSnapshotEdge.isDefined)
- assert(edgeMutate.edgesToInsert.nonEmpty)
- assert(edgeMutate.edgesToDelete.isEmpty)
- }
-
- test("buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests") {
- val schemaVersion = "v2"
- val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = Vertex(vertexId)
- val tgtVertex = srcVertex
-
- val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
-
- val snapshotEdge = None
- val propsWithTs = Map(timestampProp)
- val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
- val newVersion = 0L
-
- val newPropsWithTs = propsWithTs
-
- val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, propsWithTs, newPropsWithTs)
- logger.info(edgeMutate.toLogString)
-
- assert(edgeMutate.newSnapshotEdge.isEmpty)
- assert(edgeMutate.edgesToInsert.isEmpty)
- assert(edgeMutate.edgesToDelete.isEmpty)
- }
-
- test("buildMutation: All props older than snapshotEdge's LastDeletedAt") {
- val schemaVersion = "v2"
- val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = Vertex(vertexId)
- val tgtVertex = srcVertex
-
- val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
- val oldPropsWithTs = Map(
- timestampProp,
- LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
- )
-
- val propsWithTs = Map(
- timestampProp,
- 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 2),
- LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
- )
-
- val snapshotEdge =
- Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs))
-
- val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
- val newVersion = 0L
- val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
- logger.info(edgeMutate.toLogString)
-
- assert(edgeMutate.newSnapshotEdge.nonEmpty)
- assert(edgeMutate.edgesToInsert.isEmpty)
- assert(edgeMutate.edgesToDelete.isEmpty)
- }
-
- test("buildMutation: All props newer than snapshotEdge's LastDeletedAt") {
- val schemaVersion = "v2"
- val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = Vertex(vertexId)
- val tgtVertex = srcVertex
-
- val timestampProp = LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
- val oldPropsWithTs = Map(
- timestampProp,
- LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
- )
-
- val propsWithTs = Map(
- timestampProp,
- 3.toByte -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 4),
- LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
- )
-
- val snapshotEdge =
- Option(Edge(srcVertex, tgtVertex, labelWithDirV2, op = GraphUtil.operations("delete"), propsWithTs = oldPropsWithTs))
-
- val requestEdge = Edge(srcVertex, tgtVertex, labelWithDirV2, propsWithTs = propsWithTs)
- val newVersion = 0L
- val edgeMutate = Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
- logger.info(edgeMutate.toLogString)
-
- assert(edgeMutate.newSnapshotEdge.nonEmpty)
- assert(edgeMutate.edgesToInsert.nonEmpty)
- assert(edgeMutate.edgesToDelete.isEmpty)
- }
-}
-
-//import com.kakao.s2graph.core.types._
-//import org.hbase.async.PutRequest
-//import org.scalatest.{FunSuite, Matchers}
-//
-///**
-// * Created by shon on 5/29/15.
-// */
-//class EdgeTest extends FunSuite with Matchers with TestCommon with TestCommonWithModels {
-//
-//
-// import HBaseType.{VERSION1, VERSION2}
-//
-//
-// def srcVertex(innerVal: InnerValLike)(version: String) = {
-// val colId = if (version == VERSION1) column.id.get else columnV2.id.get
-// Vertex(SourceVertexId(colId, innerVal), ts)
-// }
-//
-// def tgtVertex(innerVal: InnerValLike)(version: String) = {
-// val colId = if (version == VERSION1) column.id.get else columnV2.id.get
-// Vertex(TargetVertexId(colId, innerVal), ts)
-// }
-//
-//
-// def testEdges(version: String) = {
-// val innerVals = if (version == VERSION1) intInnerVals else intInnerValsV2
-// val tgtV = tgtVertex(innerVals.head)(version)
-// innerVals.tail.map { intInnerVal =>
-// val labelWithDirection = if (version == VERSION1) labelWithDir else labelWithDirV2
-// val idxPropsList = if (version == VERSION1) idxPropsLs else idxPropsLsV2
-// idxPropsList.map { idxProps =>
-// val currentTs = idxProps.toMap.get(0.toByte).get.toString.toLong
-// val idxPropsWithTs = idxProps.map { case (k, v) => k -> InnerValLikeWithTs(v, currentTs) }
-//
-// Edge(srcVertex(intInnerVal)(version), tgtV, labelWithDirection, op, currentTs, currentTs, idxPropsWithTs.toMap)
-// }
-// }
-// }
-//
-// def testPropsUpdate(oldProps: Map[Byte, InnerValLikeWithTs],
-// newProps: Map[Byte, InnerValLikeWithTs],
-// expected: Map[Byte, Any],
-// expectedShouldUpdate: Boolean)
-// (f: PropsPairWithTs => (Map[Byte, InnerValLikeWithTs], Boolean))(version: String) = {
-//
-// val timestamp = newProps.toList.head._2.ts
-// val (updated, shouldUpdate) = f((oldProps, newProps, timestamp, version))
-// val rets = for {
-// (k, v) <- expected
-// } yield {
-// v match {
-// case v: String =>
-// v match {
-// case "left" => updated.get(k).isDefined && updated(k) == oldProps(k)
-// case "right" => updated.get(k).isDefined && updated(k) == newProps(k)
-// case "none" => updated.get(k).isEmpty
-// }
-// case value: InnerValLikeWithTs => updated.get(k).get == value
-// case _ => throw new RuntimeException(s"not supported keyword: $v")
-// }
-// }
-// println(rets)
-// rets.forall(x => x) && shouldUpdate == expectedShouldUpdate
-// }
-//
-// def testEdgeWithIndex(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = {
-// val rets = for {
-// edgeForSameTgtVertex <- edges
-// } yield {
-// val head = edgeForSameTgtVertex.head
-// val start = head
-// var prev = head
-// val rets = for {
-// edge <- edgeForSameTgtVertex.tail
-// } yield {
-// println(s"prevEdge: $prev")
-// println(s"currentEdge: $edge")
-// val prevEdgeWithIndex = edge.edgesWithIndex
-// val edgesWithIndex = edge.edgesWithIndex
-//
-// /** test encodeing decoding */
-// for {
-// edgeWithIndex <- edge.edgesWithIndex
-// put <- edgeWithIndex.buildPutsAsync()
-// kv <- putToKeyValues(put.asInstanceOf[PutRequest])
-// } {
-// val decoded = Edge.toEdge(kv, queryParam, None, Seq())
-// val comp = decoded.isDefined && decoded.get == edge
-// println(s"${decoded.get}")
-// println(s"$edge")
-// println(s"${decoded.get == edge}")
-// comp shouldBe true
-//
-// /** test order
-// * same source, target vertex. same indexProps keys.
-// * only difference is indexProps values so comparing qualifier is good enough
-// * */
-// for {
-// prevEdgeWithIndex <- prev.edgesWithIndex
-// } {
-// println(edgeWithIndex.qualifier)
-// println(prevEdgeWithIndex.qualifier)
-// println(edgeWithIndex.qualifier.bytes.toList)
-// println(prevEdgeWithIndex.qualifier.bytes.toList)
-// /** since index of this test label only use 0, 1 as indexProps
-// * if 0, 1 is not different then qualifier bytes should be same
-// * */
-// val comp = lessThanEqual(edgeWithIndex.qualifier.bytes, prevEdgeWithIndex.qualifier.bytes)
-// comp shouldBe true
-// }
-// }
-// prev = edge
-// }
-// }
-// }
-//
-// def testInvertedEdge(edges: Seq[Seq[Edge]])(queryParam: QueryParam) = {
-// val rets = for {
-// edgeForSameTgtVertex <- edges
-// } yield {
-// val head = edgeForSameTgtVertex.head
-// val start = head
-// var prev = head
-// val rets = for {
-// edge <- edgeForSameTgtVertex.tail
-// } yield {
-// println(s"prevEdge: $prev")
-// println(s"currentEdge: $edge")
-// val prevEdgeWithIndexInverted = prev.toSnapshotEdge
-// val edgeWithInvertedIndex = edge.toSnapshotEdge
-// /** test encode decoding */
-//
-// val put = edgeWithInvertedIndex.buildPutAsync()
-// for {
-// kv <- putToKeyValues(put)
-// } yield {
-// val decoded = Edge.toSnapshotEdge(kv, queryParam, None, isInnerCall = false, Seq())
-// val comp = decoded.isDefined && decoded.get == edge
-// println(s"${decoded.get}")
-// println(s"$edge")
-// println(s"${decoded.get == edge}")
-// comp shouldBe true
-//
-// /** no need to test ordering because qualifier only use targetVertexId */
-// }
-// prev = edge
-// }
-// }
-// }
-//
-// test("insert for edgesWithIndex version 2") {
-// val version = VERSION2
-// testEdgeWithIndex(testEdges(version))(queryParamV2)
-// }
-// test("insert for edgesWithIndex version 1") {
-// val version = VERSION1
-// testEdgeWithIndex(testEdges(version))(queryParam)
-// }
-//
-// test("insert for edgeWithInvertedIndex version 1") {
-// val version = VERSION1
-// testInvertedEdge(testEdges(version))(queryParam)
-// }
-//
-// test("insert for edgeWithInvertedIndex version 2") {
-// val version = VERSION2
-// testInvertedEdge(testEdges(version))(queryParamV2)
-// }
-//
-//
-// // /** test cases for each operation */
-//
-// def oldProps(timestamp: Long, version: String) = {
-// Map(
-// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp - 2, timestamp - 2, version),
-// 1.toByte -> InnerValLikeWithTs.withLong(0L, timestamp, version),
-// 2.toByte -> InnerValLikeWithTs.withLong(1L, timestamp - 1, version),
-// 4.toByte -> InnerValLikeWithTs.withStr("old", timestamp - 1, version)
-// )
-// }
-//
-// def newProps(timestamp: Long, version: String) = {
-// Map(
-// 2.toByte -> InnerValLikeWithTs.withLong(-10L, timestamp, version),
-// 3.toByte -> InnerValLikeWithTs.withLong(20L, timestamp, version)
-// )
-// }
-//
-// def deleteProps(timestamp: Long, version: String) = Map(
-// labelMeta.lastDeletedAt -> InnerValLikeWithTs.withLong(timestamp, timestamp, version)
-// )
-//
-// /** upsert */
-// test("Edge.buildUpsert") {
-// val shouldUpdate = true
-// val oldState = oldProps(ts, VERSION2)
-// val newState = newProps(ts + 1, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "none",
-// 2.toByte -> "right",
-// 3.toByte -> "right",
-// 4.toByte -> "none")
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true
-// }
-// test("Edge.buildUpsert shouldUpdate false") {
-// val shouldUpdate = false
-// val oldState = oldProps(ts, VERSION2)
-// val newState = newProps(ts - 10, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> "left",
-// 3.toByte -> "none",
-// 4.toByte -> "left")
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpsert)(VERSION2) shouldBe true
-// }
-//
-// /** update */
-// test("Edge.buildUpdate") {
-// val shouldUpdate = true
-// val oldState = oldProps(ts, VERSION2)
-// val newState = newProps(ts + 1, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> "right",
-// 3.toByte -> "right",
-// 4.toByte -> "left"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true
-// }
-// test("Edge.buildUpdate shouldUpdate false") {
-// val shouldUpdate = false
-// val oldState = oldProps(ts, VERSION2)
-// val newState = newProps(ts - 10, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> "left",
-// 3.toByte -> "none",
-// 4.toByte -> "left"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildUpdate)(VERSION2) shouldBe true
-// }
-//
-// /** delete */
-// test("Edge.buildDelete") {
-// val shouldUpdate = true
-// val oldState = oldProps(ts, VERSION2)
-// val newState = deleteProps(ts + 1, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "right",
-// 1.toByte -> "none",
-// 2.toByte -> "none",
-// 4.toByte -> "none"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true
-// }
-// test("Edge.buildDelete shouldUpdate false") {
-// val shouldUpdate = false
-// val oldState = oldProps(ts, VERSION2)
-// val newState = deleteProps(ts - 10, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> "left",
-// 4.toByte -> "left"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildDelete)(VERSION2) shouldBe true
-// }
-//
-// /** increment */
-// test("Edge.buildIncrement") {
-// val shouldUpdate = true
-// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte)
-// val newState = newProps(ts + 1, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> InnerValLikeWithTs.withLong(-9L, ts - 1, VERSION2),
-// 3.toByte -> "right"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true
-// }
-// test("Edge.buildIncrement shouldRepalce false") {
-// val shouldUpdate = false
-// val oldState = oldProps(ts, VERSION2).filterNot(kv => kv._1 == 4.toByte)
-// val newState = newProps(ts - 10, VERSION2)
-// val expected = Map(
-// labelMeta.lastDeletedAt -> "left",
-// 1.toByte -> "left",
-// 2.toByte -> "left"
-// )
-// testPropsUpdate(oldState, newState, expected, shouldUpdate)(Edge.buildIncrement)(VERSION2) shouldBe true
-// }
-//
-// test("Edge`s srcVertex") {
-//
-// val version = VERSION2
-// val srcId = InnerVal.withLong(10, version)
-// val tgtId = InnerVal.withStr("abc", version)
-// val srcColumn = columnV2
-// val tgtColumn = tgtColumnV2
-// val srcVertexId = VertexId(srcColumn.id.get, srcId)
-// val tgtVertexId = VertexId(tgtColumn.id.get, tgtId)
-//
-// val srcVertex = Vertex(srcVertexId)
-// val tgtVertex = Vertex(tgtVertexId)
-//
-// val labelId = undirectedLabelV2.id.get
-//
-// val outDir = LabelWithDirection(labelId, GraphUtil.directions("out"))
-// val inDir = LabelWithDirection(labelId, GraphUtil.directions("in"))
-// val bothDir = LabelWithDirection(labelId, GraphUtil.directions("undirected"))
-//
-// val op = GraphUtil.operations("insert")
-//
-//
-// val bothEdge = Edge(srcVertex, tgtVertex, bothDir)
-// println(s"edge: $bothEdge")
-// bothEdge.relatedEdges.foreach { edge =>
-// println(edge)
-// }
-//
-// }
-// test("edge buildIncrementBulk") {
-//
-// /**
-// * 172567371 List(97, 74, 2, 117, -74, -44, -76, 0, 0, 4, 8, 2)
-//169116518 List(68, -110, 2, 117, -21, 124, -103, 0, 0, 4, 9, 2)
-//11646834 List(17, 33, 2, 127, 78, 72, -115, 0, 0, 4, 9, 2)
-//148171217 List(62, 54, 2, 119, 43, 22, 46, 0, 0, 4, 9, 2)
-//116315188 List(41, 86, 2, 121, 17, 43, -53, 0, 0, 4, 9, 2)
-//180667876 List(48, -82, 2, 117, 59, 58, 27, 0, 0, 4, 8, 2)
-//4594410 List(82, 29, 2, 127, -71, -27, 21, 0, 0, 4, 8, 2)
-//151435444 List(1, 105, 2, 118, -7, 71, 75, 0, 0, 4, 8, 2)
-//168460895 List(67, -35, 2, 117, -11, 125, -96, 0, 0, 4, 9, 2)
-//7941614 List(115, 67, 2, 127, -122, -46, 17, 0, 0, 4, 8, 2)
-//171169732 List(61, -42, 2, 117, -52, 40, 59, 0, 0, 4, 9, 2)
-//174381375 List(91, 2, 2, 117, -101, 38, -64, 0, 0, 4, 9, 2)
-//12754019 List(9, -80, 2, 127, 61, 99, -100, 0, 0, 4, 9, 2)
-//175518092 List(111, 32, 2, 117, -119, -50, 115, 0, 0, 4, 8, 2)
-//174748531 List(28, -81, 2, 117, -107, -116, -116, 0, 0, 4, 8, 2)
-//
-// */
-// // val incrementsOpt = Edge.buildIncrementDegreeBulk("169116518", "talk_friend_long_term_agg_test", "out", 10)
-// //
-// // for {
-// // increments <- incrementsOpt
-// // increment <- increments
-// // (cf, qs) <- increment.getFamilyMapOfLongs
-// // (q, v) <- qs
-// // } {
-// // println(increment.getRow.toList)
-// // println(q.toList)
-// // println(v)
-// // }
-// //List(68, -110, -29, -4, 116, -24, 124, -37, 0, 0, 52, -44, 2)
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
deleted file mode 100644
index 1c09778..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
+++ /dev/null
@@ -1,226 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import com.kakao.s2graph.core.mysqls._
-import play.api.libs.json.{JsObject, Json}
-
-class CrudTest extends IntegrateCommon {
- import CrudHelper._
- import TestUtil._
-
- test("test CRUD") {
- var tcNum = 0
- var tcString = ""
- var bulkQueries = List.empty[(Long, String, String)]
- var expected = Map.empty[String, String]
-
- val curTime = System.currentTimeMillis
- val t1 = curTime + 0
- val t2 = curTime + 1
- val t3 = curTime + 2
- val t4 = curTime + 3
- val t5 = curTime + 4
-
- val tcRunner = new CrudTestRunner()
- tcNum = 1
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
-
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 2
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 3
- tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 4
- tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 5
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 6
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 7
- tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- tcNum = 8
- tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- tcNum = 9
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- tcNum = 10
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- tcNum = 11
- tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- tcNum = 12
- tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 13
- tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
- bulkQueries = List(
- (t5, "update", "{\"is_blocked\": true}"),
- (t1, "insert", "{\"is_hidden\": false}"),
- (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
- (t2, "delete", ""),
- (t4, "update", "{\"time\": 1, \"weight\": -10}"))
- expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
-
- tcRunner.run(tcNum, tcString, bulkQueries, expected)
- }
-
-
- object CrudHelper {
-
- class CrudTestRunner {
- var seed = 0
-
- def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
- for {
- labelName <- List(testLabelName, testLabelName2)
- i <- 0 until NumOfEachTest
- } {
- seed += 1
- val srcId = seed.toString
- val tgtId = srcId
-
- val maxTs = opWithProps.map(t => t._1).max
-
- /** insert edges */
- println(s"---- TC${tcNum}_init ----")
- val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
- TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
- })
-
- TestUtil.insertEdgesSync(bulkEdges: _*)
-
- for {
- label <- Label.findByName(labelName)
- direction <- List("out", "in")
- cacheTTL <- List(-1L)
- } {
- val (serviceName, columnName, id, otherId) = direction match {
- case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
- case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
- }
-
- val qId = if (labelName == testLabelName) id else "\"" + id + "\""
- val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
-
- val jsResult = TestUtil.getEdgesSync(query)
-
- val results = jsResult \ "results"
- val deegrees = (jsResult \ "degrees").as[List[JsObject]]
- val propsLs = (results \\ "props").seq
- (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
-
- val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
- val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
-
- from should be(id.toString)
- to should be(otherId.toString)
- (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
-
- for ((key, expectedVal) <- expected) {
- propsLs.last.as[JsObject].keys.contains(key) should be(true)
- (propsLs.last \ key).toString should be(expectedVal)
- }
- }
- }
- }
-
- def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = Json.parse(
- s""" { "srcVertices": [
- { "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id } ],
- "steps": [ [ {
- "label": "$labelName",
- "direction": "$dir",
- "offset": 0,
- "limit": 10,
- "cacheTTL": $cacheTTL }]]}""")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
deleted file mode 100644
index f8bf7af..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
+++ /dev/null
@@ -1,309 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.Label
-import com.kakao.s2graph.core.rest.{RequestParser, RestHandler}
-import com.kakao.s2graph.core.utils.logger
-import com.typesafe.config._
-import org.scalatest._
-import play.api.libs.json.{JsValue, Json}
-
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext, Future}
-
-trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
-
- import TestUtil._
-
- var graph: Graph = _
- var parser: RequestParser = _
- var management: Management = _
- var config: Config = _
-
- override def beforeAll = {
- config = ConfigFactory.load()
- graph = new Graph(config)(ExecutionContext.Implicits.global)
- management = new Management(graph)
- parser = new RequestParser(graph.config)
- initTestData()
- }
-
- override def afterAll(): Unit = {
- graph.shutdown()
- }
-
- /**
- * Make Service, Label, Vertex for integrate test
- */
- def initTestData() = {
- println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- Management.deleteService(testServiceName)
-
- // 1. createService
- val jsValue = Json.parse(createService)
- val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) =
- parser.toServiceElements(jsValue)
-
- val tryRes =
- management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
- println(s">> Service created : $createService, $tryRes")
-
- val labelNames = Map(testLabelName -> testLabelNameCreate,
- testLabelName2 -> testLabelName2Create,
- testLabelNameV1 -> testLabelNameV1Create,
- testLabelNameWeak -> testLabelNameWeakCreate)
-
- for {
- (labelName, create) <- labelNames
- } {
- Management.deleteLabel(labelName)
- Label.findByName(labelName, useCache = false) match {
- case None =>
- val json = Json.parse(create)
- val tryRes = for {
- labelArgs <- parser.toLabelElements(json)
- label <- (management.createLabel _).tupled(labelArgs)
- } yield label
-
- tryRes.get
- case Some(label) =>
- println(s">> Label already exist: $create, $label")
- }
- }
-
- val vertexPropsKeys = List("age" -> "int")
-
- vertexPropsKeys.map { case (key, keyType) =>
- Management.addVertexProp(testServiceName, testColumnName, key, keyType)
- }
-
- println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- }
-
-
- /**
- * Test Helpers
- */
- object TestUtil {
- implicit def ec = scala.concurrent.ExecutionContext.global
-
- // def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
- // val arr = for {
- // (label, dir, from, to) <- params
- // } yield {
- // Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
- // }
- //
- // val s = Json.toJson(arr)
- // s
- // }
-
- def deleteAllSync(jsValue: JsValue) = {
- val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
- val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json)
- val future = graph.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
-
- future
- })
-
- Await.result(future, HttpRequestWaitingTime)
- }
-
- def getEdgesSync(queryJson: JsValue): JsValue = {
- logger.info(Json.prettyPrint(queryJson))
- val restHandler = new RestHandler(graph)
- Await.result(restHandler.getEdgesAsync(queryJson)(PostProcess.toSimpleVertexArrJson), HttpRequestWaitingTime)
- }
-
- def insertEdgesSync(bulkEdges: String*) = {
- val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
- val jsResult = Await.result(req, HttpRequestWaitingTime)
-
- jsResult
- }
-
- def insertEdgesAsync(bulkEdges: String*) = {
- val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
- req
- }
-
- def toEdge(elems: Any*): String = elems.mkString("\t")
-
- // common tables
- val testServiceName = "s2graph"
- val testLabelName = "s2graph_label_test"
- val testLabelName2 = "s2graph_label_test_2"
- val testLabelNameV1 = "s2graph_label_test_v1"
- val testLabelNameWeak = "s2graph_label_test_weak"
- val testColumnName = "user_id_test"
- val testColumnType = "long"
- val testTgtColumnName = "item_id_test"
- val testHTableName = "test-htable"
- val newHTableName = "new-htable"
- val index1 = "idx_1"
- val index2 = "idx_2"
-
- val NumOfEachTest = 30
- val HttpRequestWaitingTime = Duration("60 seconds")
-
- val createService = s"""{"serviceName" : "$testServiceName"}"""
-
- val testLabelNameCreate =
- s"""
- {
- "label": "$testLabelName",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testColumnName",
- "tgtColumnType": "long",
- "indices": [
- {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
- {"name": "$index2", "propNames": ["_timestamp"]}
- ],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "schemaVersion": "v4",
- "compressionAlgorithm": "gz",
- "hTableName": "$testHTableName"
- }"""
-
- val testLabelName2Create =
- s"""
- {
- "label": "$testLabelName2",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": false,
- "schemaVersion": "v3",
- "compressionAlgorithm": "gz"
- }"""
-
- val testLabelNameV1Create =
- s"""
- {
- "label": "$testLabelNameV1",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "${testTgtColumnName}_v1",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": true,
- "schemaVersion": "v1",
- "compressionAlgorithm": "gz"
- }"""
-
- val testLabelNameWeakCreate =
- s"""
- {
- "label": "$testLabelNameWeak",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "weak",
- "isDirected": true,
- "compressionAlgorithm": "gz"
- }"""
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
deleted file mode 100644
index 0b26608..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
+++ /dev/null
@@ -1,909 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core.utils.logger
-import org.scalatest.BeforeAndAfterEach
-import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
-
-import scala.util.{Success, Try}
-
-class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
-
- import TestUtil._
-
- val insert = "insert"
- val e = "e"
- val weight = "weight"
- val is_hidden = "is_hidden"
-
- test("interval") {
- def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "$testLabelName",
- "index": "$index",
- "interval": {
- "from": [ { "$prop": $fromVal } ],
- "to": [ { "$prop": $toVal } ]
- }
- }
- ]]
- }
- """)
-
- var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
- (edges \ "size").toString should be("1")
-
- edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
- (edges \ "size").toString should be("2")
-
- edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
- (edges \ "size").toString should be("1")
-
- edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
- (edges \ "size").toString should be("2")
- }
-
- test("get edge with where condition") {
- def queryWhere(id: Int, where: String) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "where": "${where}"
- }
- ]]
- }""")
-
- var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
- (result \ "results").as[List[JsValue]].size should be(1)
-
- result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)"))
- (result \ "results").as[List[JsValue]].size should be(1)
-
- result = getEdgesSync(queryWhere(0, "_from=0"))
- (result \ "results").as[List[JsValue]].size should be(2)
-
- result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)"))
- (result \ "results").as[List[JsValue]].size should be(2)
-
- result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)"))
- (result \ "results").as[List[JsValue]].size should be(2)
- }
-
- test("get edge exclude") {
- def queryExclude(id: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- },
- {
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": 2,
- "exclude": true
- }
- ]]
- }""")
-
- val result = getEdgesSync(queryExclude(0))
- (result \ "results").as[List[JsValue]].size should be(1)
- }
-
- test("get edge groupBy property") {
- def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
- Json.obj(
- "groupBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- val result = getEdgesSync(queryGroupBy(0, Seq("weight")))
- (result \ "size").as[Int] should be(2)
- val weights = (result \ "results" \\ "groupBy").map { js =>
- (js \ "weight").as[Int]
- }
- weights should contain(30)
- weights should contain(40)
-
- weights should not contain (10)
- }
-
- test("edge transform") {
- def queryTransform(id: Int, transforms: String) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "transform": $transforms
- }
- ]]
- }""")
-
- var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]"))
- (result \ "results").as[List[JsValue]].size should be(2)
-
- result = getEdgesSync(queryTransform(0, "[[\"weight\"]]"))
- (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "weight").map(_.toString).sorted)
-
- result = getEdgesSync(queryTransform(0, "[[\"_from\"]]"))
- (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "from").map(_.toString).sorted)
- }
-
- test("index") {
- def queryIndex(ids: Seq[Int], indexName: String) = {
- val $from = Json.arr(
- Json.obj("serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "ids" -> ids))
-
- val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName))
- val $steps = Json.arr(Json.obj("step" -> $step))
-
- val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps)
- js
- }
-
- // weight order
- var result = getEdgesSync(queryIndex(Seq(0), "idx_1"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40))
-
- // timestamp order
- result = getEdgesSync(queryIndex(Seq(0), "idx_2"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30))
- }
-
- // "checkEdges" in {
- // running(FakeApplication()) {
- // val json = Json.parse( s"""
- // [{"from": 0, "to": 1, "label": "$testLabelName"},
- // {"from": 0, "to": 2, "label": "$testLabelName"}]
- // """)
- //
- // def checkEdges(queryJson: JsValue): JsValue = {
- // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
- // contentAsJson(ret)
- // }
- //
- // val res = checkEdges(json)
- // val typeRes = res.isInstanceOf[JsArray]
- // typeRes must equalTo(true)
- //
- // val fst = res.as[Seq[JsValue]].head \ "to"
- // fst.as[Int] must equalTo(1)
- //
- // val snd = res.as[Seq[JsValue]].last \ "to"
- // snd.as[Int] must equalTo(2)
- // }
- // }
-
-
-
- test("duration") {
- def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
- val $from = Json.arr(
- Json.obj("serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "ids" -> ids))
-
- val $step = Json.arr(Json.obj(
- "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100,
- "duration" -> Json.obj("from" -> from, "to" -> to)))
-
- val $steps = Json.arr(Json.obj("step" -> $step))
-
- Json.obj("srcVertices" -> $from, "steps" -> $steps)
- }
-
- // get all
- var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size should be(4)
- // inclusive, exclusive
- result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size should be(3)
-
- result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size should be(1)
-
- val bulkEdges = Seq(
- toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
- toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
- toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
- toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
- )
- insertEdgesSync(bulkEdges: _*)
-
- // duration test after udpate
- // get all
- result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size should be(4)
-
- // inclusive, exclusive
- result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size should be(3)
-
- result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size should be(1)
-
- }
-
-
- test("return tree") {
- def queryParents(id: Long) = Json.parse(
- s"""
- {
- "returnTree": true,
- "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 2
- }
- ],[{
- "label": "$testLabelName",
- "direction": "in",
- "offset": 0,
- "limit": -1
- }
- ]]
- }""".stripMargin)
-
- val src = 100
- val tgt = 200
-
- insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
-
- val result = TestUtil.getEdgesSync(queryParents(src))
- val parents = (result \ "results").as[Seq[JsValue]]
- val ret = parents.forall {
- edge => (edge \ "parents").as[Seq[JsValue]].size == 1
- }
-
- ret should be(true)
- }
-
-
-
-// test("pagination and _to") {
-// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
-// s"""
-// { "srcVertices": [
-// { "serviceName": "${testServiceName}",
-// "columnName": "${testColumnName}",
-// "id": ${id}
-// }],
-// "steps": [
-// [ {
-// "label": "${testLabelName}",
-// "direction": "out",
-// "offset": $offset,
-// "limit": $limit,
-// "_to": $to
-// }
-// ]]
-// }
-// """)
-//
-// val src = System.currentTimeMillis().toInt
-//
-// val bulkEdges = Seq(
-// toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
-// toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
-// toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
-// toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
-// )
-// insertEdgesSync(bulkEdges: _*)
-//
-// var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
-// var edges = (result \ "results").as[List[JsValue]]
-//
-// edges.size should be(2)
-// (edges(0) \ "to").as[Long] should be(4)
-// (edges(1) \ "to").as[Long] should be(3)
-//
-// result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
-//
-// edges = (result \ "results").as[List[JsValue]]
-// edges.size should be(2)
-// (edges(0) \ "to").as[Long] should be(3)
-// (edges(1) \ "to").as[Long] should be(2)
-//
-// result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
-// edges = (result \ "results").as[List[JsValue]]
-// edges.size should be(1)
-// }
-
- test("order by") {
- def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj(
- "srcVertices" -> Json.arr(
- Json.obj(
- "serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "id" -> id
- )
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj(
- "orderBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
-
- val bulkEdges = Seq(
- toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
- toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
- toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
- toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
- )
-
- insertEdgesSync(bulkEdges: _*)
-
- // get edges
- val edges = getEdgesSync(queryScore(0, Map("weight" -> 1)))
- val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
- val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
-
- val edgesTo = edges \ "results" \\ "to"
- val orderByTo = orderByScore \ "results" \\ "to"
- val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
-
- edgesTo should be(Seq(JsNumber(2), JsNumber(1)))
- edgesTo should be(orderByTo)
- ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2)))
- edgesTo.reverse should be(ascOrderByTo)
- }
-
-
- test("query with sampling") {
- def queryWithSampling(id: Int, sample: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- {
- "step": [{
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": $sample
- }]
- }
- ]
- }""")
-
- def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- {
- "step": [{
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": $sample
- }]
- },
- {
- "step": [{
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": $sample
- }]
- }
- ]
- }""")
-
- def twoQueryWithSampling(id: Int, sample: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- {
- "step": [{
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 50,
- "sample": $sample
- },
- {
- "label": "$testLabelName2",
- "direction": "out",
- "offset": 0,
- "limit": 50
- }]
- }
- ]
- }""")
-
- val sampleSize = 2
- val ts = "1442985659166"
- val testId = 22
-
- val bulkEdges = Seq(
- toEdge(ts, insert, e, testId, 122, testLabelName),
- toEdge(ts, insert, e, testId, 222, testLabelName),
- toEdge(ts, insert, e, testId, 322, testLabelName),
-
- toEdge(ts, insert, e, testId, 922, testLabelName2),
- toEdge(ts, insert, e, testId, 222, testLabelName2),
- toEdge(ts, insert, e, testId, 322, testLabelName2),
-
- toEdge(ts, insert, e, 122, 1122, testLabelName),
- toEdge(ts, insert, e, 122, 1222, testLabelName),
- toEdge(ts, insert, e, 122, 1322, testLabelName),
- toEdge(ts, insert, e, 222, 2122, testLabelName),
- toEdge(ts, insert, e, 222, 2222, testLabelName),
- toEdge(ts, insert, e, 222, 2322, testLabelName),
- toEdge(ts, insert, e, 322, 3122, testLabelName),
- toEdge(ts, insert, e, 322, 3222, testLabelName),
- toEdge(ts, insert, e, 322, 3322, testLabelName)
- )
-
- insertEdgesSync(bulkEdges: _*)
-
- val result1 = getEdgesSync(queryWithSampling(testId, sampleSize))
- (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
-
- val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
- (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
-
- val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
- (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
- }
- test("test query with filterOut query") {
- def queryWithFilterOut(id1: String, id2: String) = Json.parse(
- s"""{
- | "limit": 10,
- | "filterOut": {
- | "srcVertices": [{
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id1
- | }],
- | "steps": [{
- | "step": [{
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 10
- | }]
- | }]
- | },
- | "srcVertices": [{
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id2
- | }],
- | "steps": [{
- | "step": [{
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 5
- | }]
- | }]
- |}
- """.stripMargin
- )
-
- val testId1 = "-23"
- val testId2 = "-25"
-
- val bulkEdges = Seq(
- toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
- toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
- toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
- toEdge(4, insert, e, testId2, 111, testLabelName, Json.obj(weight -> 1)),
- toEdge(5, insert, e, testId2, 333, testLabelName, Json.obj(weight -> 1)),
- toEdge(6, insert, e, testId2, 555, testLabelName, Json.obj(weight -> 1))
- )
- logger.debug(s"${bulkEdges.mkString("\n")}")
- insertEdgesSync(bulkEdges: _*)
-
- val rs = getEdgesSync(queryWithFilterOut(testId1, testId2))
- logger.debug(Json.prettyPrint(rs))
- val results = (rs \ "results").as[List[JsValue]]
- results.size should be(1)
- (results(0) \ "to").toString should be("555")
- }
-
-
- /** note that this merge two different label result into one */
- test("weighted union") {
- def queryWithWeightedUnion(id1: String, id2: String) = Json.parse(
- s"""
- |{
- | "limit": 10,
- | "weights": [
- | 10,
- | 1
- | ],
- | "groupBy": ["weight"],
- | "queries": [
- | {
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id1
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 5
- | }
- | ]
- | }
- | ]
- | },
- | {
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id2
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName2",
- | "direction": "out",
- | "offset": 0,
- | "limit": 5
- | }
- | ]
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
- )
-
- val testId1 = "1"
- val testId2 = "2"
-
- val bulkEdges = Seq(
- toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
- toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
- toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
- toEdge(4, insert, e, testId2, 444, testLabelName2, Json.obj(weight -> 1)),
- toEdge(5, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)),
- toEdge(6, insert, e, testId2, 666, testLabelName2, Json.obj(weight -> 1))
- )
-
- insertEdgesSync(bulkEdges: _*)
-
- val rs = getEdgesSync(queryWithWeightedUnion(testId1, testId2))
- logger.debug(Json.prettyPrint(rs))
- val results = (rs \ "results").as[List[JsValue]]
- results.size should be(2)
- (results(0) \ "scoreSum").as[Float] should be(30.0)
- (results(0) \ "agg").as[List[JsValue]].size should be(3)
- (results(1) \ "scoreSum").as[Float] should be(3.0)
- (results(1) \ "agg").as[List[JsValue]].size should be(3)
- }
-
- test("weighted union with options") {
- def queryWithWeightedUnionWithOptions(id1: String, id2: String) = Json.parse(
- s"""
- |{
- | "limit": 10,
- | "weights": [
- | 10,
- | 1
- | ],
- | "groupBy": ["to"],
- | "select": ["to", "weight"],
- | "filterOut": {
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id1
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 10
- | }
- | ]
- | }
- | ]
- | },
- | "queries": [
- | {
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id1
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 5
- | }
- | ]
- | }
- | ]
- | },
- | {
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id2
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName2",
- | "direction": "out",
- | "offset": 0,
- | "limit": 5
- | }
- | ]
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
- )
-
- val testId1 = "-192848"
- val testId2 = "-193849"
-
- val bulkEdges = Seq(
- toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
- toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
- toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
- toEdge(4, insert, e, testId2, 111, testLabelName2, Json.obj(weight -> 1)),
- toEdge(5, insert, e, testId2, 333, testLabelName2, Json.obj(weight -> 1)),
- toEdge(6, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1))
- )
-
- insertEdgesSync(bulkEdges: _*)
-
- val rs = getEdgesSync(queryWithWeightedUnionWithOptions(testId1, testId2))
- logger.debug(Json.prettyPrint(rs))
- val results = (rs \ "results").as[List[JsValue]]
- results.size should be(1)
-
- }
-
- test("scoreThreshold") {
- def queryWithScoreThreshold(id: String, scoreThreshold: Int) = Json.parse(
- s"""{
- | "limit": 10,
- | "scoreThreshold": $scoreThreshold,
- | "groupBy": ["to"],
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "id": $id
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 10
- | }
- | ]
- | },
- | {
- | "step": [
- | {
- | "label": "$testLabelName",
- | "direction": "out",
- | "offset": 0,
- | "limit": 10
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
- )
-
- val testId = "-23903"
-
- val bulkEdges = Seq(
- toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 101, 102, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 101, 103, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 101, 104, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 102, 103, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 102, 104, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, 103, 105, testLabelName, Json.obj(weight -> 10))
- )
- // expected: 104 -> 2, 103 -> 2, 102 -> 1,, 105 -> 1
- insertEdgesSync(bulkEdges: _*)
-
- var rs = getEdgesSync(queryWithScoreThreshold(testId, 2))
- logger.debug(Json.prettyPrint(rs))
- var results = (rs \ "results").as[List[JsValue]]
- results.size should be(2)
-
- rs = getEdgesSync(queryWithScoreThreshold(testId, 1))
- logger.debug(Json.prettyPrint(rs))
-
- results = (rs \ "results").as[List[JsValue]]
- results.size should be(4)
- }
-
- def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "$testLabelName",
- "direction": "out",
- "offset": $offset,
- "limit": $limit
- }
- ]]
- }
- """)
-
- def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj(
- "limit" -> limit,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
-
-
- // called by each test, each
- override def beforeEach = initTestData()
-
- // called by start test, once
- override def initTestData(): Unit = {
- super.initTestData()
-
- insertEdgesSync(
- toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
- toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
- toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
- toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)),
- toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)),
- toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)),
- toEdge(1, insert, e, -1, 1000, testLabelName),
- toEdge(1, insert, e, -1, 2000, testLabelName),
- toEdge(1, insert, e, -1, 3000, testLabelName),
- toEdge(1, insert, e, 1000, 10000, testLabelName),
- toEdge(1, insert, e, 1000, 11000, testLabelName),
- toEdge(1, insert, e, 2000, 11000, testLabelName),
- toEdge(1, insert, e, 2000, 12000, testLabelName),
- toEdge(1, insert, e, 3000, 12000, testLabelName),
- toEdge(1, insert, e, 3000, 13000, testLabelName),
- toEdge(1, insert, e, 10000, 100000, testLabelName),
- toEdge(2, insert, e, 11000, 200000, testLabelName),
- toEdge(3, insert, e, 12000, 300000, testLabelName)
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
deleted file mode 100644
index f4da49d..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
+++ /dev/null
@@ -1,283 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import java.util.concurrent.TimeUnit
-
-import play.api.libs.json.{JsValue, Json}
-
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, Future}
-import scala.util.Random
-
-class StrongLabelDeleteTest extends IntegrateCommon {
-
- import StrongDeleteUtil._
- import TestUtil._
-
- test("Strong consistency select") {
- insertEdgesSync(bulkEdges(): _*)
-
- var result = getEdgesSync(query(0))
- (result \ "results").as[List[JsValue]].size should be(2)
- result = getEdgesSync(query(10))
- (result \ "results").as[List[JsValue]].size should be(2)
- }
-
- test("Strong consistency deleteAll") {
- val deletedAt = 100
- var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
-
- println(result)
- (result \ "results").as[List[JsValue]].size should be(3)
-
- val deleteParam = Json.arr(
- Json.obj("label" -> testLabelName2,
- "direction" -> "in",
- "ids" -> Json.arr("20"),
- "timestamp" -> deletedAt))
-
- deleteAllSync(deleteParam)
-
- result = getEdgesSync(query(11, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- result = getEdgesSync(query(12, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- result = getEdgesSync(query(10, direction = "out"))
- println(result)
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size should be(1)
- (result \\ "to").size should be(1)
- (result \\ "to").head.as[String] should be("21")
-
- result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
-
- result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
-
- (result \ "results").as[List[JsValue]].size should be(3)
- }
-
-
- test("update delete") {
- val ret = for {
- i <- 0 until testNum
- } yield {
- val src = (i + 1) * 10000
-// val src = System.currentTimeMillis()
-
- val (ret, last) = testInner(i, src)
- ret should be(true)
- ret
- }
-
- ret.forall(identity)
- }
-
- test("update delete 2") {
- val src = System.currentTimeMillis()
- var ts = 0L
-
- val ret = for {
- i <- 0 until testNum
- } yield {
- val (ret, lastTs) = testInner(ts, src)
- val deletedAt = lastTs + 1
- val deletedAt2 = lastTs + 2
- ts = deletedAt2 + 1 // nex start ts
-
- ret should be(true)
-
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
- val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
-
- val deleteRet = deleteAllSync(deleteAllRequest)
- val deleteRet2 = deleteAllSync(deleteAllRequest2)
-
- val result = getEdgesSync(query(id = src))
- println(result)
-
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty should be(true)
-
- val degreeAfterDeleteAll = getDegree(result)
-
- degreeAfterDeleteAll should be(0)
- degreeAfterDeleteAll === (0)
- }
-
- ret.forall(identity)
- }
-
- /** This test stress out test on degree
- * when contention is low but number of adjacent edges are large
- * Large set of contention test
- */
- test("large degrees") {
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- // val allRequests = insertRequests ++ deleteRequests
- val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
- insertEdgesAsync(bulkRequests: _*)
- }
-
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = insertRequests.size - deleteRequests.size
- val queryJson = query(id = src)
- val result = getEdgesSync(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- // println(result)
-
- val ret = resultSize == expectedDegree && resultDegree == resultSize
- println(s"[MaxSize]: $maxSize")
- println(s"[DeleteSize]: $deleteSize")
- println(s"[ResultDegree]: $resultDegree")
- println(s"[ExpectedDegree]: $expectedDegree")
- println(s"[ResultSize]: $resultSize")
- ret should be(true)
- }
-
- test("deleteAll") {
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
- insertEdgesAsync(bulkRequests: _*)
- }
-
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val deletedAt = System.currentTimeMillis()
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
-
- deleteAllSync(deleteAllRequest)
-
- val result = getEdgesSync(query(id = src))
- println(result)
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty should be(true)
-
- val degreeAfterDeleteAll = getDegree(result)
- degreeAfterDeleteAll should be(0)
- }
-
- object StrongDeleteUtil {
-
- val labelName = testLabelName2
-// val labelName = testLabelName
- val maxTgtId = 10
- val batchSize = 10
- val testNum = 100
- val numOfBatch = 10
-
- def testInner(startTs: Long, src: Long) = {
- val lastOps = Array.fill(maxTgtId)("none")
- var currentTs = startTs
-
- val allRequests = for {
- ith <- 0 until numOfBatch
- jth <- 0 until batchSize
- } yield {
- currentTs += 1
-
- val tgt = Random.nextInt(maxTgtId)
- val op = if (Random.nextDouble() < 0.5) "delete" else "update"
-
- lastOps(tgt) = op
- Seq(currentTs, op, "e", src, tgt, labelName, "{}").mkString("\t")
- }
-
- allRequests.foreach(println(_))
-
- val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequests =>
- insertEdgesAsync(bulkRequests: _*)
- }
-
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
- val queryJson = query(id = src)
- val result = getEdgesSync(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- println(lastOps.toList)
- println(result)
-
- val ret = resultDegree == expectedDegree && resultSize == resultDegree
- if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
-
- (ret, currentTs)
- }
-
- def bulkEdges(startTs: Int = 0) = Seq(
- toEdge(startTs + 1, "insert", "e", "0", "1", labelName, s"""{"time": 10}"""),
- toEdge(startTs + 2, "insert", "e", "0", "1", labelName, s"""{"time": 11}"""),
- toEdge(startTs + 3, "insert", "e", "0", "1", labelName, s"""{"time": 12}"""),
- toEdge(startTs + 4, "insert", "e", "0", "2", labelName, s"""{"time": 10}"""),
- toEdge(startTs + 5, "insert", "e", "10", "20", labelName, s"""{"time": 10}"""),
- toEdge(startTs + 6, "insert", "e", "10", "21", labelName, s"""{"time": 11}"""),
- toEdge(startTs + 7, "insert", "e", "11", "20", labelName, s"""{"time": 12}"""),
- toEdge(startTs + 8, "insert", "e", "12", "20", labelName, s"""{"time": 13}""")
- )
-
- def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
- _labelName: String = labelName, direction: String = "out") = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "${_labelName}",
- "direction": "${direction}",
- "offset": 0,
- "limit": -1,
- "duplicate": "raw"
- }
- ]]
- }""")
-
- def getDegree(jsValue: JsValue): Long = {
- ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
deleted file mode 100644
index ffbec11..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
+++ /dev/null
@@ -1,71 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import com.kakao.s2graph.core.PostProcess
-import play.api.libs.json.{JsValue, Json}
-
-import scala.concurrent.Await
-import scala.util.Random
-
-
-class VertexTestHelper extends IntegrateCommon {
-
- import TestUtil._
- import VertexTestHelper._
-
- test("vertex") {
- val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
- val (serviceName, columnName) = (testServiceName, testColumnName)
-
- val data = vertexInsertsPayload(serviceName, columnName, ids)
- val payload = Json.parse(Json.toJson(data).toString)
- println(payload)
-
- val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName))
- Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime)
-
- val res = graph.getVertices(vertices).map { vertices =>
- PostProcess.verticesToJson(vertices)
- }
-
- val ret = Await.result(res, HttpRequestWaitingTime)
- val fetched = ret.as[Seq[JsValue]]
- for {
- (d, f) <- data.zip(fetched)
- } yield {
- (d \ "id") should be(f \ "id")
- ((d \ "props") \ "age") should be((f \ "props") \ "age")
- }
- }
-
- object VertexTestHelper {
- def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
- Json.parse(
- s"""
- |[
- |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
- ]}
- |]
- """.stripMargin)
- }
-
- def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
- ids.map { id =>
- Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
- }
- }
-
- val vertexPropsKeys = List(
- ("age", "int")
- )
-
- def randomProps() = {
- (for {
- (propKey, propType) <- vertexPropsKeys
- } yield {
- propKey -> Random.nextInt(100)
- }).toMap
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
deleted file mode 100644
index 2028c44..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-package com.kakao.s2graph.core.Integrate
-
-import java.util.concurrent.TimeUnit
-
-import org.scalatest.BeforeAndAfterEach
-import play.api.libs.json.{JsObject, JsValue, Json}
-
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-
-class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
-
- import TestUtil._
- import WeakLabelDeleteHelper._
-
- test("test weak consistency select") {
- var result = getEdgesSync(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(4)
- result = getEdgesSync(query(10))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(2)
- }
-
- test("test weak consistency delete") {
- var result = getEdgesSync(query(0))
- println(result)
-
- /** expect 4 edges */
- (result \ "results").as[List[JsValue]].size should be(4)
- val edges = (result \ "results").as[List[JsObject]]
- val edgesToStore = parser.toEdges(Json.toJson(edges), "delete")
- val rets = graph.mutateEdges(edgesToStore, withWait = true)
- Await.result(rets, Duration(20, TimeUnit.MINUTES))
-
- /** expect noting */
- result = getEdgesSync(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- /** insert should be ignored */
- /**
- * I am wondering if this is right test case
- * This makes sense because hbase think cell is deleted when there are
- * insert/delete with same timestamp(version) on same cell.
- * This can be different on different storage system so I think
- * this test should be removed.
- */
-// val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert")
-// val rets2 = graph.mutateEdges(edgesToStore2, withWait = true)
-// Await.result(rets2, Duration(20, TimeUnit.MINUTES))
-//
-// result = getEdgesSync(query(0))
-// (result \ "results").as[List[JsValue]].size should be(0)
- }
-
-
- test("test weak consistency deleteAll") {
- val deletedAt = 100
- var result = getEdgesSync(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(3)
-
- val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
- "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
- println(json)
- deleteAllSync(json)
-
- result = getEdgesSync(query(11, "out"))
- (result \ "results").as[List[JsValue]].size should be(0)
-
- result = getEdgesSync(query(12, "out"))
- (result \ "results").as[List[JsValue]].size should be(0)
-
- result = getEdgesSync(query(10, "out"))
-
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size should be(1)
- (result \\ "to").size should be(1)
- (result \\ "to").head.as[String] should be("21")
-
- result = getEdgesSync(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
-
- result = getEdgesSync(query(20, "in", testTgtColumnName))
- (result \ "results").as[List[JsValue]].size should be(3)
- }
-
-
- // called by each test, each
- override def beforeEach = initTestData()
-
- // called by start test, once
-
- override def initTestData(): Unit = {
- super.initTestData()
- insertEdgesSync(bulkEdges(): _*)
- }
-
- object WeakLabelDeleteHelper {
-
- def bulkEdges(startTs: Int = 0) = Seq(
- toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""),
- toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""),
- toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""),
- toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""),
- toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""),
- toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""),
- toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""),
- toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""")
- )
-
- def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$columnName",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelNameWeak}",
- "direction": "${direction}",
- "offset": 0,
- "limit": 10,
- "duplicate": "raw"
- }
- ]]
- }""")
- }
-
-}
-
[14/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
new file mode 100644
index 0000000..dacc37f
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
@@ -0,0 +1,53 @@
+package org.apache.s2graph.core.storage.serde.indexedge.tall
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
+import org.apache.s2graph.core.types.VertexId
+import org.apache.s2graph.core.{GraphUtil, IndexEdge}
+
+
+class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] {
+ import StorageSerializable._
+
+ val label = indexEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ val idxPropsMap = indexEdge.orders.toMap
+ val idxPropsBytes = propsToBytes(indexEdge.orders)
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+
+ val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+ // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
+
+ val qualifier =
+ if (indexEdge.degreeEdge) Array.empty[Byte]
+ else
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => Bytes.add(idxPropsBytes, VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes)
+ case Some(vId) => idxPropsBytes
+ }
+
+ /** TODO search usage of op byte. if there is no, then remove opByte */
+ val rowBytes = Bytes.add(row, Array.fill(1)(GraphUtil.defaultOpByte), qualifier)
+ // val qualifierBytes = Array.fill(1)(indexEdge.op)
+ val qualifierBytes = Array.empty[Byte]
+
+ val value =
+ if (indexEdge.degreeEdge)
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.degreeSeq).innerVal.toString().toLong)
+ else if (indexEdge.op == GraphUtil.operations("incrementCount"))
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.countSeq).innerVal.toString().toLong)
+ else propsToKeyValues(indexEdge.metas.toSeq)
+
+ val kv = SKeyValue(table, rowBytes, cf, qualifierBytes, value, indexEdge.version)
+
+ // logger.debug(s"[Ser]: ${kv.row.toList}, ${kv.qualifier.toList}, ${kv.value.toList}")
+ Seq(kv)
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
new file mode 100644
index 0000000..8b540cd
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
@@ -0,0 +1,116 @@
+package org.apache.s2graph.core.storage.serde.indexedge.wide
+
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.storage.StorageDeserializable._
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.{GraphUtil, IndexEdge, QueryParam, Vertex}
+
+class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] {
+ import StorageDeserializable._
+
+ type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
+ type ValueRaw = (Array[(Byte, InnerValLike)], Int)
+
+ private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+ // val degree = Bytes.toLong(kv.value)
+ val degree = bytesToLongFunc(kv.value, 0)
+ val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
+ val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
+ (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
+ }
+
+ private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+ var qualifierLen = 0
+ var pos = 0
+ val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
+ val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
+ pos = endAt
+ qualifierLen += endAt
+ val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
+ (HBaseType.defaultTgtVertexId, 0)
+ } else {
+ TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
+ }
+ qualifierLen += tgtVertexIdLen
+ (props, endAt, tgtVertexId, tgtVertexIdLen)
+ }
+ val (op, opLen) =
+ if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
+ else (kv.qualifier(qualifierLen), 1)
+
+ qualifierLen += opLen
+
+ (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
+ }
+
+ private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
+ val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
+ (props, endAt)
+ }
+
+ private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
+ (Array.empty[(Byte, InnerValLike)], 0)
+ }
+
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[IndexEdge]): IndexEdge = {
+ assert(_kvs.size == 1)
+
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+
+ val kv = kvs.head
+ val (srcVertexId, labelWithDir, labelIdxSeq, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.id, e.labelWithDir, e.labelIndexSeq, false, 0)
+ }.getOrElse(parseRow(kv, version))
+
+ val (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdInQualifier, _) =
+ if (kv.qualifier.isEmpty) parseDegreeQualifier(kv, version)
+ else parseQualifier(kv, version)
+
+ val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
+ // val countVal = Bytes.toLong(kv.value)
+ val countVal = bytesToLongFunc(kv.value, 0)
+ val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
+ (dummyProps, 8)
+ } else if (kv.qualifier.isEmpty) {
+ parseDegreeValue(kv, version)
+ } else {
+ parseValue(kv, version)
+ }
+
+ val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${queryParam.label.id.get}, ${labelIdxSeq}"))
+
+
+ // assert(kv.qualifier.nonEmpty && index.metaSeqs.size == idxPropsRaw.size)
+
+ val idxProps = for {
+ (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
+ } yield {
+ if (k == LabelMeta.degreeSeq) k -> v
+ else seq -> v
+ }
+
+ val idxPropsMap = idxProps.toMap
+ val tgtVertexId = if (tgtVertexIdInQualifier) {
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => tgtVertexIdRaw
+ case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
+ }
+ } else tgtVertexIdRaw
+
+ val _mergedProps = (idxProps ++ props).toMap
+ val mergedProps =
+ if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
+ else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
+
+ // logger.error(s"$mergedProps")
+ // val ts = mergedProps(LabelMeta.timeStampSeq).toString().toLong
+
+ val ts = kv.timestamp
+ IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
new file mode 100644
index 0000000..6c70ae1
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
@@ -0,0 +1,52 @@
+package org.apache.s2graph.core.storage.serde.indexedge.wide
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
+import org.apache.s2graph.core.types.VertexId
+import org.apache.s2graph.core.{GraphUtil, IndexEdge}
+
+class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] {
+ import StorageSerializable._
+
+ val label = indexEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ val idxPropsMap = indexEdge.orders.toMap
+ val idxPropsBytes = propsToBytes(indexEdge.orders)
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+
+ val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+ // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
+ val tgtIdBytes = VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes
+ val qualifier =
+ if (indexEdge.degreeEdge) Array.empty[Byte]
+ else {
+ if (indexEdge.op == GraphUtil.operations("incrementCount")) {
+ Bytes.add(idxPropsBytes, tgtIdBytes, Array.fill(1)(indexEdge.op))
+ } else {
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => Bytes.add(idxPropsBytes, tgtIdBytes)
+ case Some(vId) => idxPropsBytes
+ }
+ }
+ }
+
+
+ val value =
+ if (indexEdge.degreeEdge)
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.degreeSeq).innerVal.toString().toLong)
+ else if (indexEdge.op == GraphUtil.operations("incrementCount"))
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.countSeq).innerVal.toString().toLong)
+ else propsToKeyValues(indexEdge.metas.toSeq)
+
+ val kv = SKeyValue(table, row, cf, qualifier, value, indexEdge.version)
+
+ Seq(kv)
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
new file mode 100644
index 0000000..37d5910
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
@@ -0,0 +1,84 @@
+package org.apache.s2graph.core.storage.serde.snapshotedge.tall
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta}
+import org.apache.s2graph.core.storage.StorageDeserializable._
+import org.apache.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue, StorageDeserializable}
+import org.apache.s2graph.core.types.{HBaseType, LabelWithDirection, SourceAndTargetVertexIdPair, SourceVertexId}
+import org.apache.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
+
+class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] {
+
+ def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
+ val statusCode = byte >> 4
+ val op = byte & ((1 << 4) - 1)
+ (statusCode.toByte, op.toByte)
+ }
+
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+ assert(kvs.size == 1)
+
+ val kv = kvs.head
+ val schemaVer = queryParam.label.schemaVersion
+ val cellVersion = kv.timestamp
+ /** rowKey */
+ def parseRowV3(kv: SKeyValue, version: String) = {
+ var pos = 0
+ val (srcIdAndTgtId, srcIdAndTgtIdLen) = SourceAndTargetVertexIdPair.fromBytes(kv.row, pos, kv.row.length, version)
+ pos += srcIdAndTgtIdLen
+ val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
+ pos += 4
+ val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
+
+ val rowLen = srcIdAndTgtIdLen + 4 + 1
+ (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isInverted, rowLen)
+
+ }
+ val (srcInnerId, tgtInnerId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
+ }.getOrElse(parseRowV3(kv, schemaVer))
+
+ val srcVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, srcInnerId)
+ val tgtVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, tgtInnerId)
+
+ val (props, op, ts, statusCode, _pendingEdgeOpt) = {
+ var pos = 0
+ val (statusCode, op) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ val kvsMap = props.toMap
+ val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
+
+ pos = endAt
+ val _pendingEdgeOpt =
+ if (pos == kv.value.length) None
+ else {
+ val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ // val versionNum = Bytes.toLong(kv.value, pos, 8)
+ // pos += 8
+ val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ pos = endAt
+ val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
+
+ val pendingEdge =
+ Edge(Vertex(srcVertexId, cellVersion),
+ Vertex(tgtVertexId, cellVersion),
+ labelWithDir, pendingEdgeOp,
+ cellVersion, pendingEdgeProps.toMap,
+ statusCode = pendingEdgeStatusCode, lockTs = lockTs)
+ Option(pendingEdge)
+ }
+
+ (kvsMap, op, ts, statusCode, _pendingEdgeOpt)
+ }
+
+ SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
+ labelWithDir, op, cellVersion, props, statusCode = statusCode,
+ pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
new file mode 100644
index 0000000..f018827
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
@@ -0,0 +1,47 @@
+package org.apache.s2graph.core.storage.serde.snapshotedge.tall
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.SnapshotEdge
+import org.apache.s2graph.core.mysqls.LabelIndex
+import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
+import org.apache.s2graph.core.types.SourceAndTargetVertexIdPair
+
+
+class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] {
+ import StorageSerializable._
+
+ val label = snapshotEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
+ val byte = (((statusCode << 4) | op).toByte)
+ Array.fill(1)(byte.toByte)
+ }
+ def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
+ propsToKeyValuesWithTs(snapshotEdge.props.toList))
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes
+ val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
+
+ val row = Bytes.add(srcIdAndTgtIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+
+ val qualifier = Array.empty[Byte]
+
+ val value = snapshotEdge.pendingEdgeOpt match {
+ case None => valueBytes()
+ case Some(pendingEdge) =>
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val versionBytes = Array.empty[Byte]
+ val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
+ val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+
+ Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
+ }
+
+ val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
+ Seq(kv)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
new file mode 100644
index 0000000..68eb125
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
@@ -0,0 +1,70 @@
+package org.apache.s2graph.core.storage.serde.snapshotedge.wide
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.mysqls.{LabelIndex, LabelMeta}
+import org.apache.s2graph.core.storage.StorageDeserializable._
+import org.apache.s2graph.core.storage.{CanSKeyValue, Deserializable, StorageDeserializable}
+import org.apache.s2graph.core.types.TargetVertexId
+import org.apache.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
+
+class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] {
+
+ def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
+ val statusCode = byte >> 4
+ val op = byte & ((1 << 4) - 1)
+ (statusCode.toByte, op.toByte)
+ }
+
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+ assert(kvs.size == 1)
+
+ val kv = kvs.head
+ val schemaVer = queryParam.label.schemaVersion
+ val cellVersion = kv.timestamp
+
+ val (srcVertexId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.id, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
+ }.getOrElse(parseRow(kv, schemaVer))
+
+ val (tgtVertexId, props, op, ts, statusCode, _pendingEdgeOpt) = {
+ val (tgtVertexId, _) = TargetVertexId.fromBytes(kv.qualifier, 0, kv.qualifier.length, schemaVer)
+ var pos = 0
+ val (statusCode, op) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ val kvsMap = props.toMap
+ val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
+
+ pos = endAt
+ val _pendingEdgeOpt =
+ if (pos == kv.value.length) None
+ else {
+ val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ // val versionNum = Bytes.toLong(kv.value, pos, 8)
+ // pos += 8
+ val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ pos = endAt
+ val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
+
+ val pendingEdge =
+ Edge(Vertex(srcVertexId, cellVersion),
+ Vertex(tgtVertexId, cellVersion),
+ labelWithDir, pendingEdgeOp,
+ cellVersion, pendingEdgeProps.toMap,
+ statusCode = pendingEdgeStatusCode, lockTs = lockTs)
+ Option(pendingEdge)
+ }
+
+ (tgtVertexId, kvsMap, op, ts, statusCode, _pendingEdgeOpt)
+ }
+
+ SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
+ labelWithDir, op, cellVersion, props, statusCode = statusCode,
+ pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
new file mode 100644
index 0000000..e4d0ac1
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
@@ -0,0 +1,51 @@
+package org.apache.s2graph.core.storage.serde.snapshotedge.wide
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.SnapshotEdge
+import org.apache.s2graph.core.mysqls.LabelIndex
+import org.apache.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
+import org.apache.s2graph.core.types.VertexId
+
+
+
+/**
+ * this class serialize
+ * @param snapshotEdge
+ */
+class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] {
+ import StorageSerializable._
+
+ val label = snapshotEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
+ val byte = (((statusCode << 4) | op).toByte)
+ Array.fill(1)(byte.toByte)
+ }
+ def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
+ propsToKeyValuesWithTs(snapshotEdge.props.toList))
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val srcIdBytes = VertexId.toSourceVertexId(snapshotEdge.srcVertex.id).bytes
+ val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
+
+ val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+ val tgtIdBytes = VertexId.toTargetVertexId(snapshotEdge.tgtVertex.id).bytes
+
+ val qualifier = tgtIdBytes
+
+ val value = snapshotEdge.pendingEdgeOpt match {
+ case None => valueBytes()
+ case Some(pendingEdge) =>
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val versionBytes = Array.empty[Byte]
+ val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
+ val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+ Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
+ }
+ val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
+ Seq(kv)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
new file mode 100644
index 0000000..00a5dc2
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
@@ -0,0 +1,47 @@
+package org.apache.s2graph.core.storage.serde.vertex
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.storage.{CanSKeyValue, Deserializable}
+import org.apache.s2graph.core.types.{InnerVal, InnerValLike, VertexId}
+import org.apache.s2graph.core.{QueryParam, Vertex}
+
+import scala.collection.mutable.ListBuffer
+
+class VertexDeserializable extends Deserializable[Vertex] {
+ def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[Vertex]): Vertex = {
+
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+
+ val kv = kvs.head
+ val (vertexId, _) = VertexId.fromBytes(kv.row, 0, kv.row.length, version)
+
+ var maxTs = Long.MinValue
+ val propsMap = new collection.mutable.HashMap[Int, InnerValLike]
+ val belongLabelIds = new ListBuffer[Int]
+
+ for {
+ kv <- kvs
+ } {
+ val propKey =
+ if (kv.qualifier.length == 1) kv.qualifier.head.toInt
+ else Bytes.toInt(kv.qualifier)
+
+ val ts = kv.timestamp
+ if (ts > maxTs) maxTs = ts
+
+ if (Vertex.isLabelId(propKey)) {
+ belongLabelIds += Vertex.toLabelId(propKey)
+ } else {
+ val v = kv.value
+ val (value, _) = InnerVal.fromBytes(v, 0, v.length, version)
+ propsMap += (propKey -> value)
+ }
+ }
+ assert(maxTs != Long.MinValue)
+ Vertex(vertexId, maxTs, propsMap.toMap, belongLabelIds = belongLabelIds)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
new file mode 100644
index 0000000..a81a86e
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
@@ -0,0 +1,19 @@
+package org.apache.s2graph.core.storage.serde.vertex
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.Vertex
+import org.apache.s2graph.core.storage.{SKeyValue, Serializable}
+
+case class VertexSerializable(vertex: Vertex) extends Serializable[Vertex] {
+
+ val cf = Serializable.vertexCf
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val row = vertex.id.bytes
+ val base = for ((k, v) <- vertex.props ++ vertex.defaultProps) yield Bytes.toBytes(k) -> v.bytes
+ val belongsTo = vertex.belongLabelIds.map { labelId => Bytes.toBytes(Vertex.toPropKey(labelId)) -> Array.empty[Byte] }
+ (base ++ belongsTo).map { case (qualifier, value) =>
+ SKeyValue(vertex.hbaseTableName.getBytes, row, cf, qualifier, value, vertex.ts)
+ } toSeq
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala
new file mode 100644
index 0000000..a207547
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/HBaseType.scala
@@ -0,0 +1,160 @@
+package org.apache.s2graph.core.types
+
+import org.apache.hadoop.hbase.util.Bytes
+
+object HBaseType {
+ val VERSION4 = "v4"
+ val VERSION3 = "v3"
+ val VERSION2 = "v2"
+ val VERSION1 = "v1"
+// val DEFAULT_VERSION = VERSION2
+ val DEFAULT_VERSION = VERSION3
+ val EMPTY_SEQ_BYTE = Byte.MaxValue
+ val DEFAULT_COL_ID = 0
+ val bitsForDir = 2
+ val maxBytes = Bytes.toBytes(Int.MaxValue)
+ val toSeqByte = -5.toByte
+ val defaultTgtVertexId = null
+}
+
+object HBaseDeserializable {
+
+ import HBaseType._
+
+ // 6 bits is used for index sequence so total index per label is limited to 2^6
+ def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
+ val byte = bytes(offset)
+ val isInverted = if ((byte & 1) != 0) true else false
+ val labelOrderSeq = byte >> 1
+ (labelOrderSeq.toByte, isInverted)
+ }
+
+ def bytesToKeyValues(bytes: Array[Byte],
+ offset: Int,
+ length: Int,
+ version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLike)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLike)](len)
+ var i = 0
+ while (i < len) {
+ val k = bytes(pos)
+ pos += 1
+ val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ val ret = (kvs, pos)
+ // logger.debug(s"bytesToProps: $ret")
+ ret
+ }
+
+ def bytesToKeyValuesWithTs(bytes: Array[Byte],
+ offset: Int,
+ version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLikeWithTs)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLikeWithTs)](len)
+ var i = 0
+ while (i < len) {
+ val k = bytes(pos)
+ pos += 1
+ val (v, numOfBytesUsed) = InnerValLikeWithTs.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ val ret = (kvs, pos)
+ // logger.debug(s"bytesToProps: $ret")
+ ret
+ }
+
+ def bytesToProps(bytes: Array[Byte],
+ offset: Int,
+ version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLike)], Int) = {
+ var pos = offset
+ val len = bytes(pos)
+ pos += 1
+ val kvs = new Array[(Byte, InnerValLike)](len)
+ var i = 0
+ while (i < len) {
+ val k = EMPTY_SEQ_BYTE
+ val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
+ pos += numOfBytesUsed
+ kvs(i) = (k -> v)
+ i += 1
+ }
+ // logger.error(s"bytesToProps: $kvs")
+ val ret = (kvs, pos)
+
+ ret
+ }
+}
+
+object HBaseSerializable {
+ def propsToBytes(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, v.bytes)
+ bytes
+ }
+
+ def propsToKeyValues(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
+ bytes
+ }
+
+ def propsToKeyValuesWithTs(props: Seq[(Byte, InnerValLikeWithTs)]): Array[Byte] = {
+ val len = props.length
+ assert(len < Byte.MaxValue)
+ var bytes = Array.fill(1)(len.toByte)
+ for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
+ bytes
+ }
+
+ def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
+ assert(labelOrderSeq < (1 << 6))
+ val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
+ Array.fill(1)(byte.toByte)
+ }
+}
+
+trait HBaseSerializable {
+ def bytes: Array[Byte]
+}
+
+trait HBaseDeserializable {
+
+ import HBaseType._
+
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (HBaseSerializable, Int)
+
+ // def fromBytesWithIndex(bytes: Array[Byte],
+ // offset: Int,
+ // len: Int,
+ // version: String = DEFAULT_VERSION): (HBaseSerializable, Int)
+ def notSupportedEx(version: String) = new RuntimeException(s"not supported version, $version")
+}
+
+trait HBaseDeserializableWithIsVertexId {
+
+ import HBaseType._
+
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION,
+ isVertexId: Boolean = false): (HBaseSerializable, Int)
+
+ def notSupportedEx(version: String) = new RuntimeException(s"not supported version, $version")
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala
new file mode 100644
index 0000000..dd1b833
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/InnerValLike.scala
@@ -0,0 +1,243 @@
+package org.apache.s2graph.core.types
+
+import org.apache.hadoop.hbase.util._
+
+object InnerVal extends HBaseDeserializableWithIsVertexId {
+ import HBaseType._
+
+ val order = Order.DESCENDING
+ val stringLenOffset = 7.toByte
+ val maxStringLen = Byte.MaxValue - stringLenOffset
+ val maxMetaByte = Byte.MaxValue
+ val minMetaByte = 0.toByte
+
+ /** supported data type */
+ val BLOB = "blob"
+ val STRING = "string"
+ val DOUBLE = "double"
+ val FLOAT = "float"
+ val LONG = "long"
+ val INT = "integer"
+ val SHORT = "short"
+ val BYTE = "byte"
+ val NUMERICS = List(DOUBLE, FLOAT, LONG, INT, SHORT, BYTE)
+ val BOOLEAN = "boolean"
+
+ def isNumericType(dataType: String): Boolean = {
+ dataType match {
+ case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE => true
+ case _ => false
+ }
+ }
+ def toInnerDataType(dataType: String): String = {
+ dataType match {
+ case "blob" => BLOB
+ case "string" | "str" | "s" => STRING
+ case "double" | "d" | "float64" => DOUBLE
+ case "float" | "f" | "float32" => FLOAT
+ case "long" | "l" | "int64" | "integer64" => LONG
+ case "int" | "integer" | "i" | "int32" | "integer32" => INT
+ case "short" | "int16" | "integer16" => SHORT
+ case "byte" | "b" | "tinyint" | "int8" | "integer8" => BYTE
+ case "boolean" | "bool" => BOOLEAN
+ case _ => throw new RuntimeException(s"can`t convert $dataType into InnerDataType")
+ }
+ }
+
+ def numByteRange(num: BigDecimal) = {
+// val byteLen =
+// if (num.isValidByte | num.isValidChar) 1
+// else if (num.isValidShort) 2
+// else if (num.isValidInt) 4
+// else if (num.isValidLong) 8
+// else if (num.isValidFloat) 4
+// else 12
+ val byteLen = 12
+ // else throw new RuntimeException(s"wrong data $num")
+ new SimplePositionedMutableByteRange(byteLen + 4)
+ }
+
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String,
+ isVertexId: Boolean): (InnerValLike, Int) = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal.fromBytes(bytes, offset, len, version, isVertexId)
+ case VERSION1 => v1.InnerVal.fromBytes(bytes, offset, len, version, isVertexId)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withLong(l: Long, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(l))
+ case VERSION1 => v1.InnerVal(Some(l), None, None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withInt(i: Int, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(i))
+ case VERSION1 => v1.InnerVal(Some(i.toLong), None, None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withFloat(f: Float, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(f.toDouble))
+ case VERSION1 => v1.InnerVal(Some(f.toLong), None, None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withDouble(d: Double, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(d))
+ case VERSION1 => v1.InnerVal(Some(d.toLong), None, None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withNumber(num: BigDecimal, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(num)
+ case VERSION1 => v1.InnerVal(Some(num.toLong), None, None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withBoolean(b: Boolean, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(b)
+ case VERSION1 => v1.InnerVal(None, None, Some(b))
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withBlob(blob: Array[Byte], version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(blob)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+ def withStr(s: String, version: String): InnerValLike = {
+ version match {
+ case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(s)
+ case VERSION1 => v1.InnerVal(None, Some(s), None)
+ case _ => throw notSupportedEx(version)
+ }
+ }
+
+// def withInnerVal(innerVal: InnerValLike, version: String): InnerValLike = {
+// val bytes = innerVal.bytes
+// version match {
+// case VERSION2 => v2.InnerVal.fromBytes(bytes, 0, bytes.length, version)._1
+// case VERSION1 => v1.InnerVal.fromBytes(bytes, 0, bytes.length, version)._1
+// case _ => throw notSupportedEx(version)
+// }
+// }
+
+ /** nasty implementation for backward compatability */
+ def convertVersion(innerVal: InnerValLike, dataType: String, toVersion: String): InnerValLike = {
+ val ret = toVersion match {
+ case VERSION2 | VERSION3 | VERSION4 =>
+ if (innerVal.isInstanceOf[v1.InnerVal]) {
+ val obj = innerVal.asInstanceOf[v1.InnerVal]
+ obj.valueType match {
+ case "long" => InnerVal.withLong(obj.longV.get, toVersion)
+ case "string" => InnerVal.withStr(obj.strV.get, toVersion)
+ case "boolean" => InnerVal.withBoolean(obj.boolV.get, toVersion)
+ case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
+ }
+ } else {
+ innerVal
+ }
+ case VERSION1 =>
+ if (innerVal.isInstanceOf[v2.InnerVal]) {
+ val obj = innerVal.asInstanceOf[v2.InnerVal]
+ obj.value match {
+ case str: String => InnerVal.withStr(str, toVersion)
+ case b: Boolean => InnerVal.withBoolean(b, toVersion)
+ case n: BigDecimal => InnerVal.withNumber(n, toVersion)
+ case n: Long => InnerVal.withNumber(n, toVersion)
+ case n: Double => InnerVal.withNumber(n, toVersion)
+ case n: Int => InnerVal.withNumber(n, toVersion)
+ case _ => throw notSupportedEx(s"v2 to v1: $obj -> $toVersion")
+ }
+ } else {
+ innerVal
+ }
+ case _ => throw notSupportedEx(toVersion)
+ }
+// logger.debug(s"convertVersion: $innerVal, $dataType, $toVersion, $ret, ${innerVal.bytes.toList}, ${ret.bytes.toList}")
+ ret
+ }
+
+}
+
+trait InnerValLike extends HBaseSerializable {
+
+ val value: Any
+
+ def compare(other: InnerValLike): Int
+
+ def +(other: InnerValLike): InnerValLike
+
+ def <(other: InnerValLike) = this.compare(other) < 0
+
+ def <=(other: InnerValLike) = this.compare(other) <= 0
+
+ def >(other: InnerValLike) = this.compare(other) > 0
+
+ def >=(other: InnerValLike) = this.compare(other) >= 0
+
+ override def toString(): String = value.toString
+
+ override def hashCode(): Int = value.hashCode()
+
+ override def equals(obj: Any): Boolean = {
+ obj match {
+ case other: InnerValLike =>
+ val ret = toString == obj.toString
+// logger.debug(s"InnerValLike.equals($this, $obj) => $ret")
+ ret
+ case _ => false
+ }
+ }
+ def hashKey(dataType: String): Int
+
+ def toIdString(): String
+
+}
+
+object InnerValLikeWithTs extends HBaseDeserializable {
+ import HBaseType._
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (InnerValLikeWithTs, Int) = {
+ val (innerVal, numOfBytesUsed) = InnerVal.fromBytes(bytes, offset, len, version)
+ val ts = Bytes.toLong(bytes, offset + numOfBytesUsed)
+ (InnerValLikeWithTs(innerVal, ts), numOfBytesUsed + 8)
+ }
+
+ def withLong(l: Long, ts: Long, version: String): InnerValLikeWithTs = {
+ InnerValLikeWithTs(InnerVal.withLong(l, version), ts)
+ }
+
+ def withStr(s: String, ts: Long, version: String): InnerValLikeWithTs = {
+ InnerValLikeWithTs(InnerVal.withStr(s, version), ts)
+ }
+}
+
+case class InnerValLikeWithTs(innerVal: InnerValLike, ts: Long)
+ extends HBaseSerializable {
+
+ def bytes: Array[Byte] = {
+ Bytes.add(innerVal.bytes, Bytes.toBytes(ts))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala
new file mode 100644
index 0000000..d34299b
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/LabelWithDirection.scala
@@ -0,0 +1,61 @@
+package org.apache.s2graph.core.types
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.GraphUtil
+
+object LabelWithDirection {
+
+ import HBaseType._
+
+ def apply(compositeInt: Int): LabelWithDirection = {
+ // logger.debug(s"CompositeInt: $compositeInt")
+
+ val dir = compositeInt & ((1 << bitsForDir) - 1)
+ val labelId = compositeInt >> bitsForDir
+ LabelWithDirection(labelId, dir)
+ }
+
+ def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
+ assert(labelOrderSeq < (1 << 6))
+ val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
+ Array.fill(1)(byte.toByte)
+ }
+
+ def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
+ val byte = bytes(offset)
+ val isInverted = if ((byte & 1) != 0) true else false
+ val labelOrderSeq = byte >> 1
+ (labelOrderSeq.toByte, isInverted)
+ }
+}
+
+case class LabelWithDirection(labelId: Int, dir: Int) extends HBaseSerializable {
+
+ import HBaseType._
+
+ assert(dir < (1 << bitsForDir))
+ assert(labelId < (Int.MaxValue >> bitsForDir))
+
+ lazy val labelBits = labelId << bitsForDir
+
+ lazy val compositeInt = labelBits | dir
+
+ def bytes = {
+ Bytes.toBytes(compositeInt)
+ }
+
+ lazy val dirToggled = LabelWithDirection(labelId, GraphUtil.toggleDir(dir))
+
+ def updateDir(newDir: Int) = LabelWithDirection(labelId, newDir)
+
+ def isDirected = dir == 0 || dir == 1
+
+ override def hashCode(): Int = compositeInt
+
+ override def equals(other: Any): Boolean = {
+ other match {
+ case o: LabelWithDirection => hashCode == o.hashCode()
+ case _ => false
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala
new file mode 100644
index 0000000..79c7122
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/VertexId.scala
@@ -0,0 +1,142 @@
+package org.apache.s2graph.core.types
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.core.types.HBaseType._
+
+object VertexId extends HBaseDeserializable {
+ import HBaseType._
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (VertexId, Int) = {
+ /** since murmur hash is prepended, skip numOfBytes for murmur hash */
+ var pos = offset + GraphUtil.bytesForMurMurHash
+
+ val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
+ pos += numOfBytesUsed
+ val colId = Bytes.toInt(bytes, pos, 4)
+ (VertexId(colId, innerId), GraphUtil.bytesForMurMurHash + numOfBytesUsed + 4)
+ }
+
+ def apply(colId: Int, innerId: InnerValLike): VertexId = new VertexId(colId, innerId)
+
+ def toSourceVertexId(vid: VertexId) = {
+ SourceVertexId(vid.colId, vid.innerId)
+ }
+
+ def toTargetVertexId(vid: VertexId) = {
+ TargetVertexId(vid.colId, vid.innerId)
+ }
+}
+
+class VertexId protected (val colId: Int, val innerId: InnerValLike) extends HBaseSerializable {
+ val storeHash: Boolean = true
+ val storeColId: Boolean = true
+ lazy val hashBytes =
+// if (storeHash) Bytes.toBytes(GraphUtil.murmur3(innerId.toString))
+ if (storeHash) Bytes.toBytes(GraphUtil.murmur3(innerId.toIdString()))
+ else Array.empty[Byte]
+
+ lazy val colIdBytes: Array[Byte] =
+ if (storeColId) Bytes.toBytes(colId)
+ else Array.empty[Byte]
+
+ def bytes: Array[Byte] = Bytes.add(hashBytes, innerId.bytes, colIdBytes)
+
+ override def toString(): String = {
+ colId.toString() + "," + innerId.toString()
+// s"VertexId($colId, $innerId)"
+ }
+
+ override def hashCode(): Int = {
+ val ret = if (storeColId) {
+ colId * 31 + innerId.hashCode()
+ } else {
+ innerId.hashCode()
+ }
+// logger.debug(s"VertexId.hashCode: $ret")
+ ret
+ }
+ override def equals(obj: Any): Boolean = {
+ val ret = obj match {
+ case other: VertexId => colId == other.colId && innerId.toIdString() == other.innerId.toIdString()
+ case _ => false
+ }
+// logger.debug(s"VertexId.equals: $this, $obj => $ret")
+ ret
+ }
+
+ def compareTo(other: VertexId): Int = {
+ Bytes.compareTo(bytes, other.bytes)
+ }
+ def <(other: VertexId): Boolean = compareTo(other) < 0
+ def <=(other: VertexId): Boolean = compareTo(other) <= 0
+ def >(other: VertexId): Boolean = compareTo(other) > 0
+ def >=(other: VertexId): Boolean = compareTo(other) >= 0
+}
+
+object SourceVertexId extends HBaseDeserializable {
+ import HBaseType._
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (VertexId, Int) = {
+ /** since murmur hash is prepended, skip numOfBytes for murmur hash */
+ val pos = offset + GraphUtil.bytesForMurMurHash
+ val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
+
+ (SourceVertexId(DEFAULT_COL_ID, innerId), GraphUtil.bytesForMurMurHash + numOfBytesUsed)
+ }
+
+}
+
+
+case class SourceVertexId(override val colId: Int,
+ override val innerId: InnerValLike) extends VertexId(colId, innerId) {
+ override val storeColId: Boolean = false
+}
+
+object TargetVertexId extends HBaseDeserializable {
+ import HBaseType._
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (VertexId, Int) = {
+ /** murmur has is not prepended so start from offset */
+ val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, offset, len, version, isVertexId = true)
+ (TargetVertexId(DEFAULT_COL_ID, innerId), numOfBytesUsed)
+ }
+}
+
+case class TargetVertexId(override val colId: Int,
+ override val innerId: InnerValLike)
+ extends VertexId(colId, innerId) {
+ override val storeColId: Boolean = false
+ override val storeHash: Boolean = false
+
+}
+
+object SourceAndTargetVertexIdPair extends HBaseDeserializable {
+ val delimiter = ":"
+ import HBaseType._
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION): (SourceAndTargetVertexIdPair, Int) = {
+ val pos = offset + GraphUtil.bytesForMurMurHash
+ val (srcId, srcBytesLen) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
+ val (tgtId, tgtBytesLen) = InnerVal.fromBytes(bytes, pos + srcBytesLen, len, version, isVertexId = true)
+ (SourceAndTargetVertexIdPair(srcId, tgtId), GraphUtil.bytesForMurMurHash + srcBytesLen + tgtBytesLen)
+ }
+}
+
+case class SourceAndTargetVertexIdPair(val srcInnerId: InnerValLike, val tgtInnerId: InnerValLike) extends HBaseSerializable {
+ val colId = DEFAULT_COL_ID
+ import SourceAndTargetVertexIdPair._
+ override def bytes: Array[Byte] = {
+ val hashBytes = Bytes.toBytes(GraphUtil.murmur3(srcInnerId + delimiter + tgtInnerId))
+ Bytes.add(hashBytes, srcInnerId.bytes, tgtInnerId.bytes)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala
new file mode 100644
index 0000000..c7b2c73
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/v1/InnerVal.scala
@@ -0,0 +1,224 @@
+package org.apache.s2graph.core.types.v1
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.GraphExceptions
+import org.apache.s2graph.core.GraphExceptions.IllegalDataTypeException
+import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, HBaseType, InnerValLike}
+
+object InnerVal extends HBaseDeserializableWithIsVertexId {
+ import HBaseType._
+ // val defaultVal = new InnerVal(None, None, None)
+ val stringLenOffset = 7.toByte
+ val maxStringLen = Byte.MaxValue - stringLenOffset
+ val maxMetaByte = Byte.MaxValue
+ val minMetaByte = 0.toByte
+ /**
+ * first byte encoding rule.
+ * 0 => default
+ * 1 => long
+ * 2 => int
+ * 3 => short
+ * 4 => byte
+ * 5 => true
+ * 6 => false
+ * 7 ~ 127 => string len + 7
+ */
+ val metaByte = Map("default" -> 0, "long" -> 1, "int" -> 2, "short" -> 3,
+ "byte" -> 4, "true" -> 5, "false" -> 6).map {
+ case (k, v) => (k, v.toByte)
+ }
+ val metaByteRev = metaByte.map { case (k, v) => (v.toByte, k) } ++ metaByte.map { case (k, v) => ((-v).toByte, k) }
+
+ def maxIdVal(dataType: String) = {
+ dataType match {
+ case "string" => InnerVal.withStr((0 until (Byte.MaxValue - stringLenOffset)).map("~").mkString)
+ case "long" => InnerVal.withLong(Long.MaxValue)
+ case "bool" => InnerVal.withBoolean(true)
+ case _ => throw IllegalDataTypeException(dataType)
+ }
+ }
+
+ def minIdVal(dataType: String) = {
+ dataType match {
+ case "string" => InnerVal.withStr("")
+ case "long" => InnerVal.withLong(1)
+ case "bool" => InnerVal.withBoolean(false)
+ case _ => throw IllegalDataTypeException(dataType)
+ }
+ }
+
+ def fromBytes(bytes: Array[Byte], offset: Int, len: Int, version: String = DEFAULT_VERSION, isVertexId: Boolean = false): (InnerVal, Int) = {
+ var pos = offset
+ //
+ val header = bytes(pos)
+ // logger.debug(s"${bytes(offset)}: ${bytes.toList.slice(pos, bytes.length)}")
+ pos += 1
+
+ var numOfBytesUsed = 0
+ val (longV, strV, boolV) = metaByteRev.get(header) match {
+ case Some(s) =>
+ s match {
+ case "default" =>
+ (None, None, None)
+ case "true" =>
+ numOfBytesUsed = 0
+ (None, None, Some(true))
+ case "false" =>
+ numOfBytesUsed = 0
+ (None, None, Some(false))
+ case "byte" =>
+ numOfBytesUsed = 1
+ val b = bytes(pos)
+ val value = if (b >= 0) Byte.MaxValue - b else Byte.MinValue - b - 1
+ (Some(value.toLong), None, None)
+ case "short" =>
+ numOfBytesUsed = 2
+ val b = Bytes.toShort(bytes, pos, 2)
+ val value = if (b >= 0) Short.MaxValue - b else Short.MinValue - b - 1
+ (Some(value.toLong), None, None)
+ case "int" =>
+ numOfBytesUsed = 4
+ val b = Bytes.toInt(bytes, pos, 4)
+ val value = if (b >= 0) Int.MaxValue - b else Int.MinValue - b - 1
+ (Some(value.toLong), None, None)
+ case "long" =>
+ numOfBytesUsed = 8
+ val b = Bytes.toLong(bytes, pos, 8)
+ val value = if (b >= 0) Long.MaxValue - b else Long.MinValue - b - 1
+ (Some(value.toLong), None, None)
+ }
+ case _ => // string
+ val strLen = header - stringLenOffset
+ numOfBytesUsed = strLen
+ (None, Some(Bytes.toString(bytes, pos, strLen)), None)
+ }
+
+ (InnerVal(longV, strV, boolV), numOfBytesUsed + 1)
+ }
+
+ def withLong(l: Long): InnerVal = {
+ // if (l < 0) throw new IllegalDataRangeException("value shoudl be >= 0")
+ InnerVal(Some(l), None, None)
+ }
+
+ def withStr(s: String): InnerVal = {
+ InnerVal(None, Some(s), None)
+ }
+
+ def withBoolean(b: Boolean): InnerVal = {
+ InnerVal(None, None, Some(b))
+ }
+
+ /**
+ * In natural order
+ * -129, -128 , -2, -1 < 0 < 1, 2, 127, 128
+ *
+ * In byte order
+ * 0 < 1, 2, 127, 128 < -129, -128, -2, -1
+ *
+ */
+ def transform(l: Long): (Byte, Array[Byte]) = {
+ if (Byte.MinValue <= l && l <= Byte.MaxValue) {
+ // val value = if (l < 0) l - Byte.MinValue else l + Byte.MinValue
+ val key = if (l >= 0) metaByte("byte") else -metaByte("byte")
+ val value = if (l >= 0) Byte.MaxValue - l else Byte.MinValue - l - 1
+ val valueBytes = Array.fill(1)(value.toByte)
+ (key.toByte, valueBytes)
+ } else if (Short.MinValue <= l && l <= Short.MaxValue) {
+ val key = if (l >= 0) metaByte("short") else -metaByte("short")
+ val value = if (l >= 0) Short.MaxValue - l else Short.MinValue - l - 1
+ val valueBytes = Bytes.toBytes(value.toShort)
+ (key.toByte, valueBytes)
+ } else if (Int.MinValue <= l && l <= Int.MaxValue) {
+ val key = if (l >= 0) metaByte("int") else -metaByte("int")
+ val value = if (l >= 0) Int.MaxValue - l else Int.MinValue - l - 1
+ val valueBytes = Bytes.toBytes(value.toInt)
+ (key.toByte, valueBytes)
+ } else if (Long.MinValue <= l && l <= Long.MaxValue) {
+ val key = if (l >= 0) metaByte("long") else -metaByte("long")
+ val value = if (l >= 0) Long.MaxValue - l else Long.MinValue - l - 1
+ val valueBytes = Bytes.toBytes(value.toLong)
+ (key.toByte, valueBytes)
+ } else {
+ throw new Exception(s"InnerVal range is out: $l")
+ }
+ }
+}
+
+case class InnerVal(longV: Option[Long], strV: Option[String], boolV: Option[Boolean])
+ extends HBaseSerializable with InnerValLike {
+
+ import InnerVal._
+
+ lazy val isDefault = longV.isEmpty && strV.isEmpty && boolV.isEmpty
+ val value = (longV, strV, boolV) match {
+ case (Some(l), None, None) => l
+ case (None, Some(s), None) => s
+ case (None, None, Some(b)) => b
+ case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
+ }
+ def valueType = (longV, strV, boolV) match {
+ case (Some(l), None, None) => "long"
+ case (None, Some(s), None) => "string"
+ case (None, None, Some(b)) => "boolean"
+ case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
+ }
+
+ def compare(other: InnerValLike): Int = {
+ if (!other.isInstanceOf[InnerVal]) {
+ throw new RuntimeException(s"compare between $this vs $other is not supported")
+ } else {
+// (value, other.value) match {
+// case (v1: Long, v2: Long) => v1.compare(v2)
+// case (b1: Boolean, b2: Boolean) => b1.compare(b2)
+// case (s1: String, s2: String) => s1.compare(s2)
+// case _ => throw new Exception("Please check a type of the compare operands")
+// }
+ Bytes.compareTo(bytes, other.bytes) * -1
+ }
+ }
+
+ def +(other: InnerValLike) = {
+ if (!other.isInstanceOf[InnerVal]) {
+ throw new RuntimeException(s"+ between $this vs $other is not supported")
+ } else {
+ (value, other.value) match {
+ case (v1: Long, v2: Long) => InnerVal.withLong(v1 + v2)
+ case (b1: Boolean, b2: Boolean) => InnerVal.withBoolean(if (b2) !b1 else b1)
+ case _ => throw new Exception("Please check a type of the incr operands")
+ }
+ }
+ }
+
+ def bytes = {
+ val (meta, valBytes) = (longV, strV, boolV) match {
+ case (None, None, None) =>
+ (metaByte("default"), Array.empty[Byte])
+ case (Some(l), None, None) =>
+ transform(l)
+ case (None, None, Some(b)) =>
+ val meta = if (b) metaByte("true") else metaByte("false")
+ (meta, Array.empty[Byte])
+ case (None, Some(s), None) =>
+ val sBytes = Bytes.toBytes(s)
+ if (sBytes.length > maxStringLen) {
+ throw new IllegalDataTypeException(s"string in innerVal maxSize is $maxStringLen, given ${sBytes.length}")
+ }
+ assert(sBytes.length <= maxStringLen)
+ val meta = (stringLenOffset + sBytes.length).toByte
+ (meta, sBytes)
+ case _ => throw new IllegalDataTypeException("innerVal data type should be [long/string/bool]")
+ }
+ Bytes.add(Array.fill(1)(meta.toByte), valBytes)
+ }
+
+ override def toString(): String = {
+ value.toString
+ }
+ override def hashKey(dataType: String): Int = {
+ value.toString.hashCode()
+ }
+ override def toIdString(): String = {
+ value.toString
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala b/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala
new file mode 100644
index 0000000..a511f17
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/types/v2/InnerVal.scala
@@ -0,0 +1,160 @@
+package org.apache.s2graph.core.types.v2
+
+import org.apache.hadoop.hbase.util._
+import org.apache.s2graph.core.types
+import org.apache.s2graph.core.types.{HBaseDeserializableWithIsVertexId, HBaseSerializable, HBaseType, InnerValLike}
+
+object InnerVal extends HBaseDeserializableWithIsVertexId {
+
+ import HBaseType._
+
+ val order = Order.DESCENDING
+
+ def fromBytes(bytes: Array[Byte],
+ offset: Int,
+ len: Int,
+ version: String = DEFAULT_VERSION,
+ isVertexId: Boolean = false): (InnerVal, Int) = {
+ val pbr = new SimplePositionedByteRange(bytes)
+ pbr.setPosition(offset)
+ val startPos = pbr.getPosition
+ if (bytes(offset) == -1 | bytes(offset) == 0) {
+ /** simple boolean */
+ val boolean = order match {
+ case Order.DESCENDING => bytes(offset) == 0
+ case _ => bytes(offset) == -1
+ }
+ (InnerVal(boolean), 1)
+ }
+ else {
+ if (OrderedBytes.isNumeric(pbr)) {
+ val numeric = OrderedBytes.decodeNumericAsBigDecimal(pbr)
+ if (isVertexId) (InnerVal(numeric.longValue()), pbr.getPosition - startPos)
+ else (InnerVal(BigDecimal(numeric)), pbr.getPosition - startPos)
+// (InnerVal(numeric.doubleValue()), pbr.getPosition - startPos)
+// (InnerVal(BigDecimal(numeric)), pbr.getPosition - startPos)
+ } else if (OrderedBytes.isText(pbr)) {
+ val str = OrderedBytes.decodeString(pbr)
+ (InnerVal(str), pbr.getPosition - startPos)
+ } else if (OrderedBytes.isBlobVar(pbr)) {
+ val blobVar = OrderedBytes.decodeBlobVar(pbr)
+ (InnerVal(blobVar), pbr.getPosition - startPos)
+ } else {
+ throw new RuntimeException("!!")
+ }
+ }
+ }
+}
+
+case class InnerVal(value: Any) extends HBaseSerializable with InnerValLike {
+
+ import types.InnerVal._
+
+ def bytes: Array[Byte] = {
+ val ret = value match {
+ case b: Boolean =>
+
+ /** since OrderedBytes header start from 0x05, it is safe to use -1, 0
+ * for decreasing order (true, false) */
+ // Bytes.toBytes(b)
+ order match {
+ case Order.DESCENDING => if (b) Array(0.toByte) else Array(-1.toByte)
+ case _ => if (!b) Array(0.toByte) else Array(-1.toByte)
+ }
+ case d: Double =>
+ val num = BigDecimal(d)
+ val pbr = numByteRange(num)
+ val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
+ pbr.getBytes().take(len)
+ case l: Long =>
+ val num = BigDecimal(l)
+ val pbr = numByteRange(num)
+ val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
+ pbr.getBytes().take(len)
+ case i: Int =>
+ val num = BigDecimal(i)
+ val pbr = numByteRange(num)
+ val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
+ pbr.getBytes().take(len)
+ case sh: Short =>
+ val num = BigDecimal(sh)
+ val pbr = numByteRange(num)
+ val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
+ pbr.getBytes().take(len)
+ case b: Byte =>
+ val num = BigDecimal(b)
+ val pbr = numByteRange(num)
+ val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
+ pbr.getBytes().take(len)
+
+
+ case b: BigDecimal =>
+ val pbr = numByteRange(b)
+ val len = OrderedBytes.encodeNumeric(pbr, b.bigDecimal, order)
+ pbr.getBytes().take(len)
+ case s: String =>
+ val pbr = new SimplePositionedMutableByteRange(s.getBytes.length + 3)
+ val len = OrderedBytes.encodeString(pbr, s, order)
+ pbr.getBytes().take(len)
+ case blob: Array[Byte] =>
+ val len = OrderedBytes.blobVarEncodedLength(blob.length)
+ val pbr = new SimplePositionedMutableByteRange(len)
+ val totalLen = OrderedBytes.encodeBlobVar(pbr, blob, order)
+ pbr.getBytes().take(totalLen)
+ }
+ // println(s"$value => ${ret.toList}, ${ret.length}")
+ ret
+ }
+
+//
+ override def hashKey(dataType: String): Int = {
+ if (value.isInstanceOf[String]) {
+ // since we use dummy stringn value for degree edge.
+ value.toString.hashCode()
+ } else {
+ dataType match {
+ case BYTE => value.asInstanceOf[BigDecimal].bigDecimal.byteValue().hashCode()
+ case FLOAT => value.asInstanceOf[BigDecimal].bigDecimal.floatValue().hashCode()
+ case DOUBLE => value.asInstanceOf[BigDecimal].bigDecimal.doubleValue().hashCode()
+ case LONG => value.asInstanceOf[BigDecimal].bigDecimal.longValue().hashCode()
+ case INT => value.asInstanceOf[BigDecimal].bigDecimal.intValue().hashCode()
+ case SHORT => value.asInstanceOf[BigDecimal].bigDecimal.shortValue().hashCode()
+ case STRING => value.toString.hashCode
+ case _ => throw new RuntimeException(s"NotSupportede type: $dataType")
+ }
+ }
+ }
+
+ def compare(other: InnerValLike): Int = {
+ if (!other.isInstanceOf[InnerValLike])
+ throw new RuntimeException(s"compare $this vs $other")
+ Bytes.compareTo(bytes, other.bytes) * -1
+ }
+
+ def +(other: InnerValLike): InnerValLike = {
+ if (!other.isInstanceOf[InnerValLike])
+ throw new RuntimeException(s"+ $this, $other")
+
+ (value, other.value) match {
+ case (v1: BigDecimal, v2: BigDecimal) => new InnerVal(BigDecimal(v1.bigDecimal.add(v2.bigDecimal)))
+ case _ => throw new RuntimeException("+ operation on inner val is for big decimal pair")
+ }
+ }
+
+ //need to be removed ??
+ override def toString(): String = {
+// value.toString()
+ value match {
+ case n: BigDecimal => n.bigDecimal.toPlainString
+ case _ => value.toString
+ }
+ }
+
+ override def toIdString(): String = {
+ value match {
+ case n: BigDecimal => n.bigDecimal.longValue().toString()
+ case _ => value.toString
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/utils/DeferCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/DeferCache.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/DeferCache.scala
new file mode 100644
index 0000000..59452df
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/DeferCache.scala
@@ -0,0 +1,82 @@
+package org.apache.s2graph.core.utils
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.CacheBuilder
+import com.stumbleupon.async.Deferred
+import com.typesafe.config.Config
+
+import scala.concurrent.ExecutionContext
+
+class DeferCache[R](config: Config)(implicit ex: ExecutionContext) {
+
+ import Extensions.DeferOps
+
+ type Value = (Long, Deferred[R])
+
+ private val maxSize = config.getInt("future.cache.max.size")
+ private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+ private val futureCache = CacheBuilder.newBuilder()
+ .initialCapacity(maxSize)
+ .concurrencyLevel(Runtime.getRuntime.availableProcessors())
+ .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
+ .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
+ .maximumSize(maxSize).build[java.lang.Long, (Long, Deferred[R])]()
+
+
+ def asMap() = futureCache.asMap()
+
+ def getIfPresent(cacheKey: Long): Value = futureCache.getIfPresent(cacheKey)
+
+ private def checkAndExpire(cacheKey: Long,
+ cachedAt: Long,
+ cacheTTL: Long,
+ oldDefer: Deferred[R])(op: => Deferred[R]): Deferred[R] = {
+ if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
+ // future is too old. so need to expire and fetch new data from storage.
+ futureCache.asMap().remove(cacheKey)
+
+ val newPromise = new Deferred[R]()
+ val now = System.currentTimeMillis()
+
+ futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
+ case null =>
+ // only one thread succeed to come here concurrently
+ // initiate fetch to storage then add callback on complete to finish promise.
+ op withCallback { value =>
+ newPromise.callback(value)
+ value
+ }
+ newPromise
+ case (cachedAt, oldDefer) => oldDefer
+ }
+ } else {
+ // future is not to old so reuse it.
+ oldDefer
+ }
+ }
+ def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Deferred[R]): Deferred[R] = {
+ val cacheVal = futureCache.getIfPresent(cacheKey)
+ cacheVal match {
+ case null =>
+ val promise = new Deferred[R]()
+ val now = System.currentTimeMillis()
+ val (cachedAt, defer) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
+ case null =>
+ op.withCallback { value =>
+ promise.callback(value)
+ value
+ }
+ (now, promise)
+ case oldVal => oldVal
+ }
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
+
+ case (cachedAt, defer) =>
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/utils/Extentions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/Extentions.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/Extentions.scala
new file mode 100644
index 0000000..5805c32
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/Extentions.scala
@@ -0,0 +1,73 @@
+package org.apache.s2graph.core.utils
+
+import com.stumbleupon.async.{Callback, Deferred}
+import com.typesafe.config.Config
+
+import scala.concurrent.{ExecutionContext, Future, Promise}
+
+object Extensions {
+
+
+ def retryOnSuccess[T](maxRetryNum: Int, n: Int = 1)(fn: => Future[T])(shouldStop: T => Boolean)(implicit ex: ExecutionContext): Future[T] = n match {
+ case i if n <= maxRetryNum =>
+ fn.flatMap { result =>
+ if (!shouldStop(result)) {
+ logger.info(s"retryOnSuccess $n")
+ retryOnSuccess(maxRetryNum, n + 1)(fn)(shouldStop)
+ } else {
+ Future.successful(result)
+ }
+ }
+ case _ => fn
+ }
+
+ def retryOnFailure[T](maxRetryNum: Int, n: Int = 1)(fn: => Future[T])(fallback: => T)(implicit ex: ExecutionContext): Future[T] = n match {
+ case i if n <= maxRetryNum =>
+ fn recoverWith { case t: Throwable =>
+ logger.info(s"retryOnFailure $n $t")
+ retryOnFailure(maxRetryNum, n + 1)(fn)(fallback)
+ }
+ case _ =>
+ Future.successful(fallback)
+ }
+
+
+ implicit class DeferOps[T](d: Deferred[T])(implicit ex: ExecutionContext) {
+
+ def withCallback[R](op: T => R): Deferred[R] = {
+ d.addCallback(new Callback[R, T] {
+ override def call(arg: T): R = op(arg)
+ })
+ }
+
+ def recoverWith(op: Exception => T): Deferred[T] = {
+ d.addErrback(new Callback[Deferred[T], Exception] {
+ override def call(e: Exception): Deferred[T] = Deferred.fromResult(op(e))
+ })
+ }
+
+
+ def toFuture: Future[T] = {
+ val promise = Promise[T]
+
+ d.addBoth(new Callback[Unit, T] {
+ def call(arg: T) = arg match {
+ case e: Exception => promise.failure(e)
+ case _ => promise.success(arg)
+ }
+ })
+
+ promise.future
+ }
+
+ def toFutureWith(fallback: => T): Future[T] = {
+ toFuture recoverWith { case t: Throwable => Future.successful(fallback) }
+ }
+
+ }
+
+ implicit class ConfigOps(config: Config) {
+ def getBooleanWithFallback(key: String, defaultValue: Boolean): Boolean =
+ if (config.hasPath(key)) config.getBoolean(key) else defaultValue
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/utils/FutureCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/FutureCache.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/FutureCache.scala
new file mode 100644
index 0000000..b23566e
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/FutureCache.scala
@@ -0,0 +1,82 @@
+package org.apache.s2graph.core.utils
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.CacheBuilder
+import com.typesafe.config.Config
+
+import scala.concurrent.{ExecutionContext, Future, Promise}
+
+
+class FutureCache[R](config: Config)(implicit ex: ExecutionContext) {
+
+ type Value = (Long, Future[R])
+
+ private val maxSize = config.getInt("future.cache.max.size")
+ private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+ private val futureCache = CacheBuilder.newBuilder()
+ .initialCapacity(maxSize)
+ .concurrencyLevel(Runtime.getRuntime.availableProcessors())
+ .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
+ .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
+ .maximumSize(maxSize).build[java.lang.Long, (Long, Promise[R])]()
+
+
+ def asMap() = futureCache.asMap()
+
+ def getIfPresent(cacheKey: Long): Value = {
+ val (cachedAt, promise) = futureCache.getIfPresent(cacheKey)
+ (cachedAt, promise.future)
+ }
+
+ private def checkAndExpire(cacheKey: Long,
+ cachedAt: Long,
+ cacheTTL: Long,
+ oldFuture: Future[R])(op: => Future[R]): Future[R] = {
+ if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
+ // future is too old. so need to expire and fetch new data from storage.
+ futureCache.asMap().remove(cacheKey)
+
+ val newPromise = Promise[R]
+ val now = System.currentTimeMillis()
+
+ futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
+ case null =>
+ // only one thread succeed to come here concurrently
+ // initiate fetch to storage then add callback on complete to finish promise.
+ op.onSuccess { case value =>
+ newPromise.success(value)
+ value
+ }
+ newPromise.future
+ case (cachedAt, oldPromise) => oldPromise.future
+ }
+ } else {
+ // future is not to old so reuse it.
+ oldFuture
+ }
+ }
+ def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Future[R]): Future[R] = {
+ val cacheVal = futureCache.getIfPresent(cacheKey)
+ cacheVal match {
+ case null =>
+ val promise = Promise[R]
+ val now = System.currentTimeMillis()
+ val (cachedAt, cachedPromise) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
+ case null =>
+ op.onSuccess { case value =>
+ promise.success(value)
+ value
+ }
+ (now, promise)
+ case oldVal => oldVal
+ }
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
+
+ case (cachedAt, cachedPromise) =>
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/utils/Logger.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/Logger.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/Logger.scala
new file mode 100644
index 0000000..6b02a00
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/Logger.scala
@@ -0,0 +1,44 @@
+package org.apache.s2graph.core.utils
+
+import org.slf4j.LoggerFactory
+import play.api.libs.json.JsValue
+
+import scala.language.{higherKinds, implicitConversions}
+
+object logger {
+
+ trait Loggable[T] {
+ def toLogMessage(msg: T): String
+ }
+
+ object Loggable {
+ implicit val stringLoggable = new Loggable[String] {
+ def toLogMessage(msg: String) = msg
+ }
+
+ implicit def numericLoggable[T: Numeric] = new Loggable[T] {
+ def toLogMessage(msg: T) = msg.toString
+ }
+
+ implicit val jsonLoggable = new Loggable[JsValue] {
+ def toLogMessage(msg: JsValue) = msg.toString()
+ }
+
+ implicit val booleanLoggable = new Loggable[Boolean] {
+ def toLogMessage(msg: Boolean) = msg.toString()
+ }
+ }
+
+ private val logger = LoggerFactory.getLogger("application")
+ private val errorLogger = LoggerFactory.getLogger("error")
+
+ def info[T: Loggable](msg: => T) = logger.info(implicitly[Loggable[T]].toLogMessage(msg))
+
+ def debug[T: Loggable](msg: => T) = logger.debug(implicitly[Loggable[T]].toLogMessage(msg))
+
+ def error[T: Loggable](msg: => T, exception: => Throwable) = errorLogger.error(implicitly[Loggable[T]].toLogMessage(msg), exception)
+
+ def error[T: Loggable](msg: => T) = errorLogger.error(implicitly[Loggable[T]].toLogMessage(msg))
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
new file mode 100644
index 0000000..3c67e42
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
@@ -0,0 +1,62 @@
+package org.apache.s2graph.core.utils
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+import com.google.common.cache.CacheBuilder
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+object SafeUpdateCache {
+
+ case class CacheKey(key: String)
+
+}
+
+class SafeUpdateCache[T](prefix: String, maxSize: Int, ttl: Int)(implicit executionContext: ExecutionContext) {
+
+ import SafeUpdateCache._
+
+ implicit class StringOps(key: String) {
+ def toCacheKey = new CacheKey(prefix + ":" + key)
+ }
+
+ def toTs() = (System.currentTimeMillis() / 1000).toInt
+
+ private val cache = CacheBuilder.newBuilder().maximumSize(maxSize).build[CacheKey, (T, Int, AtomicBoolean)]()
+
+ def put(key: String, value: T) = cache.put(key.toCacheKey, (value, toTs, new AtomicBoolean(false)))
+
+ def invalidate(key: String) = cache.invalidate(key.toCacheKey)
+
+ def withCache(key: String)(op: => T): T = {
+ val cacheKey = key.toCacheKey
+ val cachedValWithTs = cache.getIfPresent(cacheKey)
+
+ if (cachedValWithTs == null) {
+ // fetch and update cache.
+ val newValue = op
+ cache.put(cacheKey, (newValue, toTs(), new AtomicBoolean(false)))
+ newValue
+ } else {
+ val (cachedVal, updatedAt, isUpdating) = cachedValWithTs
+ if (toTs() < updatedAt + ttl) cachedVal // in cache TTL
+ else {
+ val running = isUpdating.getAndSet(true)
+ if (running) cachedVal
+ else {
+ Future(op)(executionContext) onComplete {
+ case Failure(ex) =>
+ cache.put(cacheKey, (cachedVal, toTs(), new AtomicBoolean(false))) // keep old value
+ logger.error(s"withCache update failed: $cacheKey")
+ case Success(newValue) =>
+ cache.put(cacheKey, (newValue, toTs(), new AtomicBoolean(false))) // update new value
+ logger.info(s"withCache update success: $cacheKey")
+ }
+ cachedVal
+ }
+ }
+ }
+ }
+}
+
[09/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/TimedQualifier.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/TimedQualifier.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/TimedQualifier.scala
new file mode 100644
index 0000000..ad2bea2
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/TimedQualifier.scala
@@ -0,0 +1,215 @@
+package org.apache.s2graph.counter.core
+
+import java.text.SimpleDateFormat
+import java.util.Calendar
+
+case class TimedQualifier(q: TimedQualifier.IntervalUnit.Value, ts: Long) {
+ import TimedQualifier.IntervalUnit._
+
+ def dateTime: Long = {
+ val dateFormat = new SimpleDateFormat("yyyyMMddHHmm")
+ dateFormat.format(ts).toLong
+ }
+
+ def add(amount: Int): TimedQualifier = {
+ val cal = Calendar.getInstance()
+ cal.setTimeInMillis(ts)
+ q match {
+ case MINUTELY =>
+ cal.add(Calendar.MINUTE, amount)
+ case HOURLY =>
+ cal.add(Calendar.HOUR, amount)
+ case DAILY =>
+ cal.add(Calendar.DAY_OF_MONTH, amount)
+ case MONTHLY =>
+ cal.add(Calendar.MONTH, amount)
+ case TOTAL =>
+ }
+ copy(ts = cal.getTimeInMillis)
+ }
+}
+
+object TimedQualifier {
+ object IntervalUnit extends Enumeration {
+ type IntervalUnit = Value
+ val TOTAL = Value("t")
+ val MONTHLY = Value("M")
+ val DAILY = Value("d")
+ val HOURLY = Value("H")
+ val MINUTELY = Value("m")
+ }
+
+ def apply(q: String, ts: Long): TimedQualifier = TimedQualifier(IntervalUnit.withName(q), ts)
+
+ import IntervalUnit._
+
+ def getTsUnit(intervalUnit: IntervalUnit.IntervalUnit): Long = {
+ intervalUnit match {
+ case MINUTELY => 1 * 60 * 1000l
+ case HOURLY => 60 * 60 * 1000l
+ case DAILY => 24 * 60 * 60 * 1000l
+ case MONTHLY => 31 * 24 * 60 * 60 * 1000l
+ case v: IntervalUnit.IntervalUnit =>
+ throw new RuntimeException(s"unsupported operation for ${v.toString}")
+ }
+ }
+
+ def getQualifiers(intervals: Seq[IntervalUnit], millis: Long): Seq[TimedQualifier] = {
+ val cal = Calendar.getInstance()
+ cal.setTimeInMillis(millis)
+
+ val newCal = Calendar.getInstance()
+ newCal.set(cal.get(Calendar.YEAR), cal.get(Calendar.MONTH), 1, 0, 0, 0)
+ newCal.set(Calendar.MILLISECOND, 0)
+ val month = newCal.getTimeInMillis
+ val Seq(day, hour, minute) = {
+ for {
+ field <- Seq(Calendar.DATE, Calendar.HOUR_OF_DAY, Calendar.MINUTE)
+ } yield {
+ newCal.set(field, cal.get(field))
+ newCal.getTimeInMillis
+ }
+ }
+
+ for {
+ interval <- intervals
+ } yield {
+ val ts = interval match {
+ case MINUTELY => minute
+ case HOURLY => hour
+ case DAILY => day
+ case MONTHLY => month
+ case TOTAL => 0L
+ }
+ TimedQualifier(interval, ts)
+ }
+ }
+
+ // descending order
+ def getQualifiersToLimit(intervals: Seq[IntervalUnit], limit: Int, tsOpt: Option[Long] = None): Seq[TimedQualifier] = {
+ val ts = tsOpt.getOrElse(System.currentTimeMillis())
+ for {
+ interval <- intervals
+ newLimit = if (interval == TOTAL) 1 else limit
+ i <- 0 until (-newLimit, -1)
+ } yield {
+ val newMillis = nextTime(interval, ts, i)
+ TimedQualifier(interval, newMillis)
+ }
+ }
+
+ private def nextTime(interval: IntervalUnit, ts: Long, i: Int): Long = {
+ val newCal = Calendar.getInstance()
+ newCal.setTimeInMillis(ts)
+ newCal.set(Calendar.MILLISECOND, 0)
+ interval match {
+ case MINUTELY =>
+ newCal.set(Calendar.SECOND, 0)
+ newCal.add(Calendar.MINUTE, i)
+ newCal.getTimeInMillis
+ case HOURLY =>
+ newCal.set(Calendar.SECOND, 0)
+ newCal.set(Calendar.MINUTE, 0)
+ newCal.add(Calendar.HOUR_OF_DAY, i)
+ newCal.getTimeInMillis
+ case DAILY =>
+ newCal.set(Calendar.SECOND, 0)
+ newCal.set(Calendar.MINUTE, 0)
+ newCal.set(Calendar.HOUR_OF_DAY, 0)
+ newCal.add(Calendar.DAY_OF_MONTH, i)
+ newCal.getTimeInMillis
+ case MONTHLY =>
+ newCal.set(Calendar.SECOND, 0)
+ newCal.set(Calendar.MINUTE, 0)
+ newCal.set(Calendar.HOUR_OF_DAY, 0)
+ newCal.set(Calendar.DAY_OF_MONTH, 1)
+ newCal.add(Calendar.MONTH, i)
+ newCal.getTimeInMillis
+ case TOTAL =>
+ 0L
+ }
+ }
+
+ def getTimeList(interval: IntervalUnit, from: Long, to: Long, rst: List[Long] = Nil): List[Long] = {
+ interval match {
+ case TOTAL => List(0)
+ case _ =>
+ val next = nextTime(interval, from, 1)
+ if (next < from) {
+ // ignore
+ getTimeList(interval, next, to, rst)
+ }
+ else if (next < to) {
+ // recall
+ getTimeList(interval, next, to, rst :+ next)
+ } else {
+ // end condition
+ rst :+ next
+ }
+ }
+ }
+
+ // for reader
+ def getQualifiersToLimit(intervals: Seq[IntervalUnit],
+ limit: Int,
+ optFrom: Option[Long],
+ optTo: Option[Long]): Seq[List[TimedQualifier]] = {
+ val newLimit = limit - 1
+ for {
+ interval <- intervals
+ } yield {
+ {
+ (optFrom, optTo) match {
+ case (Some(from), Some(to)) =>
+ getTimeList(interval, from, to)
+ case (Some(from), None) =>
+ getTimeList(interval, from, nextTime(interval, from, newLimit))
+ case (None, Some(to)) =>
+ getTimeList(interval, nextTime(interval, to, -newLimit), to)
+ case (None, None) =>
+ val current = System.currentTimeMillis()
+ getTimeList(interval, nextTime(interval, current, -newLimit), current)
+ }
+ }.map { ts =>
+ TimedQualifier(interval, ts)
+ }
+ }
+ }
+
+ def getTimeRange(intervals: Seq[IntervalUnit],
+ limit: Int,
+ optFrom: Option[Long],
+ optTo: Option[Long]): Seq[(TimedQualifier, TimedQualifier)] = {
+ val newLimit = limit - 1
+ val maxInterval = intervals.maxBy {
+ case MINUTELY => 0
+ case HOURLY => 1
+ case DAILY => 2
+ case MONTHLY => 3
+ case TOTAL => 4
+ }
+ val minInterval = intervals.minBy {
+ case MINUTELY => 0
+ case HOURLY => 1
+ case DAILY => 2
+ case MONTHLY => 3
+ case TOTAL => 4
+ }
+ val (from, to) = (optFrom, optTo) match {
+ case (Some(f), Some(t)) =>
+ (f, t)
+ case (Some(f), None) =>
+ (f, nextTime(minInterval, f, newLimit))
+ case (None, Some(t)) =>
+ (nextTime(maxInterval, t, -newLimit), t)
+ case (None, None) =>
+ val current = System.currentTimeMillis()
+ (nextTime(maxInterval, current, -newLimit), nextTime(minInterval, current, 0))
+ }
+ for {
+ interval <- intervals
+ } yield {
+ (TimedQualifier(interval, from), TimedQualifier(interval, to))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/BytesUtilV1.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/BytesUtilV1.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/BytesUtilV1.scala
new file mode 100644
index 0000000..eaef60d
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/BytesUtilV1.scala
@@ -0,0 +1,60 @@
+package org.apache.s2graph.counter.core.v1
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core.{TimedQualifier, ExactQualifier, ExactKeyTrait, BytesUtil}
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.apache.s2graph.counter.util.Hashes
+import scala.collection.mutable.ArrayBuffer
+
+object BytesUtilV1 extends BytesUtil {
+ // ExactKey: [hash(2b)][policy(4b)][item(variable)]
+ val BUCKET_BYTE_SIZE = Bytes.SIZEOF_SHORT
+ val POLICY_ID_SIZE = Bytes.SIZEOF_INT
+ val INTERVAL_SIZE = Bytes.SIZEOF_BYTE
+ val TIMESTAMP_SIZE = Bytes.SIZEOF_LONG
+ val TIMED_QUALIFIER_SIZE = INTERVAL_SIZE + TIMESTAMP_SIZE
+
+ override def getRowKeyPrefix(id: Int): Array[Byte] = {
+ Bytes.toBytes(id)
+ }
+
+ override def toBytes(key: ExactKeyTrait): Array[Byte] = {
+ val buff = new ArrayBuffer[Byte]
+ // hash key (2 byte)
+ buff ++= Bytes.toBytes(Hashes.murmur3(key.itemKey)).take(BUCKET_BYTE_SIZE)
+
+ buff ++= getRowKeyPrefix(key.policyId)
+ buff ++= {
+ key.itemType match {
+ case ItemType.INT => Bytes.toBytes(key.itemKey.toInt)
+ case ItemType.LONG => Bytes.toBytes(key.itemKey.toLong)
+ case ItemType.STRING | ItemType.BLOB => Bytes.toBytes(key.itemKey)
+ }
+ }
+ buff.toArray
+ }
+
+ override def toBytes(eq: ExactQualifier): Array[Byte] = {
+ toBytes(eq.tq) ++ eq.dimension.getBytes
+ }
+
+ override def toBytes(tq: TimedQualifier): Array[Byte] = {
+ Bytes.toBytes(tq.q.toString) ++ Bytes.toBytes(tq.ts)
+ }
+
+ override def toExactQualifier(bytes: Array[Byte]): ExactQualifier = {
+ // qualifier: interval, ts, dimension 순서
+ val tq = toTimedQualifier(bytes)
+
+ val dimension = Bytes.toString(bytes, TIMED_QUALIFIER_SIZE, bytes.length - TIMED_QUALIFIER_SIZE)
+ ExactQualifier(tq, dimension)
+ }
+
+ override def toTimedQualifier(bytes: Array[Byte]): TimedQualifier = {
+ val interval = Bytes.toString(bytes, 0, INTERVAL_SIZE)
+ val ts = Bytes.toLong(bytes, INTERVAL_SIZE)
+
+ TimedQualifier(IntervalUnit.withName(interval), ts)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageAsyncHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageAsyncHBase.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageAsyncHBase.scala
new file mode 100644
index 0000000..340986f
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageAsyncHBase.scala
@@ -0,0 +1,317 @@
+package org.apache.s2graph.counter.core.v1
+
+import java.util
+import com.stumbleupon.async.{Callback, Deferred}
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.CellUtil
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter.helper.{Management, WithAsyncHBase, WithHBase}
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.hbase.async.{KeyValue, ColumnRangeFilter, FilterList, GetRequest}
+import org.slf4j.LoggerFactory
+import scala.collection.JavaConversions._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+class ExactStorageAsyncHBase(config: Config) extends ExactStorage {
+ import ExactStorageHBase._
+
+ private val log = LoggerFactory.getLogger(getClass)
+
+ lazy val s2config = new S2CounterConfig(config)
+
+ private[counter] val withHBase = new WithHBase(config)
+ private[counter] val withAsyncHBase = new WithAsyncHBase(config)
+ private[counter] val hbaseManagement = new Management(config)
+
+ private def getTableName(policy: Counter): String = {
+ policy.hbaseTable.getOrElse(s2config.HBASE_TABLE_NAME)
+ }
+
+ override def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
+
+ val tableName = getTableName(policy)
+
+ lazy val messageForLog = s"${policy.service}.${policy.action} $items $timeRange"
+
+ val keys = {
+ for {
+ item <- items
+ } yield {
+ ExactKey(policy, item, checkItemType = true)
+ }
+ }
+
+ val gets = {
+ for {
+ cf <- timeRange.map(t => intervalsMap(t._1.q)).distinct
+ key <- keys
+ } yield {
+ val get = new GetRequest(tableName, BytesUtilV1.toBytes(key))
+ get.family(cf.toString)
+ get.setFilter(new FilterList({
+ for {
+ (from, to) <- timeRange
+ } yield {
+ new ColumnRangeFilter(
+ BytesUtilV1.toBytes(from), true,
+ BytesUtilV1.toBytes(to.copy(ts = to.ts + 1)), false)
+ }
+ }, FilterList.Operator.MUST_PASS_ONE))
+ (key, cf, get)
+ }
+ }
+
+// println(s"$messageForLog $gets")
+
+ withAsyncHBase[Seq[FetchedCounts]] { client =>
+ val deferreds: Seq[Deferred[FetchedCounts]] = {
+ for {
+ (key, cf, get) <- gets
+ } yield {
+ client.get(get).addCallback { new Callback[FetchedCounts, util.ArrayList[KeyValue]] {
+ override def call(kvs: util.ArrayList[KeyValue]): FetchedCounts = {
+ val qualifierWithCounts = {
+ for {
+ kv <- kvs
+ eq = BytesUtilV1.toExactQualifier(kv.qualifier())
+ } yield {
+ eq -> Bytes.toLong(kv.value())
+ }
+ }.toMap
+// println(s"$key $qualifierWithCounts")
+ FetchedCounts(key, qualifierWithCounts)
+ }
+ }}
+ }
+ }
+ Deferred.group(deferreds).addCallback { new Callback[Seq[FetchedCounts], util.ArrayList[FetchedCounts]] {
+ override def call(arg: util.ArrayList[FetchedCounts]): Seq[FetchedCounts] = {
+ for {
+ (key, fetchedGroup) <- Seq(arg: _*).groupBy(_.exactKey)
+ } yield {
+ fetchedGroup.reduce[FetchedCounts] { case (f1, f2) =>
+ FetchedCounts(key, f1.qualifierWithCountMap ++ f2.qualifierWithCountMap)
+ }
+ }
+ }.toSeq
+ }}
+ }
+ }
+
+ override def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
+ get(policy, items, timeRange).map { fetchedLs =>
+ for {
+ FetchedCounts(exactKey, qualifierWithCountMap) <- fetchedLs
+ } yield {
+ val intervalWithCountMap = qualifierWithCountMap
+ .filter { case (eq, v) => eq.checkDimensionEquality(dimQuery) }
+ .groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) }
+ FetchedCountsGrouped(exactKey, intervalWithCountMap)
+ }
+ }
+ }
+
+ override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
+ // increment mutation to hbase
+ val increments = {
+ for {
+ (exactKey, values) <- counts
+ inc = new Increment(BytesUtilV1.toBytes(exactKey))
+ } yield {
+ for {
+ (eq, value) <- values
+ } {
+ inc.addColumn(intervalsMap.apply(eq.tq.q).toString.getBytes, BytesUtilV1.toBytes(eq), value)
+ }
+ // add column by dimension
+ inc
+ }
+ }
+
+ val results: Array[Object] = Array.fill(increments.size)(null)
+
+ withHBase(getTableName(policy)) { table =>
+ table.batch(increments, results)
+ } match {
+ case Failure(ex) =>
+ log.error(s"${ex.getMessage}")
+ case _ =>
+ }
+
+ assert(counts.length == results.length)
+
+ for {
+ ((exactKey, eqWithValue), result) <- counts.zip(results)
+ } yield {
+ val eqWithResult = result match {
+ case r: Result =>
+ for {
+ (eq, value) <- eqWithValue
+ } yield {
+ val interval = eq.tq.q
+ val cf = intervalsMap(interval)
+ val result = Option(r.getColumnLatestCell(cf.toString.getBytes, BytesUtilV1.toBytes(eq))).map { cell =>
+ Bytes.toLong(CellUtil.cloneValue(cell))
+ }.getOrElse(-1l)
+ eq -> result
+ }
+ case ex: Throwable =>
+ log.error(s"${ex.getMessage}: $exactKey")
+ Nil
+ case _ =>
+ log.error(s"result is null: $exactKey")
+ Nil
+ }
+ (exactKey, eqWithResult.toMap)
+ }
+ }.toMap
+
+ override def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
+ withHBase(getTableName(policy)) { table =>
+ table.delete {
+ for {
+ key <- keys
+ } yield {
+ new Delete(BytesUtilV1.toBytes(key))
+ }
+ }
+ } match {
+ case Failure(ex) =>
+ log.error(ex.getMessage)
+ case _ =>
+ }
+ }
+
+ override def get(policy: Counter,
+ queries: Seq[(ExactKeyTrait, Seq[core.ExactQualifier])])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
+
+ val tableName = getTableName(policy)
+
+ val gets = {
+ for {
+ (key, eqs) <- queries
+ (cf, eqsGrouped) <- eqs.groupBy(eq => intervalsMap(eq.tq.q))
+ } yield {
+// println(s"$key $eqsGrouped")
+ val get = new GetRequest(tableName, BytesUtilV1.toBytes(key))
+ get.family(cf.toString)
+ get.qualifiers(eqsGrouped.map(BytesUtilV1.toBytes).toArray)
+ (key, cf, get)
+ }
+ }
+
+ withAsyncHBase[Seq[FetchedCounts]] { client =>
+ val deferreds: Seq[Deferred[FetchedCounts]] = {
+ for {
+ (key, cf, get) <- gets
+ } yield {
+ client.get(get).addCallback { new Callback[FetchedCounts, util.ArrayList[KeyValue]] {
+ override def call(kvs: util.ArrayList[KeyValue]): FetchedCounts = {
+ val qualifierWithCounts = {
+ for {
+ kv <- kvs
+ eq = BytesUtilV1.toExactQualifier(kv.qualifier())
+ } yield {
+ eq -> Bytes.toLong(kv.value())
+ }
+ }.toMap
+ FetchedCounts(key, qualifierWithCounts)
+ }
+ }}
+ }
+ }
+ Deferred.group(deferreds).addCallback { new Callback[Seq[FetchedCounts], util.ArrayList[FetchedCounts]] {
+ override def call(arg: util.ArrayList[FetchedCounts]): Seq[FetchedCounts] = arg
+ }}
+ }
+ }
+
+ override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
+ val results: Array[Object] = Array.fill(keys.size)(null)
+
+ val puts = keys.map { key =>
+ val put = new Put(BytesUtilV1.toBytes(key))
+ put.addColumn(blobCF, blobColumn, key.itemId.getBytes)
+ }
+
+ withHBase(getTableName(policy)) { table =>
+ table.batch(puts, results)
+ } match {
+ case Failure(ex) =>
+ log.error(s"${ex.getMessage}")
+ case _ =>
+ }
+
+ for {
+ (result, key) <- results.zip(keys)
+ } yield {
+ Option(result).map(_ => true).getOrElse {
+ log.error(s"fail to insert blob value: $key")
+ false
+ }
+ }
+ }
+
+ override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
+ lazy val messageForLog = s"${policy.service}.${policy.action}.$blobId"
+
+ policy.itemType match {
+ case ItemType.BLOB =>
+ withHBase(getTableName(policy)) { table =>
+ val rowKey = BytesUtilV1.toBytes(ExactKey(policy.id, policy.version, policy.itemType, blobId))
+ val get = new Get(rowKey)
+ get.addColumn(blobCF, blobColumn)
+ table.get(get)
+ } match {
+ case Success(result) =>
+ Option(result).filter(!_.isEmpty).map { rst =>
+ Bytes.toString(rst.getValue(blobCF, blobColumn))
+ }
+ case Failure(ex) =>
+ throw ex
+ }
+ case _ =>
+ log.warn(s"is not blob type counter. $messageForLog")
+ throw new Exception(s"is not blob type counter. $messageForLog")
+ }
+ }
+
+ override def prepare(policy: Counter): Unit = {
+ // create hbase table
+ policy.hbaseTable.foreach { table =>
+ if (!hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)) {
+ hbaseManagement.createTable(s2config.HBASE_ZOOKEEPER_QUORUM, table,
+ ColumnFamily.values.map(_.toString).toList, 1)
+ hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.SHORT.toString, policy.ttl)
+ policy.dailyTtl.foreach { i =>
+ hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.LONG.toString, i * 24 * 60 * 60)
+ }
+ }
+ }
+ }
+
+ override def destroy(policy: Counter): Unit = {
+
+ }
+
+ override def ready(policy: Counter): Boolean = {
+ policy.hbaseTable.map { table =>
+ hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)
+ }.getOrElse(true)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageHBase.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageHBase.scala
new file mode 100644
index 0000000..d6d87d3
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/ExactStorageHBase.scala
@@ -0,0 +1,326 @@
+package org.apache.s2graph.counter.core.v1
+
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.CellUtil
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.filter.{ColumnRangeFilter, FilterList}
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter.helper.{Management, WithHBase}
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.slf4j.LoggerFactory
+import scala.collection.JavaConversions._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+
+class ExactStorageHBase(config: Config) extends ExactStorage {
+ import ExactStorageHBase._
+
+ private val log = LoggerFactory.getLogger(getClass)
+
+ lazy val s2config = new S2CounterConfig(config)
+
+ private[counter] val withHBase = new WithHBase(config)
+ private[counter] val hbaseManagement = new Management(config)
+
+
+
+ private def getTableName(policy: Counter): String = {
+ policy.hbaseTable.getOrElse(s2config.HBASE_TABLE_NAME)
+ }
+
+ override def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
+ lazy val messageForLog = s"${policy.service}.${policy.action} $items $timeRange"
+
+ val keys = {
+ for {
+ item <- items
+ } yield {
+ ExactKey(policy, item, checkItemType = true)
+ }
+ }
+
+ val gets = {
+ for {
+ key <- keys
+ } yield {
+ val get = new Get(BytesUtilV1.toBytes(key))
+ timeRange.map(t => intervalsMap(t._1.q)).distinct.foreach { cf =>
+ get.addFamily(cf.toString.getBytes)
+ }
+ get.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ONE, {
+ for {
+ (from, to) <- timeRange
+ } yield {
+ new ColumnRangeFilter(
+ BytesUtilV1.toBytes(from), true,
+ BytesUtilV1.toBytes(to.copy(ts = to.ts + 1)), false)
+ }
+ }))
+ }
+ }
+
+ // println(s"$messageForLog $gets")
+ Future {
+ withHBase(getTableName(policy)) { table =>
+ for {
+ (rst, key) <- table.get(gets).zip(keys) if !rst.isEmpty
+ } yield {
+ val qualifierWithCounts = {
+ for {
+ cell <- rst.listCells()
+ eq = BytesUtilV1.toExactQualifier(CellUtil.cloneQualifier(cell))
+ } yield {
+ eq -> Bytes.toLong(CellUtil.cloneValue(cell))
+ }
+ }.toMap
+ FetchedCounts(key, qualifierWithCounts)
+ }
+ } match {
+ case Success(rst) => rst
+ case Failure(ex) =>
+ log.error(s"$ex: $messageForLog")
+ Nil
+ }
+ }
+ }
+
+ override def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
+ get(policy, items, timeRange).map { fetchedLs =>
+ for {
+ FetchedCounts(exactKey, qualifierWithCountMap) <- fetchedLs
+ } yield {
+ val intervalWithCountMap = qualifierWithCountMap
+ .filter { case (eq, v) => eq.checkDimensionEquality(dimQuery) }
+ .groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) }
+ FetchedCountsGrouped(exactKey, intervalWithCountMap)
+ }
+ }
+ }
+
+ override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
+ // increment mutation to hbase
+ val increments = {
+ for {
+ (exactKey, values) <- counts
+ inc = new Increment(BytesUtilV1.toBytes(exactKey))
+ } yield {
+ for {
+ (eq, value) <- values
+ } {
+ inc.addColumn(intervalsMap.apply(eq.tq.q).toString.getBytes, BytesUtilV1.toBytes(eq), value)
+ }
+ // add column by dimension
+ inc
+ }
+ }
+
+ val results: Array[Object] = Array.fill(increments.size)(null)
+
+ withHBase(getTableName(policy)) { table =>
+ table.batch(increments, results)
+ } match {
+ case Failure(ex) =>
+ log.error(s"${ex.getMessage}")
+ case _ =>
+ }
+
+ assert(counts.length == results.length)
+
+ for {
+ ((exactKey, eqWithValue), result) <- counts.zip(results)
+ } yield {
+ val eqWithResult = result match {
+ case r: Result =>
+ for {
+ (eq, value) <- eqWithValue
+ } yield {
+ val interval = eq.tq.q
+ val cf = intervalsMap(interval)
+ val result = Option(r.getColumnLatestCell(cf.toString.getBytes, BytesUtilV1.toBytes(eq))).map { cell =>
+ Bytes.toLong(CellUtil.cloneValue(cell))
+ }.getOrElse(-1l)
+ eq -> result
+ }
+ case ex: Throwable =>
+ log.error(s"${ex.getMessage}: $exactKey")
+ Nil
+ case _ =>
+ log.error(s"result is null: $exactKey")
+ Nil
+ }
+ (exactKey, eqWithResult.toMap)
+ }
+ }.toMap
+
+ override def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
+ withHBase(getTableName(policy)) { table =>
+ table.delete {
+ for {
+ key <- keys
+ } yield {
+ new Delete(BytesUtilV1.toBytes(key))
+ }
+ }
+ } match {
+ case Failure(ex) =>
+ log.error(ex.getMessage)
+ case _ =>
+ }
+ }
+
+ override def get(policy: Counter,
+ queries: Seq[(ExactKeyTrait, Seq[core.ExactQualifier])])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
+ lazy val messageForLog = s"${policy.service}.${policy.action} $queries"
+
+ val gets = {
+ for {
+ (key, eqs) <- queries
+ } yield {
+ // println(s"$key $eqsGrouped")
+ val get = new Get(BytesUtilV1.toBytes(key))
+
+ for {
+ eq <- eqs
+ } {
+ val cf = intervalsMap(eq.tq.q)
+ get.addColumn(cf.toString.getBytes, BytesUtilV1.toBytes(eq))
+ }
+ get
+ }
+ }
+
+ Future {
+ withHBase(getTableName(policy)) { table =>
+ for {
+ (rst, key) <- table.get(gets).zip(queries.map(_._1)) if !rst.isEmpty
+ } yield {
+ val qualifierWithCounts = {
+ for {
+ cell <- rst.listCells()
+ eq = BytesUtilV1.toExactQualifier(CellUtil.cloneQualifier(cell))
+ } yield {
+ eq -> Bytes.toLong(CellUtil.cloneValue(cell))
+ }
+ }.toMap
+ FetchedCounts(key, qualifierWithCounts)
+ }
+ } match {
+ case Success(rst) => rst.toSeq
+ case Failure(ex) =>
+ log.error(s"$ex: $messageForLog")
+ Nil
+ }
+ }
+ }
+
+ override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
+ val results: Array[Object] = Array.fill(keys.size)(null)
+
+ val puts = keys.map { key =>
+ val put = new Put(BytesUtilV1.toBytes(key))
+ put.addColumn(blobCF, blobColumn, key.itemId.getBytes)
+ }
+
+ withHBase(getTableName(policy)) { table =>
+ table.batch(puts, results)
+ } match {
+ case Failure(ex) =>
+ log.error(s"${ex.getMessage}")
+ case _ =>
+ }
+
+ for {
+ (result, key) <- results.zip(keys)
+ } yield {
+ Option(result).map(_ => true).getOrElse {
+ log.error(s"fail to insert blob value: $key")
+ false
+ }
+ }
+ }
+
+ override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
+ lazy val messageForLog = s"${policy.service}.${policy.action}.$blobId"
+
+ policy.itemType match {
+ case ItemType.BLOB =>
+ withHBase(getTableName(policy)) { table =>
+ val rowKey = BytesUtilV1.toBytes(ExactKey(policy.id, policy.version, policy.itemType, blobId))
+ val get = new Get(rowKey)
+ get.addColumn(blobCF, blobColumn)
+ table.get(get)
+ } match {
+ case Success(result) =>
+ Option(result).filter(!_.isEmpty).map { rst =>
+ Bytes.toString(rst.getValue(blobCF, blobColumn))
+ }
+ case Failure(ex) =>
+ throw ex
+ }
+ case _ =>
+ log.warn(s"is not blob type counter. $messageForLog")
+ throw new Exception(s"is not blob type counter. $messageForLog")
+ }
+ }
+
+ override def prepare(policy: Counter): Unit = {
+ // create hbase table
+ policy.hbaseTable.foreach { table =>
+
+ if (!hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)) {
+ hbaseManagement.createTable(s2config.HBASE_ZOOKEEPER_QUORUM, table,
+ ColumnFamily.values.map(_.toString).toList, 1)
+ hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.SHORT.toString, policy.ttl)
+ policy.dailyTtl.foreach { i =>
+ hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.LONG.toString, i * 24 * 60 * 60)
+ }
+ }
+ }
+ }
+
+ override def destroy(policy: Counter): Unit = {
+
+ }
+
+ override def ready(policy: Counter): Boolean = {
+ policy.hbaseTable.map { table =>
+ hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)
+ }.getOrElse(true)
+ }
+}
+
+object ExactStorageHBase {
+ import core.TimedQualifier.IntervalUnit._
+
+ object ColumnFamily extends Enumeration {
+ type ColumnFamily = Value
+
+ val SHORT = Value("s")
+ val LONG = Value("l")
+ }
+
+ val blobCF = ColumnFamily.LONG.toString.getBytes
+ val blobColumn = "b".getBytes
+
+ val intervalsMap = Map(
+ MINUTELY -> ColumnFamily.SHORT,
+ HOURLY -> ColumnFamily.SHORT,
+ DAILY -> ColumnFamily.LONG,
+ MONTHLY -> ColumnFamily.LONG,
+ TOTAL -> ColumnFamily.LONG
+ )
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/RankingStorageRedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/RankingStorageRedis.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/RankingStorageRedis.scala
new file mode 100644
index 0000000..15ff380
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v1/RankingStorageRedis.scala
@@ -0,0 +1,185 @@
+package org.apache.s2graph.counter.core.v1
+
+import java.lang
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core.{RankingResult, RankingKey, RankingStorage}
+import org.apache.s2graph.counter.helper.WithRedis
+import org.apache.s2graph.counter.models.{Counter, CounterModel}
+import org.slf4j.LoggerFactory
+import redis.clients.jedis.Pipeline
+import scala.collection.JavaConversions._
+import scala.util.{Failure, Success}
+
+class RankingStorageRedis(config: Config) extends RankingStorage {
+ private[counter] val log = LoggerFactory.getLogger(this.getClass)
+ private[counter] val withRedis = new WithRedis(config)
+
+ val counterModel = new CounterModel(config)
+
+ val TOTAL = "_total_"
+
+ /**
+ * ex1)
+ * dimension = "age.32"
+ * ex2)
+ * dimension = "age.gender.32.m"
+ *
+ */
+ private def makeBucket(rankingKey: RankingKey): String = {
+ val policyId = rankingKey.policyId
+ val q = rankingKey.eq.tq.q
+ val ts = rankingKey.eq.tq.ts
+ val dimension = rankingKey.eq.dimension
+ if (dimension.nonEmpty) {
+ s"$policyId.$q.$ts.$dimension"
+ }
+ else {
+ s"$policyId.$q.$ts"
+ }
+ }
+
+ override def getTopK(rankingKey: RankingKey, k: Int): Option[RankingResult] = {
+ val bucket = makeBucket(rankingKey)
+ withRedis.doBlockWithKey(bucket) { jedis =>
+ jedis.zrevrangeByScoreWithScores(bucket, "+inf", "-inf", 0, k + 1).toSeq.map(t => (t.getElement, t.getScore))
+ } match {
+ case Success(values) =>
+ if (values.nonEmpty) {
+// println(values)
+ Some(RankingResult(values.find(_._1 == TOTAL).map(_._2).getOrElse(-1d), values.filter(_._1 != TOTAL).take(k)))
+ }
+ else {
+ None
+ }
+ case Failure(ex) =>
+ log.error(s"fail to get top k($ex). $rankingKey")
+ None
+ }
+ }
+
+ private def getTTL(policyId: Int, intervalUnit: IntervalUnit.IntervalUnit): Option[Int] = {
+ counterModel.findById(policyId).flatMap { policy =>
+ intervalUnit match {
+ case IntervalUnit.MINUTELY => Some(policy.ttl)
+ case IntervalUnit.HOURLY => Some(policy.ttl)
+ // default daily ttl 31 day
+ case IntervalUnit.DAILY => Some(policy.dailyTtl.getOrElse(31) * 24 * 3600)
+ case IntervalUnit.MONTHLY => policy.dailyTtl
+ case IntervalUnit.TOTAL => policy.dailyTtl
+ }
+ }
+ }
+
+ override def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
+ // update ranking by score
+ val bucket = makeBucket(key)
+ withRedis.doBlockWithKey(bucket) { jedis =>
+ val pipeline = jedis.pipelined()
+ updateItem(pipeline, bucket, key, value, k)
+ pipeline.sync()
+ } match {
+ case Failure(ex) =>
+ log.error(s"fail to update $key $value: $ex")
+ case _ =>
+ }
+ }
+
+ private def updateItem(pipeline: Pipeline, bucket: String, key: RankingKey, value: RankingValueMap, k: Int): Unit = {
+ val topSeq = value.map { case (item, rv) =>
+ // jedis client accept only java's double
+ item -> rv.score.asInstanceOf[lang.Double]
+ }.toSeq.sortBy(_._2).takeRight(k)
+ pipeline.zadd(bucket, topSeq.toMap[String, lang.Double])
+ pipeline.zincrby(bucket, value.mapValues(_.increment).values.sum, TOTAL)
+ pipeline.zremrangeByRank(bucket, 0, -(k + 1))
+ // if ttl defined, set expire time to bucket
+ getTTL(key.policyId, key.eq.tq.q).foreach { ttl =>
+ pipeline.expire(bucket, ttl)
+ }
+ }
+
+ override def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
+ values.map { case (key, value) =>
+ (makeBucket(key), key, value)
+ }.groupBy { case (bucket, key, value) =>
+ withRedis.getBucketIdx(bucket)
+ }.foreach { case (idx, seq) =>
+ withRedis.doBlockWithIndex(idx) { jedis =>
+ val pipeline = jedis.pipelined()
+ for {
+ (bucket, key, value) <- seq
+ } {
+ updateItem(pipeline, bucket, key, value, k)
+ }
+ pipeline.sync()
+ } match {
+ case Failure(ex) =>
+ log.error(s"fail to update multi $idx: $ex")
+ case _ =>
+ }
+ }
+ }
+
+ override def delete(key: RankingKey): Unit = {
+ val bucket = makeBucket(key)
+ withRedis.doBlockWithKey(bucket) { jedis =>
+ jedis.del(bucket)
+ } match {
+ case Success(deleted) =>
+ log.info(s"success to delete $key")
+ case Failure(ex) =>
+ log.error(s"fail to delete $key")
+ }
+ }
+
+ override def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)] = {
+ keys.map { key =>
+ (makeBucket(key), key)
+ }.groupBy { case (bucket, key) =>
+ withRedis.getBucketIdx(bucket)
+ }.toSeq.par.flatMap { case (idx, seq) =>
+ withRedis.doBlockWithIndex(idx) { jedis =>
+ val pipeline = jedis.pipelined()
+ val keyWithRespLs = {
+ for {
+ (bucket, rankingKey) <- seq
+ } yield {
+ (rankingKey, pipeline.zrevrangeByScoreWithScores(bucket, "+inf", "-inf", 0, k + 1))
+ }
+ }
+ pipeline.sync()
+ for {
+ (rankingKey, resp) <- keyWithRespLs
+ } yield {
+ (rankingKey, resp.get().toSeq.map { t => (t.getElement, t.getScore)})
+ }
+ } match {
+ case Success(keyWithValues) =>
+ for {
+ (rankingKey, values) <- keyWithValues
+ } yield {
+ val result = RankingResult(values.find(_._1 == TOTAL).map(_._2).getOrElse(-1d), values.filter(_._1 != TOTAL).take(k))
+ (rankingKey, result)
+ }
+ case Failure(ex) =>
+ Nil
+ }
+ }
+ }.seq
+
+ override def prepare(policy: Counter): Unit = {
+ // do nothing
+ }
+
+ override def destroy(policy: Counter): Unit = {
+
+ }
+
+ override def ready(policy: Counter): Boolean = {
+ // always return true
+ true
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/BytesUtilV2.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/BytesUtilV2.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/BytesUtilV2.scala
new file mode 100644
index 0000000..a37506c
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/BytesUtilV2.scala
@@ -0,0 +1,88 @@
+package org.apache.s2graph.counter.core.v2
+
+import org.apache.hadoop.hbase.util._
+import org.apache.s2graph.counter
+import org.apache.s2graph.counter.core.TimedQualifier.IntervalUnit
+import org.apache.s2graph.counter.core.{TimedQualifier, ExactQualifier, ExactKeyTrait, BytesUtil}
+import org.apache.s2graph.counter.models.Counter.ItemType
+import org.apache.s2graph.counter.util.Hashes
+import scala.collection.mutable.ArrayBuffer
+
+object BytesUtilV2 extends BytesUtil {
+ // ExactKey: [hash(1b)][version(1b)][policy(4b)][item(variable)]
+ val BUCKET_BYTE_SIZE = Bytes.SIZEOF_BYTE
+ val VERSION_BYTE_SIZE = Bytes.SIZEOF_BYTE
+ val POLICY_ID_SIZE = Bytes.SIZEOF_INT
+
+ val INTERVAL_SIZE = Bytes.SIZEOF_BYTE
+ val TIMESTAMP_SIZE = Bytes.SIZEOF_LONG
+ val TIMED_QUALIFIER_SIZE = INTERVAL_SIZE + TIMESTAMP_SIZE
+
+ override def getRowKeyPrefix(id: Int): Array[Byte] = {
+ Array(counter.VERSION_2) ++ Bytes.toBytes(id)
+ }
+
+ override def toBytes(key: ExactKeyTrait): Array[Byte] = {
+ val buff = new ArrayBuffer[Byte]
+ // hash byte
+ buff ++= Bytes.toBytes(Hashes.murmur3(key.itemKey)).take(BUCKET_BYTE_SIZE)
+
+ // row key prefix
+ // version + policy id
+ buff ++= getRowKeyPrefix(key.policyId)
+
+ buff ++= {
+ key.itemType match {
+ case ItemType.INT => Bytes.toBytes(key.itemKey.toInt)
+ case ItemType.LONG => Bytes.toBytes(key.itemKey.toLong)
+ case ItemType.STRING | ItemType.BLOB => Bytes.toBytes(key.itemKey)
+ }
+ }
+ buff.toArray
+ }
+
+ override def toBytes(eq: ExactQualifier): Array[Byte] = {
+ val len = eq.dimKeyValues.map { case (k, v) => k.length + 2 + v.length + 2 }.sum
+ val pbr = new SimplePositionedMutableByteRange(len)
+ for {
+ v <- ExactQualifier.makeSortedDimension(eq.dimKeyValues)
+ } {
+ OrderedBytes.encodeString(pbr, v, Order.ASCENDING)
+ }
+ toBytes(eq.tq) ++ pbr.getBytes
+ }
+
+ override def toBytes(tq: TimedQualifier): Array[Byte] = {
+ val pbr = new SimplePositionedMutableByteRange(INTERVAL_SIZE + 2 + TIMESTAMP_SIZE + 1)
+ OrderedBytes.encodeString(pbr, tq.q.toString, Order.ASCENDING)
+ OrderedBytes.encodeInt64(pbr, tq.ts, Order.DESCENDING)
+ pbr.getBytes
+ }
+
+ private def decodeString(pbr: PositionedByteRange): Stream[String] = {
+ if (pbr.getRemaining > 0) {
+ Stream.cons(OrderedBytes.decodeString(pbr), decodeString(pbr))
+ }
+ else {
+ Stream.empty
+ }
+ }
+
+ override def toExactQualifier(bytes: Array[Byte]): ExactQualifier = {
+ val pbr = new SimplePositionedByteRange(bytes)
+ ExactQualifier(toTimedQualifier(pbr), {
+ val seqStr = decodeString(pbr).toSeq
+ val (keys, values) = seqStr.splitAt(seqStr.length / 2)
+ keys.zip(values).toMap
+ })
+ }
+
+ override def toTimedQualifier(bytes: Array[Byte]): TimedQualifier = {
+ val pbr = new SimplePositionedByteRange(bytes)
+ toTimedQualifier(pbr)
+ }
+
+ def toTimedQualifier(pbr: PositionedByteRange): TimedQualifier = {
+ TimedQualifier(IntervalUnit.withName(OrderedBytes.decodeString(pbr)), OrderedBytes.decodeInt64(pbr))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
new file mode 100644
index 0000000..522cf18
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
@@ -0,0 +1,343 @@
+package org.apache.s2graph.counter.core.v2
+
+import com.typesafe.config.Config
+import org.apache.http.HttpStatus
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core
+import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
+import org.apache.s2graph.counter.core._
+import org.apache.s2graph.counter.models.Counter
+import org.apache.s2graph.counter.util.CartesianProduct
+import org.slf4j.LoggerFactory
+import play.api.libs.json._
+import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext, Future}
+
+object ExactStorageGraph {
+ case class RespGraph(success: Boolean, result: Long)
+ implicit val respGraphFormat = Json.format[RespGraph]
+
+ // using play-ws without play app
+ private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
+}
+
+case class ExactStorageGraph(config: Config) extends ExactStorage {
+ private val log = LoggerFactory.getLogger(this.getClass)
+ private val s2config = new S2CounterConfig(config)
+
+ private val SERVICE_NAME = "s2counter"
+ private val COLUMN_NAME = "bucket"
+ private val labelPostfix = "_counts"
+
+ val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
+ val graphOp = new GraphOperation(config)
+
+ import ExactStorageGraph._
+
+ override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
+ import scala.concurrent.ExecutionContext.Implicits.global
+
+ val (keyWithEq, reqJsLs) = toIncrementCountRequests(policy, counts).unzip(x => ((x._1, x._2), x._3))
+
+ val future = wsClient.url(s"$s2graphUrl/graphs/edges/incrementCount").post(Json.toJson(reqJsLs)).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val respSeq = resp.json.as[Seq[RespGraph]]
+
+ val keyWithEqResult = {
+ for {
+ ((key, eq), RespGraph(success, result)) <- keyWithEq.zip(respSeq)
+ } yield {
+ (key, (eq, result))
+ }
+ }.groupBy(_._1).mapValues{ seq => seq.map(_._2).toMap }
+ keyWithEqResult
+ case _ =>
+ throw new RuntimeException(s"update failed: $policy $counts")
+ }
+ }
+ Await.result(future, 10 second)
+ }
+
+ def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
+
+ }
+
+ private def toIncrementCountRequests(policy: Counter,
+ counts: Seq[(ExactKeyTrait, ExactValueMap)])
+ : Seq[(ExactKeyTrait, core.ExactQualifier, JsValue)] = {
+ val labelName = policy.action + labelPostfix
+ val timestamp = System.currentTimeMillis()
+ for {
+ (exactKey, values) <- counts
+ (eq, value) <- values
+ } yield {
+ val from = exactKey.itemKey
+ val to = eq.dimension
+ val json = Json.obj(
+ "timestamp" -> timestamp,
+ "operation" -> "incrementCount",
+ "from" -> from,
+ "to" -> to,
+ "label" -> labelName,
+ "props" -> Json.obj(
+ "_count" -> value,
+ "time_unit" -> eq.tq.q.toString,
+ "time_value" -> eq.tq.ts
+ )
+ )
+ (exactKey, eq, json)
+ }
+ }
+
+ override def get(policy: Counter,
+ items: Seq[String],
+ timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)],
+ dimQuery: Map[String, Set[String]])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
+ val labelName = policy.action + labelPostfix
+ val label = Label.findByName(labelName).get
+// val label = labelModel.findByName(labelName).get
+
+ val ids = Json.toJson(items)
+
+ val dimensions = {
+ for {
+ values <- CartesianProduct(dimQuery.values.map(ss => ss.toList).toList)
+ } yield {
+ dimQuery.keys.zip(values).toMap
+ }
+ }
+
+ val stepJsLs = {
+ for {
+ (tqFrom, tqTo) <- timeRange
+ dimension <- dimensions
+ } yield {
+ val eqFrom = core.ExactQualifier(tqFrom, dimension)
+ val eqTo = core.ExactQualifier(tqTo, dimension)
+ val intervalJs =
+ s"""
+ |{
+ | "from": {
+ | "_to": "${eqFrom.dimension}",
+ | "time_unit": "${eqFrom.tq.q}",
+ | "time_value": ${eqFrom.tq.ts}
+ | },
+ | "to": {
+ | "_to": "${eqTo.dimension}",
+ | "time_unit": "${eqTo.tq.q}",
+ | "time_value": ${eqTo.tq.ts + 1}
+ | }
+ |}
+ """.stripMargin
+ val stepJs =
+ s"""
+ |{
+ | "direction": "out",
+ | "limit": -1,
+ | "duplicate": "raw",
+ | "label": "$labelName",
+ | "interval": $intervalJs
+ |}
+ """.stripMargin
+ stepJs
+ }
+ }
+
+ val reqJsStr =
+ s"""
+ |{
+ | "srcVertices": [
+ | {"serviceName": "${policy.service}", "columnName": "${label.srcColumnName}", "ids": $ids}
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | ${stepJsLs.mkString(",")}
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+
+ val reqJs = Json.parse(reqJsStr)
+// log.warn(s"query: ${reqJs.toString()}")
+
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(reqJs).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val respJs = resp.json
+// println(respJs)
+ val keyWithValues = (respJs \ "results").as[Seq[JsValue]].map { result =>
+// println(s"result: $result")
+ resultToExactKeyValues(policy, result)
+ }.groupBy(_._1).mapValues(seq => seq.map(_._2).toMap.groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) })
+ for {
+ (k, v) <- keyWithValues.toSeq
+ } yield {
+ FetchedCountsGrouped(k, v)
+ }
+ case n: Int =>
+ log.warn(s"getEdges status($n): $reqJsStr")
+// println(s"getEdges status($n): $reqJsStr")
+ Nil
+ }
+ }
+ }
+
+ private def resultToExactKeyValues(policy: Counter, result: JsValue): (ExactKeyTrait, (core.ExactQualifier, Long)) = {
+ val from = result \ "from" match {
+ case s: JsString => s.as[String]
+ case n: JsNumber => n.as[Long].toString
+ case x: JsValue => throw new RuntimeException(s"$x's type must be string or number")
+ }
+ val dimension = (result \ "to").as[String]
+ val props = result \ "props"
+ val count = (props \ "_count").as[Long]
+ val timeUnit = (props \ "time_unit").as[String]
+ val timeValue = (props \ "time_value").as[Long]
+ (ExactKey(policy, from, checkItemType = true), (core.ExactQualifier(core.TimedQualifier(timeUnit, timeValue), dimension), count))
+ }
+
+ private def getInner(policy: Counter, key: ExactKeyTrait, eqs: Seq[core.ExactQualifier])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
+ val labelName = policy.action + labelPostfix
+
+ Label.findByName(labelName) match {
+ case Some(label) =>
+
+ val src = Json.obj("serviceName" -> policy.service, "columnName" -> label.srcColumnName, "id" -> key.itemKey)
+ val step = {
+ val stepLs = {
+ for {
+ eq <- eqs
+ } yield {
+ val from = Json.obj("_to" -> eq.dimension, "time_unit" -> eq.tq.q.toString, "time_value" -> eq.tq.ts)
+ val to = Json.obj("_to" -> eq.dimension, "time_unit" -> eq.tq.q.toString, "time_value" -> eq.tq.ts)
+ val interval = Json.obj("from" -> from, "to" -> to)
+ Json.obj("limit" -> 1, "label" -> labelName, "interval" -> interval)
+ }
+ }
+ Json.obj("step" -> stepLs)
+ }
+ val query = Json.obj("srcVertices" -> Json.arr(src), "steps" -> Json.arr(step))
+ // println(s"query: ${query.toString()}")
+
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(query).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val respJs = resp.json
+ val keyWithValues = (respJs \ "results").as[Seq[JsValue]].map { result =>
+ resultToExactKeyValues(policy, result)
+ }.groupBy(_._1).mapValues(seq => seq.map(_._2).toMap)
+ for {
+ (key, eqWithValues) <- keyWithValues.toSeq
+ } yield {
+ FetchedCounts(key, eqWithValues)
+ }
+ case _ =>
+ Nil
+ }
+ }
+ case None =>
+ Future.successful(Nil)
+ }
+ }
+
+ // for query exact qualifier
+ override def get(policy: Counter, queries: Seq[(ExactKeyTrait, Seq[core.ExactQualifier])])
+ (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
+ val futures = {
+ for {
+ (key, eqs) <- queries
+ } yield {
+// println(s"$key $eqs")
+ getInner(policy, key, eqs)
+ }
+ }
+ Future.sequence(futures).map(_.flatten)
+ }
+
+ override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
+ throw new RuntimeException("unsupported getBlobValue operation")
+ }
+
+ override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
+ throw new RuntimeException("unsupported insertBlobValue operation")
+ }
+
+ private def existsLabel(policy: Counter, useCache: Boolean = true): Boolean = {
+ val action = policy.action
+ val counterLabelName = action + labelPostfix
+
+ Label.findByName(counterLabelName, useCache).nonEmpty
+ }
+
+ override def prepare(policy: Counter): Unit = {
+ val service = policy.service
+ val action = policy.action
+
+ val graphLabel = Label.findByName(action)
+ if (graphLabel.isEmpty) {
+ throw new Exception(s"label not found. $service.$action")
+ }
+
+ if (!existsLabel(policy, useCache = false)) {
+ val label = Label.findByName(action, useCache = false).get
+
+ val counterLabelName = action + labelPostfix
+ val defaultJson =
+ s"""
+ |{
+ | "label": "$counterLabelName",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "${label.tgtColumnName}",
+ | "srcColumnType": "${label.tgtColumnType}",
+ | "tgtServiceName": "$SERVICE_NAME",
+ | "tgtColumnName": "$COLUMN_NAME",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | {"name": "time", "propNames": ["_to", "time_unit", "time_value"]}
+ | ],
+ | "props": [
+ | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
+ | {"name": "time_value", "dataType": "long", "defaultValue": 0}
+ | ],
+ | "hTableName": "${policy.hbaseTable.get}"
+ |}
+ """.stripMargin
+ val json = policy.dailyTtl.map(ttl => ttl * 24 * 60 * 60) match {
+ case Some(ttl) =>
+ Json.parse(defaultJson).as[JsObject] + ("hTableTTL" -> Json.toJson(ttl))
+ case None =>
+ Json.parse(defaultJson)
+ }
+
+ graphOp.createLabel(json)
+ }
+ }
+
+ override def destroy(policy: Counter): Unit = {
+ val action = policy.action
+
+ if (existsLabel(policy, useCache = false)) {
+ val counterLabelName = action + labelPostfix
+
+ graphOp.deleteLabel(counterLabelName)
+ }
+ }
+
+ override def ready(policy: Counter): Boolean = {
+ existsLabel(policy)
+ }
+
+ // for range query
+ override def get(policy: Counter, items: Seq[String], timeRange: Seq[(core.TimedQualifier, core.TimedQualifier)])
+ (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
+ throw new NotImplementedError("Not implemented")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
new file mode 100644
index 0000000..d1bb2ef
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
@@ -0,0 +1,48 @@
+package org.apache.s2graph.counter.core.v2
+
+import com.typesafe.config.Config
+import org.apache.http.HttpStatus
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.slf4j.LoggerFactory
+import play.api.libs.json.{JsObject, JsValue, Json}
+import scala.concurrent.Await
+import scala.concurrent.duration._
+
+class GraphOperation(config: Config) {
+ // using play-ws without play app
+ private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
+ private val s2config = new S2CounterConfig(config)
+ val s2graphUrl = s2config.GRAPH_URL
+ private[counter] val log = LoggerFactory.getLogger(this.getClass)
+
+ import scala.concurrent.ExecutionContext.Implicits.global
+
+ def createLabel(json: JsValue): Boolean = {
+ // fix counter label's schemaVersion
+ val newJson = json.as[JsObject] ++ Json.obj("schemaVersion" -> "v2")
+ val future = wsClient.url(s"$s2graphUrl/graphs/createLabel").post(newJson).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ true
+ case _ =>
+ throw new RuntimeException(s"failed createLabel. errCode: ${resp.status} body: ${resp.body} query: $json")
+ }
+ }
+
+ Await.result(future, 10 second)
+ }
+
+ def deleteLabel(label: String): Boolean = {
+ val future = wsClient.url(s"$s2graphUrl/graphs/deleteLabel/$label").put("").map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ true
+ case _ =>
+ throw new RuntimeException(s"failed deleteLabel. errCode: ${resp.status} body: ${resp.body}")
+ }
+ }
+
+ Await.result(future, 10 second)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
new file mode 100644
index 0000000..add4b04
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
@@ -0,0 +1,395 @@
+package org.apache.s2graph.counter.core.v2
+
+import com.typesafe.config.Config
+import org.apache.commons.httpclient.HttpStatus
+import org.apache.s2graph.core.GraphUtil
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.core.{RankingResult, RankingKey, RankingStorage}
+import org.apache.s2graph.counter.models.{Counter, CounterModel}
+import org.apache.s2graph.counter.util.{CollectionCacheConfig, CollectionCache}
+import org.slf4j.LoggerFactory
+import play.api.libs.json.{JsObject, JsString, JsValue, Json}
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Future}
+import scala.util.hashing.MurmurHash3
+
+object RankingStorageGraph {
+ // using play-ws without play app
+ private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
+}
+
+class RankingStorageGraph(config: Config) extends RankingStorage {
+ import RankingStorageGraph._
+
+ private[counter] val log = LoggerFactory.getLogger(this.getClass)
+ private val s2config = new S2CounterConfig(config)
+
+ private val BUCKET_SHARD_COUNT = 53
+ private val SERVICE_NAME = "s2counter"
+ private val BUCKET_COLUMN_NAME = "bucket"
+ private val counterModel = new CounterModel(config)
+ private val labelPostfix = "_topK"
+
+ val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
+
+ val prepareCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(10000, 600))
+ val graphOp = new GraphOperation(config)
+ import scala.concurrent.ExecutionContext.Implicits.global
+
+ private def makeBucketKey(rankingKey: RankingKey): String = {
+ val eq = rankingKey.eq
+ val tq = eq.tq
+ s"${tq.q}.${tq.ts}.${eq.dimension}"
+ }
+
+ // "", "age.32", "age.gender.32.M"
+ private def makeBucketShardKey(shardIdx: Int, rankingKey: RankingKey): String = {
+ s"$shardIdx.${makeBucketKey(rankingKey)}"
+ }
+
+ /**
+ * indexProps: ["time_unit", "time_value", "score"]
+ */
+ override def getTopK(key: RankingKey, k: Int): Option[RankingResult] = {
+ getTopK(Seq(key), k).headOption.map(_._2)
+ }
+
+ override def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)] = {
+ val futures = for {
+ key <- keys
+ } yield {
+ getEdges(key).map { edges =>
+ key -> RankingResult(0d, toWithScoreLs(edges).take(k))
+ }
+ }
+
+ Await.result(Future.sequence(futures), 10 seconds)
+ }
+
+ override def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
+ update(Seq((key, value)), k)
+ }
+
+ override def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
+ val futures = {
+ for {
+ (key, value) <- values
+ } yield {
+ // prepare dimension bucket edge
+ checkAndPrepareDimensionBucket(key)
+
+ val future = getEdges(key, "raw").flatMap { edges =>
+ val prevRankingSeq = toWithScoreLs(edges)
+ val prevRankingMap: Map[String, Double] = prevRankingSeq.groupBy(_._1).map(_._2.sortBy(-_._2).head)
+ val currentRankingMap: Map[String, Double] = value.mapValues(_.score)
+ val mergedRankingSeq = (prevRankingMap ++ currentRankingMap).toSeq.sortBy(-_._2).take(k)
+ val mergedRankingMap = mergedRankingSeq.toMap
+
+ val bucketRankingSeq = mergedRankingSeq.groupBy { case (itemId, score) =>
+ // 0-index
+ GraphUtil.transformHash(MurmurHash3.stringHash(itemId)) % BUCKET_SHARD_COUNT
+ }.map { case (shardIdx, groupedRanking) =>
+ shardIdx -> groupedRanking.filter { case (itemId, _) => currentRankingMap.contains(itemId) }
+ }.toSeq
+
+ insertBulk(key, bucketRankingSeq).flatMap { _ =>
+ val duplicatedItems = prevRankingMap.filterKeys(s => currentRankingMap.contains(s))
+ val cutoffItems = prevRankingMap.filterKeys(s => !mergedRankingMap.contains(s))
+ val deleteItems = duplicatedItems ++ cutoffItems
+
+ val keyWithEdgesLs = prevRankingSeq.map(_._1).zip(edges)
+ val deleteEdges = keyWithEdgesLs.filter{ case (s, _) => deleteItems.contains(s) }.map(_._2)
+
+ deleteAll(deleteEdges)
+ }
+ }
+
+ future
+ }
+ }
+
+ Await.result(Future.sequence(futures), 10 seconds)
+ }
+
+ private def toWithScoreLs(edges: List[JsValue]): List[(String, Double)] = {
+ for {
+ edgeJson <- edges
+ to = (edgeJson \ "to").as[JsValue]
+ score = (edgeJson \ "score").as[JsValue].toString().toDouble
+ } yield {
+ val toValue = to match {
+ case s: JsString => s.as[String]
+ case _ => to.toString()
+ }
+ toValue -> score
+ }
+ }
+
+ private def insertBulk(key: RankingKey, newRankingSeq: Seq[(Int, Seq[(String, Double)])]): Future[Boolean] = {
+ val labelName = counterModel.findById(key.policyId).get.action + labelPostfix
+ val timestamp: Long = System.currentTimeMillis
+ val payload = Json.toJson {
+ for {
+ (shardIdx, rankingSeq) <- newRankingSeq
+ (itemId, score) <- rankingSeq
+ } yield {
+ val srcId = makeBucketShardKey(shardIdx, key)
+ Json.obj(
+ "timestamp" -> timestamp,
+ "from" -> srcId,
+ "to" -> itemId,
+ "label" -> labelName,
+ "props" -> Json.obj(
+ "time_unit" -> key.eq.tq.q.toString,
+ "time_value" -> key.eq.tq.ts,
+ "date_time" -> key.eq.tq.dateTime,
+ "score" -> score
+ )
+ )
+ }
+ }
+
+ wsClient.url(s"$s2graphUrl/graphs/edges/insertBulk").post(payload).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ true
+ case _ =>
+ throw new RuntimeException(s"failed insertBulk. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
+ }
+ }
+ }
+
+ private def deleteAll(edges: List[JsValue]): Future[Boolean] = {
+ // /graphs/edges/delete
+ val futures = {
+ for {
+ groupedEdges <- edges.grouped(50)
+ } yield {
+ val payload = Json.toJson(groupedEdges)
+ wsClient.url(s"$s2graphUrl/graphs/edges/delete").post(payload).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ true
+ case _ =>
+ log.error(s"failed delete. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
+ false
+ }
+ }
+ }
+ }.toSeq
+
+ Future.sequence(futures).map { seq =>
+ seq.forall(x => x)
+ }
+ }
+
+ /** select and delete */
+ override def delete(key: RankingKey): Unit = {
+ val future = getEdges(key).flatMap { edges =>
+ deleteAll(edges)
+ }
+ Await.result(future, 10 second)
+ }
+
+ private def getEdges(key: RankingKey, duplicate: String="first"): Future[List[JsValue]] = {
+ val labelName = counterModel.findById(key.policyId).get.action + labelPostfix
+
+ val ids = {
+ (0 until BUCKET_SHARD_COUNT).map { shardIdx =>
+ s""""${makeBucketShardKey(shardIdx, key)}""""
+ }
+ }.mkString(",")
+
+ val strJs =
+ s"""
+ |{
+ | "srcVertices": [
+ | {
+ | "serviceName": "$SERVICE_NAME",
+ | "columnName": "$BUCKET_COLUMN_NAME",
+ | "ids": [$ids]
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$labelName",
+ | "duplicate": "$duplicate",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": -1,
+ | "interval": {
+ | "from": {"time_unit": "${key.eq.tq.q.toString}", "time_value": ${key.eq.tq.ts}},
+ | "to": {"time_unit": "${key.eq.tq.q.toString}", "time_value": ${key.eq.tq.ts}}
+ | },
+ | "scoring": {"score": 1}
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ log.debug(strJs)
+
+ val payload = Json.parse(strJs)
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ (resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
+ case _ =>
+ throw new RuntimeException(s"failed getEdges. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
+ }
+ }
+ }
+
+ private def existsLabel(policy: Counter, useCache: Boolean = true): Boolean = {
+ val action = policy.action
+ val counterLabelName = action + labelPostfix
+
+ Label.findByName(counterLabelName, useCache).nonEmpty
+ }
+
+ private def checkAndPrepareDimensionBucket(rankingKey: RankingKey): Boolean = {
+ val dimension = rankingKey.eq.dimension
+ val bucketKey = makeBucketKey(rankingKey)
+ val labelName = "s2counter_topK_bucket"
+
+ val prepared = prepareCache.withCache(s"$dimension:$bucketKey") {
+ val checkReqJs = Json.arr(
+ Json.obj(
+ "label" -> labelName,
+ "direction" -> "out",
+ "from" -> dimension,
+ "to" -> makeBucketShardKey(BUCKET_SHARD_COUNT - 1, rankingKey)
+ )
+ )
+
+ val future = wsClient.url(s"$s2graphReadOnlyUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ val checkRespJs = resp.json
+ if (checkRespJs.as[Seq[JsValue]].nonEmpty) {
+ true
+ } else {
+ false
+ }
+ case _ =>
+ // throw exception
+ throw new Exception(s"failed checkEdges. ${resp.body} ${resp.status}")
+ }
+ }.flatMap {
+ case true => Future.successful(Some(true))
+ case false =>
+ val insertReqJsLs = {
+ for {
+ i <- 0 until BUCKET_SHARD_COUNT
+ } yield {
+ Json.obj(
+ "timestamp" -> rankingKey.eq.tq.ts,
+ "from" -> dimension,
+ "to" -> makeBucketShardKey(i, rankingKey),
+ "label" -> labelName,
+ "props" -> Json.obj(
+ "time_unit" -> rankingKey.eq.tq.q.toString,
+ "date_time" -> rankingKey.eq.tq.dateTime
+ )
+ )
+ }
+ }
+ wsClient.url(s"$s2graphUrl/graphs/edges/insert").post(Json.toJson(insertReqJsLs)).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ Some(true)
+ case _ =>
+ // throw exception
+ throw new Exception(s"failed insertEdges. ${resp.body} ${resp.status}")
+ }
+ }
+ }.recover {
+ case e: Exception =>
+ None
+ }
+
+ Await.result(future, 10 second)
+ }
+ prepared.getOrElse(false)
+ }
+
+ override def prepare(policy: Counter): Unit = {
+ val service = policy.service
+ val action = policy.action
+
+ val graphLabel = {
+ policy.rateActionId match {
+ case Some(rateId) =>
+ counterModel.findById(rateId, useCache = false).flatMap { ratePolicy =>
+ Label.findByName(ratePolicy.action)
+ }
+ case None =>
+ Label.findByName(action)
+ }
+ }
+ if (graphLabel.isEmpty) {
+ throw new Exception(s"label not found. $service.$action")
+ }
+
+ if (!existsLabel(policy, useCache = false)) {
+ // find input label to specify target column
+ val inputLabelName = policy.rateActionId.flatMap { id =>
+ counterModel.findById(id, useCache = false).map(_.action)
+ }.getOrElse(action)
+ val label = graphLabel.get
+
+ val counterLabelName = action + labelPostfix
+ val defaultJson =
+ s"""
+ |{
+ | "label": "$counterLabelName",
+ | "srcServiceName": "$SERVICE_NAME",
+ | "srcColumnName": "$BUCKET_COLUMN_NAME",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "${label.tgtColumnName}",
+ | "tgtColumnType": "${label.tgtColumnType}",
+ | "indices": [
+ | {"name": "time", "propNames": ["time_unit", "time_value", "score"]}
+ | ],
+ | "props": [
+ | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
+ | {"name": "time_value", "dataType": "long", "defaultValue": 0},
+ | {"name": "date_time", "dataType": "long", "defaultValue": 0},
+ | {"name": "score", "dataType": "float", "defaultValue": 0.0}
+ | ],
+ | "hTableName": "${policy.hbaseTable.get}"
+ |}
+ """.stripMargin
+ val json = policy.dailyTtl.map(ttl => ttl * 24 * 60 * 60) match {
+ case Some(ttl) =>
+ Json.parse(defaultJson).as[JsObject] + ("hTableTTL" -> Json.toJson(ttl))
+ case None =>
+ Json.parse(defaultJson)
+ }
+
+ graphOp.createLabel(json)
+ }
+ }
+
+ override def destroy(policy: Counter): Unit = {
+ val action = policy.action
+
+ if (existsLabel(policy, useCache = false)) {
+ val counterLabelName = action + labelPostfix
+
+ graphOp.deleteLabel(counterLabelName)
+ }
+ }
+
+ override def ready(policy: Counter): Boolean = {
+ existsLabel(policy)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/DecayFormula.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/DecayFormula.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/DecayFormula.scala
new file mode 100644
index 0000000..4e5da90
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/DecayFormula.scala
@@ -0,0 +1,6 @@
+package org.apache.s2graph.counter.decay
+
+trait DecayFormula {
+ def apply(value: Double, millis: Long): Double
+ def apply(value: Double, seconds: Int): Double
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/ExpDecayFormula.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/ExpDecayFormula.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/ExpDecayFormula.scala
new file mode 100644
index 0000000..1075421
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/decay/ExpDecayFormula.scala
@@ -0,0 +1,25 @@
+package org.apache.s2graph.counter.decay
+
+case class ExpDecayFormula(halfLifeInMillis: Double) extends DecayFormula {
+ val decayRate = - Math.log(2) / halfLifeInMillis
+
+ override def apply(value: Double, millis: Long): Double = {
+ value * Math.pow(Math.E, decayRate * millis)
+ }
+
+ override def apply(value: Double, seconds: Int): Double = {
+ apply(value, seconds * 1000L)
+ }
+}
+
+object ExpDecayFormula {
+ @deprecated("do not use. just experimental", "0.14")
+ def byWindowTime(windowInMillis: Long, pct: Double): ExpDecayFormula = {
+ val halfLife = windowInMillis * Math.log(0.5) / Math.log(pct)
+ ExpDecayFormula(halfLife)
+ }
+
+ def byMeanLifeTime(millis: Long): ExpDecayFormula = {
+ ExpDecayFormula(millis * Math.log(2))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/CounterAdmin.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/CounterAdmin.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/CounterAdmin.scala
new file mode 100644
index 0000000..cae2245
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/CounterAdmin.scala
@@ -0,0 +1,113 @@
+package org.apache.s2graph.counter.helper
+
+import com.typesafe.config.Config
+import org.apache
+import org.apache.s2graph.core.Graph
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.{RankingCounter, ExactCounter}
+import org.apache.s2graph.counter.core.v1.{RankingStorageRedis, ExactStorageAsyncHBase}
+import org.apache.s2graph.counter.core.v2.{RankingStorageGraph, ExactStorageGraph, GraphOperation}
+import org.apache.s2graph.counter.models.{Counter, CounterModel}
+import play.api.libs.json.Json
+
+import scala.util.Try
+
+class CounterAdmin(config: Config) {
+ val s2config = new S2CounterConfig(config)
+ val counterModel = new CounterModel(config)
+ val graphOp = new GraphOperation(config)
+ val s2graph = new Graph(config)(scala.concurrent.ExecutionContext.global)
+ val storageManagement = new org.apache.s2graph.core.Management(s2graph)
+
+ def setupCounterOnGraph(): Unit = {
+ // create s2counter service
+ val service = "s2counter"
+ storageManagement.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"$service-${config.getString("phase")}", 1, None, "gz")
+ // create bucket label
+ val label = "s2counter_topK_bucket"
+ if (Label.findByName(label, useCache = false).isEmpty) {
+ val strJs =
+ s"""
+ |{
+ | "label": "$label",
+ | "srcServiceName": "s2counter",
+ | "srcColumnName": "dimension",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "s2counter",
+ | "tgtColumnName": "bucket",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | {"name": "time", "propNames": ["time_unit", "date_time"]}
+ | ],
+ | "props": [
+ | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
+ | {"name": "date_time", "dataType": "long", "defaultValue": 0}
+ | ],
+ | "hTableName": "s2counter_60",
+ | "hTableTTL": 5184000
+ |}
+ """.stripMargin
+ graphOp.createLabel(Json.parse(strJs))
+ }
+ }
+
+ def createCounter(policy: Counter): Unit = {
+ val newPolicy = policy.copy(hbaseTable = Some(makeHTableName(policy)))
+ prepareStorage(newPolicy)
+ counterModel.createServiceAction(newPolicy)
+ }
+
+ def deleteCounter(service: String, action: String): Option[Try[Unit]] = {
+ for {
+ policy <- counterModel.findByServiceAction(service, action, useCache = false)
+ } yield {
+ Try {
+ exactCounter(policy).destroy(policy)
+ if (policy.useRank) {
+ rankingCounter(policy).destroy(policy)
+ }
+ counterModel.deleteServiceAction(policy)
+ }
+ }
+ }
+
+ def prepareStorage(policy: Counter): Unit = {
+ if (policy.rateActionId.isEmpty) {
+ // if defined rate action, do not use exact counter
+ exactCounter(policy).prepare(policy)
+ }
+ if (policy.useRank) {
+ rankingCounter(policy).prepare(policy)
+ }
+ }
+
+ def prepareStorage(policy: Counter, version: Byte): Unit = {
+ // this function to prepare storage by version parameter instead of policy.version
+ prepareStorage(policy.copy(version = version))
+ }
+
+ private val exactCounterMap = Map(
+ counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
+ counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
+ )
+ private val rankingCounterMap = Map(
+ apache.s2graph.counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
+ apache.s2graph.counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
+ )
+
+ private val tablePrefixMap = Map (
+ apache.s2graph.counter.VERSION_1 -> "s2counter",
+ apache.s2graph.counter.VERSION_2 -> "s2counter_v2"
+ )
+
+ def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
+ def exactCounter(policy: Counter): ExactCounter = exactCounter(policy.version)
+ def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
+ def rankingCounter(policy: Counter): RankingCounter = rankingCounter(policy.version)
+
+ def makeHTableName(policy: Counter): String = {
+ Seq(tablePrefixMap(policy.version), policy.service, policy.ttl) ++ policy.dailyTtl mkString "_"
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
new file mode 100644
index 0000000..c06cabe
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
@@ -0,0 +1,70 @@
+package org.apache.s2graph.counter.helper
+
+import java.util
+import java.util.Comparator
+
+import com.google.common.primitives.SignedBytes
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.util.Bytes
+
+object DistributedScanner {
+ val BUCKET_BYTE_SIZE = Bytes.SIZEOF_BYTE
+
+ def getRealRowKey(result: Result): Array[Byte] = {
+ result.getRow.drop(BUCKET_BYTE_SIZE)
+ }
+ }
+
+class DistributedScanner(table: Table, scan: Scan) extends AbstractClientScanner {
+ import DistributedScanner._
+
+ private val BYTE_MAX = BigInt(256)
+
+ private[helper] val scanners = {
+ for {
+ i <- 0 until BYTE_MAX.pow(BUCKET_BYTE_SIZE).toInt
+ } yield {
+ val bucketBytes: Array[Byte] = Bytes.toBytes(i).takeRight(BUCKET_BYTE_SIZE)
+ val newScan = new Scan(scan).setStartRow(bucketBytes ++ scan.getStartRow).setStopRow(bucketBytes ++ scan.getStopRow)
+ table.getScanner(newScan)
+ }
+ }
+
+ val resultCache = new util.TreeMap[Result, java.util.Iterator[Result]](new Comparator[Result] {
+ val comparator = SignedBytes.lexicographicalComparator()
+ override def compare(o1: Result, o2: Result): Int = {
+ comparator.compare(getRealRowKey(o1), getRealRowKey(o2))
+ }
+ })
+
+ lazy val initialized = {
+ val iterators = scanners.map(_.iterator()).filter(_.hasNext)
+ iterators.foreach { it =>
+ resultCache.put(it.next(), it)
+ }
+ iterators.nonEmpty
+ }
+
+ override def next(): Result = {
+ if (initialized) {
+ Option(resultCache.pollFirstEntry()).map { entry =>
+ val it = entry.getValue
+ if (it.hasNext) {
+ // fill cache
+ resultCache.put(it.next(), it)
+ }
+ entry.getKey
+ }.orNull
+ } else {
+ null
+ }
+ }
+
+ override def close(): Unit = {
+ for {
+ scanner <- scanners
+ } {
+ scanner.close()
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/HashShardingJedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/HashShardingJedis.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/HashShardingJedis.scala
new file mode 100644
index 0000000..f970c34
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/HashShardingJedis.scala
@@ -0,0 +1,153 @@
+package org.apache.s2graph.counter.helper
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.util.Hashes
+import org.slf4j.LoggerFactory
+import redis.clients.jedis.exceptions.JedisException
+import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
+
+class HashShardingJedis(config: Config) {
+ lazy val s2config = new S2CounterConfig(config)
+
+ private val log = LoggerFactory.getLogger(getClass)
+
+ val poolConfig = new JedisPoolConfig()
+ poolConfig.setMaxTotal(150)
+ poolConfig.setMaxIdle(50)
+ poolConfig.setMaxWaitMillis(200)
+
+ val jedisPools = s2config.REDIS_INSTANCES.map { case (host, port) =>
+ new JedisPool(poolConfig, host, port)
+ }
+ val jedisPoolSize = jedisPools.size
+
+ def getJedisPool(idx: Int): JedisPool = {
+ if(idx >= jedisPoolSize)
+ null
+ else
+ jedisPools(idx)
+ }
+
+ def getJedisPoolWithBucketname2(bucket: String): JedisPool = {
+ val hashedValue = Hashes.murmur3(bucket)
+ val idx = hashedValue % jedisPoolSize
+ getJedisPool(idx)
+ }
+
+ def getJedisPoolWithBucketname(bucket: String): (JedisPool, JedisPool) = {
+ val hashedValue = Hashes.murmur3(bucket)
+ val idx = hashedValue % jedisPoolSize
+ val secondaryIdx = if (jedisPoolSize <= 1) {
+ throw new Exception("too small sharding pool <= 1")
+ } else {
+ val newIdx = (hashedValue / jedisPoolSize) % (jedisPoolSize -1)
+ if(newIdx < idx) {
+ newIdx
+ } else {
+ newIdx +1
+ }
+ }
+ (getJedisPool(idx), getJedisPool(secondaryIdx))
+ }
+
+ def doBlockWithJedisInstace(f : Jedis => Any, fallBack : => Any, jedis : Jedis) = {
+ try {
+ f(jedis)
+ }
+ catch {
+ case e:JedisException => {
+ fallBack
+ }
+ }
+ }
+
+ def doBlockWithBucketName(f : Jedis => Any, fallBack : => Any, bucket : String) = {
+ // Logger.debug(s"start jedis do block")
+ //val (jedis_pool1, jedis_pool2) = getJedisPoolWithBucketname(bucket)
+ val jedis_pool1= getJedisPoolWithBucketname2(bucket)
+ // if(jedis_pool1 != null && jedis_pool2 != null) {
+ if(jedis_pool1 != null) {
+ var jedis1: Jedis = null
+ // var jedis2: Jedis = null
+ try {
+ jedis1 = jedis_pool1.getResource()
+ // jedis2 = jedis_pool2.getResource()
+ log.info(s">> Jedis Pool Active Num : ${jedis_pool1.getNumActive}")
+
+ /* val f1 = Future(f(jedis1))
+ val f2 = Future(f(jedis2))
+
+ val mixedFuture = Future.sequence(List(f1,f2)) */
+
+ val r1 = f(jedis1)
+ //val r2 = f(jedis2)
+
+ r1
+ }
+ catch {
+ case e:JedisException => {
+ // Logger.debug(s"following exception catched")
+ // Logger.debug(s"${e}")
+ jedis_pool1.returnBrokenResource(jedis1)
+ // jedis_pool2.returnBrokenResource(jedis2)
+
+ jedis1 = null
+ // jedis2 = null
+ fallBack
+ }
+ }
+ finally {
+ if (jedis1 != null) jedis_pool1.returnResource(jedis1)
+ // if (jedis2 != null) jedis_pool2.returnResource(jedis2)
+ }
+ }
+ else{
+ // Logger.debug(s"fallback executed")
+ fallBack
+ }
+ }
+
+ def doBlockWithKey[T](key: String)(f: Jedis => T)(fallBack: => T) = {
+ // Logger.debug(s"start jedis do block")
+ val (jedis_pool1, jedis_pool2) = getJedisPoolWithBucketname(key)
+ if(jedis_pool1 != null && jedis_pool2 != null) {
+ var jedis1: Jedis = null
+ var jedis2: Jedis = null
+ try {
+ jedis1 = jedis_pool1.getResource()
+ jedis2 = jedis_pool2.getResource()
+
+ /* val f1 = Future(f(jedis1))
+ val f2 = Future(f(jedis2))
+
+ val mixedFuture = Future.sequence(List(f1,f2)) */
+
+ val r1 = f(jedis1)
+ //val r2 = f(jedis2)
+
+ r1
+ }
+ catch {
+ case e:JedisException => {
+ // Logger.debug(s"following exception catched")
+ // Logger.debug(s"${e}")
+ jedis_pool1.returnBrokenResource(jedis1)
+ jedis_pool2.returnBrokenResource(jedis2)
+
+ jedis1 = null
+ jedis2 = null
+ fallBack
+ }
+ }
+ finally {
+ if (jedis1 != null) jedis_pool1.returnResource(jedis1)
+ if (jedis2 != null) jedis_pool2.returnResource(jedis2)
+ }
+ }
+ else{
+ // Logger.debug(s"fallback executed")
+ fallBack
+ }
+ }
+ }
[26/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/JavaHBaseContext.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/JavaHBaseContext.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/JavaHBaseContext.scala
new file mode 100644
index 0000000..edfc635
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/JavaHBaseContext.scala
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.client.{Connection, Delete, Get, Put, Result, Scan}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
+import org.apache.spark.api.java.function.{FlatMapFunction, Function, VoidFunction}
+import org.apache.spark.streaming.api.java.JavaDStream
+
+import scala.collection.JavaConversions._
+import scala.reflect.ClassTag
+
+/**
+ * This is the Java Wrapper over HBaseContext which is written in
+ * Scala. This class will be used by developers that want to
+ * work with Spark or Spark Streaming in Java
+ *
+ * @param jsc This is the JavaSparkContext that we will wrap
+ * @param config This is the config information to out HBase cluster
+ */
+class JavaHBaseContext(@transient jsc: JavaSparkContext,
+ @transient config: Configuration) extends Serializable {
+ val hbaseContext = new HBaseContext(jsc.sc, config)
+
+ /**
+ * A simple enrichment of the traditional Spark javaRdd foreachPartition.
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * @param javaRdd Original javaRdd with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the RDD values and a HConnection object to interact
+ * with HBase
+ */
+ def foreachPartition[T](javaRdd: JavaRDD[T],
+ f: VoidFunction[(java.util.Iterator[T], Connection)]) = {
+
+ hbaseContext.foreachPartition(javaRdd.rdd,
+ (it: Iterator[T], conn: Connection) => {
+ f.call((it, conn))
+ })
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark Streaming dStream foreach
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * @param javaDstream Original DStream with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the JavaDStream values and a HConnection object to
+ * interact with HBase
+ */
+ def foreachPartition[T](javaDstream: JavaDStream[T],
+ f: VoidFunction[(Iterator[T], Connection)]) = {
+ hbaseContext.foreachPartition(javaDstream.dstream,
+ (it: Iterator[T], conn: Connection) => f.call(it, conn))
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark JavaRDD mapPartition.
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * Note: Make sure to partition correctly to avoid memory issue when
+ * getting data from HBase
+ *
+ * @param javaRdd Original JavaRdd with data to iterate over
+ * @param f Function to be given a iterator to iterate through
+ * the RDD values and a HConnection object to interact
+ * with HBase
+ * @return Returns a new RDD generated by the user definition
+ * function just like normal mapPartition
+ */
+ def mapPartitions[T, R](javaRdd: JavaRDD[T],
+ f: FlatMapFunction[(java.util.Iterator[T],
+ Connection), R]): JavaRDD[R] = {
+
+ def fn = (it: Iterator[T], conn: Connection) =>
+ asScalaIterator(
+ f.call((asJavaIterator(it), conn)).iterator()
+ )
+
+ JavaRDD.fromRDD(hbaseContext.mapPartitions(javaRdd.rdd,
+ (iterator: Iterator[T], connection: Connection) =>
+ fn(iterator, connection))(fakeClassTag[R]))(fakeClassTag[R])
+ }
+
+ /**
+ * A simple enrichment of the traditional Spark Streaming JavaDStream
+ * mapPartition.
+ *
+ * This function differs from the original in that it offers the
+ * developer access to a already connected HConnection object
+ *
+ * Note: Do not close the HConnection object. All HConnection
+ * management is handled outside this method
+ *
+ * Note: Make sure to partition correctly to avoid memory issue when
+ * getting data from HBase
+ *
+ * @param javaDstream Original JavaDStream with data to iterate over
+ * @param mp Function to be given a iterator to iterate through
+ * the JavaDStream values and a HConnection object to
+ * interact with HBase
+ * @return Returns a new JavaDStream generated by the user
+ * definition function just like normal mapPartition
+ */
+ def streamMap[T, U](javaDstream: JavaDStream[T],
+ mp: Function[(Iterator[T], Connection), Iterator[U]]):
+ JavaDStream[U] = {
+ JavaDStream.fromDStream(hbaseContext.streamMapPartitions(javaDstream.dstream,
+ (it: Iterator[T], conn: Connection) =>
+ mp.call(it, conn))(fakeClassTag[U]))(fakeClassTag[U])
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.foreachPartition method.
+ *
+ * It allow addition support for a user to take JavaRDD
+ * and generate puts and send them to HBase.
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param javaRdd Original JavaRDD with data to iterate over
+ * @param tableName The name of the table to put into
+ * @param f Function to convert a value in the JavaRDD
+ * to a HBase Put
+ */
+ def bulkPut[T](javaRdd: JavaRDD[T],
+ tableName: TableName,
+ f: Function[(T), Put]) {
+
+ hbaseContext.bulkPut(javaRdd.rdd, tableName, (t: T) => f.call(t))
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamMapPartition method.
+ *
+ * It allow addition support for a user to take a JavaDStream and
+ * generate puts and send them to HBase.
+ *
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param javaDstream Original DStream with data to iterate over
+ * @param tableName The name of the table to put into
+ * @param f Function to convert a value in
+ * the JavaDStream to a HBase Put
+ */
+ def streamBulkPut[T](javaDstream: JavaDStream[T],
+ tableName: TableName,
+ f: Function[T, Put]) = {
+ hbaseContext.streamBulkPut(javaDstream.dstream,
+ tableName,
+ (t: T) => f.call(t))
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.foreachPartition method.
+ *
+ * It allow addition support for a user to take a JavaRDD and
+ * generate delete and send them to HBase.
+ *
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param javaRdd Original JavaRDD with data to iterate over
+ * @param tableName The name of the table to delete from
+ * @param f Function to convert a value in the JavaRDD to a
+ * HBase Deletes
+ * @param batchSize The number of deletes to batch before sending to HBase
+ */
+ def bulkDelete[T](javaRdd: JavaRDD[T], tableName: TableName,
+ f: Function[T, Delete], batchSize: Integer) {
+ hbaseContext.bulkDelete(javaRdd.rdd, tableName, (t: T) => f.call(t), batchSize)
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamBulkMutation method.
+ *
+ * It allow addition support for a user to take a JavaDStream and
+ * generate Delete and send them to HBase.
+ *
+ * The complexity of managing the HConnection is
+ * removed from the developer
+ *
+ * @param javaDStream Original DStream with data to iterate over
+ * @param tableName The name of the table to delete from
+ * @param f Function to convert a value in the JavaDStream to a
+ * HBase Delete
+ * @param batchSize The number of deletes to be sent at once
+ */
+ def streamBulkDelete[T](javaDStream: JavaDStream[T],
+ tableName: TableName,
+ f: Function[T, Delete],
+ batchSize: Integer) = {
+ hbaseContext.streamBulkDelete(javaDStream.dstream, tableName,
+ (t: T) => f.call(t),
+ batchSize)
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.mapPartition method.
+ *
+ * It allow addition support for a user to take a JavaRDD and generates a
+ * new RDD based on Gets and the results they bring back from HBase
+ *
+ * @param tableName The name of the table to get from
+ * @param batchSize batch size of how many gets to retrieve in a single fetch
+ * @param javaRdd Original JavaRDD with data to iterate over
+ * @param makeGet Function to convert a value in the JavaRDD to a
+ * HBase Get
+ * @param convertResult This will convert the HBase Result object to
+ * what ever the user wants to put in the resulting
+ * JavaRDD
+ * @return New JavaRDD that is created by the Get to HBase
+ */
+ def bulkGet[T, U](tableName: TableName,
+ batchSize: Integer,
+ javaRdd: JavaRDD[T],
+ makeGet: Function[T, Get],
+ convertResult: Function[Result, U]): JavaRDD[U] = {
+
+ JavaRDD.fromRDD(hbaseContext.bulkGet[T, U](tableName,
+ batchSize,
+ javaRdd.rdd,
+ (t: T) => makeGet.call(t),
+ (r: Result) => {
+ convertResult.call(r)
+ })(fakeClassTag[U]))(fakeClassTag[U])
+
+ }
+
+ /**
+ * A simple abstraction over the HBaseContext.streamMap method.
+ *
+ * It allow addition support for a user to take a DStream and
+ * generates a new DStream based on Gets and the results
+ * they bring back from HBase
+ *
+
+ * @param tableName The name of the table to get from
+ * @param batchSize The number of gets to be batched together
+ * @param javaDStream Original DStream with data to iterate over
+ * @param makeGet Function to convert a value in the JavaDStream to a
+ * HBase Get
+ * @param convertResult This will convert the HBase Result object to
+ * what ever the user wants to put in the resulting
+ * JavaDStream
+ * @return New JavaDStream that is created by the Get to HBase
+ */
+ def streamBulkGet[T, U](tableName: TableName,
+ batchSize: Integer,
+ javaDStream: JavaDStream[T],
+ makeGet: Function[T, Get],
+ convertResult: Function[Result, U]) {
+ JavaDStream.fromDStream(hbaseContext.streamBulkGet(tableName,
+ batchSize,
+ javaDStream.dstream,
+ (t: T) => makeGet.call(t),
+ (r: Result) => convertResult.call(r))(fakeClassTag[U]))(fakeClassTag[U])
+ }
+
+ /**
+ * This function will use the native HBase TableInputFormat with the
+ * given scan object to generate a new JavaRDD
+ *
+ * @param tableName The name of the table to scan
+ * @param scans The HBase scan object to use to read data from HBase
+ * @param f Function to convert a Result object from HBase into
+ * What the user wants in the final generated JavaRDD
+ * @return New JavaRDD with results from scan
+ */
+ def hbaseRDD[U](tableName: TableName,
+ scans: Scan,
+ f: Function[(ImmutableBytesWritable, Result), U]):
+ JavaRDD[U] = {
+ JavaRDD.fromRDD(
+ hbaseContext.hbaseRDD[U](tableName,
+ scans,
+ (v: (ImmutableBytesWritable, Result)) =>
+ f.call(v._1, v._2))(fakeClassTag[U]))(fakeClassTag[U])
+ }
+
+ /**
+ * A overloaded version of HBaseContext hbaseRDD that define the
+ * type of the resulting JavaRDD
+ *
+ * @param tableName The name of the table to scan
+ * @param scans The HBase scan object to use to read data from HBase
+ * @return New JavaRDD with results from scan
+ */
+ def hbaseRDD(tableName: TableName,
+ scans: Scan):
+ JavaRDD[(ImmutableBytesWritable, Result)] = {
+ JavaRDD.fromRDD(hbaseContext.hbaseRDD(tableName, scans))
+ }
+
+ /**
+ * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+ *
+ * This method is used to keep ClassTags out of the external Java API, as the Java compiler
+ * cannot produce them automatically. While this ClassTag-faking does please the compiler,
+ * it can cause problems at runtime if the Scala API relies on ClassTags for correctness.
+ *
+ * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
+ * just worse performance or security issues.
+ * For instance, an Array[AnyRef] can hold any type T,
+ * but may lose primitive
+ * specialization.
+ */
+ private[spark]
+ def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/spark/KeyFamilyQualifier.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/KeyFamilyQualifier.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/KeyFamilyQualifier.scala
new file mode 100644
index 0000000..d7c6277
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/KeyFamilyQualifier.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.loader.spark
+
+import java.io.Serializable
+
+import org.apache.hadoop.hbase.util.Bytes
+
+/**
+ * This is the key to be used for sorting and shuffling.
+ *
+ * We will only partition on the rowKey but we will sort on all three
+ *
+ * @param rowKey Record RowKey
+ * @param family Record ColumnFamily
+ * @param qualifier Cell Qualifier
+ */
+class KeyFamilyQualifier(val rowKey:Array[Byte], val family:Array[Byte], val qualifier:Array[Byte])
+ extends Comparable[KeyFamilyQualifier] with Serializable {
+ override def compareTo(o: KeyFamilyQualifier): Int = {
+ var result = Bytes.compareTo(rowKey, o.rowKey)
+ if (result == 0) {
+ result = Bytes.compareTo(family, o.family)
+ if (result == 0) result = Bytes.compareTo(qualifier, o.qualifier)
+ }
+ result
+ }
+ override def toString: String = {
+ Bytes.toString(rowKey) + ":" + Bytes.toString(family) + ":" + Bytes.toString(qualifier)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
new file mode 100644
index 0000000..7838593
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
@@ -0,0 +1,335 @@
+package org.apache.s2graph.loader.subscriber
+
+import com.typesafe.config.{Config, ConfigFactory}
+import kafka.javaapi.producer.Producer
+import kafka.producer.KeyedMessage
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.client._
+import org.apache.s2graph.core._
+import org.apache.s2graph.spark.spark.WithKafka
+import org.apache.spark.{Accumulable, SparkContext}
+import scala.collection.JavaConversions._
+import scala.collection.mutable.HashMap
+import scala.concurrent.ExecutionContext
+
+object GraphConfig {
+ var database = ""
+ var zkQuorum = ""
+ var kafkaBrokers = ""
+ var cacheTTL = s"${60 * 60 * 1}"
+ def apply(phase: String, dbUrl: Option[String], zkAddr: Option[String], kafkaBrokerList: Option[String]): Config = {
+ database = dbUrl.getOrElse("jdbc:mysql://localhost:3306/graph_dev")
+ zkQuorum = zkAddr.getOrElse("localhost")
+
+// val newConf = new util.HashMap[String, Object]()
+// newConf.put("hbase.zookeeper.quorum", zkQuorum)
+// newConf.put("db.default.url", database)
+// newConf.put("kafka.metadata.broker.list", kafkaBrokers)
+ val newConf =
+ if (kafkaBrokerList.isEmpty) Map("hbase.zookeeper.quorum" -> zkQuorum, "db.default.url" -> database, "cache.ttl.seconds" -> cacheTTL)
+ else Map("hbase.zookeeper.quorum" -> zkQuorum, "db.default.url" -> database, "kafka.metadata.broker.list" -> kafkaBrokers, "cache.ttl.seconds" -> cacheTTL)
+
+ ConfigFactory.parseMap(newConf).withFallback(Graph.DefaultConfig)
+ }
+}
+
+object GraphSubscriberHelper extends WithKafka {
+
+
+ type HashMapAccumulable = Accumulable[HashMap[String, Long], (String, Long)]
+
+
+ lazy val producer = new Producer[String, String](kafkaConf(GraphConfig.kafkaBrokers))
+ var config: Config = _
+ private val writeBufferSize = 1024 * 1024 * 8
+ private val sleepPeriod = 10000
+ private val maxTryNum = 10
+
+ var g: Graph = null
+ var management: Management = null
+ val conns = new scala.collection.mutable.HashMap[String, Connection]()
+
+ def toOption(s: String) = {
+ s match {
+ case "" | "none" => None
+ case _ => Some(s)
+ }
+ }
+
+ def apply(phase: String, dbUrl: String, zkQuorum: String, kafkaBrokerList: String): Unit = {
+ config = GraphConfig(phase, toOption(dbUrl), toOption(zkQuorum), toOption(kafkaBrokerList))
+
+ if (g == null) {
+ val ec = ExecutionContext.Implicits.global
+ g = new Graph(config)(ec)
+ management = new Management(g)
+ }
+ }
+
+ def getConn(zkQuorum: String): Connection = {
+ conns.getOrElseUpdate(zkQuorum, {
+ val hbaseConf = HBaseConfiguration.create()
+ hbaseConf.set("hbase.zookeeper.quorum", zkQuorum)
+ ConnectionFactory.createConnection(hbaseConf)
+ })
+ conns(zkQuorum)
+ }
+ // def apply(phase: String, dbUrl: Option[String], zkQuorum: Option[String], kafkaBrokerList: Option[String]): Unit = {
+ // Graph.apply(GraphConfig(phase, dbUrl, zkQuorum, kafkaBrokerList))(ExecutionContext.Implicits.global)
+ // }
+ def report(key: String, value: Option[String], topic: String = "report") = {
+ val msg = Seq(Some(key), value).flatten.mkString("\t")
+ val kafkaMsg = new KeyedMessage[String, String](topic, msg)
+ producer.send(kafkaMsg)
+ }
+
+ def toGraphElements(msgs: Seq[String], labelMapping: Map[String, String] = Map.empty)
+ (statFunc: (String, Int) => Unit): Iterable[GraphElement] = {
+ (for (msg <- msgs) yield {
+ statFunc("total", 1)
+ Graph.toGraphElement(msg, labelMapping) match {
+ case Some(e) if e.isInstanceOf[Edge] =>
+ statFunc("EdgeParseOk", 1)
+ e.asInstanceOf[Edge]
+ case Some(v) if v.isInstanceOf[Vertex] =>
+ statFunc("VertexParseOk", 1)
+ v.asInstanceOf[Vertex]
+ case Some(x) =>
+ throw new RuntimeException(s">>>>> GraphSubscriber.toGraphElements: parsing failed. ${x.serviceName}")
+ case None =>
+ throw new RuntimeException(s"GraphSubscriber.toGraphElements: parsing failed. $msg")
+ }
+
+ }).toList
+ }
+
+// private def storeRec(zkQuorum: String, tableName: String, puts: List[Put], elementsSize: Int, tryNum: Int)
+// (statFunc: (String, Int) => Unit, statPrefix: String = "edge"): Unit = {
+// if (tryNum <= 0) {
+// statFunc("errorStore", elementsSize)
+// throw new RuntimeException(s"retry failed after $maxTryNum")
+// }
+// val conn = getConn(zkQuorum)
+// val mutator = conn.getBufferedMutator(TableName.valueOf(tableName))
+// // val table = conn.getTable(TableName.valueOf(tableName))
+// // table.setAutoFlush(false, false)
+//
+// try {
+// puts.foreach { put => put.setDurability(Durability.ASYNC_WAL) }
+// mutator.mutate(puts)
+// // table.put(puts)
+// statFunc(s"$statPrefix:storeOk", elementsSize)
+// } catch {
+// case e: Throwable =>
+// e.printStackTrace()
+// Thread.sleep(sleepPeriod)
+// storeRec(zkQuorum, tableName, puts, elementsSize, tryNum - 1)(statFunc)
+// } finally {
+// mutator.close()
+// // table.close()
+// }
+// }
+//
+// def storeDegreeBulk(zkQuorum: String, tableName: String)
+// (degrees: Iterable[(String, String, String, Int)], labelMapping: Map[String, String] = Map.empty)
+// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
+// val counts = HashMap[String, Long]()
+// val statFunc = storeStat(counts)(mapAccOpt) _
+//
+// for {
+// (vertexId, labelName, direction, degreeVal) <- degrees
+// incrementRequests <- TransferToHFile.buildDegreePutRequests(vertexId, labelName, direction, degreeVal)
+// } {
+// storeRec(zkQuorum, tableName, incrementRequests, degrees.size, maxTryNum)(statFunc, "degree")
+// }
+// counts
+// }
+// def storeBulk(zkQuorum: String, tableName: String)
+// (msgs: Seq[String], labelMapping: Map[String, String] = Map.empty, autoCreateEdge: Boolean = false)
+// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
+//
+// val counts = HashMap[String, Long]()
+// val statFunc = storeStat(counts)(mapAccOpt) _
+// val elements = toGraphElements(msgs, labelMapping)(statFunc)
+//
+// val puts = elements.flatMap { element =>
+// element match {
+// case v: Vertex if v.op == GraphUtil.operations("insert") || v.op == GraphUtil.operations("insertBulk") =>
+// v.buildPuts()
+// case e: Edge if e.op == GraphUtil.operations("insert") || e.op == GraphUtil.operations("insertBulk") =>
+// EdgeWriter(e).insertBulkForLoader(autoCreateEdge)
+// case _ => Nil
+// }
+// } toList
+//
+// storeRec(zkQuorum, tableName, puts, msgs.size, maxTryNum)(statFunc)
+// counts
+// }
+
+ def storeStat(counts: HashMap[String, Long])(mapAccOpt: Option[HashMapAccumulable])(key: String, value: Int) = {
+ counts.put(key, counts.getOrElse(key, 0L) + value)
+ mapAccOpt match {
+ case None =>
+ case Some(mapAcc) => mapAcc += (key -> value)
+ }
+ }
+
+ def toLabelMapping(lableMapping: String): Map[String, String] = {
+ (for {
+ token <- lableMapping.split(",")
+ inner = token.split(":") if inner.length == 2
+ } yield {
+ (inner.head, inner.last)
+ }).toMap
+ }
+
+ def isValidQuorum(quorum: String) = {
+ quorum.split(",").size > 1
+ }
+}
+
+//object GraphSubscriber extends SparkApp with WithKafka {
+// val sleepPeriod = 5000
+// val usages =
+// s"""
+// |/**
+// |* this job read edge format(TSV) from HDFS file system then bulk load edges into s2graph. assumes that newLabelName is already created by API.
+// |* params:
+// |* 0. hdfsPath: where is your data in hdfs. require full path with hdfs:// predix
+// |* 1. dbUrl: jdbc database connection string to specify database for meta.
+// |* 2. labelMapping: oldLabel:newLabel delimited by ,
+// |* 3. zkQuorum: target hbase zkQuorum where this job will publish data to.
+// |* 4. hTableName: target hbase physical table name where this job will publish data to.
+// |* 5. batchSize: how many edges will be batched for Put request to target hbase.
+// |* 6. kafkaBrokerList: using kafka as fallback queue. when something goes wrong during batch, data needs to be replay will be stored in kafka.
+// |* 7. kafkaTopic: fallback queue topic.
+// |* 8. edgeAutoCreate: true if need to create reversed edge automatically.
+// |*
+// |* after this job finished, s2graph will have data with sequence corresponding newLabelName.
+// |* change this newLabelName to ogirinalName if you want to online replace of label.
+// |*
+// |*/
+// """.stripMargin
+//
+// override def run() = {
+// /**
+// * Main function
+// */
+// println(args.toList)
+//// if (args.length != 10) {
+//// System.err.println(usages)
+//// System.exit(1)
+//// }
+// val hdfsPath = args(0)
+// val dbUrl = args(1)
+// val labelMapping = GraphSubscriberHelper.toLabelMapping(args(2))
+//
+// val zkQuorum = args(3)
+// val hTableName = args(4)
+// val batchSize = args(5).toInt
+// val kafkaBrokerList = args(6)
+// val kafkaTopic = args(7)
+// val edgeAutoCreate = args(8).toBoolean
+// val vertexDegreePathOpt = if (args.length >= 10) GraphSubscriberHelper.toOption(args(9)) else None
+//
+// val conf = sparkConf(s"$hdfsPath: GraphSubscriber")
+// val sc = new SparkContext(conf)
+// val mapAcc = sc.accumulable(HashMap.empty[String, Long], "counter")(HashMapParam[String, Long](_ + _))
+//
+//
+// if (!GraphSubscriberHelper.isValidQuorum(zkQuorum)) throw new RuntimeException(s"$zkQuorum is not valid.")
+//
+// /** this job expect only one hTableName. all labels in this job will be stored in same physical hbase table */
+// try {
+//
+// import GraphSubscriberHelper._
+// // set local driver setting.
+// val phase = System.getProperty("phase")
+// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
+//
+// /** copy when oldLabel exist and newLabel done exist. otherwise ignore. */
+//
+// if (labelMapping.isEmpty) {
+// // pass
+// } else {
+// for {
+// (oldLabelName, newLabelName) <- labelMapping
+// } {
+// Management.copyLabel(oldLabelName, newLabelName, toOption(hTableName))
+// }
+// }
+//
+// vertexDegreePathOpt.foreach { vertexDegreePath =>
+// val vertexDegrees = sc.textFile(vertexDegreePath).filter(line => line.split("\t").length == 4).map { line =>
+// val tokens = line.split("\t")
+// (tokens(0), tokens(1), tokens(2), tokens(3).toInt)
+// }
+// vertexDegrees.foreachPartition { partition =>
+//
+// // init Graph
+// val phase = System.getProperty("phase")
+// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
+//
+// partition.grouped(batchSize).foreach { msgs =>
+// try {
+// val start = System.currentTimeMillis()
+// val counts = GraphSubscriberHelper.storeDegreeBulk(zkQuorum, hTableName)(msgs, labelMapping)(Some(mapAcc))
+// for ((k, v) <- counts) {
+// mapAcc +=(k, v)
+// }
+// val duration = System.currentTimeMillis() - start
+// println(s"[Success]: store, $mapAcc, $duration, $zkQuorum, $hTableName")
+// } catch {
+// case e: Throwable =>
+// println(s"[Failed]: store $e")
+//
+// msgs.foreach { msg =>
+// GraphSubscriberHelper.report(msg.toString(), Some(e.getMessage()), topic = kafkaTopic)
+// }
+// }
+// }
+// }
+// }
+//
+//
+// val msgs = sc.textFile(hdfsPath)
+// msgs.foreachPartition(partition => {
+// // set executor setting.
+// val phase = System.getProperty("phase")
+// GraphSubscriberHelper.apply(phase, dbUrl, zkQuorum, kafkaBrokerList)
+//
+// partition.grouped(batchSize).foreach { msgs =>
+// try {
+// val start = System.currentTimeMillis()
+// // val counts =
+// // GraphSubscriberHelper.store(msgs, GraphSubscriberHelper.toOption(newLabelName))(Some(mapAcc))
+// val counts =
+// GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping, edgeAutoCreate)(Some(mapAcc))
+//
+// for ((k, v) <- counts) {
+// mapAcc +=(k, v)
+// }
+// val duration = System.currentTimeMillis() - start
+// println(s"[Success]: store, $mapAcc, $duration, $zkQuorum, $hTableName")
+// } catch {
+// case e: Throwable =>
+// println(s"[Failed]: store $e")
+//
+// msgs.foreach { msg =>
+// GraphSubscriberHelper.report(msg, Some(e.getMessage()), topic = kafkaTopic)
+// }
+// }
+// }
+// })
+//
+// logInfo(s"counter: $mapAcc")
+// println(s"Stats: ${mapAcc}")
+//
+// } catch {
+// case e: Throwable =>
+// println(s"job failed with exception: $e")
+// throw e
+// }
+//
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
new file mode 100644
index 0000000..60b43ca
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
@@ -0,0 +1,194 @@
+package org.apache.s2graph.loader.subscriber
+
+import org.apache.hadoop.hbase.client.Put
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.mapreduce.{TableOutputFormat}
+import org.apache.hadoop.hbase._
+import org.apache.hadoop.hbase.regionserver.BloomType
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.types.{InnerValLikeWithTs, SourceVertexId, LabelWithDirection}
+import org.apache.s2graph.loader.spark.{KeyFamilyQualifier, HBaseContext, FamilyHFileWriteOptions}
+import org.apache.s2graph.spark.spark.SparkApp
+import org.apache.spark.{SparkContext}
+import org.apache.spark.rdd.RDD
+import org.hbase.async.{PutRequest}
+import play.api.libs.json.Json
+import scala.collection.JavaConversions._
+
+
+object TransferToHFile extends SparkApp with JSONParser {
+
+ val usages =
+ s"""
+ |create HFiles for hbase table on zkQuorum specified.
+ |note that hbase table is created already and pre-splitted properly.
+ |
+ |params:
+ |0. input: hdfs path for tsv file(bulk format).
+ |1. output: hdfs path for storing HFiles.
+ |2. zkQuorum: running hbase cluster zkQuorum.
+ |3. tableName: table name for this bulk upload.
+ |4. dbUrl: db url for parsing to graph element.
+ """.stripMargin
+
+ //TODO: Process AtomicIncrementRequest too.
+ /** build key values */
+ case class DegreeKey(vertexIdStr: String, labelName: String, direction: String)
+
+ private def insertBulkForLoaderAsync(edge: Edge, createRelEdges: Boolean = true): List[PutRequest] = {
+ val relEdges = if (createRelEdges) edge.relatedEdges else List(edge)
+ buildPutRequests(edge.toSnapshotEdge) ++ relEdges.toList.flatMap { e =>
+ e.edgesWithIndex.flatMap { indexEdge => buildPutRequests(indexEdge) }
+ }
+ }
+
+ def buildDegrees(msgs: RDD[String], labelMapping: Map[String, String], edgeAutoCreate: Boolean) = {
+ for {
+ msg <- msgs
+ tokens = GraphUtil.split(msg)
+ if tokens(2) == "e" || tokens(2) == "edge"
+ tempDirection = if (tokens.length == 7) "out" else tokens(7)
+ direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
+ reverseDirection = if (direction == "out") "in" else "out"
+ convertedLabelName = labelMapping.get(tokens(5)).getOrElse(tokens(5))
+ (vertexIdStr, vertexIdStrReversed) = (tokens(3), tokens(4))
+ degreeKey = DegreeKey(vertexIdStr, convertedLabelName, direction)
+ degreeKeyReversed = DegreeKey(vertexIdStrReversed, convertedLabelName, reverseDirection)
+ extra = if (edgeAutoCreate) List(degreeKeyReversed -> 1L) else Nil
+ output <- List(degreeKey -> 1L) ++ extra
+ } yield output
+ }
+ def buildPutRequests(snapshotEdge: SnapshotEdge): List[PutRequest] = {
+ val kvs = GraphSubscriberHelper.g.storage.snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
+ kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
+ }
+ def buildPutRequests(indexEdge: IndexEdge): List[PutRequest] = {
+ val kvs = GraphSubscriberHelper.g.storage.indexEdgeSerializer(indexEdge).toKeyValues.toList
+ kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
+ }
+ def buildDegreePutRequests(vertexId: String, labelName: String, direction: String, degreeVal: Long): List[PutRequest] = {
+ val label = Label.findByName(labelName).getOrElse(throw new RuntimeException(s"$labelName is not found in DB."))
+ val dir = GraphUtil.directions(direction)
+ val innerVal = jsValueToInnerVal(Json.toJson(vertexId), label.srcColumnWithDir(dir).columnType, label.schemaVersion).getOrElse {
+ throw new RuntimeException(s"$vertexId can not be converted into innerval")
+ }
+ val vertex = Vertex(SourceVertexId(label.srcColumn.id.get, innerVal))
+
+ val ts = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
+ val labelWithDir = LabelWithDirection(label.id.get, dir)
+ val edge = Edge(vertex, vertex, labelWithDir, propsWithTs=propsWithTs)
+
+ edge.edgesWithIndex.flatMap { indexEdge =>
+ GraphSubscriberHelper.g.storage.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
+ new PutRequest(kv.table, kv.row, kv.cf, Array.empty[Byte], Bytes.toBytes(degreeVal), kv.timestamp)
+ }
+ }
+ }
+
+ def toKeyValues(degreeKeyVals: Seq[(DegreeKey, Long)]): Iterator[KeyValue] = {
+ val kvs = for {
+ (key, value) <- degreeKeyVals
+ putRequest <- buildDegreePutRequests(key.vertexIdStr, key.labelName, key.direction, value)
+ } yield {
+ val p = putRequest
+ val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+ kv
+ }
+ kvs.toIterator
+ }
+
+ def toKeyValues(strs: Seq[String], labelMapping: Map[String, String], autoEdgeCreate: Boolean): Iterator[KeyValue] = {
+ val kvs = for {
+ s <- strs
+ element <- Graph.toGraphElement(s, labelMapping).toSeq if element.isInstanceOf[Edge]
+ edge = element.asInstanceOf[Edge]
+ putRequest <- insertBulkForLoaderAsync(edge, autoEdgeCreate)
+ } yield {
+ val p = putRequest
+ val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+
+ // println(s"[Edge]: $edge\n[Put]: $p\n[KeyValue]: ${kv.getRow.toList}, ${kv.getQualifier.toList}, ${kv.getValue.toList}, ${kv.getTimestamp}")
+
+ kv
+ }
+ kvs.toIterator
+ }
+
+
+ override def run() = {
+ val input = args(0)
+ val tmpPath = args(1)
+ val zkQuorum = args(2)
+ val tableName = args(3)
+ val dbUrl = args(4)
+ val maxHFilePerResionServer = args(5).toInt
+ val labelMapping = if (args.length >= 7) GraphSubscriberHelper.toLabelMapping(args(6)) else Map.empty[String, String]
+ val autoEdgeCreate = if (args.length >= 8) args(7).toBoolean else false
+ val buildDegree = if (args.length >= 9) args(8).toBoolean else true
+ val compressionAlgorithm = if (args.length >= 10) args(9) else "lz4"
+ val conf = sparkConf(s"$input: TransferToHFile")
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+ conf.set("spark.kryoserializer.buffer.mb", "24")
+
+ val sc = new SparkContext(conf)
+
+ GraphSubscriberHelper.management.createTable(zkQuorum, tableName, List("e", "v"), maxHFilePerResionServer, None, compressionAlgorithm)
+
+ /** set up hbase init */
+ val hbaseConf = HBaseConfiguration.create()
+ hbaseConf.set("hbase.zookeeper.quorum", zkQuorum)
+ hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, tableName)
+ hbaseConf.set("hadoop.tmp.dir", s"/tmp/$tableName")
+
+
+ val rdd = sc.textFile(input)
+
+
+ val kvs = rdd.mapPartitions { iter =>
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
+ toKeyValues(iter.toSeq, labelMapping, autoEdgeCreate)
+ }
+ //
+ // val newRDD = if (!buildDegree) new HFileRDD(kvs)
+ // else {
+ // val degreeKVs = buildDegrees(rdd, labelMapping, autoEdgeCreate).reduceByKey { (agg, current) =>
+ // agg + current
+ // }.mapPartitions { iter =>
+ // val phase = System.getProperty("phase")
+ // GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
+ // toKeyValues(iter.toSeq)
+ // }
+ // new HFileRDD(kvs ++ degreeKVs)
+ // }
+ //
+ // newRDD.toHFile(hbaseConf, zkQuorum, tableName, maxHFilePerResionServer, tmpPath)
+ val merged = if (!buildDegree) kvs
+ else {
+ kvs ++ buildDegrees(rdd, labelMapping, autoEdgeCreate).reduceByKey { (agg, current) =>
+ agg + current
+ }.mapPartitions { iter =>
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
+ toKeyValues(iter.toSeq)
+ }
+ }
+
+ val hbaseSc = new HBaseContext(sc, hbaseConf)
+ def flatMap(kv: KeyValue): Iterator[(KeyFamilyQualifier, Array[Byte])] = {
+ val k = new KeyFamilyQualifier(kv.getRow(), kv.getFamily(), kv.getQualifier())
+ val v = kv.getValue()
+ Seq((k -> v)).toIterator
+ }
+ val familyOptions = new FamilyHFileWriteOptions(Algorithm.LZ4.getName.toUpperCase,
+ BloomType.ROW.name().toUpperCase, 32768, DataBlockEncoding.FAST_DIFF.name().toUpperCase)
+ val familyOptionsMap = Map("e".getBytes() -> familyOptions, "v".getBytes() -> familyOptions)
+
+ hbaseSc.bulkLoad(merged, TableName.valueOf(tableName), flatMap, tmpPath, familyOptionsMap)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
new file mode 100644
index 0000000..3b54e1c
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
@@ -0,0 +1,90 @@
+package org.apache.s2graph.loader.subscriber
+
+import kafka.producer.KeyedMessage
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.core.Graph
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp}
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.HasOffsetRanges
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.language.postfixOps
+
+object WalLogStat extends SparkApp with WithKafka {
+
+ override def run() = {
+
+ validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "statTopic")
+
+ val kafkaZkQuorum = args(0)
+ val brokerList = args(1)
+ val topics = args(2)
+ val intervalInSec = seconds(args(3).toLong)
+ val dbUrl = args(4)
+ val statTopic = args(5)
+
+
+ val conf = sparkConf(s"$topics: ${getClass.getSimpleName}")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val groupId = topics.replaceAll(",", "_") + "_stat"
+
+ val kafkaParams = Map(
+ "zookeeper.connect" -> kafkaZkQuorum,
+ "group.id" -> groupId,
+ "metadata.broker.list" -> brokerList,
+ "zookeeper.connection.timeout.ms" -> "10000",
+ "auto.offset.reset" -> "largest")
+
+ val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
+ val statProducer = getProducer[String, String](brokerList)
+
+ stream.foreachRDD { (rdd, time) =>
+
+ val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
+
+ val ts = time.milliseconds
+
+ val elements = rdd.mapPartitions { partition =>
+ // set executor setting.
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
+ partition.map { case (key, msg) =>
+ Graph.toGraphElement(msg) match {
+ case Some(elem) =>
+ val serviceName = elem.serviceName
+ msg.split("\t", 7) match {
+ case Array(_, operation, log_type, _, _, label, _*) =>
+ Seq(serviceName, label, operation, log_type).mkString("\t")
+ case _ =>
+ Seq("no_service_name", "no_label", "no_operation", "parsing_error").mkString("\t")
+ }
+ case None =>
+ Seq("no_service_name", "no_label", "no_operation", "no_element_error").mkString("\t")
+ }
+ }
+ }
+
+ val countByKey = elements.map(_ -> 1L).reduceByKey(_ + _).collect()
+ val totalCount = countByKey.map(_._2).sum
+ val keyedMessage = countByKey.map { case (key, value) =>
+ new KeyedMessage[String, String](statTopic, s"$ts\t$key\t$value\t$totalCount")
+ }
+
+ statProducer.send(keyedMessage: _*)
+
+ elements.mapPartitionsWithIndex { (i, part) =>
+ // commit offset range
+ val osr = offsets(i)
+ getStreamHelper(kafkaParams).commitConsumerOffset(osr)
+ Iterator.empty
+ }.foreach {
+ (_: Nothing) => ()
+ }
+
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
new file mode 100644
index 0000000..f51a148
--- /dev/null
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
@@ -0,0 +1,148 @@
+package org.apache.s2graph.loader.subscriber
+
+import java.text.SimpleDateFormat
+import java.util.Date
+import kafka.serializer.StringDecoder
+import org.apache.s2graph.core.Graph
+import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
+import org.apache.spark.sql.hive.HiveContext
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.HasOffsetRanges
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.language.postfixOps
+
+object WalLogToHDFS extends SparkApp with WithKafka {
+
+ override def run() = {
+
+ validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "outputPath", "hiveDatabase", "hiveTable", "splitListPath")
+
+ val kafkaZkQuorum = args(0)
+ val brokerList = args(1)
+ val topics = args(2)
+ val intervalInSec = seconds(args(3).toLong)
+ val dbUrl = args(4)
+ val outputPath = args(5)
+ val hiveDatabase = args(6)
+ val hiveTable = args(7)
+ val splitListPath = args(8)
+
+ val conf = sparkConf(s"$topics: WalLogToHDFS")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val groupId = topics.replaceAll(",", "_") + "_stream"
+ val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
+
+ val kafkaParams = Map(
+ "zookeeper.connect" -> kafkaZkQuorum,
+ "group.id" -> groupId,
+ "metadata.broker.list" -> brokerList,
+ "zookeeper.connection.timeout.ms" -> "10000",
+ "auto.offset.reset" -> "largest")
+
+ val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
+
+ val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
+
+ val hdfsBlockSize = 134217728 // 128M
+ val hiveContext = new HiveContext(sc)
+ var splits = Array[String]()
+ var excludeLabels = Set[String]()
+ var excludeServices = Set[String]()
+ stream.foreachRDD { (rdd, time) =>
+ try {
+ val read = sc.textFile(splitListPath).collect().map(_.split("=")).flatMap {
+ case Array(value) => Some(("split", value))
+ case Array(key, value) => Some((key, value))
+ case _ => None
+ }
+ splits = read.filter(_._1 == "split").map(_._2)
+ excludeLabels = read.filter(_._1 == "exclude_label").map(_._2).toSet
+ excludeServices = read.filter(_._1 == "exclude_service").map(_._2).toSet
+ } catch {
+ case _: Throwable => // use previous information
+ }
+
+ val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
+
+ val elements = rdd.mapPartitions { partition =>
+ // set executor setting.
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
+
+ partition.flatMap { case (key, msg) =>
+ val optMsg = Graph.toGraphElement(msg).flatMap { element =>
+ val arr = msg.split("\t", 7)
+ val service = element.serviceName
+ val label = arr(5)
+ val n = arr.length
+
+ if (excludeServices.contains(service) || excludeLabels.contains(label)) {
+ None
+ } else if(n == 6) {
+ Some(Seq(msg, "{}", service).mkString("\t"))
+ }
+ else if(n == 7) {
+ Some(Seq(msg, service).mkString("\t"))
+ }
+ else {
+ None
+ }
+ }
+ optMsg
+ }
+ }
+
+ val ts = time.milliseconds
+ val dateId = new SimpleDateFormat("yyyy-MM-dd").format(new Date(ts))
+
+ /** make sure that `elements` are not running at the same time */
+ val elementsWritten = {
+ elements.cache()
+ (Array("all") ++ splits).foreach {
+ case split if split == "all" =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ elements.saveAsTextFile(path)
+ case split =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ val strlen = split.length
+ val splitData = elements.filter(_.takeRight(strlen) == split).cache()
+ val totalSize = splitData
+ .mapPartitions { iterator =>
+ val s = iterator.map(_.length.toLong).sum
+ Iterator.single(s)
+ }
+ .sum
+ .toLong
+ val numPartitions = math.max(1, (totalSize / hdfsBlockSize.toDouble).toInt)
+ splitData.coalesce(math.min(splitData.partitions.length, numPartitions)).saveAsTextFile(path)
+ splitData.unpersist()
+ }
+ elements.unpersist()
+ elements
+ }
+
+ elementsWritten.mapPartitionsWithIndex { (i, part) =>
+ // commit offset range
+ val osr = offsets(i)
+ getStreamHelper(kafkaParams).commitConsumerOffset(osr)
+ Iterator.empty
+ }.foreach {
+ (_: Nothing) => ()
+ }
+
+ (Array("all") ++ splits).foreach { split =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ hiveContext.sql(s"use $hiveDatabase")
+ hiveContext.sql(s"alter table $hiveTable add partition (split='$split', date_id='$dateId', ts='$ts') location '$path'")
+ }
+ }
+
+ logInfo(s"counter: ${mapAcc.value}")
+ println(s"counter: ${mapAcc.value}")
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/BulkLoadPartitioner.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/BulkLoadPartitioner.scala b/loader/src/main/scala/spark/BulkLoadPartitioner.scala
deleted file mode 100644
index ed1587a..0000000
--- a/loader/src/main/scala/spark/BulkLoadPartitioner.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package spark
-
-import java.util
-import java.util.Comparator
-
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.spark.Partitioner
-
-/**
- * A Partitioner implementation that will separate records to different
- * HBase Regions based on region splits
- *
- * @param startKeys The start keys for the given table
- */
-class BulkLoadPartitioner(startKeys:Array[Array[Byte]])
- extends Partitioner {
-
- override def numPartitions: Int = startKeys.length
-
- override def getPartition(key: Any): Int = {
-
- val rowKey:Array[Byte] =
- key match {
- case qualifier: KeyFamilyQualifier =>
- qualifier.rowKey
- case _ =>
- key.asInstanceOf[Array[Byte]]
- }
-
- val comparator: Comparator[Array[Byte]] = new Comparator[Array[Byte]] {
- override def compare(o1: Array[Byte], o2: Array[Byte]): Int = {
- Bytes.compareTo(o1, o2)
- }
- }
- val partition = util.Arrays.binarySearch(startKeys, rowKey, comparator)
- if (partition < 0) partition * -1 + -2
- else partition
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/loader/src/main/scala/spark/ColumnFamilyQualifierMapKeyWrapper.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/spark/ColumnFamilyQualifierMapKeyWrapper.scala b/loader/src/main/scala/spark/ColumnFamilyQualifierMapKeyWrapper.scala
deleted file mode 100644
index ab34b76..0000000
--- a/loader/src/main/scala/spark/ColumnFamilyQualifierMapKeyWrapper.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package spark
-//
-//import org.apache.hadoop.hbase.util.Bytes
-//
-///**
-// * A wrapper class that will allow both columnFamily and qualifier to
-// * be the key of a hashMap. Also allow for finding the value in a hashmap
-// * with out cloning the HBase value from the HBase Cell object
-// * @param columnFamily ColumnFamily byte array
-// * @param columnFamilyOffSet Offset of columnFamily value in the array
-// * @param columnFamilyLength Length of the columnFamily value in the columnFamily array
-// * @param qualifier Qualifier byte array
-// * @param qualifierOffSet Offset of qualifier value in the array
-// * @param qualifierLength Length of the qualifier value with in the array
-// */
-//class ColumnFamilyQualifierMapKeyWrapper(val columnFamily:Array[Byte],
-// val columnFamilyOffSet:Int,
-// val columnFamilyLength:Int,
-// val qualifier:Array[Byte],
-// val qualifierOffSet:Int,
-// val qualifierLength:Int)
-// extends Serializable{
-//
-// override def equals(other:Any): Boolean = {
-// val otherWrapper = other.asInstanceOf[ColumnFamilyQualifierMapKeyWrapper]
-//
-// Bytes.compareTo(columnFamily,
-// columnFamilyOffSet,
-// columnFamilyLength,
-// otherWrapper.columnFamily,
-// otherWrapper.columnFamilyOffSet,
-// otherWrapper.columnFamilyLength) == 0 && Bytes.compareTo(qualifier,
-// qualifierOffSet,
-// qualifierLength,
-// otherWrapper.qualifier,
-// otherWrapper.qualifierOffSet,
-// otherWrapper.qualifierLength) == 0
-// }
-//
-// override def hashCode():Int = {
-// Bytes.hashCode(columnFamily, columnFamilyOffSet, columnFamilyLength) +
-// Bytes.hashCode(qualifier, qualifierOffSet, qualifierLength)
-// }
-//
-// def cloneColumnFamily():Array[Byte] = {
-// val resultArray = new Array[Byte](columnFamilyLength)
-// System.arraycopy(columnFamily, columnFamilyOffSet, resultArray, 0, columnFamilyLength)
-// resultArray
-// }
-//
-// def cloneQualifier():Array[Byte] = {
-// val resultArray = new Array[Byte](qualifierLength)
-// System.arraycopy(qualifier, qualifierOffSet, resultArray, 0, qualifierLength)
-// resultArray
-// }
-//}
[08/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/Management.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/Management.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/Management.scala
new file mode 100644
index 0000000..c507075
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/Management.scala
@@ -0,0 +1,143 @@
+package org.apache.s2graph.counter.helper
+
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.client.{Admin, ConnectionFactory, Durability}
+import org.apache.hadoop.hbase.io.compress.Compression
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.regionserver.BloomType
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
+import org.slf4j.LoggerFactory
+import redis.clients.jedis.ScanParams
+
+import scala.collection.JavaConversions._
+import scala.util.Random
+
+class Management(config: Config) {
+ val withRedis = new HashShardingJedis(config)
+
+ val log = LoggerFactory.getLogger(this.getClass)
+
+ def describe(zkAddr: String, tableName: String) = {
+ val admin = getAdmin(zkAddr)
+ val table = admin.getTableDescriptor(TableName.valueOf(tableName))
+
+ table.getColumnFamilies.foreach { cf =>
+ println(s"columnFamily: ${cf.getNameAsString}")
+ cf.getValues.foreach { case (k, v) =>
+ println(s"${Bytes.toString(k.get())} ${Bytes.toString(v.get())}")
+ }
+ }
+ }
+
+ def setTTL(zkAddr: String, tableName: String, cfName: String, ttl: Int) = {
+ val admin = getAdmin(zkAddr)
+ val tableNameObj = TableName.valueOf(tableName)
+ val table = admin.getTableDescriptor(tableNameObj)
+
+ val cf = table.getFamily(cfName.getBytes)
+ cf.setTimeToLive(ttl)
+
+ admin.modifyColumn(tableNameObj, cf)
+ }
+
+ def getAdmin(zkAddr: String): Admin = {
+ val conf = HBaseConfiguration.create()
+ conf.set("hbase.zookeeper.quorum", zkAddr)
+ val conn = ConnectionFactory.createConnection(conf)
+ conn.getAdmin
+ }
+
+ def tableExists(zkAddr: String, tableName: String): Boolean = {
+ getAdmin(zkAddr).tableExists(TableName.valueOf(tableName))
+ }
+
+ def createTable(zkAddr: String, tableName: String, cfs: List[String], regionMultiplier: Int) = {
+ log.info(s"create table: $tableName on $zkAddr, $cfs, $regionMultiplier")
+ val admin = getAdmin(zkAddr)
+ val regionCount = admin.getClusterStatus.getServersSize * regionMultiplier
+ try {
+ val desc = new HTableDescriptor(TableName.valueOf(tableName))
+ desc.setDurability(Durability.ASYNC_WAL)
+ for (cf <- cfs) {
+ val columnDesc = new HColumnDescriptor(cf)
+ .setCompressionType(Compression.Algorithm.LZ4)
+ .setBloomFilterType(BloomType.ROW)
+ .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)
+ .setMaxVersions(1)
+ .setMinVersions(0)
+ .setBlocksize(32768)
+ .setBlockCacheEnabled(true)
+ desc.addFamily(columnDesc)
+ }
+
+ if (regionCount <= 1) admin.createTable(desc)
+ else admin.createTable(desc, getStartKey(regionCount), getEndKey(regionCount), regionCount)
+ } catch {
+ case e: Throwable =>
+ log.error(s"$zkAddr, $tableName failed with $e", e)
+ throw e
+ }
+ }
+
+ // we only use murmur hash to distribute row key.
+ private def getStartKey(regionCount: Int) = {
+ Bytes.toBytes(Int.MaxValue / regionCount)
+ }
+
+ private def getEndKey(regionCount: Int) = {
+ Bytes.toBytes(Int.MaxValue / regionCount * (regionCount - 1))
+ }
+
+ case class RedisScanIterator(scanParams: ScanParams = new ScanParams().count(100)) extends Iterator[String] {
+ val nextCursorId: collection.mutable.Map[Int, String] = collection.mutable.Map.empty[Int, String]
+ var innerIterator: Iterator[String] = _
+
+ for {
+ i <- 0 until withRedis.jedisPoolSize
+ } {
+ nextCursorId.put(i, "0")
+ }
+
+ def callScan(): Unit = {
+ if (nextCursorId.nonEmpty) {
+ // println(s"callScan: idx: $nextIdx, cursor: $nextCursorId")
+ val idx = Random.shuffle(nextCursorId.keys).head
+ val cursorId = nextCursorId(idx)
+ val pool = withRedis.getJedisPool(idx)
+ val conn = pool.getResource
+ try {
+ val result = conn.scan(cursorId, scanParams)
+ result.getStringCursor match {
+ case "0" =>
+ log.debug(s"end scan: idx: $idx, cursor: $cursorId")
+ nextCursorId.remove(idx)
+ case x: String =>
+ nextCursorId.put(idx, x)
+ }
+ innerIterator = result.getResult.toIterator
+ } finally {
+ pool.returnResource(conn)
+ }
+ }
+ else {
+ innerIterator = List.empty[String].toIterator
+ }
+ }
+
+ // initialize
+ callScan()
+
+ override def hasNext: Boolean = {
+ innerIterator.hasNext match {
+ case true =>
+ true
+ case false =>
+ callScan()
+ innerIterator.hasNext
+ }
+ }
+
+ override def next(): String = innerIterator.next()
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithHBase.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithHBase.scala
new file mode 100644
index 0000000..108c47a
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithHBase.scala
@@ -0,0 +1,98 @@
+package org.apache.s2graph.counter.helper
+
+import com.stumbleupon.async.{Callback, Deferred}
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.hbase.async.HBaseClient
+import org.slf4j.LoggerFactory
+import scala.concurrent.{Future, Promise}
+import scala.util.Try
+
+
+class WithHBase(config: Config) {
+ lazy val logger = LoggerFactory.getLogger(this.getClass)
+ lazy val s2config = new S2CounterConfig(config)
+
+ lazy val zkQuorum = s2config.HBASE_ZOOKEEPER_QUORUM
+ lazy val defaultTableName = s2config.HBASE_TABLE_NAME
+
+ logger.info(s"$zkQuorum, $defaultTableName")
+
+ val hbaseConfig = HBaseConfiguration.create()
+ s2config.getConfigMap("hbase").foreach { case (k, v) =>
+ hbaseConfig.set(k, v)
+ }
+
+// lazy val conn: HConnection = HConnectionManager.createConnection(hbaseConfig)
+ lazy val conn: Connection = ConnectionFactory.createConnection(hbaseConfig)
+
+ val writeBufferSize = 1024 * 1024 * 2 // 2MB
+
+// def apply[T](op: Table => T): Try[T] = {
+// Try {
+// val table = conn.getTable(TableName.valueOf(defaultTableName))
+// // do not keep failed operation in writer buffer
+// table.setWriteBufferSize(writeBufferSize)
+// try {
+// op(table)
+// } catch {
+// case e: Throwable =>
+// logger.error(s"Operation to table($defaultTableName) is failed: ${e.getMessage}")
+// throw e
+// } finally {
+// table.close()
+// }
+// }
+// }
+
+ def apply[T](tableName: String)(op: Table => T): Try[T] = {
+ Try {
+ val table = conn.getTable(TableName.valueOf(tableName))
+ // do not keep failed operation in writer buffer
+ table.setWriteBufferSize(writeBufferSize)
+ try {
+ op(table)
+ } catch {
+ case ex: Exception =>
+ logger.error(s"$ex: Operation to table($tableName) is failed")
+ throw ex
+ } finally {
+ table.close()
+ }
+ }
+ }
+}
+
+case class WithAsyncHBase(config: Config) {
+ lazy val logger = LoggerFactory.getLogger(this.getClass)
+ lazy val s2config = new S2CounterConfig(config)
+
+ lazy val zkQuorum = s2config.HBASE_ZOOKEEPER_QUORUM
+
+ val hbaseConfig = HBaseConfiguration.create()
+ s2config.getConfigMap("hbase").foreach { case (k, v) =>
+ hbaseConfig.set(k, v)
+ }
+
+// lazy val conn: HConnection = HConnectionManager.createConnection(hbaseConfig)
+ lazy val client: HBaseClient = new HBaseClient(zkQuorum)
+
+ val writeBufferSize = 1024 * 1024 * 2 // 2MB
+
+ def apply[T](op: HBaseClient => Deferred[T]): Future[T] = {
+ val promise = Promise[T]()
+
+ op(client).addCallback(new Callback[Unit, T] {
+ def call(arg: T): Unit = {
+ promise.success(arg)
+ }
+ }).addErrback(new Callback[Unit, Exception] {
+ def call(ex: Exception): Unit = {
+ promise.failure(ex)
+ }
+ })
+ promise.future
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithRedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithRedis.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithRedis.scala
new file mode 100644
index 0000000..a7b99c8
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/WithRedis.scala
@@ -0,0 +1,59 @@
+package org.apache.s2graph.counter.helper
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.util.Hashes
+import org.slf4j.LoggerFactory
+import redis.clients.jedis.exceptions.JedisException
+import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
+
+import scala.util.Try
+
+class WithRedis(config: Config) {
+ lazy val s2config = new S2CounterConfig(config)
+
+ private val log = LoggerFactory.getLogger(getClass)
+
+ val poolConfig = new JedisPoolConfig()
+ poolConfig.setMaxTotal(150)
+ poolConfig.setMaxIdle(50)
+ poolConfig.setMaxWaitMillis(200)
+
+ val jedisPools = s2config.REDIS_INSTANCES.map { case (host, port) =>
+ new JedisPool(poolConfig, host, port)
+ }
+
+ def getBucketIdx(key: String): Int = {
+ Hashes.murmur3(key) % jedisPools.size
+ }
+
+ def doBlockWithIndex[T](idx: Int)(f: Jedis => T): Try[T] = {
+ Try {
+ val pool = jedisPools(idx)
+
+ var jedis: Jedis = null
+
+ try {
+ jedis = pool.getResource
+
+ f(jedis)
+ }
+ catch {
+ case e: JedisException =>
+ pool.returnBrokenResource(jedis)
+
+ jedis = null
+ throw e
+ }
+ finally {
+ if (jedis != null) {
+ pool.returnResource(jedis)
+ }
+ }
+ }
+ }
+
+ def doBlockWithKey[T](key: String)(f: Jedis => T): Try[T] = {
+ doBlockWithIndex(getBucketIdx(key))(f)
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/CachedDBModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/CachedDBModel.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/CachedDBModel.scala
new file mode 100644
index 0000000..6e06caf
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/CachedDBModel.scala
@@ -0,0 +1,11 @@
+package org.apache.s2graph.counter.models
+
+import org.apache.s2graph.counter.util.{CollectionCache, CollectionCacheConfig}
+import scalikejdbc.AutoSession
+
+trait CachedDBModel[T] {
+ implicit val s = AutoSession
+
+ val cacheConfig: CollectionCacheConfig
+ lazy val cache = new CollectionCache[Option[T]](cacheConfig)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/Counter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/Counter.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/Counter.scala
new file mode 100644
index 0000000..467e110
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/Counter.scala
@@ -0,0 +1,210 @@
+package org.apache.s2graph.counter.models
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.util.{CollectionCache, CollectionCacheConfig}
+import scalikejdbc._
+
+case class Counter(id: Int, useFlag: Boolean, version: Byte, service: String, action: String,
+ itemType: Counter.ItemType.ItemType, autoComb: Boolean, dimension: String,
+ useProfile: Boolean, bucketImpId: Option[String],
+ useRank: Boolean,
+ ttl: Int, dailyTtl: Option[Int], hbaseTable: Option[String],
+ intervalUnit: Option[String],
+ rateActionId: Option[Int], rateBaseId: Option[Int], rateThreshold: Option[Int]) {
+ val intervals: Array[String] = intervalUnit.map(s => s.split(',')).getOrElse(Array("t", "M", "d", "H"))
+ val dimensionSp = if (dimension.isEmpty) Array.empty[String] else dimension.split(',').sorted
+
+ val dimensionList: List[Array[String]] = {
+ autoComb match {
+ case true =>
+ for {
+ i <- (0 to math.min(4, dimensionSp.length)).toList
+ combines <- dimensionSp.combinations(i)
+ } yield {
+ combines
+ }
+ case false =>
+ dimensionSp isEmpty match {
+ case true => List(Array())
+ case false => dimensionSp.toList.map(sp => sp.split('.'))
+ }
+ }
+ }
+
+ val dimensionSet: Set[Set[String]] = {
+ for {
+ arr <- dimensionList
+ } yield {
+ arr.toSet
+ }
+ }.toSet
+
+ val isRateCounter: Boolean = rateActionId.isDefined && rateBaseId.isDefined && rateActionId != rateBaseId
+ val isTrendCounter: Boolean = rateActionId.isDefined && rateBaseId.isDefined && rateActionId == rateBaseId
+}
+
+object Counter extends SQLSyntaxSupport[Counter] {
+ object ItemType extends Enumeration {
+ type ItemType = Value
+ val INT, LONG, STRING, BLOB = Value
+ }
+
+ def apply(c: SyntaxProvider[Counter])(rs: WrappedResultSet): Counter = apply(c.resultName)(rs)
+ def apply(r: ResultName[Counter])(rs: WrappedResultSet): Counter = {
+ lazy val itemType = Counter.ItemType(rs.int(r.itemType))
+ Counter(rs.int(r.id), rs.boolean(r.useFlag), rs.byte(r.version), rs.string(r.service), rs.string(r.action),
+ itemType, rs.boolean(r.autoComb), rs.string(r.dimension),
+ rs.boolean(r.useProfile), rs.stringOpt(r.bucketImpId),
+ rs.boolean(r.useRank),
+ rs.int(r.ttl), rs.intOpt(r.dailyTtl), rs.stringOpt(r.hbaseTable), rs.stringOpt(r.intervalUnit),
+ rs.intOpt(r.rateActionId), rs.intOpt(r.rateBaseId), rs.intOpt(r.rateThreshold))
+ }
+
+ def apply(useFlag: Boolean, version: Byte, service: String, action: String, itemType: Counter.ItemType.ItemType,
+ autoComb: Boolean, dimension: String, useProfile: Boolean = false, bucketImpId: Option[String] = None,
+ useRank: Boolean = false, ttl: Int = 259200, dailyTtl: Option[Int] = None,
+ hbaseTable: Option[String] = None, intervalUnit: Option[String] = None,
+ rateActionId: Option[Int] = None, rateBaseId: Option[Int] = None, rateThreshold: Option[Int] = None): Counter = {
+ Counter(-1, useFlag, version, service, action, itemType, autoComb, dimension,
+ useProfile, bucketImpId,
+ useRank, ttl, dailyTtl, hbaseTable,
+ intervalUnit, rateActionId, rateBaseId, rateThreshold)
+ }
+ }
+
+class CounterModel(config: Config) extends CachedDBModel[Counter] {
+ private lazy val s2Config = new S2CounterConfig(config)
+ // enable negative cache
+ override val cacheConfig: CollectionCacheConfig =
+ new CollectionCacheConfig(s2Config.CACHE_MAX_SIZE, s2Config.CACHE_TTL_SECONDS,
+ negativeCache = true, s2Config.CACHE_NEGATIVE_TTL_SECONDS)
+
+ val c = Counter.syntax("c")
+ val r = c.result
+
+ val multiCache = new CollectionCache[Seq[Counter]](cacheConfig)
+
+ def findById(id: Int, useCache: Boolean = true): Option[Counter] = {
+ lazy val sql = withSQL {
+ selectFrom(Counter as c).where.eq(c.id, id).and.eq(c.useFlag, 1)
+ }.map(Counter(c))
+
+ if (useCache) {
+ cache.withCache(s"_id:$id") {
+ sql.single().apply()
+ }
+ } else {
+ sql.single().apply()
+ }
+ }
+
+ def findByServiceAction(service: String, action: String, useCache: Boolean = true): Option[Counter] = {
+ lazy val sql = withSQL {
+ selectFrom(Counter as c).where.eq(c.service, service).and.eq(c.action, action).and.eq(c.useFlag, 1)
+ }.map(Counter(c))
+
+ if (useCache) {
+ cache.withCache(s"$service.$action") {
+ sql.single().apply()
+ }
+ }
+ else {
+ sql.single().apply()
+ }
+ }
+
+ def findByRateActionId(rateActionId: Int, useCache: Boolean = true): Seq[Counter] = {
+ lazy val sql = withSQL {
+ selectFrom(Counter as c).where.eq(c.rateActionId, rateActionId).and.ne(c.rateBaseId, rateActionId).and.eq(c.useFlag, 1)
+ }.map(Counter(c))
+
+ if (useCache) {
+ multiCache.withCache(s"_rate_action_id.$rateActionId") {
+ sql.list().apply()
+ }
+ } else {
+ sql.list().apply()
+ }
+ }
+
+ def findByRateBaseId(rateBaseId: Int, useCache: Boolean = true): Seq[Counter] = {
+ lazy val sql = withSQL {
+ selectFrom(Counter as c).where.eq(c.rateBaseId, rateBaseId).and.ne(c.rateActionId, rateBaseId).and.eq(c.useFlag, 1)
+ }.map(Counter(c))
+
+ if (useCache) {
+ multiCache.withCache(s"_rate_base_id.$rateBaseId") {
+ sql.list().apply()
+ }
+ } else {
+ sql.list().apply()
+ }
+ }
+
+ def findByTrendActionId(trendActionId: Int, useCache: Boolean = true): Seq[Counter] = {
+ lazy val sql = withSQL {
+ selectFrom(Counter as c).where.eq(c.rateActionId, trendActionId).and.eq(c.rateBaseId, trendActionId).and.eq(c.useFlag, 1)
+ }.map(Counter(c))
+
+ if (useCache) {
+ multiCache.withCache(s"_trend_action_id.$trendActionId") {
+ sql.list().apply()
+ }
+ } else {
+ sql.list().apply()
+ }
+ }
+
+ def createServiceAction(policy: Counter): Unit = {
+ withSQL {
+ val c = Counter.column
+ insert.into(Counter).namedValues(
+ c.useFlag -> policy.useFlag,
+ c.version -> policy.version,
+ c.service -> policy.service,
+ c.action -> policy.action,
+ c.itemType -> policy.itemType.id,
+ c.autoComb -> policy.autoComb,
+ c.dimension -> policy.dimension,
+ c.useProfile -> policy.useProfile,
+ c.bucketImpId -> policy.bucketImpId,
+ c.useRank -> policy.useRank,
+ c.ttl -> policy.ttl,
+ c.dailyTtl -> policy.dailyTtl,
+ c.hbaseTable -> policy.hbaseTable,
+ c.intervalUnit -> policy.intervalUnit,
+ c.rateActionId -> policy.rateActionId,
+ c.rateBaseId -> policy.rateBaseId,
+ c.rateThreshold -> policy.rateThreshold
+ )
+ }.update().apply()
+ }
+
+ def updateServiceAction(policy: Counter): Unit = {
+ withSQL {
+ val c = Counter.column
+ update(Counter).set(
+ c.autoComb -> policy.autoComb,
+ c.dimension -> policy.dimension,
+ c.useProfile -> policy.useProfile,
+ c.bucketImpId -> policy.bucketImpId,
+ c.useRank -> policy.useRank,
+ c.intervalUnit -> policy.intervalUnit,
+ c.rateActionId -> policy.rateActionId,
+ c.rateBaseId -> policy.rateBaseId,
+ c.rateThreshold -> policy.rateThreshold
+ ).where.eq(c.id, policy.id)
+ }.update().apply()
+ }
+
+ def deleteServiceAction(policy: Counter): Unit = {
+ withSQL {
+ val c = Counter.column
+ update(Counter).set(
+ c.action -> s"deleted_${System.currentTimeMillis()}_${policy.action}",
+ c.useFlag -> false
+ ).where.eq(c.id, policy.id)
+ }.update().apply()
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/DBModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/DBModel.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/DBModel.scala
new file mode 100644
index 0000000..1757a7f
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/models/DBModel.scala
@@ -0,0 +1,24 @@
+package org.apache.s2graph.counter.models
+
+import com.typesafe.config.Config
+import org.apache.s2graph.counter.config.S2CounterConfig
+import scalikejdbc._
+
+object DBModel {
+ private var initialized = false
+
+ def initialize(config: Config): Unit = {
+ if (!initialized) {
+ this synchronized {
+ if (!initialized) {
+ val s2Config = new S2CounterConfig(config)
+ Class.forName(s2Config.DB_DEFAULT_DRIVER)
+ val settings = ConnectionPoolSettings(initialSize = 0, maxSize = 10, connectionTimeoutMillis = 5000L, validationQuery = "select 1;")
+
+ ConnectionPool.singleton(s2Config.DB_DEFAULT_URL, s2Config.DB_DEFAULT_USER, s2Config.DB_DEFAULT_PASSWORD, settings)
+ initialized = true
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/package.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/package.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/package.scala
new file mode 100644
index 0000000..01bff0c
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/package.scala
@@ -0,0 +1,8 @@
+package org.apache.s2graph
+
+package object counter {
+ val VERSION_1: Byte = 1
+ val VERSION_2: Byte = 2
+
+ case class MethodNotSupportedException(message: String, cause: Throwable = null) extends Exception(message, cause)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CartesianProduct.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CartesianProduct.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CartesianProduct.scala
new file mode 100644
index 0000000..b1712fa
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CartesianProduct.scala
@@ -0,0 +1,8 @@
+package org.apache.s2graph.counter.util
+
+object CartesianProduct {
+ def apply[T](xss: List[List[T]]): List[List[T]] = xss match {
+ case Nil => List(Nil)
+ case h :: t => for(xh <- h; xt <- apply(t)) yield xh :: xt
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CollectionCache.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CollectionCache.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CollectionCache.scala
new file mode 100644
index 0000000..74b5238
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/CollectionCache.scala
@@ -0,0 +1,66 @@
+package org.apache.s2graph.counter.util
+
+import java.net.InetAddress
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.{Cache, CacheBuilder}
+import org.slf4j.LoggerFactory
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.{postfixOps, reflectiveCalls}
+
+case class CollectionCacheConfig(maxSize: Int, ttl: Int, negativeCache: Boolean = false, negativeTTL: Int = 600)
+
+class CollectionCache[C <: { def nonEmpty: Boolean; def isEmpty: Boolean } ](config: CollectionCacheConfig) {
+ private val cache: Cache[String, C] = CacheBuilder.newBuilder()
+ .expireAfterWrite(config.ttl, TimeUnit.SECONDS)
+ .maximumSize(config.maxSize)
+ .build[String, C]()
+
+// private lazy val cache = new SynchronizedLruMap[String, (C, Int)](config.maxSize)
+ private lazy val className = this.getClass.getSimpleName
+
+ private lazy val log = LoggerFactory.getLogger(this.getClass)
+ val localHostname = InetAddress.getLocalHost.getHostName
+
+ def size = cache.size
+ val maxSize = config.maxSize
+
+ // cache statistics
+ def getStatsString: String = {
+ s"$localHostname ${cache.stats().toString}"
+ }
+
+ def withCache(key: String)(op: => C): C = {
+ Option(cache.getIfPresent(key)) match {
+ case Some(r) => r
+ case None =>
+ val r = op
+ if (r.nonEmpty || config.negativeCache) {
+ cache.put(key, r)
+ }
+ r
+ }
+ }
+
+ def withCacheAsync(key: String)(op: => Future[C])(implicit ec: ExecutionContext): Future[C] = {
+ Option(cache.getIfPresent(key)) match {
+ case Some(r) => Future.successful(r)
+ case None =>
+ op.map { r =>
+ if (r.nonEmpty || config.negativeCache) {
+ cache.put(key, r)
+ }
+ r
+ }
+ }
+ }
+
+ def purgeKey(key: String) = {
+ cache.invalidate(key)
+ }
+
+ def contains(key: String): Boolean = {
+ Option(cache.getIfPresent(key)).nonEmpty
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/FunctionParser.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/FunctionParser.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/FunctionParser.scala
new file mode 100644
index 0000000..e65910f
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/FunctionParser.scala
@@ -0,0 +1,21 @@
+package org.apache.s2graph.counter.util
+
+object FunctionParser {
+ val funcRe = """([a-zA-Z_]+)(\((\d+)?\))?""".r
+
+ def apply(str: String): Option[(String, String)] = {
+ str match {
+ case funcRe(funcName, funcParam, funcArg) =>
+ funcName match {
+ case x: String =>
+ Some((funcName, funcArg match {
+ case x: String => funcArg
+ case null => ""
+ }))
+ case null => None
+ }
+ case _ =>
+ None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Hashes.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Hashes.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Hashes.scala
new file mode 100644
index 0000000..634b723
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Hashes.scala
@@ -0,0 +1,21 @@
+package org.apache.s2graph.counter.util
+
+import org.apache.hadoop.hbase.util.Bytes
+
+import scala.util.hashing.MurmurHash3
+
+object Hashes {
+ def sha1(s: String): String = {
+ val md = java.security.MessageDigest.getInstance("SHA-1")
+ Bytes.toHex(md.digest(s.getBytes("UTF-8")))
+ }
+
+ private def positiveHash(h: Int): Int = {
+ if (h < 0) -1 * (h + 1) else h
+ }
+
+ def murmur3(s: String): Int = {
+ val hash = MurmurHash3.stringHash(s)
+ positiveHash(hash)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/ReduceMapValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/ReduceMapValue.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/ReduceMapValue.scala
new file mode 100644
index 0000000..037813b
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/ReduceMapValue.scala
@@ -0,0 +1,9 @@
+package org.apache.s2graph.counter.util
+
+class ReduceMapValue[T, U](op: (U, U) => U, default: U) {
+ def apply(m1: Map[T, U], m2: Map[T, U]): Map[T, U] = {
+ m1 ++ m2.map { case (k, v) =>
+ k -> op(m1.getOrElse(k, default), v)
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Retry.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Retry.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Retry.scala
new file mode 100644
index 0000000..f49a231
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/Retry.scala
@@ -0,0 +1,44 @@
+package org.apache.s2graph.counter.util
+
+import scala.annotation.tailrec
+import scala.concurrent.{ExecutionContext, Future, Promise}
+import scala.util.{Failure, Success, Try}
+
+object Retry {
+ @tailrec
+ def apply[T](n: Int, withSleep: Boolean = true, tryCount: Int = 0)(fn: => T): T = {
+ Try { fn } match {
+ case Success(x) => x
+ case Failure(e) if e.isInstanceOf[RetryStopException] => throw e.getCause
+ case _ if n > 1 =>
+ // backoff
+ if (withSleep) Thread.sleep(tryCount * 1000)
+ apply(n - 1, withSleep, tryCount + 1)(fn)
+ case Failure(e) => throw e
+ }
+ }
+}
+
+object RetryAsync {
+ def apply[T](n: Int, withSleep: Boolean = true, tryCount: Int = 0)(fn: => Future[T])(implicit ex: ExecutionContext): Future[T] = {
+ val promise = Promise[T]()
+ fn onComplete {
+ case Success(x) => promise.success(x)
+ case Failure(e) if e.isInstanceOf[RetryStopException] => promise.failure(e.getCause)
+ case _ if n > 1 =>
+ // backoff
+ if (withSleep) Thread.sleep(tryCount * 1000)
+ apply(n - 1, withSleep, tryCount + 1)(fn)
+ case Failure(e) => promise.failure(e)
+ }
+ promise.future
+ }
+}
+
+class RetryStopException(message: String, cause: Throwable)
+ extends Exception(message, cause) {
+
+ def this(message: String) = this(message, null)
+
+ def this(cause: Throwable) = this(cause.toString, cause)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/SplitBytes.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/SplitBytes.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/SplitBytes.scala
new file mode 100644
index 0000000..34f4cf4
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/SplitBytes.scala
@@ -0,0 +1,21 @@
+package org.apache.s2graph.counter.util
+
+object SplitBytes {
+ def apply(bytes: Array[Byte], sizes: Seq[Int]): Seq[Array[Byte]] = {
+ if (sizes.sum > bytes.length) {
+ throw new Exception(s"sizes.sum bigger than bytes.length ${sizes.sum} > ${bytes.length}} ")
+ }
+
+ var position = 0
+ val rtn = {
+ for {
+ size <- sizes
+ } yield {
+ val slice = bytes.slice(position, position + size)
+ position += size
+ slice
+ }
+ }
+ rtn ++ Seq(bytes.drop(position))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/UnitConverter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/UnitConverter.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/UnitConverter.scala
new file mode 100644
index 0000000..af6bc0c
--- /dev/null
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/util/UnitConverter.scala
@@ -0,0 +1,25 @@
+package org.apache.s2graph.counter.util
+
+object UnitConverter {
+ def toMillis(ts: Int): Long = {
+ ts * 1000L
+ }
+
+ def toMillis(ts: Long): Long = {
+ if (ts <= Int.MaxValue) {
+ ts * 1000
+ } else {
+ ts
+ }
+ }
+
+ def toMillis(s: String): Long = {
+ toMillis(s.toLong)
+ }
+
+ def toHours(ts: Long): Long = {
+ toMillis(ts) / HOUR_MILLIS * HOUR_MILLIS
+ }
+
+ val HOUR_MILLIS = 60 * 60 * 1000
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/config/ConfigFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/config/ConfigFunctions.scala b/s2counter_core/src/main/scala/s2/config/ConfigFunctions.scala
deleted file mode 100644
index 20d07ce..0000000
--- a/s2counter_core/src/main/scala/s2/config/ConfigFunctions.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package s2.config
-
-import com.typesafe.config.Config
-
-import scala.collection.JavaConversions._
-import scala.reflect.ClassTag
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 3. 2..
- */
-abstract class ConfigFunctions(conf: Config) {
- def getOrElse[T: ClassTag](key: String, default: T): T = {
- val ret = if (conf.hasPath(key)) (default match {
- case _: String => conf.getString(key)
- case _: Int | _: Integer => conf.getInt(key)
- case _: Float | _: Double => conf.getDouble(key)
- case _: Boolean => conf.getBoolean(key)
- case _ => default
- }).asInstanceOf[T]
- else default
- println(s"${this.getClass.getName}: $key -> $ret")
- ret
- }
-
- def getConfigMap(path: String): Map[String, String] = {
- conf.getConfig(path).entrySet().map { entry =>
- val key = s"$path.${entry.getKey}"
- val value = conf.getString(key)
- println(s"${this.getClass.getName}: $key -> $value")
- key -> value
- }.toMap
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
deleted file mode 100644
index fcd0e6a..0000000
--- a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-package s2.config
-
-import com.typesafe.config.Config
-
-import scala.collection.JavaConversions._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 3. 2..
- */
-class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
- // HBase
- lazy val HBASE_ZOOKEEPER_QUORUM = getOrElse("hbase.zookeeper.quorum", "")
- lazy val HBASE_TABLE_NAME = getOrElse("hbase.table.name", "s2counter")
- lazy val HBASE_TABLE_POOL_SIZE = getOrElse("hbase.table.pool.size", 100)
- lazy val HBASE_CONNECTION_POOL_SIZE = getOrElse("hbase.connection.pool.size", 10)
-
- lazy val HBASE_CLIENT_IPC_POOL_SIZE = getOrElse("hbase.client.ipc.pool.size", 5)
- lazy val HBASE_CLIENT_MAX_TOTAL_TASKS = getOrElse("hbase.client.max.total.tasks", 100)
- lazy val HBASE_CLIENT_MAX_PERSERVER_TASKS = getOrElse("hbase.client.max.perserver.tasks", 5)
- lazy val HBASE_CLIENT_MAX_PERREGION_TASKS = getOrElse("hbase.client.max.perregion.tasks", 1)
- lazy val HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD = getOrElse("hbase.client.scanner.timeout.period", 300)
- lazy val HBASE_CLIENT_OPERATION_TIMEOUT = getOrElse("hbase.client.operation.timeout", 100)
- lazy val HBASE_CLIENT_RETRIES_NUMBER = getOrElse("hbase.client.retries.number", 1)
-
- // MySQL
- lazy val DB_DEFAULT_DRIVER = getOrElse("db.default.driver", "com.mysql.jdbc.Driver")
- lazy val DB_DEFAULT_URL = getOrElse("db.default.url", "")
- lazy val DB_DEFAULT_USER = getOrElse("db.default.user", "graph")
- lazy val DB_DEFAULT_PASSWORD = getOrElse("db.default.password", "graph")
-
- // Redis
- lazy val REDIS_INSTANCES = (for {
- s <- config.getStringList("redis.instances")
- } yield {
- val sp = s.split(':')
- (sp(0), if (sp.length > 1) sp(1).toInt else 6379)
- }).toList
-
- // Graph
- lazy val GRAPH_URL = getOrElse("s2graph.url", "http://localhost:9000")
- lazy val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
-
- // Cache
- lazy val CACHE_TTL_SECONDS = getOrElse("cache.ttl.seconds", 600)
- lazy val CACHE_MAX_SIZE = getOrElse("cache.max.size", 10000)
- lazy val CACHE_NEGATIVE_TTL_SECONDS = getOrElse("cache.negative.ttl.seconds", CACHE_TTL_SECONDS)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/TrxLog.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/TrxLog.scala b/s2counter_core/src/main/scala/s2/counter/TrxLog.scala
deleted file mode 100644
index c1db356..0000000
--- a/s2counter_core/src/main/scala/s2/counter/TrxLog.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-package s2.counter
-
-import play.api.libs.json.Json
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 4. 7..
- */
-// item1 -> likedCount -> month:2015-10, 1
-// edge
- // policyId = Label.findByName(likedCount).id.get
- // item = edge.srcVertexId
- // results =
-case class TrxLog(success: Boolean, policyId: Int, item: String, results: Iterable[TrxLogResult])
-
-// interval = m, ts = 2015-10, "age.gender.20.M", 1, 2
-case class TrxLogResult(interval: String, ts: Long, dimension: String, value: Long, result: Long = -1)
-
-object TrxLogResult {
- implicit val writes = Json.writes[TrxLogResult]
- implicit val reads = Json.reads[TrxLogResult]
- implicit val formats = Json.format[TrxLogResult]
-}
-
-object TrxLog {
- implicit val writes = Json.writes[TrxLog]
- implicit val reads = Json.reads[TrxLog]
- implicit val formats = Json.format[TrxLog]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/BytesUtil.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/BytesUtil.scala b/s2counter_core/src/main/scala/s2/counter/core/BytesUtil.scala
deleted file mode 100644
index 1d945ed..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/BytesUtil.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package s2.counter.core
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 11..
- */
-trait BytesUtil {
- def getRowKeyPrefix(id: Int): Array[Byte]
-
- def toBytes(key: ExactKeyTrait): Array[Byte]
- def toBytes(eq: ExactQualifier): Array[Byte]
- def toBytes(tq: TimedQualifier): Array[Byte]
-
- def toExactQualifier(bytes: Array[Byte]): ExactQualifier
- def toTimedQualifier(bytes: Array[Byte]): TimedQualifier
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/ExactCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/ExactCounter.scala b/s2counter_core/src/main/scala/s2/counter/core/ExactCounter.scala
deleted file mode 100644
index 7f36681..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/ExactCounter.scala
+++ /dev/null
@@ -1,247 +0,0 @@
-package s2.counter.core
-
-import com.typesafe.config.Config
-import org.slf4j.LoggerFactory
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core.TimedQualifier.IntervalUnit.IntervalUnit
-import s2.counter.decay.ExpDecayFormula
-import s2.counter.{TrxLog, TrxLogResult}
-import s2.models.Counter
-import s2.util.{CollectionCache, CollectionCacheConfig, FunctionParser}
-
-import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 11..
- */
-
-case class ExactCounterRow(key: ExactKeyTrait, value: Map[ExactQualifier, Long])
-
-case class FetchedCounts(exactKey: ExactKeyTrait, qualifierWithCountMap: Map[ExactQualifier, Long])
-case class DecayedCounts(exactKey: ExactKeyTrait, qualifierWithCountMap: Map[ExactQualifier, Double])
-
-case class FetchedCountsGrouped(exactKey: ExactKeyTrait, intervalWithCountMap: Map[(IntervalUnit, Map[String, String]), Map[ExactQualifier, Long]])
-
-class ExactCounter(config: Config, storage: ExactStorage) {
- import ExactCounter._
-
- val syncDuration = Duration(10, SECONDS)
- private val log = LoggerFactory.getLogger(getClass)
-
- val storageStatusCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(1000, 60, negativeCache = false, 60))
-
- // dimension: age, value of ages
- def getCountAsync(policy: Counter,
- itemId: String,
- intervals: Seq[IntervalUnit],
- from: Long,
- to: Long,
- dimension: Map[String, Set[String]])
- (implicit ex: ExecutionContext): Future[Option[FetchedCountsGrouped]] = {
- for {
- fetchedCounts <- getCountsAsync(policy, Seq(itemId),
- intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))), dimension)
- } yield {
- fetchedCounts.headOption
- }
- }
-
- // multi item, time range and multi dimension
- def getCountsAsync(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
- storage.get(policy, items, timeRange, dimQuery)
- }
-
- def getCount(policy: Counter,
- itemId: String,
- intervals: Seq[IntervalUnit],
- from: Long,
- to: Long,
- dimension: Map[String, Set[String]])
- (implicit ex: ExecutionContext): Option[FetchedCountsGrouped] = {
- getCounts(policy, Seq(itemId),
- intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))), dimension).headOption
- }
-
- def getCount(policy: Counter,
- itemId: String,
- intervals: Seq[IntervalUnit],
- from: Long,
- to: Long)
- (implicit ex: ExecutionContext): Option[FetchedCounts] = {
- val future = storage.get(policy,
- Seq(itemId),
- intervals.map(interval => (TimedQualifier(interval, from), TimedQualifier(interval, to))))
- Await.result(future, syncDuration).headOption
- }
-
- // multi item, time range and multi dimension
- def getCounts(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ex: ExecutionContext): Seq[FetchedCountsGrouped] = {
- Await.result(storage.get(policy, items, timeRange, dimQuery), syncDuration)
- }
-
- def getRelatedCounts(policy: Counter, keyWithQualifiers: Seq[(String, Seq[ExactQualifier])])
- (implicit ex: ExecutionContext): Map[String, Map[ExactQualifier, Long]] = {
- val queryKeyWithQualifiers = {
- for {
- (itemKey, qualifiers) <- keyWithQualifiers
- } yield {
- val relKey = ExactKey(policy.id, policy.version, policy.itemType, itemKey)
- (relKey, qualifiers)
- }
- }
- val future = storage.get(policy, queryKeyWithQualifiers)
-
- for {
- FetchedCounts(exactKey, exactQualifierToLong) <- Await.result(future, syncDuration)
- } yield {
- exactKey.itemKey -> exactQualifierToLong
- }
- }.toMap
-
- def getPastCounts(policy: Counter, keyWithQualifiers: Seq[(String, Seq[ExactQualifier])])
- (implicit ex: ExecutionContext): Map[String, Map[ExactQualifier, Long]] = {
- // query paste count
- val queryKeyWithQualifiers = {
- for {
- (itemKey, qualifiers) <- keyWithQualifiers
- } yield {
- val relKey = ExactKey(policy.id, policy.version, policy.itemType, itemKey)
- (relKey, qualifiers.map(eq => eq.copy(tq = eq.tq.add(-1))))
- }
- }
- val future = storage.get(policy, queryKeyWithQualifiers)
-
- for {
- FetchedCounts(exactKey, exactQualifierToLong) <- Await.result(future, syncDuration)
- } yield {
- // restore tq
- exactKey.itemKey -> exactQualifierToLong.map { case (eq, v) =>
- eq.copy(tq = eq.tq.add(1)) -> v
- }
- }
- }.toMap
-
- def getDecayedCountsAsync(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]],
- qsSum: Option[String])(implicit ex: ExecutionContext): Future[Seq[DecayedCounts]] = {
- val groupedTimeRange = timeRange.groupBy(_._1.q)
- getCountsAsync(policy, items, timeRange, dimQuery).map { seq =>
- for {
- FetchedCountsGrouped(k, intervalWithCountMap) <- seq
- } yield {
- DecayedCounts(k, {
- for {
- ((interval, dimKeyValues), grouped) <- intervalWithCountMap
- } yield {
- val (tqFrom, tqTo) = groupedTimeRange(interval).head
- val formula = {
- for {
- strSum <- qsSum
- (func, arg) <- FunctionParser(strSum)
- } yield {
- // apply function
- func.toLowerCase match {
- case "exp_decay" => ExpDecayFormula.byMeanLifeTime(arg.toLong * TimedQualifier.getTsUnit(interval))
- case _ => throw new UnsupportedOperationException(s"unknown function: $strSum")
- }
- }
- }
- ExactQualifier(tqFrom, dimKeyValues) -> {
- grouped.map { case (eq, count) =>
- formula match {
- case Some(decay) =>
- decay(count, tqTo.ts - eq.tq.ts)
- case None =>
- count
- }
- }.sum
- }
- }
- })
- }
- }
- }
-
- def updateCount(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Seq[TrxLog] = {
- ready(policy) match {
- case true =>
- val updateResults = storage.update(policy, counts)
- for {
- (exactKey, values) <- counts
- results = updateResults.getOrElse(exactKey, Nil.toMap)
- } yield {
- TrxLog(results.nonEmpty, exactKey.policyId, exactKey.itemKey, makeTrxLogResult(values, results))
- }
- case false =>
- Nil
- }
- }
-
- def deleteCount(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
- storage.delete(policy, keys)
- }
-
- private def makeTrxLogResult(values: ExactValueMap, results: ExactValueMap): Seq[TrxLogResult] = {
- for {
- (eq, value) <- values
- } yield {
- val result = results.getOrElse(eq, -1l)
- TrxLogResult(eq.tq.q.toString, eq.tq.ts, eq.dimension, value, result)
- }
- }.toSeq
-
- def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
- storage.insertBlobValue(policy, keys)
- }
-
- def getBlobValue(policy: Counter, blobId: String): Option[String] = {
- storage.getBlobValue(policy, blobId)
- }
-
- def prepare(policy: Counter) = {
- storage.prepare(policy)
- }
-
- def destroy(policy: Counter) = {
- storage.destroy(policy)
- }
-
- def ready(policy: Counter): Boolean = {
- storageStatusCache.withCache(s"${policy.id}") {
- val ready = storage.ready(policy)
- if (!ready) {
- // if key is not in cache, log message
- log.warn(s"${policy.service}.${policy.action} storage is not ready.")
- }
- Some(ready)
- }.getOrElse(false)
- }
-}
-
-object ExactCounter {
- object ColumnFamily extends Enumeration {
- type ColumnFamily = Value
-
- val SHORT = Value("s")
- val LONG = Value("l")
- }
- import IntervalUnit._
- val intervalsMap = Map(MINUTELY -> ColumnFamily.SHORT, HOURLY -> ColumnFamily.SHORT,
- DAILY -> ColumnFamily.LONG, MONTHLY -> ColumnFamily.LONG, TOTAL -> ColumnFamily.LONG)
-
- val blobCF = ColumnFamily.LONG.toString.getBytes
- val blobColumn = "b".getBytes
-
- type ExactValueMap = Map[ExactQualifier, Long]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/ExactKey.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/ExactKey.scala b/s2counter_core/src/main/scala/s2/counter/core/ExactKey.scala
deleted file mode 100644
index 63aca51..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/ExactKey.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-package s2.counter.core
-
-import s2.models.Counter
-import s2.models.Counter.ItemType
-import s2.models.Counter.ItemType.ItemType
-import s2.util.Hashes
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 27..
- */
-trait ExactKeyTrait {
- def policyId: Int
- def version: Byte
- def itemType: ItemType
- def itemKey: String
-}
-
-case class ExactKey(policyId: Int, version: Byte, itemType: ItemType, itemKey: String) extends ExactKeyTrait
-case class BlobExactKey(policyId: Int, version: Byte, itemType: ItemType, itemKey: String, itemId: String) extends ExactKeyTrait
-
-object ExactKey {
- def apply(policy: Counter, itemId: String, checkItemType: Boolean): ExactKeyTrait = {
- if (checkItemType && policy.itemType == ItemType.BLOB) {
- BlobExactKey(policy.id, policy.version, ItemType.BLOB, Hashes.sha1(itemId), itemId)
- } else {
- ExactKey(policy.id, policy.version, policy.itemType, itemId)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/ExactQualifier.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/ExactQualifier.scala b/s2counter_core/src/main/scala/s2/counter/core/ExactQualifier.scala
deleted file mode 100644
index f4ac708..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/ExactQualifier.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package s2.counter.core
-
-import java.util
-
-import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
-import s2.counter.core.TimedQualifier.IntervalUnit.IntervalUnit
-
-import scala.collection.JavaConversions._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 27..
- */
-case class ExactQualifier(tq: TimedQualifier, dimKeyValues: Map[String, String], dimension: String) {
- def checkDimensionEquality(dimQuery: Map[String, Set[String]]): Boolean = {
-// println(s"self: $dimKeyValues, query: $dimQuery")
- dimQuery.size == dimKeyValues.size && {
- for {
- (k, v) <- dimKeyValues
- } yield {
- dimQuery.get(k).exists(qv => qv.isEmpty || qv.contains(v))
- }
- }.forall(x => x)
- }
-}
-
-object ExactQualifier {
- val cache: LoadingCache[String, Map[String, String]] = CacheBuilder.newBuilder()
- .maximumSize(10000)
- .build(
- new CacheLoader[String, Map[String, String]]() {
- def load(s: String): Map[String, String] = {
- strToDimensionMap(s)
- }
- }
- )
-
- def apply(tq: TimedQualifier, dimension: String): ExactQualifier = {
- ExactQualifier(tq, cache.get(dimension), dimension)
- }
-
- def apply(tq: TimedQualifier, dimKeyValues: Map[String, String]): ExactQualifier = {
- ExactQualifier(tq, dimKeyValues, makeDimensionStr(dimKeyValues))
- }
-
- def makeSortedDimension(dimKeyValues: Map[String, String]): Iterator[String] = {
- val sortedDimKeyValues = new util.TreeMap[String, String](dimKeyValues)
- sortedDimKeyValues.keysIterator ++ sortedDimKeyValues.valuesIterator
- }
-
- def makeDimensionStr(dimKeyValues: Map[String, String]): String = {
- makeSortedDimension(dimKeyValues).mkString(".")
- }
-
- def getQualifiers(intervals: Seq[IntervalUnit], ts: Long, dimKeyValues: Map[String, String]): Seq[ExactQualifier] = {
- for {
- tq <- TimedQualifier.getQualifiers(intervals, ts)
- } yield {
- ExactQualifier(tq, dimKeyValues, makeDimensionStr(dimKeyValues))
- }
- }
-
- def strToDimensionMap(dimension: String): Map[String, String] = {
- val dimSp = {
- val sp = dimension.split('.')
- if (dimension == ".") {
- Array("", "")
- }
- else if (dimension.nonEmpty && dimension.last == '.') {
- sp ++ Array("")
- } else {
- sp
- }
- }
- val dimKey = dimSp.take(dimSp.length / 2)
- val dimVal = dimSp.takeRight(dimSp.length / 2)
- dimKey.zip(dimVal).toMap
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/ExactStorage.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/ExactStorage.scala b/s2counter_core/src/main/scala/s2/counter/core/ExactStorage.scala
deleted file mode 100644
index 6a81f41..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/ExactStorage.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-package s2.counter.core
-
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.models.Counter
-
-import scala.concurrent.{ExecutionContext, Future}
-
-/**
- * Created by shon on 8/12/15.
- */
-trait ExactStorage {
- // for range query and check dimension
- def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]]
- // for range query
- def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]]
- // for query exact qualifier
- def get(policy: Counter,
- queries: Seq[(ExactKeyTrait, Seq[ExactQualifier])])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]]
- def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap]
- def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit
- def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean]
- def getBlobValue(policy: Counter, blobId: String): Option[String]
-
- def prepare(policy: Counter): Unit
- def destroy(policy: Counter): Unit
- def ready(policy: Counter): Boolean
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RankingCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RankingCounter.scala b/s2counter_core/src/main/scala/s2/counter/core/RankingCounter.scala
deleted file mode 100644
index b98ef30..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RankingCounter.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-package s2.counter.core
-
-import java.util.concurrent.TimeUnit
-
-import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
-import com.typesafe.config.Config
-import org.slf4j.LoggerFactory
-import s2.counter.core.RankingCounter.RankingValueMap
-import s2.models.Counter
-import s2.util.{CollectionCache, CollectionCacheConfig}
-
-import scala.collection.JavaConversions._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-case class RankingRow(key: RankingKey, value: Map[String, RankingValue])
-case class RateRankingRow(key: RankingKey, value: Map[String, RateRankingValue])
-
-class RankingCounter(config: Config, storage: RankingStorage) {
- private val log = LoggerFactory.getLogger(getClass)
-
- val storageStatusCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(1000, 60, negativeCache = false, 60))
-
- val cache: LoadingCache[RankingKey, RankingResult] = CacheBuilder.newBuilder()
- .maximumSize(1000000)
- .expireAfterWrite(10l, TimeUnit.MINUTES)
- .build(
- new CacheLoader[RankingKey, RankingResult]() {
- def load(rankingKey: RankingKey): RankingResult = {
-// log.warn(s"cache load: $rankingKey")
- storage.getTopK(rankingKey, Int.MaxValue).getOrElse(RankingResult(-1, Nil))
- }
- }
- )
-
- def getTopK(rankingKey: RankingKey, k: Int = Int.MaxValue): Option[RankingResult] = {
- val tq = rankingKey.eq.tq
- if (TimedQualifier.getQualifiers(Seq(tq.q), System.currentTimeMillis()).head == tq) {
- // do not use cache
- storage.getTopK(rankingKey, k)
- }
- else {
- val result = cache.get(rankingKey)
- if (result.values.nonEmpty) {
- Some(result.copy(values = result.values.take(k)))
- }
- else {
- None
- }
- }
- }
-
- def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
- storage.update(key, value, k)
- }
-
- def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
- storage.update(values, k)
- }
-
- def delete(key: RankingKey): Unit = {
- storage.delete(key)
- }
-
- def getAllItems(keys: Seq[RankingKey], k: Int = Int.MaxValue): Seq[String] = {
- val oldKeys = keys.filter(key => TimedQualifier.getQualifiers(Seq(key.eq.tq.q), System.currentTimeMillis()).head != key.eq.tq)
- val cached = cache.getAllPresent(oldKeys)
- val missed = keys.diff(cached.keys.toSeq)
- val found = storage.getTopK(missed, k)
-
-// log.warn(s"cached: ${cached.size()}, missed: ${missed.size}")
-
- for {
- (key, result) <- found
- } {
- cache.put(key, result)
- }
-
- for {
- (key, RankingResult(totalScore, values)) <- cached ++ found
- (item, score) <- values
- } yield {
- item
- }
- }.toSeq.distinct
-
- def prepare(policy: Counter): Unit = {
- storage.prepare(policy)
- }
-
- def destroy(policy: Counter): Unit = {
- storage.destroy(policy)
- }
-
- def ready(policy: Counter): Boolean = {
- storageStatusCache.withCache(s"${policy.id}") {
- Some(storage.ready(policy))
- }.getOrElse(false)
- }
-}
-
-object RankingCounter {
- type RankingValueMap = Map[String, RankingValue]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RankingKey.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RankingKey.scala b/s2counter_core/src/main/scala/s2/counter/core/RankingKey.scala
deleted file mode 100644
index 7d70625..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RankingKey.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package s2.counter.core
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-case class RankingKey(policyId: Int, version: Byte, eq: ExactQualifier)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RankingResult.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RankingResult.scala b/s2counter_core/src/main/scala/s2/counter/core/RankingResult.scala
deleted file mode 100644
index 42fae3e..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RankingResult.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package s2.counter.core
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-case class RankingResult(totalScore: Double, values: Seq[(String, Double)])
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RankingStorage.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RankingStorage.scala b/s2counter_core/src/main/scala/s2/counter/core/RankingStorage.scala
deleted file mode 100644
index b643bd8..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RankingStorage.scala
+++ /dev/null
@@ -1,19 +0,0 @@
-package s2.counter.core
-
-import s2.counter.core.RankingCounter.RankingValueMap
-import s2.models.Counter
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 22..
- */
-trait RankingStorage {
- def getTopK(key: RankingKey, k: Int): Option[RankingResult]
- def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)]
- def update(key: RankingKey, value: RankingValueMap, k: Int): Unit
- def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit
- def delete(key: RankingKey)
-
- def prepare(policy: Counter): Unit
- def destroy(policy: Counter): Unit
- def ready(policy: Counter): Boolean
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RankingValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RankingValue.scala b/s2counter_core/src/main/scala/s2/counter/core/RankingValue.scala
deleted file mode 100644
index 4b67633..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RankingValue.scala
+++ /dev/null
@@ -1,19 +0,0 @@
-package s2.counter.core
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 22..
- */
-
-/**
- * ranking score and increment value
- * @param score ranking score
- * @param increment increment value for v1
- */
-case class RankingValue(score: Double, increment: Double)
-
-object RankingValue {
- def reduce(r1: RankingValue, r2: RankingValue): RankingValue = {
- // maximum score and sum of increment
- RankingValue(math.max(r1.score, r2.score), r1.increment + r2.increment)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/RateRankingValue.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/RateRankingValue.scala b/s2counter_core/src/main/scala/s2/counter/core/RateRankingValue.scala
deleted file mode 100644
index eca3abc..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/RateRankingValue.scala
+++ /dev/null
@@ -1,18 +0,0 @@
-package s2.counter.core
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 7. 2..
- */
-case class RateRankingValue(actionScore: Double, baseScore: Double) {
- // increment score do not use.
- lazy val rankingValue: RankingValue = {
- RankingValue(actionScore / math.max(1d, baseScore), 0)
- }
-}
-
-object RateRankingValue {
- def reduce(r1: RateRankingValue, r2: RateRankingValue): RateRankingValue = {
- // maximum score and sum of increment
- RateRankingValue(math.max(r1.actionScore, r2.actionScore), math.max(r1.baseScore, r2.baseScore))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/TimedQualifier.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/TimedQualifier.scala b/s2counter_core/src/main/scala/s2/counter/core/TimedQualifier.scala
deleted file mode 100644
index b763bc2..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/TimedQualifier.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-package s2.counter.core
-
-import java.text.SimpleDateFormat
-import java.util.Calendar
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 8..
- */
-case class TimedQualifier(q: TimedQualifier.IntervalUnit.Value, ts: Long) {
- import TimedQualifier.IntervalUnit._
-
- def dateTime: Long = {
- val dateFormat = new SimpleDateFormat("yyyyMMddHHmm")
- dateFormat.format(ts).toLong
- }
-
- def add(amount: Int): TimedQualifier = {
- val cal = Calendar.getInstance()
- cal.setTimeInMillis(ts)
- q match {
- case MINUTELY =>
- cal.add(Calendar.MINUTE, amount)
- case HOURLY =>
- cal.add(Calendar.HOUR, amount)
- case DAILY =>
- cal.add(Calendar.DAY_OF_MONTH, amount)
- case MONTHLY =>
- cal.add(Calendar.MONTH, amount)
- case TOTAL =>
- }
- copy(ts = cal.getTimeInMillis)
- }
-}
-
-object TimedQualifier {
- object IntervalUnit extends Enumeration {
- type IntervalUnit = Value
- val TOTAL = Value("t")
- val MONTHLY = Value("M")
- val DAILY = Value("d")
- val HOURLY = Value("H")
- val MINUTELY = Value("m")
- }
-
- def apply(q: String, ts: Long): TimedQualifier = TimedQualifier(IntervalUnit.withName(q), ts)
-
- import IntervalUnit._
-
- def getTsUnit(intervalUnit: IntervalUnit.IntervalUnit): Long = {
- intervalUnit match {
- case MINUTELY => 1 * 60 * 1000l
- case HOURLY => 60 * 60 * 1000l
- case DAILY => 24 * 60 * 60 * 1000l
- case MONTHLY => 31 * 24 * 60 * 60 * 1000l
- case v: IntervalUnit.IntervalUnit =>
- throw new RuntimeException(s"unsupported operation for ${v.toString}")
- }
- }
-
- def getQualifiers(intervals: Seq[IntervalUnit], millis: Long): Seq[TimedQualifier] = {
- val cal = Calendar.getInstance()
- cal.setTimeInMillis(millis)
-
- val newCal = Calendar.getInstance()
- newCal.set(cal.get(Calendar.YEAR), cal.get(Calendar.MONTH), 1, 0, 0, 0)
- newCal.set(Calendar.MILLISECOND, 0)
- val month = newCal.getTimeInMillis
- val Seq(day, hour, minute) = {
- for {
- field <- Seq(Calendar.DATE, Calendar.HOUR_OF_DAY, Calendar.MINUTE)
- } yield {
- newCal.set(field, cal.get(field))
- newCal.getTimeInMillis
- }
- }
-
- for {
- interval <- intervals
- } yield {
- val ts = interval match {
- case MINUTELY => minute
- case HOURLY => hour
- case DAILY => day
- case MONTHLY => month
- case TOTAL => 0L
- }
- TimedQualifier(interval, ts)
- }
- }
-
- // descending order
- def getQualifiersToLimit(intervals: Seq[IntervalUnit], limit: Int, tsOpt: Option[Long] = None): Seq[TimedQualifier] = {
- val ts = tsOpt.getOrElse(System.currentTimeMillis())
- for {
- interval <- intervals
- newLimit = if (interval == TOTAL) 1 else limit
- i <- 0 until (-newLimit, -1)
- } yield {
- val newMillis = nextTime(interval, ts, i)
- TimedQualifier(interval, newMillis)
- }
- }
-
- private def nextTime(interval: IntervalUnit, ts: Long, i: Int): Long = {
- val newCal = Calendar.getInstance()
- newCal.setTimeInMillis(ts)
- newCal.set(Calendar.MILLISECOND, 0)
- interval match {
- case MINUTELY =>
- newCal.set(Calendar.SECOND, 0)
- newCal.add(Calendar.MINUTE, i)
- newCal.getTimeInMillis
- case HOURLY =>
- newCal.set(Calendar.SECOND, 0)
- newCal.set(Calendar.MINUTE, 0)
- newCal.add(Calendar.HOUR_OF_DAY, i)
- newCal.getTimeInMillis
- case DAILY =>
- newCal.set(Calendar.SECOND, 0)
- newCal.set(Calendar.MINUTE, 0)
- newCal.set(Calendar.HOUR_OF_DAY, 0)
- newCal.add(Calendar.DAY_OF_MONTH, i)
- newCal.getTimeInMillis
- case MONTHLY =>
- newCal.set(Calendar.SECOND, 0)
- newCal.set(Calendar.MINUTE, 0)
- newCal.set(Calendar.HOUR_OF_DAY, 0)
- newCal.set(Calendar.DAY_OF_MONTH, 1)
- newCal.add(Calendar.MONTH, i)
- newCal.getTimeInMillis
- case TOTAL =>
- 0L
- }
- }
-
- def getTimeList(interval: IntervalUnit, from: Long, to: Long, rst: List[Long] = Nil): List[Long] = {
- interval match {
- case TOTAL => List(0)
- case _ =>
- val next = nextTime(interval, from, 1)
- if (next < from) {
- // ignore
- getTimeList(interval, next, to, rst)
- }
- else if (next < to) {
- // recall
- getTimeList(interval, next, to, rst :+ next)
- } else {
- // end condition
- rst :+ next
- }
- }
- }
-
- // for reader
- def getQualifiersToLimit(intervals: Seq[IntervalUnit],
- limit: Int,
- optFrom: Option[Long],
- optTo: Option[Long]): Seq[List[TimedQualifier]] = {
- val newLimit = limit - 1
- for {
- interval <- intervals
- } yield {
- {
- (optFrom, optTo) match {
- case (Some(from), Some(to)) =>
- getTimeList(interval, from, to)
- case (Some(from), None) =>
- getTimeList(interval, from, nextTime(interval, from, newLimit))
- case (None, Some(to)) =>
- getTimeList(interval, nextTime(interval, to, -newLimit), to)
- case (None, None) =>
- val current = System.currentTimeMillis()
- getTimeList(interval, nextTime(interval, current, -newLimit), current)
- }
- }.map { ts =>
- TimedQualifier(interval, ts)
- }
- }
- }
-
- def getTimeRange(intervals: Seq[IntervalUnit],
- limit: Int,
- optFrom: Option[Long],
- optTo: Option[Long]): Seq[(TimedQualifier, TimedQualifier)] = {
- val newLimit = limit - 1
- val maxInterval = intervals.maxBy {
- case MINUTELY => 0
- case HOURLY => 1
- case DAILY => 2
- case MONTHLY => 3
- case TOTAL => 4
- }
- val minInterval = intervals.minBy {
- case MINUTELY => 0
- case HOURLY => 1
- case DAILY => 2
- case MONTHLY => 3
- case TOTAL => 4
- }
- val (from, to) = (optFrom, optTo) match {
- case (Some(f), Some(t)) =>
- (f, t)
- case (Some(f), None) =>
- (f, nextTime(minInterval, f, newLimit))
- case (None, Some(t)) =>
- (nextTime(maxInterval, t, -newLimit), t)
- case (None, None) =>
- val current = System.currentTimeMillis()
- (nextTime(maxInterval, current, -newLimit), nextTime(minInterval, current, 0))
- }
- for {
- interval <- intervals
- } yield {
- (TimedQualifier(interval, from), TimedQualifier(interval, to))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v1/BytesUtilV1.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v1/BytesUtilV1.scala b/s2counter_core/src/main/scala/s2/counter/core/v1/BytesUtilV1.scala
deleted file mode 100644
index 1b70f63..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v1/BytesUtilV1.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-package s2.counter.core.v1
-
-import org.apache.hadoop.hbase.util.Bytes
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.models.Counter.ItemType
-import s2.util.Hashes
-
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 11..
- */
-object BytesUtilV1 extends BytesUtil {
- // ExactKey: [hash(2b)][policy(4b)][item(variable)]
- val BUCKET_BYTE_SIZE = Bytes.SIZEOF_SHORT
- val POLICY_ID_SIZE = Bytes.SIZEOF_INT
- val INTERVAL_SIZE = Bytes.SIZEOF_BYTE
- val TIMESTAMP_SIZE = Bytes.SIZEOF_LONG
- val TIMED_QUALIFIER_SIZE = INTERVAL_SIZE + TIMESTAMP_SIZE
-
- override def getRowKeyPrefix(id: Int): Array[Byte] = {
- Bytes.toBytes(id)
- }
-
- override def toBytes(key: ExactKeyTrait): Array[Byte] = {
- val buff = new ArrayBuffer[Byte]
- // hash key (2 byte)
- buff ++= Bytes.toBytes(Hashes.murmur3(key.itemKey)).take(BUCKET_BYTE_SIZE)
-
- buff ++= getRowKeyPrefix(key.policyId)
- buff ++= {
- key.itemType match {
- case ItemType.INT => Bytes.toBytes(key.itemKey.toInt)
- case ItemType.LONG => Bytes.toBytes(key.itemKey.toLong)
- case ItemType.STRING | ItemType.BLOB => Bytes.toBytes(key.itemKey)
- }
- }
- buff.toArray
- }
-
- override def toBytes(eq: ExactQualifier): Array[Byte] = {
- toBytes(eq.tq) ++ eq.dimension.getBytes
- }
-
- override def toBytes(tq: TimedQualifier): Array[Byte] = {
- Bytes.toBytes(tq.q.toString) ++ Bytes.toBytes(tq.ts)
- }
-
- override def toExactQualifier(bytes: Array[Byte]): ExactQualifier = {
- // qualifier: interval, ts, dimension 순서
- val tq = toTimedQualifier(bytes)
-
- val dimension = Bytes.toString(bytes, TIMED_QUALIFIER_SIZE, bytes.length - TIMED_QUALIFIER_SIZE)
- ExactQualifier(tq, dimension)
- }
-
- override def toTimedQualifier(bytes: Array[Byte]): TimedQualifier = {
- val interval = Bytes.toString(bytes, 0, INTERVAL_SIZE)
- val ts = Bytes.toLong(bytes, INTERVAL_SIZE)
-
- TimedQualifier(IntervalUnit.withName(interval), ts)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageAsyncHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageAsyncHBase.scala b/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageAsyncHBase.scala
deleted file mode 100644
index 6aae3cd..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageAsyncHBase.scala
+++ /dev/null
@@ -1,321 +0,0 @@
-package s2.counter.core.v1
-
-import java.util
-
-import com.stumbleupon.async.{Callback, Deferred}
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.CellUtil
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.{ColumnRangeFilter, FilterList, GetRequest, KeyValue}
-import org.slf4j.LoggerFactory
-import s2.config.S2CounterConfig
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.counter.core._
-import s2.helper.{Management, WithAsyncHBase, WithHBase}
-import s2.models.Counter
-import s2.models.Counter.ItemType
-
-import scala.collection.JavaConversions._
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 8. 19..
- */
-class ExactStorageAsyncHBase(config: Config) extends ExactStorage {
- import ExactStorageHBase._
-
- private val log = LoggerFactory.getLogger(getClass)
-
- lazy val s2config = new S2CounterConfig(config)
-
- private[counter] val withHBase = new WithHBase(config)
- private[counter] val withAsyncHBase = new WithAsyncHBase(config)
- private[counter] val hbaseManagement = new Management(config)
-
- private def getTableName(policy: Counter): String = {
- policy.hbaseTable.getOrElse(s2config.HBASE_TABLE_NAME)
- }
-
- override def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
-
- val tableName = getTableName(policy)
-
- lazy val messageForLog = s"${policy.service}.${policy.action} $items $timeRange"
-
- val keys = {
- for {
- item <- items
- } yield {
- ExactKey(policy, item, checkItemType = true)
- }
- }
-
- val gets = {
- for {
- cf <- timeRange.map(t => intervalsMap(t._1.q)).distinct
- key <- keys
- } yield {
- val get = new GetRequest(tableName, BytesUtilV1.toBytes(key))
- get.family(cf.toString)
- get.setFilter(new FilterList({
- for {
- (from, to) <- timeRange
- } yield {
- new ColumnRangeFilter(
- BytesUtilV1.toBytes(from), true,
- BytesUtilV1.toBytes(to.copy(ts = to.ts + 1)), false)
- }
- }, FilterList.Operator.MUST_PASS_ONE))
- (key, cf, get)
- }
- }
-
-// println(s"$messageForLog $gets")
-
- withAsyncHBase[Seq[FetchedCounts]] { client =>
- val deferreds: Seq[Deferred[FetchedCounts]] = {
- for {
- (key, cf, get) <- gets
- } yield {
- client.get(get).addCallback { new Callback[FetchedCounts, util.ArrayList[KeyValue]] {
- override def call(kvs: util.ArrayList[KeyValue]): FetchedCounts = {
- val qualifierWithCounts = {
- for {
- kv <- kvs
- eq = BytesUtilV1.toExactQualifier(kv.qualifier())
- } yield {
- eq -> Bytes.toLong(kv.value())
- }
- }.toMap
-// println(s"$key $qualifierWithCounts")
- FetchedCounts(key, qualifierWithCounts)
- }
- }}
- }
- }
- Deferred.group(deferreds).addCallback { new Callback[Seq[FetchedCounts], util.ArrayList[FetchedCounts]] {
- override def call(arg: util.ArrayList[FetchedCounts]): Seq[FetchedCounts] = {
- for {
- (key, fetchedGroup) <- Seq(arg: _*).groupBy(_.exactKey)
- } yield {
- fetchedGroup.reduce[FetchedCounts] { case (f1, f2) =>
- FetchedCounts(key, f1.qualifierWithCountMap ++ f2.qualifierWithCountMap)
- }
- }
- }.toSeq
- }}
- }
- }
-
- override def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
- get(policy, items, timeRange).map { fetchedLs =>
- for {
- FetchedCounts(exactKey, qualifierWithCountMap) <- fetchedLs
- } yield {
- val intervalWithCountMap = qualifierWithCountMap
- .filter { case (eq, v) => eq.checkDimensionEquality(dimQuery) }
- .groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) }
- FetchedCountsGrouped(exactKey, intervalWithCountMap)
- }
- }
- }
-
- override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
- // increment mutation to hbase
- val increments = {
- for {
- (exactKey, values) <- counts
- inc = new Increment(BytesUtilV1.toBytes(exactKey))
- } yield {
- for {
- (eq, value) <- values
- } {
- inc.addColumn(intervalsMap.apply(eq.tq.q).toString.getBytes, BytesUtilV1.toBytes(eq), value)
- }
- // add column by dimension
- inc
- }
- }
-
- val results: Array[Object] = Array.fill(increments.size)(null)
-
- withHBase(getTableName(policy)) { table =>
- table.batch(increments, results)
- } match {
- case Failure(ex) =>
- log.error(s"${ex.getMessage}")
- case _ =>
- }
-
- assert(counts.length == results.length)
-
- for {
- ((exactKey, eqWithValue), result) <- counts.zip(results)
- } yield {
- val eqWithResult = result match {
- case r: Result =>
- for {
- (eq, value) <- eqWithValue
- } yield {
- val interval = eq.tq.q
- val cf = intervalsMap(interval)
- val result = Option(r.getColumnLatestCell(cf.toString.getBytes, BytesUtilV1.toBytes(eq))).map { cell =>
- Bytes.toLong(CellUtil.cloneValue(cell))
- }.getOrElse(-1l)
- eq -> result
- }
- case ex: Throwable =>
- log.error(s"${ex.getMessage}: $exactKey")
- Nil
- case _ =>
- log.error(s"result is null: $exactKey")
- Nil
- }
- (exactKey, eqWithResult.toMap)
- }
- }.toMap
-
- override def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
- withHBase(getTableName(policy)) { table =>
- table.delete {
- for {
- key <- keys
- } yield {
- new Delete(BytesUtilV1.toBytes(key))
- }
- }
- } match {
- case Failure(ex) =>
- log.error(ex.getMessage)
- case _ =>
- }
- }
-
- override def get(policy: Counter,
- queries: Seq[(ExactKeyTrait, Seq[ExactQualifier])])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
-
- val tableName = getTableName(policy)
-
- val gets = {
- for {
- (key, eqs) <- queries
- (cf, eqsGrouped) <- eqs.groupBy(eq => intervalsMap(eq.tq.q))
- } yield {
-// println(s"$key $eqsGrouped")
- val get = new GetRequest(tableName, BytesUtilV1.toBytes(key))
- get.family(cf.toString)
- get.qualifiers(eqsGrouped.map(BytesUtilV1.toBytes).toArray)
- (key, cf, get)
- }
- }
-
- withAsyncHBase[Seq[FetchedCounts]] { client =>
- val deferreds: Seq[Deferred[FetchedCounts]] = {
- for {
- (key, cf, get) <- gets
- } yield {
- client.get(get).addCallback { new Callback[FetchedCounts, util.ArrayList[KeyValue]] {
- override def call(kvs: util.ArrayList[KeyValue]): FetchedCounts = {
- val qualifierWithCounts = {
- for {
- kv <- kvs
- eq = BytesUtilV1.toExactQualifier(kv.qualifier())
- } yield {
- eq -> Bytes.toLong(kv.value())
- }
- }.toMap
- FetchedCounts(key, qualifierWithCounts)
- }
- }}
- }
- }
- Deferred.group(deferreds).addCallback { new Callback[Seq[FetchedCounts], util.ArrayList[FetchedCounts]] {
- override def call(arg: util.ArrayList[FetchedCounts]): Seq[FetchedCounts] = arg
- }}
- }
- }
-
- override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
- val results: Array[Object] = Array.fill(keys.size)(null)
-
- val puts = keys.map { key =>
- val put = new Put(BytesUtilV1.toBytes(key))
- put.addColumn(blobCF, blobColumn, key.itemId.getBytes)
- }
-
- withHBase(getTableName(policy)) { table =>
- table.batch(puts, results)
- } match {
- case Failure(ex) =>
- log.error(s"${ex.getMessage}")
- case _ =>
- }
-
- for {
- (result, key) <- results.zip(keys)
- } yield {
- Option(result).map(_ => true).getOrElse {
- log.error(s"fail to insert blob value: $key")
- false
- }
- }
- }
-
- override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
- lazy val messageForLog = s"${policy.service}.${policy.action}.$blobId"
-
- policy.itemType match {
- case ItemType.BLOB =>
- withHBase(getTableName(policy)) { table =>
- val rowKey = BytesUtilV1.toBytes(ExactKey(policy.id, policy.version, policy.itemType, blobId))
- val get = new Get(rowKey)
- get.addColumn(blobCF, blobColumn)
- table.get(get)
- } match {
- case Success(result) =>
- Option(result).filter(!_.isEmpty).map { rst =>
- Bytes.toString(rst.getValue(blobCF, blobColumn))
- }
- case Failure(ex) =>
- throw ex
- }
- case _ =>
- log.warn(s"is not blob type counter. $messageForLog")
- throw new Exception(s"is not blob type counter. $messageForLog")
- }
- }
-
- override def prepare(policy: Counter): Unit = {
- // create hbase table
- policy.hbaseTable.foreach { table =>
- if (!hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)) {
- hbaseManagement.createTable(s2config.HBASE_ZOOKEEPER_QUORUM, table,
- ColumnFamily.values.map(_.toString).toList, 1)
- hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.SHORT.toString, policy.ttl)
- policy.dailyTtl.foreach { i =>
- hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.LONG.toString, i * 24 * 60 * 60)
- }
- }
- }
- }
-
- override def destroy(policy: Counter): Unit = {
-
- }
-
- override def ready(policy: Counter): Boolean = {
- policy.hbaseTable.map { table =>
- hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)
- }.getOrElse(true)
- }
-}
[16/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelMeta.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelMeta.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelMeta.scala
new file mode 100644
index 0000000..b641ec5
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/LabelMeta.scala
@@ -0,0 +1,150 @@
+package org.apache.s2graph.core.mysqls
+
+/**
+ * Created by shon on 6/3/15.
+ */
+
+import org.apache.s2graph.core.GraphExceptions.MaxPropSizeReachedException
+import org.apache.s2graph.core.{GraphExceptions, JSONParser}
+import play.api.libs.json.Json
+import scalikejdbc._
+
+object LabelMeta extends Model[LabelMeta] with JSONParser {
+
+ /** dummy sequences */
+
+ val fromSeq = -4.toByte
+ val toSeq = -5.toByte
+ val lastOpSeq = -3.toByte
+ val lastDeletedAt = -2.toByte
+ val timeStampSeq = 0.toByte
+ val countSeq = (Byte.MaxValue - 2).toByte
+ val degreeSeq = (Byte.MaxValue - 1).toByte
+ val maxValue = Byte.MaxValue
+ val emptyValue = Byte.MaxValue
+
+ /** reserved sequences */
+ // val deleted = LabelMeta(id = Some(lastDeletedAt), labelId = lastDeletedAt, name = "lastDeletedAt",
+ // seq = lastDeletedAt, defaultValue = "", dataType = "long")
+ val from = LabelMeta(id = Some(fromSeq), labelId = fromSeq, name = "_from",
+ seq = fromSeq, defaultValue = fromSeq.toString, dataType = "long")
+ val to = LabelMeta(id = Some(toSeq), labelId = toSeq, name = "_to",
+ seq = toSeq, defaultValue = toSeq.toString, dataType = "long")
+ val timestamp = LabelMeta(id = Some(-1), labelId = -1, name = "_timestamp",
+ seq = timeStampSeq, defaultValue = "0", dataType = "long")
+ val degree = LabelMeta(id = Some(-1), labelId = -1, name = "_degree",
+ seq = degreeSeq, defaultValue = "0", dataType = "long")
+ val count = LabelMeta(id = Some(-1), labelId = -1, name = "_count",
+ seq = countSeq, defaultValue = "-1", dataType = "long")
+
+ // Each reserved column(_timestamp, timestamp) has same seq number, starts with '_' has high priority
+ val reservedMetas = List(from, to, degree, timestamp, count).flatMap { lm => List(lm, lm.copy(name = lm.name.drop(1))) }.reverse
+ val reservedMetasInner = List(from, to, degree, timestamp, count)
+
+ def apply(rs: WrappedResultSet): LabelMeta = {
+ LabelMeta(Some(rs.int("id")), rs.int("label_id"), rs.string("name"), rs.byte("seq"), rs.string("default_value"), rs.string("data_type").toLowerCase)
+ }
+
+ def isValidSeq(seq: Byte): Boolean = seq >= 0 && seq <= countSeq
+ def isValidSeqForAdmin(seq: Byte): Boolean = seq > 0 && seq < countSeq
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession): LabelMeta = {
+ val cacheKey = "id=" + id
+
+ withCache(cacheKey) {
+ sql"""select * from label_metas where id = ${id}""".map { rs => LabelMeta(rs) }.single.apply
+ }.get
+ }
+
+ def findAllByLabelId(labelId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[LabelMeta] = {
+ val cacheKey = "labelId=" + labelId
+ lazy val labelMetas = sql"""select *
+ from label_metas
+ where label_id = ${labelId} order by seq ASC""".map(LabelMeta(_)).list.apply()
+
+ if (useCache) withCaches(cacheKey)(labelMetas)
+ else labelMetas
+ }
+
+ def findByName(labelId: Int, name: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[LabelMeta] = {
+ name match {
+ case timestamp.name => Some(timestamp)
+ case from.name => Some(from)
+ case to.name => Some(to)
+ case _ =>
+ val cacheKey = "labelId=" + labelId + ":name=" + name
+ lazy val labelMeta = sql"""
+ select *
+ from label_metas where label_id = ${labelId} and name = ${name}"""
+ .map { rs => LabelMeta(rs) }.single.apply()
+
+ if (useCache) withCache(cacheKey)(labelMeta)
+ else labelMeta
+ }
+ }
+
+ def insert(labelId: Int, name: String, defaultValue: String, dataType: String)(implicit session: DBSession = AutoSession) = {
+ val ls = findAllByLabelId(labelId, useCache = false)
+ val seq = ls.size + 1
+
+ if (seq < maxValue) {
+ sql"""insert into label_metas(label_id, name, seq, default_value, data_type)
+ select ${labelId}, ${name}, ${seq}, ${defaultValue}, ${dataType}""".updateAndReturnGeneratedKey.apply()
+ } else {
+ throw MaxPropSizeReachedException("max property size reached")
+ }
+ }
+
+ def findOrInsert(labelId: Int,
+ name: String,
+ defaultValue: String,
+ dataType: String)(implicit session: DBSession = AutoSession): LabelMeta = {
+
+ findByName(labelId, name) match {
+ case Some(c) => c
+ case None =>
+ insert(labelId, name, defaultValue, dataType)
+ val cacheKey = "labelId=" + labelId + ":name=" + name
+ val cacheKeys = "labelId=" + labelId
+ expireCache(cacheKey)
+ expireCaches(cacheKeys)
+ findByName(labelId, name, useCache = false).get
+ }
+ }
+
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val labelMeta = findById(id)
+ val (labelId, name) = (labelMeta.labelId, labelMeta.name)
+ sql"""delete from label_metas where id = ${id}""".execute.apply()
+ val cacheKeys = List(s"id=$id", s"labelId=$labelId", s"labelId=$labelId:name=$name")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ }
+
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from label_metas""".map { rs => LabelMeta(rs) }.list.apply
+ putsToCache(ls.map { x =>
+ val cacheKey = s"id=${x.id.get}"
+ cacheKey -> x
+ })
+ putsToCache(ls.map { x =>
+ val cacheKey = s"labelId=${x.labelId}:name=${x.name}"
+ cacheKey -> x
+ })
+ putsToCache(ls.map { x =>
+ val cacheKey = s"labelId=${x.labelId}:seq=${x.seq}"
+ cacheKey -> x
+ })
+
+ putsToCaches(ls.groupBy(x => x.labelId).map { case (labelId, ls) =>
+ val cacheKey = s"labelId=${labelId}"
+ cacheKey -> ls
+ }.toList)
+ }
+}
+
+case class LabelMeta(id: Option[Int], labelId: Int, name: String, seq: Byte, defaultValue: String, dataType: String) extends JSONParser {
+ lazy val toJson = Json.obj("name" -> name, "defaultValue" -> defaultValue, "dataType" -> dataType)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
new file mode 100644
index 0000000..700b8d7
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
@@ -0,0 +1,100 @@
+package org.apache.s2graph.core.mysqls
+
+import java.util.concurrent.Executors
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.utils.{SafeUpdateCache, logger}
+import scalikejdbc._
+
+import scala.concurrent.ExecutionContext
+import scala.language.{higherKinds, implicitConversions}
+import scala.util.{Failure, Try}
+
+object Model {
+ var maxSize = 10000
+ var ttl = 60
+ val numOfThread = Runtime.getRuntime.availableProcessors()
+ val threadPool = Executors.newFixedThreadPool(numOfThread)
+ val ec = ExecutionContext.fromExecutor(threadPool)
+
+ def apply(config: Config) = {
+ maxSize = config.getInt("cache.max.size")
+ ttl = config.getInt("cache.ttl.seconds")
+ Class.forName(config.getString("db.default.driver"))
+
+ val settings = ConnectionPoolSettings(
+ initialSize = 10,
+ maxSize = 10,
+ connectionTimeoutMillis = 30000L,
+ validationQuery = "select 1;")
+
+ ConnectionPool.singleton(
+ config.getString("db.default.url"),
+ config.getString("db.default.user"),
+ config.getString("db.default.password"),
+ settings)
+ }
+
+ def withTx[T](block: DBSession => T): Try[T] = {
+ using(DB(ConnectionPool.borrow())) { conn =>
+ val res = Try {
+ conn.begin()
+ val session = conn.withinTxSession()
+ val result = block(session)
+
+ conn.commit()
+
+ result
+ } recoverWith {
+ case e: Exception =>
+ conn.rollbackIfActive()
+ Failure(e)
+ }
+
+ res
+ }
+ }
+
+ def shutdown() = {
+ ConnectionPool.closeAll()
+ }
+
+ def loadCache() = {
+ Service.findAll()
+ ServiceColumn.findAll()
+ Label.findAll()
+ LabelMeta.findAll()
+ LabelIndex.findAll()
+ ColumnMeta.findAll()
+ }
+}
+
+trait Model[V] extends SQLSyntaxSupport[V] {
+
+ import Model._
+
+ implicit val ec: ExecutionContext = Model.ec
+
+ val cName = this.getClass.getSimpleName()
+ logger.info(s"LocalCache[$cName]: TTL[$ttl], MaxSize[$maxSize]")
+
+ val optionCache = new SafeUpdateCache[Option[V]](cName, maxSize, ttl)
+ val listCache = new SafeUpdateCache[List[V]](cName, maxSize, ttl)
+
+ val withCache = optionCache.withCache _
+
+ val withCaches = listCache.withCache _
+
+ val expireCache = optionCache.invalidate _
+
+ val expireCaches = listCache.invalidate _
+
+ def putsToCache(kvs: List[(String, V)]) = kvs.foreach {
+ case (key, value) => optionCache.put(key, Option(value))
+ }
+
+ def putsToCaches(kvs: List[(String, List[V])]) = kvs.foreach {
+ case (key, values) => listCache.put(key, values)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Service.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Service.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Service.scala
new file mode 100644
index 0000000..98e1866
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Service.scala
@@ -0,0 +1,94 @@
+package org.apache.s2graph.core.mysqls
+
+import java.util.UUID
+
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.json.Json
+import scalikejdbc._
+
+object Service extends Model[Service] {
+ def apply(rs: WrappedResultSet): Service = {
+ Service(rs.intOpt("id"), rs.string("service_name"), rs.string("access_token"),
+ rs.string("cluster"), rs.string("hbase_table_name"), rs.int("pre_split_size"), rs.intOpt("hbase_table_ttl"))
+ }
+
+ def findByAccessToken(accessToken: String)(implicit session: DBSession = AutoSession): Option[Service] = {
+ val cacheKey = s"accessToken=$accessToken"
+ withCache(cacheKey)( sql"""select * from services where access_token = ${accessToken}""".map { rs => Service(rs) }.single.apply)
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession): Service = {
+ val cacheKey = "id=" + id
+ withCache(cacheKey)( sql"""select * from services where id = ${id}""".map { rs => Service(rs) }.single.apply).get
+ }
+
+ def findByName(serviceName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Service] = {
+ val cacheKey = "serviceName=" + serviceName
+ lazy val serviceOpt = sql"""
+ select * from services where service_name = ${serviceName}
+ """.map { rs => Service(rs) }.single.apply()
+
+ if (useCache) withCache(cacheKey)(serviceOpt)
+ else serviceOpt
+ }
+
+ def insert(serviceName: String, cluster: String,
+ hTableName: String, preSplitSize: Int, hTableTTL: Option[Int],
+ compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Unit = {
+ logger.info(s"$serviceName, $cluster, $hTableName, $preSplitSize, $hTableTTL, $compressionAlgorithm")
+ val accessToken = UUID.randomUUID().toString()
+ sql"""insert into services(service_name, access_token, cluster, hbase_table_name, pre_split_size, hbase_table_ttl)
+ values(${serviceName}, ${accessToken}, ${cluster}, ${hTableName}, ${preSplitSize}, ${hTableTTL})""".execute.apply()
+ }
+
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val service = findById(id)
+ val serviceName = service.serviceName
+ sql"""delete from service_columns where id = ${id}""".execute.apply()
+ val cacheKeys = List(s"id=$id", s"serviceName=$serviceName")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ }
+
+ def findOrInsert(serviceName: String, cluster: String, hTableName: String,
+ preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Service = {
+ findByName(serviceName) match {
+ case Some(s) => s
+ case None =>
+ insert(serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
+ val cacheKey = "serviceName=" + serviceName
+ expireCache(cacheKey)
+ findByName(serviceName).get
+ }
+ }
+
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from services""".map { rs => Service(rs) }.list.apply
+ putsToCache(ls.map { x =>
+ val cacheKey = s"id=${x.id.get}"
+ (cacheKey -> x)
+ })
+
+ putsToCache(ls.map { x =>
+ val cacheKey = s"serviceName=${x.serviceName}"
+ (cacheKey -> x)
+ })
+ }
+
+ def findAllConn()(implicit session: DBSession = AutoSession): List[String] = {
+ sql"""select distinct(cluster) from services""".map { rs => rs.string("cluster") }.list.apply
+ }
+}
+
+case class Service(id: Option[Int], serviceName: String, accessToken: String, cluster: String, hTableName: String, preSplitSize: Int, hTableTTL: Option[Int]) {
+ lazy val toJson =
+ id match {
+ case Some(_id) =>
+ Json.obj("id" -> _id, "name" -> serviceName, "accessToken" -> accessToken, "cluster" -> cluster,
+ "hTableName" -> hTableName, "preSplitSize" -> preSplitSize, "hTableTTL" -> hTableTTL)
+ case None =>
+ Json.parse("{}")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
new file mode 100644
index 0000000..6d33fe2
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
@@ -0,0 +1,82 @@
+package org.apache.s2graph.core.mysqls
+
+/**
+ * Created by shon on 6/3/15.
+ */
+
+import org.apache.s2graph.core.JSONParser
+import play.api.libs.json.Json
+import scalikejdbc._
+object ServiceColumn extends Model[ServiceColumn] {
+
+ def apply(rs: WrappedResultSet): ServiceColumn = {
+ ServiceColumn(rs.intOpt("id"), rs.int("service_id"), rs.string("column_name"), rs.string("column_type").toLowerCase(), rs.string("schema_version"))
+ }
+
+ def findById(id: Int)(implicit session: DBSession = AutoSession): ServiceColumn = {
+// val cacheKey = s"id=$id"
+ val cacheKey = "id=" + id
+ withCache(cacheKey)(sql"""select * from service_columns where id = ${id}""".map { x => ServiceColumn(x) }.single.apply).get
+ }
+ def find(serviceId: Int, columnName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[ServiceColumn] = {
+// val cacheKey = s"serviceId=$serviceId:columnName=$columnName"
+ val cacheKey = "serviceId=" + serviceId + ":columnName=" + columnName
+ if (useCache) {
+ withCache(cacheKey) {
+ sql"""
+ select * from service_columns where service_id = ${serviceId} and column_name = ${columnName}
+ """.map { rs => ServiceColumn(rs) }.single.apply()
+ }
+ } else {
+ sql"""
+ select * from service_columns where service_id = ${serviceId} and column_name = ${columnName}
+ """.map { rs => ServiceColumn(rs) }.single.apply()
+ }
+ }
+ def insert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String)(implicit session: DBSession = AutoSession) = {
+ sql"""insert into service_columns(service_id, column_name, column_type, schema_version)
+ values(${serviceId}, ${columnName}, ${columnType}, ${schemaVersion})""".execute.apply()
+ }
+ def delete(id: Int)(implicit session: DBSession = AutoSession) = {
+ val serviceColumn = findById(id)
+ val (serviceId, columnName) = (serviceColumn.serviceId, serviceColumn.columnName)
+ sql"""delete from service_columns where id = ${id}""".execute.apply()
+ val cacheKeys = List(s"id=$id", s"serviceId=$serviceId:columnName=$columnName")
+ cacheKeys.foreach { key =>
+ expireCache(key)
+ expireCaches(key)
+ }
+ }
+ def findOrInsert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String)(implicit session: DBSession = AutoSession): ServiceColumn = {
+ find(serviceId, columnName) match {
+ case Some(sc) => sc
+ case None =>
+ insert(serviceId, columnName, columnType, schemaVersion)
+// val cacheKey = s"serviceId=$serviceId:columnName=$columnName"
+ val cacheKey = "serviceId=" + serviceId + ":columnName=" + columnName
+ expireCache(cacheKey)
+ find(serviceId, columnName).get
+ }
+ }
+ def findAll()(implicit session: DBSession = AutoSession) = {
+ val ls = sql"""select * from service_columns""".map { rs => ServiceColumn(rs) }.list.apply
+ putsToCache(ls.map { x =>
+ var cacheKey = s"id=${x.id.get}"
+ (cacheKey -> x)
+ })
+ putsToCache(ls.map { x =>
+ var cacheKey = s"serviceId=${x.serviceId}:columnName=${x.columnName}"
+ (cacheKey -> x)
+ })
+ }
+}
+case class ServiceColumn(id: Option[Int], serviceId: Int, columnName: String, columnType: String, schemaVersion: String) extends JSONParser {
+
+ lazy val service = Service.findById(serviceId)
+ lazy val metas = ColumnMeta.findAllByColumn(id.get)
+ lazy val metasInvMap = metas.map { meta => meta.name -> meta} toMap
+ lazy val metaNamesMap = (ColumnMeta.lastModifiedAtColumn :: metas).map(x => (x.seq.toInt, x.name)) toMap
+ lazy val toJson = Json.obj("serviceName" -> service.serviceName, "columnName" -> columnName, "columnType" -> columnType)
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
new file mode 100644
index 0000000..d5a3687
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
@@ -0,0 +1,207 @@
+package org.apache.s2graph.core.parsers
+
+import org.apache.s2graph.core.GraphExceptions.WhereParserException
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.types.InnerValLike
+import org.apache.s2graph.core.{Edge, GraphExceptions, JSONParser}
+
+import scala.annotation.tailrec
+import scala.util.Try
+import scala.util.parsing.combinator.JavaTokenParsers
+
+trait ExtractValue extends JSONParser {
+ val parent = "_parent."
+
+ def propToInnerVal(edge: Edge, key: String) = {
+ val (propKey, parentEdge) = findParentEdge(edge, key)
+
+ val label = parentEdge.label
+ val metaPropInvMap = label.metaPropsInvMap
+ val labelMeta = metaPropInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
+ val metaSeq = labelMeta.seq
+
+ metaSeq match {
+ case LabelMeta.from.seq => parentEdge.srcVertex.innerId
+ case LabelMeta.to.seq => parentEdge.tgtVertex.innerId
+ case _ => parentEdge.propsWithTs.get(metaSeq) match {
+ case None => toInnerVal(labelMeta.defaultValue, labelMeta.dataType, label.schemaVersion)
+ case Some(edgeVal) => edgeVal.innerVal
+ }
+ }
+ }
+
+ def valueToCompare(edge: Edge, key: String, value: String) = {
+ val label = edge.label
+ if (value.startsWith(parent) || label.metaPropsInvMap.contains(value)) propToInnerVal(edge, value)
+ else {
+ val (propKey, _) = findParentEdge(edge, key)
+
+ val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
+ val (srcColumn, tgtColumn) = label.srcTgtColumn(edge.labelWithDir.dir)
+ val dataType = propKey match {
+ case "_to" | "to" => tgtColumn.columnType
+ case "_from" | "from" => srcColumn.columnType
+ case _ => labelMeta.dataType
+ }
+ toInnerVal(value, dataType, label.schemaVersion)
+ }
+ }
+
+ @tailrec
+ private def findParent(edge: Edge, depth: Int): Edge =
+ if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1)
+ else edge
+
+ private def findParentEdge(edge: Edge, key: String): (String, Edge) = {
+ if (!key.startsWith(parent)) (key, edge)
+ else {
+ val split = key.split(parent)
+ val depth = split.length - 1
+ val propKey = split.last
+
+ val parentEdge = findParent(edge, depth)
+
+ (propKey, parentEdge)
+ }
+ }
+}
+
+trait Clause extends ExtractValue {
+ def and(otherField: Clause): Clause = And(this, otherField)
+
+ def or(otherField: Clause): Clause = Or(this, otherField)
+
+ def filter(edge: Edge): Boolean
+
+ def binaryOp(binOp: (InnerValLike, InnerValLike) => Boolean)(propKey: String, value: String)(edge: Edge): Boolean = {
+ val propValue = propToInnerVal(edge, propKey)
+ val compValue = valueToCompare(edge, propKey, value)
+
+ binOp(propValue, compValue)
+ }
+}
+
+case class Where(clauses: Seq[Clause] = Seq.empty[Clause]) {
+ def filter(edge: Edge) =
+ if (clauses.isEmpty) true else clauses.map(_.filter(edge)).forall(identity)
+}
+
+case class Gt(propKey: String, value: String) extends Clause {
+ override def filter(edge: Edge): Boolean = binaryOp(_ > _)(propKey, value)(edge)
+}
+
+case class Lt(propKey: String, value: String) extends Clause {
+ override def filter(edge: Edge): Boolean = binaryOp(_ < _)(propKey, value)(edge)
+}
+
+case class Eq(propKey: String, value: String) extends Clause {
+ override def filter(edge: Edge): Boolean = binaryOp(_ == _)(propKey, value)(edge)
+}
+
+case class InWithoutParent(label: Label, propKey: String, values: Set[String]) extends Clause {
+ val innerValLikeLs = values.map { value =>
+ val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
+ val dataType = propKey match {
+ case "_to" | "to" => label.tgtColumn.columnType
+ case "_from" | "from" => label.srcColumn.columnType
+ case _ => labelMeta.dataType
+ }
+ toInnerVal(value, dataType, label.schemaVersion)
+ }
+ override def filter(edge: Edge): Boolean = {
+ val propVal = propToInnerVal(edge, propKey)
+ innerValLikeLs.contains(propVal)
+ }
+}
+
+case class IN(propKey: String, values: Set[String]) extends Clause {
+ override def filter(edge: Edge): Boolean = {
+ val propVal = propToInnerVal(edge, propKey)
+ values.exists { value =>
+ valueToCompare(edge, propKey, value) == propVal
+ }
+ }
+}
+
+case class Between(propKey: String, minValue: String, maxValue: String) extends Clause {
+ override def filter(edge: Edge): Boolean = {
+ val propVal = propToInnerVal(edge, propKey)
+ val minVal = valueToCompare(edge, propKey, minValue)
+ val maxVal = valueToCompare(edge, propKey, maxValue)
+
+ minVal <= propVal && propVal <= maxVal
+ }
+}
+
+case class Not(self: Clause) extends Clause {
+ override def filter(edge: Edge) = !self.filter(edge)
+}
+
+case class And(left: Clause, right: Clause) extends Clause {
+ override def filter(edge: Edge) = left.filter(edge) && right.filter(edge)
+}
+
+case class Or(left: Clause, right: Clause) extends Clause {
+ override def filter(edge: Edge) = left.filter(edge) || right.filter(edge)
+}
+
+object WhereParser {
+ val success = Where()
+}
+
+case class WhereParser(label: Label) extends JavaTokenParsers with JSONParser {
+
+ val anyStr = "[^\\s(),]+".r
+
+ val and = "and|AND".r
+
+ val or = "or|OR".r
+
+ val between = "between|BETWEEN".r
+
+ val in = "in|IN".r
+
+ val notIn = "not in|NOT IN".r
+
+ def where: Parser[Where] = rep(clause) ^^ (Where(_))
+
+ def paren: Parser[Clause] = "(" ~> clause <~ ")"
+
+ def clause: Parser[Clause] = (predicate | paren) * (and ^^^ { (a: Clause, b: Clause) => And(a, b) } | or ^^^ { (a: Clause, b: Clause) => Or(a, b) })
+
+ def identWithDot: Parser[String] = repsep(ident, ".") ^^ { case values => values.mkString(".") }
+
+ def predicate = {
+ identWithDot ~ ("!=" | "=") ~ anyStr ^^ {
+ case f ~ op ~ s =>
+ if (op == "=") Eq(f, s)
+ else Not(Eq(f, s))
+ } | identWithDot ~ (">=" | "<=" | ">" | "<") ~ anyStr ^^ {
+ case f ~ op ~ s => op match {
+ case ">" => Gt(f, s)
+ case ">=" => Or(Gt(f, s), Eq(f, s))
+ case "<" => Lt(f, s)
+ case "<=" => Or(Lt(f, s), Eq(f, s))
+ }
+ } | identWithDot ~ (between ~> anyStr <~ and) ~ anyStr ^^ {
+ case f ~ minV ~ maxV => Between(f, minV, maxV)
+ } | identWithDot ~ (notIn | in) ~ ("(" ~> repsep(anyStr, ",") <~ ")") ^^ {
+ case f ~ op ~ values =>
+ val inClause =
+ if (f.startsWith("_parent")) IN(f, values.toSet)
+ else InWithoutParent(label, f, values.toSet)
+ if (op.toLowerCase == "in") inClause
+ else Not(inClause)
+
+
+ case _ => throw WhereParserException(s"Failed to parse where clause. ")
+ }
+ }
+
+ def parse(sql: String): Try[Where] = Try {
+ parseAll(where, sql) match {
+ case Success(r, q) => r
+ case fail => throw WhereParserException(s"Where parsing error: ${fail.toString}")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
new file mode 100644
index 0000000..b0af967
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -0,0 +1,610 @@
+package org.apache.s2graph.core.rest
+
+import java.util.concurrent.{Callable, TimeUnit}
+
+import com.google.common.cache.CacheBuilder
+import com.typesafe.config.Config
+import org.apache.s2graph.core.GraphExceptions.{BadQueryException, ModelNotFoundException}
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls._
+import org.apache.s2graph.core.parsers.{Where, WhereParser}
+import org.apache.s2graph.core.types._
+import play.api.libs.json._
+
+import scala.util.{Failure, Success, Try}
+
+object TemplateHelper {
+ val findVar = """\"?\$\{(.*?)\}\"?""".r
+ val num = """(next_day|next_hour|next_week|now)?\s*(-?\s*[0-9]+)?\s*(hour|day|week)?""".r
+ val hour = 60 * 60 * 1000L
+ val day = hour * 24L
+ val week = day * 7L
+ def calculate(now: Long, n: Int, unit: String): Long = {
+ val duration = unit match {
+ case "hour" | "HOUR" => n * hour
+ case "day" | "DAY" => n * day
+ case "week" | "WEEK" => n * week
+ case _ => n * day
+ }
+
+ duration + now
+ }
+
+ def replaceVariable(now: Long, body: String): String = {
+ findVar.replaceAllIn(body, m => {
+ val matched = m group 1
+
+ num.replaceSomeIn(matched, m => {
+ val (_pivot, n, unit) = (m.group(1), m.group(2), m.group(3))
+ val ts = _pivot match {
+ case null => now
+ case "now" | "NOW" => now
+ case "next_week" | "NEXT_WEEK" => now / week * week + week
+ case "next_day" | "NEXT_DAY" => now / day * day + day
+ case "next_hour" | "NEXT_HOUR" => now / hour * hour + hour
+ }
+
+ if (_pivot == null && n == null && unit == null) None
+ else if (n == null || unit == null) Option(ts.toString)
+ else Option(calculate(ts, n.replaceAll(" ", "").toInt, unit).toString)
+ })
+ })
+ }
+}
+
+class RequestParser(config: Config) extends JSONParser {
+
+ import Management.JsonModel._
+
+ val hardLimit = 100000
+ val defaultLimit = 100
+ val maxLimit = Int.MaxValue - 1
+ val DefaultRpcTimeout = config.getInt("hbase.rpc.timeout")
+ val DefaultMaxAttempt = config.getInt("hbase.client.retries.number")
+ val DefaultCluster = config.getString("hbase.zookeeper.quorum")
+ val DefaultCompressionAlgorithm = config.getString("hbase.table.compression.algorithm")
+ val DefaultPhase = config.getString("phase")
+ val parserCache = CacheBuilder.newBuilder()
+ .expireAfterAccess(10000, TimeUnit.MILLISECONDS)
+ .expireAfterWrite(10000, TimeUnit.MILLISECONDS)
+ .maximumSize(10000)
+ .initialCapacity(1000)
+ .build[String, Try[Where]]
+
+ private def extractScoring(labelId: Int, value: JsValue) = {
+ val ret = for {
+ js <- parse[Option[JsObject]](value, "scoring")
+ } yield {
+ for {
+ (k, v) <- js.fields
+ labelOrderType <- LabelMeta.findByName(labelId, k)
+ } yield {
+ val value = v match {
+ case n: JsNumber => n.as[Double]
+ case _ => throw new Exception("scoring weight should be double.")
+ }
+ (labelOrderType.seq, value)
+ }
+ }
+ ret
+ }
+
+ def extractInterval(label: Label, _jsValue: JsValue) = {
+ val replaced = TemplateHelper.replaceVariable(System.currentTimeMillis(), _jsValue.toString())
+ val jsValue = Json.parse(replaced)
+
+ def extractKv(js: JsValue) = js match {
+ case JsObject(obj) => obj
+ case JsArray(arr) => arr.flatMap {
+ case JsObject(obj) => obj
+ case _ => throw new RuntimeException(s"cannot support json type $js")
+ }
+ case _ => throw new RuntimeException(s"cannot support json type: $js")
+ }
+
+ val ret = for {
+ js <- parse[Option[JsObject]](jsValue, "interval")
+ fromJs <- (js \ "from").asOpt[JsValue]
+ toJs <- (js \ "to").asOpt[JsValue]
+ } yield {
+ val from = Management.toProps(label, extractKv(fromJs))
+ val to = Management.toProps(label, extractKv(toJs))
+ (from, to)
+ }
+
+ ret
+ }
+
+ def extractDuration(label: Label, _jsValue: JsValue) = {
+ val replaced = TemplateHelper.replaceVariable(System.currentTimeMillis(), _jsValue.toString())
+ val jsValue = Json.parse(replaced)
+
+ for {
+ js <- parse[Option[JsObject]](jsValue, "duration")
+ } yield {
+ val minTs = parse[Option[Long]](js, "from").getOrElse(Long.MaxValue)
+ val maxTs = parse[Option[Long]](js, "to").getOrElse(Long.MinValue)
+
+ if (minTs > maxTs) {
+ throw new BadQueryException("Duration error. Timestamp of From cannot be larger than To.")
+ }
+
+ (minTs, maxTs)
+ }
+ }
+
+ def extractHas(label: Label, jsValue: JsValue) = {
+ val ret = for {
+ js <- parse[Option[JsObject]](jsValue, "has")
+ } yield {
+ for {
+ (k, v) <- js.fields
+ labelMeta <- LabelMeta.findByName(label.id.get, k)
+ value <- jsValueToInnerVal(v, labelMeta.dataType, label.schemaVersion)
+ } yield {
+ labelMeta.seq -> value
+ }
+ }
+ ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
+ }
+
+ def extractWhere(label: Label, whereClauseOpt: Option[String]): Try[Where] = {
+ whereClauseOpt match {
+ case None => Success(WhereParser.success)
+ case Some(_where) =>
+ val where = TemplateHelper.replaceVariable(System.currentTimeMillis(), _where)
+ val whereParserKey = s"${label.label}_${where}"
+ parserCache.get(whereParserKey, new Callable[Try[Where]] {
+ override def call(): Try[Where] = {
+ WhereParser(label).parse(where) match {
+ case s@Success(_) => s
+ case Failure(ex) => throw BadQueryException(ex.getMessage, ex)
+ }
+ }
+ })
+ }
+ }
+
+ def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[Vertex] = {
+ val vertices = for {
+ label <- Label.findByName(labelName).toSeq
+ serviceColumn = if (direction == "out") label.srcColumn else label.tgtColumn
+ id <- ids
+ innerId <- jsValueToInnerVal(id, serviceColumn.columnType, label.schemaVersion)
+ } yield {
+ Vertex(SourceVertexId(serviceColumn.id.get, innerId), System.currentTimeMillis())
+ }
+ vertices.toSeq
+ }
+
+ def toMultiQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): MultiQuery = {
+ val queries = for {
+ queryJson <- (jsValue \ "queries").asOpt[Seq[JsValue]].getOrElse(Seq.empty)
+ } yield {
+ toQuery(queryJson, isEdgeQuery)
+ }
+ val weights = (jsValue \ "weights").asOpt[Seq[Double]].getOrElse(queries.map(_ => 1.0))
+ MultiQuery(queries = queries, weights = weights,
+ queryOption = toQueryOption(jsValue), jsonQuery = jsValue)
+ }
+
+ def toQueryOption(jsValue: JsValue): QueryOption = {
+ val filterOutFields = (jsValue \ "filterOutFields").asOpt[List[String]].getOrElse(List(LabelMeta.to.name))
+ val filterOutQuery = (jsValue \ "filterOut").asOpt[JsValue].map { v => toQuery(v) }.map { q =>
+ q.copy(queryOption = q.queryOption.copy(filterOutFields = filterOutFields))
+ }
+ val removeCycle = (jsValue \ "removeCycle").asOpt[Boolean].getOrElse(true)
+ val selectColumns = (jsValue \ "select").asOpt[List[String]].getOrElse(List.empty)
+ val groupByColumns = (jsValue \ "groupBy").asOpt[List[String]].getOrElse(List.empty)
+ val orderByColumns: List[(String, Boolean)] = (jsValue \ "orderBy").asOpt[List[JsObject]].map { jsLs =>
+ for {
+ js <- jsLs
+ (column, orderJs) <- js.fields
+ } yield {
+ val ascending = orderJs.as[String].toUpperCase match {
+ case "ASC" => true
+ case "DESC" => false
+ }
+ column -> ascending
+ }
+ }.getOrElse(List("score" -> false, "timestamp" -> false))
+ val withScore = (jsValue \ "withScore").asOpt[Boolean].getOrElse(true)
+ val returnTree = (jsValue \ "returnTree").asOpt[Boolean].getOrElse(false)
+ //TODO: Refactor this
+ val limitOpt = (jsValue \ "limit").asOpt[Int]
+ val returnAgg = (jsValue \ "returnAgg").asOpt[Boolean].getOrElse(true)
+ val scoreThreshold = (jsValue \ "scoreThreshold").asOpt[Double].getOrElse(Double.MinValue)
+ val returnDegree = (jsValue \ "returnDegree").asOpt[Boolean].getOrElse(true)
+
+ QueryOption(removeCycle = removeCycle,
+ selectColumns = selectColumns,
+ groupByColumns = groupByColumns,
+ orderByColumns = orderByColumns,
+ filterOutQuery = filterOutQuery,
+ filterOutFields = filterOutFields,
+ withScore = withScore,
+ returnTree = returnTree,
+ limitOpt = limitOpt,
+ returnAgg = returnAgg,
+ scoreThreshold = scoreThreshold,
+ returnDegree = returnDegree
+ )
+ }
+ def toQuery(jsValue: JsValue, isEdgeQuery: Boolean = true): Query = {
+ try {
+ val vertices =
+ (for {
+ value <- parse[List[JsValue]](jsValue, "srcVertices")
+ serviceName = parse[String](value, "serviceName")
+ column = parse[String](value, "columnName")
+ } yield {
+ val service = Service.findByName(serviceName).getOrElse(throw BadQueryException("service not found"))
+ val col = ServiceColumn.find(service.id.get, column).getOrElse(throw BadQueryException("bad column name"))
+ val (idOpt, idsOpt) = ((value \ "id").asOpt[JsValue], (value \ "ids").asOpt[List[JsValue]])
+ for {
+ idVal <- idOpt ++ idsOpt.toSeq.flatten
+
+ /** bug, need to use labels schemaVersion */
+ innerVal <- jsValueToInnerVal(idVal, col.columnType, col.schemaVersion)
+ } yield {
+ Vertex(SourceVertexId(col.id.get, innerVal), System.currentTimeMillis())
+ }
+ }).flatten
+
+ if (vertices.isEmpty) throw BadQueryException("srcVertices`s id is empty")
+ val steps = parse[Vector[JsValue]](jsValue, "steps")
+
+ val queryOption = toQueryOption(jsValue)
+
+ val querySteps =
+ steps.zipWithIndex.map { case (step, stepIdx) =>
+ val labelWeights = step match {
+ case obj: JsObject =>
+ val converted = for {
+ (k, v) <- (obj \ "weights").asOpt[JsObject].getOrElse(Json.obj()).fields
+ l <- Label.findByName(k)
+ } yield {
+ l.id.get -> v.toString().toDouble
+ }
+ converted.toMap
+ case _ => Map.empty[Int, Double]
+ }
+ val queryParamJsVals = step match {
+ case arr: JsArray => arr.as[List[JsValue]]
+ case obj: JsObject => (obj \ "step").as[List[JsValue]]
+ case _ => List.empty[JsValue]
+ }
+ val nextStepScoreThreshold = step match {
+ case obj: JsObject => (obj \ "nextStepThreshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
+ case _ => QueryParam.DefaultThreshold
+ }
+ val nextStepLimit = step match {
+ case obj: JsObject => (obj \ "nextStepLimit").asOpt[Int].getOrElse(-1)
+ case _ => -1
+ }
+ val cacheTTL = step match {
+ case obj: JsObject => (obj \ "cacheTTL").asOpt[Int].getOrElse(-1)
+ case _ => -1
+ }
+ val queryParams =
+ for {
+ labelGroup <- queryParamJsVals
+ queryParam <- parseQueryParam(labelGroup)
+ } yield {
+ val (_, columnName) =
+ if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) {
+ (queryParam.label.srcService.serviceName, queryParam.label.srcColumnName)
+ } else {
+ (queryParam.label.tgtService.serviceName, queryParam.label.tgtColumnName)
+ }
+ //FIXME:
+ if (stepIdx == 0 && vertices.nonEmpty && !vertices.exists(v => v.serviceColumn.columnName == columnName)) {
+ throw BadQueryException("srcVertices contains incompatiable serviceName or columnName with first step.")
+ }
+
+ queryParam
+ }
+ Step(queryParams.toList, labelWeights = labelWeights,
+ // scoreThreshold = stepThreshold,
+ nextStepScoreThreshold = nextStepScoreThreshold,
+ nextStepLimit = nextStepLimit,
+ cacheTTL = cacheTTL)
+
+ }
+
+ val ret = Query(vertices, querySteps, queryOption, jsValue)
+ // logger.debug(ret.toString)
+ ret
+ } catch {
+ case e: BadQueryException =>
+ throw e
+ case e: ModelNotFoundException =>
+ throw BadQueryException(e.getMessage, e)
+ case e: Exception =>
+ throw BadQueryException(s"$jsValue, $e", e)
+ }
+ }
+
+ private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = {
+ for {
+ labelName <- parse[Option[String]](labelGroup, "label")
+ } yield {
+ val label = Label.findByName(labelName).getOrElse(throw BadQueryException(s"$labelName not found"))
+ val direction = parse[Option[String]](labelGroup, "direction").map(GraphUtil.toDirection(_)).getOrElse(0)
+ val limit = {
+ parse[Option[Int]](labelGroup, "limit") match {
+ case None => defaultLimit
+ case Some(l) if l < 0 => maxLimit
+ case Some(l) if l >= 0 =>
+ val default = hardLimit
+ Math.min(l, default)
+ }
+ }
+ val offset = parse[Option[Int]](labelGroup, "offset").getOrElse(0)
+ val interval = extractInterval(label, labelGroup)
+ val duration = extractDuration(label, labelGroup)
+ val scoring = extractScoring(label.id.get, labelGroup).getOrElse(List.empty[(Byte, Double)]).toList
+ val exclude = parse[Option[Boolean]](labelGroup, "exclude").getOrElse(false)
+ val include = parse[Option[Boolean]](labelGroup, "include").getOrElse(false)
+ val hasFilter = extractHas(label, labelGroup)
+ val labelWithDir = LabelWithDirection(label.id.get, direction)
+ val indexNameOpt = (labelGroup \ "index").asOpt[String]
+ val indexSeq = indexNameOpt match {
+ case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq)
+ case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
+ }
+ val whereClauseOpt = (labelGroup \ "where").asOpt[String]
+ val where = extractWhere(label, whereClauseOpt)
+ val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
+ val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout)
+ val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt)
+ val tgtVertexInnerIdOpt = (labelGroup \ "_to").asOpt[JsValue].flatMap { jsVal =>
+ jsValueToInnerVal(jsVal, label.tgtColumnWithDir(direction).columnType, label.schemaVersion)
+ }
+ val cacheTTL = (labelGroup \ "cacheTTL").asOpt[Long].getOrElse(-1L)
+ val timeDecayFactor = (labelGroup \ "timeDecay").asOpt[JsObject].map { jsVal =>
+ val propName = (jsVal \ "propName").asOpt[String].getOrElse(LabelMeta.timestamp.name)
+ val propNameSeq = label.metaPropsInvMap.get(propName).map(_.seq).getOrElse(LabelMeta.timeStampSeq)
+ val initial = (jsVal \ "initial").asOpt[Double].getOrElse(1.0)
+ val decayRate = (jsVal \ "decayRate").asOpt[Double].getOrElse(0.1)
+ if (decayRate >= 1.0 || decayRate <= 0.0) throw new BadQueryException("decay rate should be 0.0 ~ 1.0")
+ val timeUnit = (jsVal \ "timeUnit").asOpt[Double].getOrElse(60 * 60 * 24.0)
+ TimeDecay(initial, decayRate, timeUnit, propNameSeq)
+ }
+ val threshold = (labelGroup \ "threshold").asOpt[Double].getOrElse(QueryParam.DefaultThreshold)
+ // TODO: refactor this. dirty
+ val duplicate = parse[Option[String]](labelGroup, "duplicate").map(s => Query.DuplicatePolicy(s))
+
+ val outputField = (labelGroup \ "outputField").asOpt[String].map(s => Json.arr(Json.arr(s)))
+ val transformer = if (outputField.isDefined) outputField else (labelGroup \ "transform").asOpt[JsValue]
+ val scorePropagateOp = (labelGroup \ "scorePropagateOp").asOpt[String].getOrElse("multiply")
+ val sample = (labelGroup \ "sample").asOpt[Int].getOrElse(-1)
+ val shouldNormalize = (labelGroup \ "normalize").asOpt[Boolean].getOrElse(false)
+ val cursorOpt = (labelGroup \ "cursor").asOpt[String]
+ // FIXME: Order of command matter
+ QueryParam(labelWithDir)
+ .sample(sample)
+ .limit(offset, limit)
+ .rank(RankParam(label.id.get, scoring))
+ .exclude(exclude)
+ .include(include)
+ .duration(duration)
+ .has(hasFilter)
+ .labelOrderSeq(indexSeq)
+ .interval(interval)
+ .where(where)
+ .duplicatePolicy(duplicate)
+ .includeDegree(includeDegree)
+ .rpcTimeout(rpcTimeout)
+ .maxAttempt(maxAttempt)
+ .tgtVertexInnerIdOpt(tgtVertexInnerIdOpt)
+ .cacheTTLInMillis(cacheTTL)
+ .timeDecay(timeDecayFactor)
+ .threshold(threshold)
+ .transformer(transformer)
+ .scorePropagateOp(scorePropagateOp)
+ .shouldNormalize(shouldNormalize)
+ .cursorOpt(cursorOpt)
+ }
+ }
+
+ private def parse[R](js: JsValue, key: String)(implicit read: Reads[R]): R = {
+ (js \ key).validate[R]
+ .fold(
+ errors => {
+ val msg = (JsError.toFlatJson(errors) \ "obj").as[List[JsValue]].map(x => x \ "msg")
+ val e = Json.obj("args" -> key, "error" -> msg)
+ throw new GraphExceptions.JsonParseException(Json.obj("error" -> key).toString)
+ },
+ r => {
+ r
+ })
+ }
+
+ def toJsValues(jsValue: JsValue): List[JsValue] = {
+ jsValue match {
+ case obj: JsObject => List(obj)
+ case arr: JsArray => arr.as[List[JsValue]]
+ case _ => List.empty[JsValue]
+ }
+
+ }
+
+ def toEdgesWithOrg(jsValue: JsValue, operation: String): (List[Edge], List[JsValue]) = {
+ val jsValues = toJsValues(jsValue)
+ val edges = jsValues.flatMap(toEdge(_, operation))
+
+ (edges, jsValues)
+ }
+
+ def toEdges(jsValue: JsValue, operation: String): List[Edge] = {
+ toJsValues(jsValue).flatMap { edgeJson =>
+ toEdge(edgeJson, operation)
+ }
+ }
+
+
+ private def toEdge(jsValue: JsValue, operation: String): List[Edge] = {
+
+ def parseId(js: JsValue) = js match {
+ case s: JsString => s.as[String]
+ case o@_ => s"${o}"
+ }
+ val srcId = (jsValue \ "from").asOpt[JsValue].toList.map(parseId(_))
+ val tgtId = (jsValue \ "to").asOpt[JsValue].toList.map(parseId(_))
+ val srcIds = (jsValue \ "froms").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ srcId
+ val tgtIds = (jsValue \ "tos").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ tgtId
+
+ val label = parse[String](jsValue, "label")
+ val timestamp = parse[Long](jsValue, "timestamp")
+ val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
+ val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
+ for {
+ srcId <- srcIds
+ tgtId <- tgtIds
+ } yield {
+ Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
+ }
+ }
+
+ def toVertices(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None) = {
+ toJsValues(jsValue).map(toVertex(_, operation, serviceName, columnName))
+ }
+
+ def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): Vertex = {
+ val id = parse[JsValue](jsValue, "id")
+ val ts = parse[Option[Long]](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
+ val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
+ val cName = if (columnName.isEmpty) parse[String](jsValue, "columnName") else columnName.get
+ val props = (jsValue \ "props").asOpt[JsObject].getOrElse(Json.obj())
+ Management.toVertex(ts, operation, id.toString, sName, cName, props.toString)
+ }
+
+ def toPropElements(jsObj: JsValue) = Try {
+ val propName = (jsObj \ "name").as[String]
+ val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
+ val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
+ case JsString(s) => s
+ case _@js => js.toString
+ }
+ Prop(propName, defaultValue, dataType)
+ }
+
+ def toPropsElements(jsValue: JsValue): Seq[Prop] = for {
+ jsObj <- jsValue.asOpt[Seq[JsValue]].getOrElse(Nil)
+ } yield {
+ val propName = (jsObj \ "name").as[String]
+ val dataType = InnerVal.toInnerDataType((jsObj \ "dataType").as[String])
+ val defaultValue = (jsObj \ "defaultValue").as[JsValue] match {
+ case JsString(s) => s
+ case _@js => js.toString
+ }
+ Prop(propName, defaultValue, dataType)
+ }
+
+ def toIndicesElements(jsValue: JsValue): Seq[Index] = for {
+ jsObj <- jsValue.as[Seq[JsValue]]
+ indexName = (jsObj \ "name").as[String]
+ propNames = (jsObj \ "propNames").as[Seq[String]]
+ } yield Index(indexName, propNames)
+
+ def toLabelElements(jsValue: JsValue) = Try {
+ val labelName = parse[String](jsValue, "label")
+ val srcServiceName = parse[String](jsValue, "srcServiceName")
+ val tgtServiceName = parse[String](jsValue, "tgtServiceName")
+ val srcColumnName = parse[String](jsValue, "srcColumnName")
+ val tgtColumnName = parse[String](jsValue, "tgtColumnName")
+ val srcColumnType = parse[String](jsValue, "srcColumnType")
+ val tgtColumnType = parse[String](jsValue, "tgtColumnType")
+ val serviceName = (jsValue \ "serviceName").asOpt[String].getOrElse(tgtServiceName)
+ val isDirected = (jsValue \ "isDirected").asOpt[Boolean].getOrElse(true)
+
+ val allProps = toPropsElements(jsValue \ "props")
+ val indices = toIndicesElements(jsValue \ "indices")
+
+ val consistencyLevel = (jsValue \ "consistencyLevel").asOpt[String].getOrElse("weak")
+
+ // expect new label don`t provide hTableName
+ val hTableName = (jsValue \ "hTableName").asOpt[String]
+ val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
+ val schemaVersion = (jsValue \ "schemaVersion").asOpt[String].getOrElse(HBaseType.DEFAULT_VERSION)
+ val isAsync = (jsValue \ "isAsync").asOpt[Boolean].getOrElse(false)
+ val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
+
+ (labelName, srcServiceName, srcColumnName, srcColumnType,
+ tgtServiceName, tgtColumnName, tgtColumnType, isDirected, serviceName,
+ indices, allProps, consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm)
+ }
+
+ def toIndexElements(jsValue: JsValue) = Try {
+ val labelName = parse[String](jsValue, "label")
+ val indices = toIndicesElements(jsValue \ "indices")
+ (labelName, indices)
+ }
+
+ def toServiceElements(jsValue: JsValue) = {
+ val serviceName = parse[String](jsValue, "serviceName")
+ val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(DefaultCluster)
+ val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${DefaultPhase}")
+ val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1)
+ val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int]
+ val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
+ (serviceName, cluster, hTableName, preSplitSize, hTableTTL, compressionAlgorithm)
+ }
+
+ def toServiceColumnElements(jsValue: JsValue) = Try {
+ val serviceName = parse[String](jsValue, "serviceName")
+ val columnName = parse[String](jsValue, "columnName")
+ val columnType = parse[String](jsValue, "columnType")
+ val props = toPropsElements(jsValue \ "props")
+ (serviceName, columnName, columnType, props)
+ }
+
+ def toCheckEdgeParam(jsValue: JsValue) = {
+ val params = jsValue.as[List[JsValue]]
+ var isReverted = false
+ val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]()
+ val quads = for {
+ param <- params
+ labelName <- (param \ "label").asOpt[String]
+ direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out"))
+ label <- Label.findByName(labelName)
+ srcId <- jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ tgtId <- jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion)
+ } yield {
+ val labelWithDir = LabelWithDirection(label.id.get, direction)
+ labelWithDirs += labelWithDir
+ val (src, tgt, dir) = if (direction == 1) {
+ isReverted = true
+ (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)),
+ Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0)
+ } else {
+ (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)),
+ Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0)
+ }
+ (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir)))
+ }
+ (quads, isReverted)
+ }
+
+ def toGraphElements(str: String): Seq[GraphElement] = {
+ val edgeStrs = str.split("\\n")
+
+ for {
+ edgeStr <- edgeStrs
+ str <- GraphUtil.parseString(edgeStr)
+ element <- Graph.toGraphElement(str)
+ } yield element
+ }
+
+ def toDeleteParam(json: JsValue) = {
+ val labelName = (json \ "label").as[String]
+ val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil).filterNot(_.isAsync)
+ val direction = (json \ "direction").asOpt[String].getOrElse("out")
+
+ val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
+ val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
+ val vertices = toVertices(labelName, direction, ids)
+ (labels, direction, ids, ts, vertices)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
new file mode 100644
index 0000000..a48bc7c
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
@@ -0,0 +1,244 @@
+package org.apache.s2graph.core.rest
+
+import java.net.URL
+
+import org.apache.s2graph.core.GraphExceptions.BadQueryException
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.{Bucket, Experiment, Service}
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+
+object RestHandler {
+ case class HandlerResult(body: Future[JsValue], headers: (String, String)*)
+}
+
+/**
+ * Public API, only return Future.successful or Future.failed
+ * Don't throw exception
+ */
+class RestHandler(graph: Graph)(implicit ec: ExecutionContext) {
+
+ import RestHandler._
+ val requestParser = new RequestParser(graph.config)
+
+ /**
+ * Public APIS
+ */
+ def doPost(uri: String, body: String, impKeyOpt: => Option[String] = None): HandlerResult = {
+ try {
+ val jsQuery = Json.parse(body)
+
+ uri match {
+ case "/graphs/getEdges" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.toSimpleVertexArrJson))
+ case "/graphs/getEdges/grouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithListFormatted))
+ case "/graphs/getEdgesExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.toSimpleVertexArrJson))
+ case "/graphs/getEdgesExcluded/grouped" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted))
+ case "/graphs/checkEdges" => checkEdges(jsQuery)
+ case "/graphs/getEdgesGrouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithList))
+ case "/graphs/getEdgesGroupedExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExclude))
+ case "/graphs/getEdgesGroupedExcludedFormatted" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted))
+ case "/graphs/getVertices" => HandlerResult(getVertices(jsQuery))
+ case uri if uri.startsWith("/graphs/experiment") =>
+ val Array(accessToken, experimentName, uuid) = uri.split("/").takeRight(3)
+ experiment(jsQuery, accessToken, experimentName, uuid, impKeyOpt)
+ case _ => throw new RuntimeException("route is not found")
+ }
+ } catch {
+ case e: Exception => HandlerResult(Future.failed(e))
+ }
+ }
+
+ // TODO: Refactor to doGet
+ def checkEdges(jsValue: JsValue): HandlerResult = {
+ try {
+ val (quads, isReverted) = requestParser.toCheckEdgeParam(jsValue)
+
+ HandlerResult(graph.checkEdges(quads).map { case queryRequestWithResultLs =>
+ val edgeJsons = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ convertedEdge = if (isReverted) edge.duplicateEdge else edge
+ edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam)
+ } yield Json.toJson(edgeJson)
+
+ Json.toJson(edgeJsons)
+ })
+ } catch {
+ case e: Exception => HandlerResult(Future.failed(e))
+ }
+ }
+
+
+ /**
+ * Private APIS
+ */
+ private def experiment(contentsBody: JsValue, accessToken: String, experimentName: String, uuid: String, impKeyOpt: => Option[String]): HandlerResult = {
+
+ try {
+ val bucketOpt = for {
+ service <- Service.findByAccessToken(accessToken)
+ experiment <- Experiment.findBy(service.id.get, experimentName)
+ bucket <- experiment.findBucket(uuid, impKeyOpt)
+ } yield bucket
+
+ val bucket = bucketOpt.getOrElse(throw new RuntimeException("bucket is not found"))
+ if (bucket.isGraphQuery) {
+ val ret = buildRequestInner(contentsBody, bucket, uuid)
+ HandlerResult(ret.body, Experiment.impressionKey -> bucket.impressionId)
+ }
+ else throw new RuntimeException("not supported yet")
+ } catch {
+ case e: Exception => HandlerResult(Future.failed(e))
+ }
+ }
+
+ private def buildRequestInner(contentsBody: JsValue, bucket: Bucket, uuid: String): HandlerResult = {
+ if (bucket.isEmpty) HandlerResult(Future.successful(PostProcess.emptyResults))
+ else {
+ val body = buildRequestBody(Option(contentsBody), bucket, uuid)
+ val url = new URL(bucket.apiPath)
+ val path = url.getPath
+
+ // dummy log for sampling
+ val experimentLog = s"POST $path took -1 ms 200 -1 $body"
+ logger.debug(experimentLog)
+
+ doPost(path, body)
+ }
+ }
+
+ private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = {
+ val filterOutQueryResultsLs = q.filterOutQuery match {
+ case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
+ case None => Future.successful(Seq.empty)
+ }
+
+ for {
+ queryResultsLs <- graph.getEdges(q)
+ filterOutResultsLs <- filterOutQueryResultsLs
+ } yield {
+ val json = post(queryResultsLs, filterOutResultsLs)
+ json
+ }
+ }
+
+ def getEdgesAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
+
+ val fetch = eachQuery(post) _
+ jsonQuery match {
+ case JsArray(arr) => Future.traverse(arr.map(requestParser.toQuery(_)))(fetch).map(JsArray)
+ case obj@JsObject(_) =>
+ (obj \ "queries").asOpt[JsValue] match {
+ case None => fetch(requestParser.toQuery(obj))
+ case _ =>
+ val multiQuery = requestParser.toMultiQuery(obj)
+ val filterOutFuture = multiQuery.queryOption.filterOutQuery match {
+ case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
+ case None => Future.successful(Seq.empty)
+ }
+ val futures = multiQuery.queries.zip(multiQuery.weights).map { case (query, weight) =>
+ val filterOutQueryResultsLs = query.queryOption.filterOutQuery match {
+ case Some(filterOutQuery) => graph.getEdges(filterOutQuery)
+ case None => Future.successful(Seq.empty)
+ }
+ for {
+ queryRequestWithResultLs <- graph.getEdges(query)
+ filterOutResultsLs <- filterOutQueryResultsLs
+ } yield {
+ val newQueryRequestWithResult = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ queryResult = queryRequestWithResult.queryResult
+ } yield {
+ val newEdgesWithScores = for {
+ edgeWithScore <- queryRequestWithResult.queryResult.edgeWithScoreLs
+ } yield {
+ edgeWithScore.copy(score = edgeWithScore.score * weight)
+ }
+ queryRequestWithResult.copy(queryResult = queryResult.copy(edgeWithScoreLs = newEdgesWithScores))
+ }
+ logger.debug(s"[Size]: ${newQueryRequestWithResult.map(_.queryResult.edgeWithScoreLs.size).sum}")
+ (newQueryRequestWithResult, filterOutResultsLs)
+ }
+ }
+ for {
+ filterOut <- filterOutFuture
+ resultWithExcludeLs <- Future.sequence(futures)
+ } yield {
+ PostProcess.toSimpleVertexArrJsonMulti(multiQuery.queryOption, resultWithExcludeLs, filterOut)
+ // val initial = (ListBuffer.empty[QueryRequestWithResult], ListBuffer.empty[QueryRequestWithResult])
+ // val (results, excludes) = resultWithExcludeLs.foldLeft(initial) { case ((prevResults, prevExcludes), (results, excludes)) =>
+ // (prevResults ++= results, prevExcludes ++= excludes)
+ // }
+ // PostProcess.toSimpleVertexArrJson(multiQuery.queryOption, results, excludes ++ filterOut)
+ }
+ }
+ case _ => throw BadQueryException("Cannot support")
+ }
+ }
+
+ private def getEdgesExcludedAsync(jsonQuery: JsValue)
+ (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[JsValue] = {
+ val q = requestParser.toQuery(jsonQuery)
+ val filterOutQuery = Query(q.vertices, Vector(q.steps.last))
+
+ val fetchFuture = graph.getEdges(q)
+ val excludeFuture = graph.getEdges(filterOutQuery)
+
+ for {
+ queryResultLs <- fetchFuture
+ exclude <- excludeFuture
+ } yield {
+ post(queryResultLs, exclude)
+ }
+ }
+
+ private def getVertices(jsValue: JsValue) = {
+ val jsonQuery = jsValue
+ val ts = System.currentTimeMillis()
+ val props = "{}"
+
+ val vertices = jsonQuery.as[List[JsValue]].flatMap { js =>
+ val serviceName = (js \ "serviceName").as[String]
+ val columnName = (js \ "columnName").as[String]
+ for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield {
+ Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props)
+ }
+ }
+
+ graph.getVertices(vertices) map { vertices => PostProcess.verticesToJson(vertices) }
+ }
+
+ private def buildRequestBody(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): String = {
+ var body = bucket.requestBody.replace("#uuid", uuid)
+
+ // // replace variable
+ // body = TemplateHelper.replaceVariable(System.currentTimeMillis(), body)
+
+ // replace param
+ for {
+ requestKeyJson <- requestKeyJsonOpt
+ jsObj <- requestKeyJson.asOpt[JsObject]
+ (key, value) <- jsObj.fieldSet
+ } {
+ val replacement = value match {
+ case JsString(s) => s
+ case _ => value.toString
+ }
+ body = body.replace(key, replacement)
+ }
+
+ body
+ }
+
+ def calcSize(js: JsValue): Int = js match {
+ case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0)
+ case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum
+ case _ => 0
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala
new file mode 100644
index 0000000..12e9547
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Deserializable.scala
@@ -0,0 +1,24 @@
+package org.apache.s2graph.core.storage
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.types.{LabelWithDirection, SourceVertexId, VertexId}
+
+
+trait Deserializable[E] extends StorageDeserializable[E] {
+ import StorageDeserializable._
+
+ type RowKeyRaw = (VertexId, LabelWithDirection, Byte, Boolean, Int)
+
+ /** version 1 and version 2 share same code for parsing row key part */
+ def parseRow(kv: SKeyValue, version: String): RowKeyRaw = {
+ var pos = 0
+ val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
+ pos += srcIdLen
+ val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
+ pos += 4
+ val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
+
+ val rowLen = srcIdLen + 4 + 1
+ (srcVertexId, labelWithDir, labelIdxSeq, isInverted, rowLen)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
new file mode 100644
index 0000000..5310248
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
@@ -0,0 +1,47 @@
+package org.apache.s2graph.core.storage
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.hbase.async.KeyValue
+
+
+object SKeyValue {
+ val Put = 1
+ val Delete = 2
+ val Increment = 3
+ val Default = Put
+}
+case class SKeyValue(table: Array[Byte],
+ row: Array[Byte],
+ cf: Array[Byte],
+ qualifier: Array[Byte],
+ value: Array[Byte],
+ timestamp: Long,
+ operation: Int = SKeyValue.Default) {
+ def toLogString = {
+ Map("table" -> table.toList, "row" -> row.toList, "cf" -> Bytes.toString(cf),
+ "qualifier" -> qualifier.toList, "value" -> value.toList, "timestamp" -> timestamp,
+ "operation" -> operation).toString
+ }
+ override def toString(): String = toLogString
+}
+
+trait CanSKeyValue[T] {
+ def toSKeyValue(from: T): SKeyValue
+}
+
+object CanSKeyValue {
+
+ // For asyncbase KeyValues
+ implicit val asyncKeyValue = new CanSKeyValue[KeyValue] {
+ def toSKeyValue(kv: KeyValue): SKeyValue = {
+ SKeyValue(Array.empty[Byte], kv.key(), kv.family(), kv.qualifier(), kv.value(), kv.timestamp())
+ }
+ }
+
+ // For asyncbase KeyValues
+ implicit val sKeyValue = new CanSKeyValue[SKeyValue] {
+ def toSKeyValue(kv: SKeyValue): SKeyValue = kv
+ }
+
+ // For hbase KeyValues
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/org/apache/s2graph/core/storage/Serializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Serializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Serializable.scala
new file mode 100644
index 0000000..08a3f73
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Serializable.scala
@@ -0,0 +1,8 @@
+package org.apache.s2graph.core.storage
+
+object Serializable {
+ val vertexCf = "v".getBytes()
+ val edgeCf = "e".getBytes()
+}
+
+trait Serializable[E] extends StorageSerializable[E]
[22/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
deleted file mode 100644
index f301d68..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
+++ /dev/null
@@ -1,554 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-
-import com.kakao.s2graph.core.mysqls.{ColumnMeta, Label, ServiceColumn, LabelMeta}
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLike}
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json.{Json, _}
-import scala.collection.mutable.{ArrayBuffer, ListBuffer}
-
-object PostProcess extends JSONParser {
-
-
- type EDGE_VALUES = Map[String, JsValue]
- type ORDER_BY_VALUES = (Any, Any, Any, Any)
- type RAW_EDGE = (EDGE_VALUES, Double, ORDER_BY_VALUES)
-
- /**
- * Result Entity score field name
- */
- val emptyDegrees = Seq.empty[JsValue]
- val timeoutResults = Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr(), "isTimeout" -> true)
- val emptyResults = Json.obj("size" -> 0, "degrees" -> Json.arr(), "results" -> Json.arr(), "isEmpty" -> true)
- def badRequestResults(ex: => Exception) = ex match {
- case ex: BadQueryException => Json.obj("message" -> ex.msg)
- case _ => Json.obj("message" -> ex.getMessage)
- }
-
- val SCORE_FIELD_NAME = "scoreSum"
- val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
-
- def groupEdgeResult(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
- // filterNot {case (edge, score) => edge.props.contains(LabelMeta.degreeSeq)}
- val groupedEdgesWithRank = (for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
- } yield {
- (queryRequest.queryParam, edge, score)
- }).groupBy {
- case (queryParam, edge, rank) if edge.labelWithDir.dir == GraphUtil.directions("in") =>
- (queryParam.label.srcColumn, queryParam.label.label, queryParam.label.tgtColumn, edge.tgtVertex.innerId, edge.isDegree)
- case (queryParam, edge, rank) =>
- (queryParam.label.tgtColumn, queryParam.label.label, queryParam.label.srcColumn, edge.tgtVertex.innerId, edge.isDegree)
- }
-
- val ret = for {
- ((tgtColumn, labelName, srcColumn, target, isDegreeEdge), edgesAndRanks) <- groupedEdgesWithRank if !isDegreeEdge
- edgesWithRanks = edgesAndRanks.groupBy(x => x._2.srcVertex).map(_._2.head)
- id <- innerValToJsValue(target, tgtColumn.columnType)
- } yield {
- Json.obj("name" -> tgtColumn.columnName, "id" -> id,
- SCORE_FIELD_NAME -> edgesWithRanks.map(_._3).sum,
- "label" -> labelName,
- "aggr" -> Json.obj(
- "name" -> srcColumn.columnName,
- "ids" -> edgesWithRanks.flatMap { case (queryParam, edge, rank) =>
- innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)
- },
- "edges" -> edgesWithRanks.map { case (queryParam, edge, rank) =>
- Json.obj("id" -> innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType),
- "props" -> propsToJson(edge),
- "score" -> rank
- )
- }
- )
- )
- }
-
- ret.toList
- }
-
- def sortWithFormatted(jsons: Seq[JsObject],
- scoreField: String = "scoreSum",
- queryRequestWithResultLs: Seq[QueryRequestWithResult],
- decrease: Boolean = true): JsObject = {
- val ordering = if (decrease) -1 else 1
- val sortedJsons = jsons.sortBy { jsObject => (jsObject \ scoreField).as[Double] * ordering }
-
- if (queryRequestWithResultLs.isEmpty) Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
- else Json.obj(
- "size" -> sortedJsons.size,
- "results" -> sortedJsons,
- "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId()
- )
- }
-
- private def toHashKey(edge: Edge, queryParam: QueryParam, fields: Seq[String], delimiter: String = ","): Int = {
- val ls = for {
- field <- fields
- } yield {
- field match {
- case "from" | "_from" => edge.srcVertex.innerId.toIdString()
- case "to" | "_to" => edge.tgtVertex.innerId.toIdString()
- case "label" => edge.labelWithDir.labelId
- case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
- case "_timestamp" | "timestamp" => edge.ts
- case _ =>
- queryParam.label.metaPropsInvMap.get(field) match {
- case None => throw new RuntimeException(s"unknow column: $field")
- case Some(labelMeta) => edge.propsWithTs.get(labelMeta.seq) match {
- case None => labelMeta.defaultValue
- case Some(propVal) => propVal
- }
- }
- }
- }
- val ret = ls.hashCode()
- ret
- }
-
- def resultInnerIds(queryRequestWithResultLs: Seq[QueryRequestWithResult], isSrcVertex: Boolean = false): Seq[Int] = {
- for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- q = queryRequest.query
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- } yield toHashKey(edge, queryRequest.queryParam, q.filterOutFields)
- }
-
- def summarizeWithListExcludeFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs, decrease = true)
- }
-
- def summarizeWithList(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
- }
-
- def summarizeWithListFormatted(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
- val jsons = groupEdgeResult(queryRequestWithResultLs, exclude)
- sortWithFormatted(jsons, queryRequestWithResultLs = queryRequestWithResultLs)
- }
-
- def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult]): JsValue = {
- toSimpleVertexArrJson(queryRequestWithResultLs, Seq.empty[QueryRequestWithResult])
- }
-
- private def orderBy(queryOption: QueryOption,
- rawEdges: ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]): ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)] = {
- import com.kakao.s2graph.core.OrderingUtil._
-
- if (queryOption.withScore && queryOption.orderByColumns.nonEmpty) {
- val ascendingLs = queryOption.orderByColumns.map(_._2)
- rawEdges.sortBy(_._3)(TupleMultiOrdering[Any](ascendingLs))
- } else {
- rawEdges
- }
- }
-
- private def getColumnValue(keyWithJs: Map[String, JsValue], score: Double, edge: Edge, column: String): Any = {
- column match {
- case "score" => score
- case "timestamp" | "_timestamp" => edge.ts
- case _ =>
- keyWithJs.get(column) match {
- case None => keyWithJs.get("props").map { js => (js \ column).as[JsValue] }.get
- case Some(x) => x
- }
- }
- }
-
- private def buildReplaceJson(jsValue: JsValue)(mapper: JsValue => JsValue): JsValue = {
- def traverse(js: JsValue): JsValue = js match {
- case JsNull => mapper(JsNull)
- case JsUndefined() => mapper(JsUndefined(""))
- case JsNumber(v) => mapper(js)
- case JsString(v) => mapper(js)
- case JsBoolean(v) => mapper(js)
- case JsArray(elements) => JsArray(elements.map { t => traverse(mapper(t)) })
- case JsObject(values) => JsObject(values.map { case (k, v) => k -> traverse(mapper(v)) })
- }
-
- traverse(jsValue)
- }
-
- /** test query with filterOut is not working since it can not diffrentate filterOut */
- private def buildNextQuery(jsonQuery: JsValue, _cursors: Seq[Seq[String]]): JsValue = {
- val cursors = _cursors.flatten.iterator
-
- buildReplaceJson(jsonQuery) {
- case js@JsObject(fields) =>
- val isStep = fields.find { case (k, _) => k == "label" } // find label group
- if (isStep.isEmpty) js
- else {
- // TODO: Order not ensured
- val withCursor = js.fieldSet | Set("cursor" -> JsString(cursors.next))
- JsObject(withCursor.toSeq)
- }
- case js => js
- }
- }
-
- private def buildRawEdges(queryOption: QueryOption,
- queryRequestWithResultLs: Seq[QueryRequestWithResult],
- excludeIds: Map[Int, Boolean],
- scoreWeight: Double = 1.0): (ListBuffer[JsValue], ListBuffer[RAW_EDGE]) = {
- val degrees = ListBuffer[JsValue]()
- val rawEdges = ListBuffer[(EDGE_VALUES, Double, ORDER_BY_VALUES)]()
- val metaPropNamesMap = scala.collection.mutable.Map[String, Int]()
- for {
- queryRequestWithResult <- queryRequestWithResultLs
- } yield {
- val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- queryRequest.queryParam.label.metaPropNames.foreach { metaPropName =>
- metaPropNamesMap.put(metaPropName, metaPropNamesMap.getOrElse(metaPropName, 0) + 1)
- }
- }
- val propsExistInAll = metaPropNamesMap.filter(kv => kv._2 == queryRequestWithResultLs.length)
- val orderByColumns = queryOption.orderByColumns.filter { case (column, _) =>
- column match {
- case "from" | "to" | "label" | "score" | "timestamp" | "_timestamp" => true
- case _ =>
- propsExistInAll.contains(column)
-// //TODO??
-// false
-// queryParam.label.metaPropNames.contains(column)
- }
- }
-
- /** build result jsons */
- for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- queryParam = queryRequest.queryParam
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- hashKey = toHashKey(edge, queryRequest.queryParam, queryOption.filterOutFields)
- if !excludeIds.contains(hashKey)
- } {
-
- // edge to json
- val (srcColumn, _) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
- val fromOpt = innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
- if (edge.isDegree && fromOpt.isDefined) {
- if (queryOption.limitOpt.isEmpty) {
- degrees += Json.obj(
- "from" -> fromOpt.get,
- "label" -> queryRequest.queryParam.label.label,
- "direction" -> GraphUtil.fromDirection(edge.labelWithDir.dir),
- LabelMeta.degree.name -> innerValToJsValue(edge.propsWithTs(LabelMeta.degreeSeq).innerVal, InnerVal.LONG)
- )
- }
- } else {
- val keyWithJs = edgeToJson(edge, score, queryRequest.query, queryRequest.queryParam)
- val orderByValues: (Any, Any, Any, Any) = orderByColumns.length match {
- case 0 =>
- (None, None, None, None)
- case 1 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, None, None, None)
- case 2 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, None, None)
- case 3 =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, v3, None)
- case _ =>
- val it = orderByColumns.iterator
- val v1 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v2 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v3 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- val v4 = getColumnValue(keyWithJs, score, edge, it.next()._1)
- (v1, v2, v3, v4)
- }
-
- val currentEdge = (keyWithJs, score * scoreWeight, orderByValues)
- rawEdges += currentEdge
- }
- }
- (degrees, rawEdges)
- }
-
- private def buildResultJsValue(queryOption: QueryOption,
- degrees: ListBuffer[JsValue],
- rawEdges: ListBuffer[RAW_EDGE]): JsValue = {
- if (queryOption.groupByColumns.isEmpty) {
- // ordering
- val filteredEdges = rawEdges.filter(t => t._2 >= queryOption.scoreThreshold)
-
- val edges = queryOption.limitOpt match {
- case None => orderBy(queryOption, filteredEdges).map(_._1)
- case Some(limit) => orderBy(queryOption, filteredEdges).map(_._1).take(limit)
- }
- val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees
-
- Json.obj(
- "size" -> edges.size,
- "degrees" -> resultDegrees,
-// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
- "results" -> edges
- )
- } else {
- val grouped = rawEdges.groupBy { case (keyWithJs, _, _) =>
- val props = keyWithJs.get("props")
-
- for {
- column <- queryOption.groupByColumns
- value <- keyWithJs.get(column) match {
- case None => props.flatMap { js => (js \ column).asOpt[JsValue] }
- case Some(x) => Some(x)
- }
- } yield column -> value
- }
-
- val groupedEdgesWithScoreSum =
- for {
- (groupByKeyVals, groupedRawEdges) <- grouped
- scoreSum = groupedRawEdges.map(x => x._2).sum if scoreSum >= queryOption.scoreThreshold
- } yield {
- // ordering
- val edges = orderBy(queryOption, groupedRawEdges).map(_._1)
-
- //TODO: refactor this
- val js = if (queryOption.returnAgg)
- Json.obj(
- "groupBy" -> Json.toJson(groupByKeyVals.toMap),
- "scoreSum" -> scoreSum,
- "agg" -> edges
- )
- else
- Json.obj(
- "groupBy" -> Json.toJson(groupByKeyVals.toMap),
- "scoreSum" -> scoreSum,
- "agg" -> Json.arr()
- )
- (js, scoreSum)
- }
-
- val groupedSortedJsons = queryOption.limitOpt match {
- case None =>
- groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1)
- case Some(limit) =>
- groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1).take(limit)
- }
- val resultDegrees = if (queryOption.returnDegree) degrees else emptyDegrees
- Json.obj(
- "size" -> groupedSortedJsons.size,
- "degrees" -> resultDegrees,
-// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
- "results" -> groupedSortedJsons
- )
- }
- }
-
- def toSimpleVertexArrJsonMulti(queryOption: QueryOption,
- resultWithExcludeLs: Seq[(Seq[QueryRequestWithResult], Seq[QueryRequestWithResult])],
- excludes: Seq[QueryRequestWithResult]): JsValue = {
- val excludeIds = (Seq((Seq.empty, excludes)) ++ resultWithExcludeLs).foldLeft(Map.empty[Int, Boolean]) { case (acc, (result, excludes)) =>
- acc ++ resultInnerIds(excludes).map(hashKey => hashKey -> true).toMap
- }
-
- val (degrees, rawEdges) = (ListBuffer.empty[JsValue], ListBuffer.empty[RAW_EDGE])
- for {
- (result, localExclude) <- resultWithExcludeLs
- } {
- val newResult = result.map { queryRequestWithResult =>
- val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- val newQuery = queryRequest.query.copy(queryOption = queryOption)
- queryRequestWithResult.copy(queryRequest = queryRequest.copy(query = newQuery))
- }
- val (_degrees, _rawEdges) = buildRawEdges(queryOption, newResult, excludeIds)
- degrees ++= _degrees
- rawEdges ++= _rawEdges
- }
- buildResultJsValue(queryOption, degrees, rawEdges)
- }
-
- def toSimpleVertexArrJson(queryOption: QueryOption,
- queryRequestWithResultLs: Seq[QueryRequestWithResult],
- exclude: Seq[QueryRequestWithResult]): JsValue = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
- val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds)
- buildResultJsValue(queryOption, degrees, rawEdges)
- }
-
-
- def toSimpleVertexArrJson(queryRequestWithResultLs: Seq[QueryRequestWithResult],
- exclude: Seq[QueryRequestWithResult]): JsValue = {
-
- queryRequestWithResultLs.headOption.map { queryRequestWithResult =>
- val (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- val query = queryRequest.query
- val queryOption = query.queryOption
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
- val (degrees, rawEdges) = buildRawEdges(queryOption, queryRequestWithResultLs, excludeIds)
- buildResultJsValue(queryOption, degrees, rawEdges)
- } getOrElse emptyResults
- }
-
- def verticesToJson(vertices: Iterable[Vertex]) = {
- Json.toJson(vertices.flatMap { v => vertexToJson(v) })
- }
-
- def propsToJson(edge: Edge, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- for {
- (seq, innerValWithTs) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
- labelMeta <- queryParam.label.metaPropsMap.get(seq)
- jsValue <- innerValToJsValue(innerValWithTs.innerVal, labelMeta.dataType)
- } yield labelMeta.name -> jsValue
- }
-
- private def edgeParent(parentEdges: Seq[EdgeWithScore], q: Query, queryParam: QueryParam): JsValue = {
- if (parentEdges.isEmpty) {
- JsNull
- } else {
- val parents = for {
- parent <- parentEdges
- (parentEdge, parentScore) = EdgeWithScore.unapply(parent).get
- parentQueryParam = QueryParam(parentEdge.labelWithDir)
- parents = edgeParent(parentEdge.parentEdges, q, parentQueryParam) if parents != JsNull
- } yield {
- val originalEdge = parentEdge.originalEdgeOpt.getOrElse(parentEdge)
- val edgeJson = edgeToJsonInner(originalEdge, parentScore, q, parentQueryParam) + ("parents" -> parents)
- Json.toJson(edgeJson)
- }
-
- Json.toJson(parents)
- }
- }
-
- /** TODO */
- def edgeToJsonInner(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- val (srcColumn, tgtColumn) = queryParam.label.srcTgtColumn(edge.labelWithDir.dir)
-
- val kvMapOpt = for {
- from <- innerValToJsValue(edge.srcVertex.id.innerId, srcColumn.columnType)
- to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType)
- } yield {
- val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props"
- val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam)
- val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap
-
- val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) =>
- val jsValue = column match {
- case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp))
- case "from" => from
- case "to" => to
- case "label" => JsString(queryParam.label.label)
- case "direction" => JsString(GraphUtil.fromDirection(edge.labelWithDir.dir))
- case "_timestamp" | "timestamp" => JsNumber(edge.ts)
- case "score" => JsNumber(score)
- case "props" if propsMap.nonEmpty => Json.toJson(propsMap)
- case _ => JsNull
- }
-
- if (jsValue == JsNull) map else map + (column -> jsValue)
- }
- kvMap
- }
-
- kvMapOpt.getOrElse(Map.empty)
- }
-
- def edgeToJson(edge: Edge, score: Double, q: Query, queryParam: QueryParam): Map[String, JsValue] = {
- val kvs = edgeToJsonInner(edge, score, q, queryParam)
- if (kvs.nonEmpty && q.returnTree) kvs + ("parents" -> Json.toJson(edgeParent(edge.parentEdges, q, queryParam)))
- else kvs
- }
-
- def vertexToJson(vertex: Vertex): Option[JsObject] = {
- val serviceColumn = ServiceColumn.findById(vertex.id.colId)
-
- for {
- id <- innerValToJsValue(vertex.innerId, serviceColumn.columnType)
- } yield {
- Json.obj("serviceName" -> serviceColumn.service.serviceName,
- "columnName" -> serviceColumn.columnName,
- "id" -> id, "props" -> propsToJson(vertex),
- "timestamp" -> vertex.ts,
- // "belongsTo" -> vertex.belongLabelIds)
- "belongsTo" -> vertex.belongLabelIds.flatMap(Label.findByIdOpt(_).map(_.label)))
- }
- }
-
- private def keysToName(seqsToNames: Map[Int, String], props: Map[Int, InnerValLike]) = {
- for {
- (seq, value) <- props
- name <- seqsToNames.get(seq)
- } yield (name, value)
- }
-
- private def propsToJson(vertex: Vertex) = {
- val serviceColumn = vertex.serviceColumn
- val props = for {
- (propKey, innerVal) <- vertex.props
- columnMeta <- ColumnMeta.findByIdAndSeq(serviceColumn.id.get, propKey.toByte, useCache = true)
- jsValue <- innerValToJsValue(innerVal, columnMeta.dataType)
- } yield {
- (columnMeta.name -> jsValue)
- }
- props.toMap
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def propsToJson(edge: Edge) = {
- for {
- (seq, v) <- edge.propsWithTs if LabelMeta.isValidSeq(seq)
- metaProp <- edge.label.metaPropsMap.get(seq)
- jsValue <- innerValToJsValue(v.innerVal, metaProp.dataType)
- } yield {
- (metaProp.name, jsValue)
- }
- }
-
- @deprecated(message = "deprecated", since = "0.2")
- def summarizeWithListExclude(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]): JsObject = {
- val excludeIds = resultInnerIds(exclude).map(innerId => innerId -> true).toMap
-
-
- val groupedEdgesWithRank = (for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- if !excludeIds.contains(toHashKey(edge, queryRequest.queryParam, queryRequest.query.filterOutFields))
- } yield {
- (edge, score)
- }).groupBy { case (edge, score) =>
- (edge.label.tgtColumn, edge.label.srcColumn, edge.tgtVertex.innerId)
- }
-
- val jsons = for {
- ((tgtColumn, srcColumn, target), edgesAndRanks) <- groupedEdgesWithRank
- (edges, ranks) = edgesAndRanks.groupBy(x => x._1.srcVertex).map(_._2.head).unzip
- tgtId <- innerValToJsValue(target, tgtColumn.columnType)
- } yield {
- Json.obj(tgtColumn.columnName -> tgtId,
- s"${srcColumn.columnName}s" ->
- edges.flatMap(edge => innerValToJsValue(edge.srcVertex.innerId, srcColumn.columnType)), "scoreSum" -> ranks.sum)
- }
- val sortedJsons = jsons.toList.sortBy { jsObj => (jsObj \ "scoreSum").as[Double] }.reverse
- if (queryRequestWithResultLs.isEmpty) {
- Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons)
- } else {
- Json.obj("size" -> sortedJsons.size, "results" -> sortedJsons,
- "impressionId" -> queryRequestWithResultLs.head.queryRequest.query.impressionId())
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
deleted file mode 100644
index 0effa07..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
+++ /dev/null
@@ -1,583 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.google.common.hash.Hashing
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.parsers.{Where, WhereParser}
-import com.kakao.s2graph.core.types._
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.ColumnRangeFilter
-import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
-
-import scala.util.hashing.MurmurHash3
-import scala.util.{Success, Try}
-
-object Query {
- val initialScore = 1.0
- lazy val empty = Query()
-
- def toQuery(srcVertices: Seq[Vertex], queryParam: QueryParam) = Query(srcVertices, Vector(Step(List(queryParam))))
-
- object DuplicatePolicy extends Enumeration {
- type DuplicatePolicy = Value
- val First, Sum, CountSum, Raw = Value
-
- def apply(policy: String): Value = {
- policy match {
- case "sum" => Query.DuplicatePolicy.Sum
- case "countSum" => Query.DuplicatePolicy.CountSum
- case "raw" => Query.DuplicatePolicy.Raw
- case _ => DuplicatePolicy.First
- }
- }
- }
-}
-
-case class MultiQuery(queries: Seq[Query],
- weights: Seq[Double],
- queryOption: QueryOption,
- jsonQuery: JsValue = JsNull)
-
-case class QueryOption(removeCycle: Boolean = false,
- selectColumns: Seq[String] = Seq.empty,
- groupByColumns: Seq[String] = Seq.empty,
- orderByColumns: Seq[(String, Boolean)] = Seq.empty,
- filterOutQuery: Option[Query] = None,
- filterOutFields: Seq[String] = Seq(LabelMeta.to.name),
- withScore: Boolean = true,
- returnTree: Boolean = false,
- limitOpt: Option[Int] = None,
- returnAgg: Boolean = true,
- scoreThreshold: Double = Double.MinValue,
- returnDegree: Boolean = true)
-
-
-case class Query(vertices: Seq[Vertex] = Seq.empty[Vertex],
- steps: IndexedSeq[Step] = Vector.empty[Step],
- queryOption: QueryOption = QueryOption(),
- jsonQuery: JsValue = JsNull) {
-
- val removeCycle = queryOption.removeCycle
- val selectColumns = queryOption.selectColumns
-// val groupBy = queryOption.groupBy
- val groupByColumns = queryOption.groupByColumns
- val orderByColumns = queryOption.orderByColumns
- val filterOutQuery = queryOption.filterOutQuery
- val filterOutFields = queryOption.filterOutFields
- val withScore = queryOption.withScore
- val returnTree = queryOption.returnTree
- val limitOpt = queryOption.limitOpt
- val returnAgg = queryOption.returnAgg
- val returnDegree = queryOption.returnDegree
-
- def cacheKeyBytes: Array[Byte] = {
- val selectBytes = Bytes.toBytes(queryOption.selectColumns.toString)
- val groupBytes = Bytes.toBytes(queryOption.groupByColumns.toString)
- val orderByBytes = Bytes.toBytes(queryOption.orderByColumns.toString)
- val filterOutBytes = queryOption.filterOutQuery.map(_.cacheKeyBytes).getOrElse(Array.empty[Byte])
- val returnTreeBytes = Bytes.toBytes(queryOption.returnTree)
-
- Seq(selectBytes, groupBytes, orderByBytes, filterOutBytes, returnTreeBytes).foldLeft(Array.empty[Byte])(Bytes.add)
- }
-
- lazy val selectColumnsSet = queryOption.selectColumns.map { c =>
- if (c == "_from") "from"
- else if (c == "_to") "to"
- else c
- }.toSet
-
- /** return logical query id without considering parameter values */
- def templateId(): JsValue = {
- Json.toJson(for {
- step <- steps
- queryParam <- step.queryParams.sortBy(_.labelWithDir.labelId)
- } yield {
- Json.obj("label" -> queryParam.label.label, "direction" -> GraphUtil.fromDirection(queryParam.labelWithDir.dir))
- })
- }
-
- def impressionId(): JsNumber = {
- val hash = MurmurHash3.stringHash(templateId().toString())
- JsNumber(hash)
- }
-
- def cursorStrings(): Seq[Seq[String]] = {
- //Don`t know how to replace all cursor keys in json
- steps.map { step =>
- step.queryParams.map { queryParam =>
- queryParam.cursorOpt.getOrElse("")
- }
- }
- }
-}
-
-object EdgeTransformer {
- val DefaultTransformField = Json.arr("_to")
- val DefaultTransformFieldAsList = Json.arr("_to").as[List[String]]
- val DefaultJson = Json.arr(DefaultTransformField)
-}
-
-/**
- * TODO: step wise outputFields should be used with nextStepLimit, nextStepThreshold.
- * @param jsValue
- */
-case class EdgeTransformer(queryParam: QueryParam, jsValue: JsValue) {
- val Delimiter = "\\$"
- val targets = jsValue.asOpt[List[Vector[String]]].toList
- val fieldsLs = for {
- target <- targets
- fields <- target
- } yield fields
- val isDefault = fieldsLs.size == 1 && fieldsLs.head.size == 1 && (fieldsLs.head.head == "_to" || fieldsLs.head.head == "to")
-
- def toHashKeyBytes: Array[Byte] = if (isDefault) Array.empty[Byte] else Bytes.toBytes(jsValue.toString)
-
- def replace(fmt: String,
- values: Seq[InnerValLike],
- nextStepOpt: Option[Step]): Seq[InnerValLike] = {
-
- val tokens = fmt.split(Delimiter)
- val _values = values.padTo(tokens.length, InnerVal.withStr("", queryParam.label.schemaVersion))
- val mergedStr = tokens.zip(_values).map { case (prefix, innerVal) => prefix + innerVal.toString }.mkString
- // logger.error(s"${tokens.toList}, ${values}, $mergedStr")
- // println(s"${tokens.toList}, ${values}, $mergedStr")
- nextStepOpt match {
- case None =>
- val columnType =
- if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) queryParam.label.tgtColumnType
- else queryParam.label.srcColumnType
-
- if (columnType == InnerVal.STRING) Seq(InnerVal.withStr(mergedStr, queryParam.label.schemaVersion))
- else Nil
- case Some(nextStep) =>
- val nextQueryParamsValid = nextStep.queryParams.filter { qParam =>
- if (qParam.labelWithDir.dir == GraphUtil.directions("out")) qParam.label.srcColumnType == "string"
- else qParam.label.tgtColumnType == "string"
- }
- for {
- nextQueryParam <- nextQueryParamsValid
- } yield {
- InnerVal.withStr(mergedStr, nextQueryParam.label.schemaVersion)
- }
- }
- }
-
- def toInnerValOpt(edge: Edge, fieldName: String): Option[InnerValLike] = {
- fieldName match {
- case LabelMeta.to.name => Option(edge.tgtVertex.innerId)
- case LabelMeta.from.name => Option(edge.srcVertex.innerId)
- case _ =>
- for {
- labelMeta <- queryParam.label.metaPropsInvMap.get(fieldName)
- value <- edge.propsWithTs.get(labelMeta.seq)
- } yield value.innerVal
- }
- }
-
- def transform(edge: Edge, nextStepOpt: Option[Step]): Seq[Edge] = {
- if (isDefault) Seq(edge)
- else {
- val edges = for {
- fields <- fieldsLs
- innerVal <- {
- if (fields.size == 1) {
- val fieldName = fields.head
- toInnerValOpt(edge, fieldName).toSeq
- } else {
- val fmt +: fieldNames = fields
- replace(fmt, fieldNames.flatMap(fieldName => toInnerValOpt(edge, fieldName)), nextStepOpt)
- }
- }
- } yield edge.updateTgtVertex(innerVal).copy(originalEdgeOpt = Option(edge))
-
-
- edges
- }
- }
-}
-
-object Step {
- val Delimiter = "|"
-}
-
-case class Step(queryParams: List[QueryParam],
- labelWeights: Map[Int, Double] = Map.empty,
- // scoreThreshold: Double = 0.0,
- nextStepScoreThreshold: Double = 0.0,
- nextStepLimit: Int = -1,
- cacheTTL: Long = -1) {
-
- lazy val excludes = queryParams.filter(_.exclude)
- lazy val includes = queryParams.filterNot(_.exclude)
- lazy val excludeIds = excludes.map(x => x.labelWithDir.labelId -> true).toMap
-
- def toCacheKey(lss: Seq[Long]): Long = Hashing.murmur3_128().hashBytes(toCacheKeyRaw(lss)).asLong()
-// MurmurHash3.bytesHash(toCacheKeyRaw(lss))
-
- def toCacheKeyRaw(lss: Seq[Long]): Array[Byte] = {
- var bytes = Array.empty[Byte]
- lss.sorted.foreach { h => bytes = Bytes.add(bytes, Bytes.toBytes(h)) }
- bytes
- }
-}
-
-case class VertexParam(vertices: Seq[Vertex]) {
- var filters: Option[Map[Byte, InnerValLike]] = None
-
- def has(what: Option[Map[Byte, InnerValLike]]): VertexParam = {
- what match {
- case None => this
- case Some(w) => has(w)
- }
- }
-
- def has(what: Map[Byte, InnerValLike]): VertexParam = {
- this.filters = Some(what)
- this
- }
-
-}
-
-//object RankParam {
-// def apply(labelId: Int, keyAndWeights: Seq[(Byte, Double)]) = {
-// new RankParam(labelId, keyAndWeights)
-// }
-//}
-
-case class RankParam(labelId: Int, var keySeqAndWeights: Seq[(Byte, Double)] = Seq.empty[(Byte, Double)]) {
- // empty => Count
- lazy val rankKeysWeightsMap = keySeqAndWeights.toMap
-
- def defaultKey() = {
- this.keySeqAndWeights = List((LabelMeta.countSeq, 1.0))
- this
- }
-
- def toHashKeyBytes(): Array[Byte] = {
- var bytes = Array.empty[Byte]
- keySeqAndWeights.map { case (key, weight) =>
- bytes = Bytes.add(bytes, Array.fill(1)(key), Bytes.toBytes(weight))
- }
- bytes
- }
-}
-
-object QueryParam {
- lazy val Empty = QueryParam(LabelWithDirection(0, 0))
- lazy val DefaultThreshold = Double.MinValue
- val Delimiter = ","
-}
-
-case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System.currentTimeMillis()) {
-
- import HBaseSerializable._
- import Query.DuplicatePolicy
- import Query.DuplicatePolicy._
-
- lazy val label = Label.findById(labelWithDir.labelId)
- val DefaultKey = LabelIndex.DefaultSeq
- val fullKey = DefaultKey
-
- var labelOrderSeq = fullKey
-
- var sample = -1
- var limit = 10
- var offset = 0
- var rank = new RankParam(labelWithDir.labelId, List(LabelMeta.countSeq -> 1))
-
- var duration: Option[(Long, Long)] = None
- var isInverted: Boolean = false
-
- var columnRangeFilter: ColumnRangeFilter = null
-
- var hasFilters: Map[Byte, InnerValLike] = Map.empty[Byte, InnerValLike]
- var where: Try[Where] = Success(WhereParser.success)
- var whereRawOpt: Option[String] = None
- var duplicatePolicy = DuplicatePolicy.First
- var rpcTimeoutInMillis = 1000
- var maxAttempt = 2
- var includeDegree = false
- var tgtVertexInnerIdOpt: Option[InnerValLike] = None
- var cacheTTLInMillis: Long = -1L
- var threshold = QueryParam.DefaultThreshold
- var timeDecay: Option[TimeDecay] = None
- var transformer: EdgeTransformer = EdgeTransformer(this, EdgeTransformer.DefaultJson)
- var scorePropagateOp: String = "multiply"
- var exclude = false
- var include = false
- var shouldNormalize= false
- var cursorOpt: Option[String] = None
-
- var columnRangeFilterMinBytes = Array.empty[Byte]
- var columnRangeFilterMaxBytes = Array.empty[Byte]
-
- lazy val srcColumnWithDir = label.srcColumnWithDir(labelWithDir.dir)
- lazy val tgtColumnWithDir = label.tgtColumnWithDir(labelWithDir.dir)
-
- def toBytes(idxSeq: Byte, offset: Int, limit: Int, isInverted: Boolean): Array[Byte] = {
- val front = Array[Byte](idxSeq, if (isInverted) 1.toByte else 0.toByte)
- Bytes.add(front, Bytes.toBytes((offset.toLong << 32 | limit)))
- }
-
- /**
- * consider only I/O specific parameters.
- * properties that is used on Graph.filterEdges should not be considered.
- * @param bytes
- * @return
- */
- def toCacheKey(bytes: Array[Byte]): Long = {
- val hashBytes = toCacheKeyRaw(bytes)
- Hashing.murmur3_128().hashBytes(hashBytes).asLong()
-// MurmurHash3.bytesHash(hashBytes)
- }
-
- def toCacheKeyRaw(bytes: Array[Byte]): Array[Byte] = {
- val transformBytes = transformer.toHashKeyBytes
- //TODO: change this to binrary format.
- val whereBytes = Bytes.toBytes(whereRawOpt.getOrElse(""))
- val durationBytes = duration.map { case (min, max) =>
- val minTs = min / cacheTTLInMillis
- val maxTs = max / cacheTTLInMillis
- Bytes.add(Bytes.toBytes(minTs), Bytes.toBytes(maxTs))
- } getOrElse Array.empty[Byte]
-// Bytes.toBytes(duration.toString)
- val conditionBytes = Bytes.add(transformBytes, whereBytes, durationBytes)
- Bytes.add(Bytes.add(bytes, labelWithDir.bytes, toBytes(labelOrderSeq, offset, limit, isInverted)), rank.toHashKeyBytes(),
- Bytes.add(columnRangeFilterMinBytes, columnRangeFilterMaxBytes, conditionBytes))
- }
-
- def isInverted(isInverted: Boolean): QueryParam = {
- this.isInverted = isInverted
- this
- }
-
- def labelOrderSeq(labelOrderSeq: Byte): QueryParam = {
- this.labelOrderSeq = labelOrderSeq
- this
- }
-
- def sample(n: Int): QueryParam = {
- this.sample = n
- this
- }
-
- def limit(offset: Int, limit: Int): QueryParam = {
- /** since degree info is located on first always */
- if (offset == 0 && this.columnRangeFilter == null) {
- this.limit = limit + 1
- this.offset = offset
- } else {
- this.limit = limit
- this.offset = offset + 1
- }
- // this.columnPaginationFilter = new ColumnPaginationFilter(this.limit, this.offset)
- this
- }
-
- def interval(fromTo: Option[(Seq[(Byte, InnerValLike)], Seq[(Byte, InnerValLike)])]): QueryParam = {
- fromTo match {
- case Some((from, to)) => interval(from, to)
- case _ => this
- }
- }
-
- def interval(from: Seq[(Byte, InnerValLike)], to: Seq[(Byte, InnerValLike)]): QueryParam = {
- // val len = label.orderTypes.size.toByte
- // val len = label.extraIndicesMap(labelOrderSeq).sortKeyTypes.size.toByte
- // logger.error(s"indicesMap: ${label.indicesMap(labelOrderSeq)}")
- val len = label.indicesMap(labelOrderSeq).sortKeyTypes.size.toByte
-
- val minMetaByte = InnerVal.minMetaByte
- // val maxMetaByte = InnerVal.maxMetaByte
- val maxMetaByte = -1.toByte
- val toVal = Bytes.add(propsToBytes(to), Array.fill(1)(minMetaByte))
- //FIXME
- val fromVal = Bytes.add(propsToBytes(from), Array.fill(10)(maxMetaByte))
- toVal(0) = len
- fromVal(0) = len
- val maxBytes = fromVal
- val minBytes = toVal
- this.columnRangeFilterMaxBytes = maxBytes
- this.columnRangeFilterMinBytes = minBytes
- val rangeFilter = new ColumnRangeFilter(minBytes, true, maxBytes, true)
- this.columnRangeFilter = rangeFilter
- this
- }
-
- def duration(minMaxTs: Option[(Long, Long)]): QueryParam = {
- minMaxTs match {
- case Some((minTs, maxTs)) => duration(minTs, maxTs)
- case _ => this
- }
- }
-
- def duration(minTs: Long, maxTs: Long): QueryParam = {
- this.duration = Some((minTs, maxTs))
- this
- }
-
- def rank(r: RankParam): QueryParam = {
- this.rank = r
- this
- }
-
- def exclude(filterOut: Boolean): QueryParam = {
- this.exclude = filterOut
- this
- }
-
- def include(filterIn: Boolean): QueryParam = {
- this.include = filterIn
- this
- }
-
- def has(hasFilters: Map[Byte, InnerValLike]): QueryParam = {
- this.hasFilters = hasFilters
- this
- }
-
- def where(whereTry: Try[Where]): QueryParam = {
- this.where = whereTry
- this
- }
-
- def duplicatePolicy(policy: Option[DuplicatePolicy]): QueryParam = {
- this.duplicatePolicy = policy.getOrElse(DuplicatePolicy.First)
- this
- }
-
- def rpcTimeout(millis: Int): QueryParam = {
- this.rpcTimeoutInMillis = millis
- this
- }
-
- def maxAttempt(attempt: Int): QueryParam = {
- this.maxAttempt = attempt
- this
- }
-
- def includeDegree(includeDegree: Boolean): QueryParam = {
- this.includeDegree = includeDegree
- this
- }
-
- def tgtVertexInnerIdOpt(other: Option[InnerValLike]): QueryParam = {
- this.tgtVertexInnerIdOpt = other
- this
- }
-
- def cacheTTLInMillis(other: Long): QueryParam = {
- this.cacheTTLInMillis = other
- this
- }
-
- def timeDecay(other: Option[TimeDecay]): QueryParam = {
- this.timeDecay = other
- this
- }
-
- def threshold(other: Double): QueryParam = {
- this.threshold = other
- this
- }
-
- def transformer(other: Option[JsValue]): QueryParam = {
- other match {
- case Some(js) => this.transformer = EdgeTransformer(this, js)
- case None =>
- }
- this
- }
-
- def scorePropagateOp(scorePropagateOp: String): QueryParam = {
- this.scorePropagateOp = scorePropagateOp
- this
- }
-
- def shouldNormalize(shouldNormalize: Boolean): QueryParam = {
- this.shouldNormalize = shouldNormalize
- this
- }
-
- def whereRawOpt(sqlOpt: Option[String]): QueryParam = {
- this.whereRawOpt = sqlOpt
- this
- }
-
- def cursorOpt(cursorOpt: Option[String]): QueryParam = {
- this.cursorOpt = cursorOpt
- this
- }
-
- def isSnapshotEdge = tgtVertexInnerIdOpt.isDefined
-
- override def toString = {
- List(label.label, labelOrderSeq, offset, limit, rank,
- duration, isInverted, exclude, include, hasFilters).mkString("\t")
- // duration, isInverted, exclude, include, hasFilters, outputFields).mkString("\t")
- }
-
- //
- // def buildGetRequest(srcVertex: Vertex) = {
- // val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
- // val (srcInnerId, tgtInnerId) = tgtVertexInnerIdOpt match {
- // case Some(tgtVertexInnerId) => // _to is given.
- // /** we use toInvertedEdgeHashLike so dont need to swap src, tgt */
- // val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- // val tgt = InnerVal.convertVersion(tgtVertexInnerId, tgtColumn.columnType, label.schemaVersion)
- // (src, tgt)
- // case None =>
- // val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- // (src, src)
- // }
- //
- // val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
- // val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
- // val edge = Edge(srcV, tgtV, labelWithDir)
- //
- // val get = if (tgtVertexInnerIdOpt.isDefined) {
- // val snapshotEdge = edge.toInvertedEdgeHashLike
- // val kv = snapshotEdge.kvs.head
- // new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
- // } else {
- // val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == labelOrderSeq)
- // assert(indexedEdgeOpt.isDefined)
- // val indexedEdge = indexedEdgeOpt.get
- // val kv = indexedEdge.kvs.head
- // val table = label.hbaseTableName.getBytes
- // //kv.table //
- // val rowKey = kv.row // indexedEdge.rowKey.bytes
- // val cf = edgeCf
- // new GetRequest(table, rowKey, cf)
- // }
- //
- // val (minTs, maxTs) = duration.getOrElse((0L, Long.MaxValue))
- //
- // get.maxVersions(1)
- // get.setFailfast(true)
- // get.setMaxResultsPerColumnFamily(limit)
- // get.setRowOffsetPerColumnFamily(offset)
- // get.setMinTimestamp(minTs)
- // get.setMaxTimestamp(maxTs)
- // get.setTimeout(rpcTimeoutInMillis)
- // if (columnRangeFilter != null) get.setFilter(columnRangeFilter)
- // // get.setMaxAttempt(maxAttempt.toByte)
- // // get.setRpcTimeout(rpcTimeoutInMillis)
- //
- // // if (columnRangeFilter != null) get.filter(columnRangeFilter)
- // // logger.debug(s"Get: $get, $offset, $limit")
- //
- // get
- // }
-}
-
-case class TimeDecay(initial: Double = 1.0,
- lambda: Double = 0.1,
- timeUnit: Double = 60 * 60 * 24,
- labelMetaSeq: Byte = LabelMeta.timeStampSeq) {
- def decay(diff: Double): Double = {
- //FIXME
- val ret = initial * Math.pow(1.0 - lambda, diff / timeUnit)
- // logger.debug(s"$initial, $lambda, $timeUnit, $diff, ${diff / timeUnit}, $ret")
- ret
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
deleted file mode 100644
index 02d9736..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-package com.kakao.s2graph.core
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLikeWithTs}
-
-import scala.collection.Seq
-
-object QueryResult {
- def fromVertices(query: Query): Seq[QueryRequestWithResult] = {
- if (query.steps.isEmpty || query.steps.head.queryParams.isEmpty) {
- Seq.empty
- } else {
- val queryParam = query.steps.head.queryParams.head
- val label = queryParam.label
- val currentTs = System.currentTimeMillis()
- val propsWithTs = Map(LabelMeta.timeStampSeq ->
- InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs))
- for {
- vertex <- query.vertices
- } yield {
- val edge = Edge(vertex, vertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
- val edgeWithScore = EdgeWithScore(edge, Graph.DefaultScore)
- QueryRequestWithResult(QueryRequest(query, -1, vertex, queryParam),
- QueryResult(edgeWithScoreLs = Seq(edgeWithScore)))
- }
- }
- }
-}
-case class QueryRequestWithResult(queryRequest: QueryRequest, queryResult: QueryResult)
-
-case class QueryRequest(query: Query,
- stepIdx: Int,
- vertex: Vertex,
- queryParam: QueryParam)
-
-
-case class QueryResult(edgeWithScoreLs: Seq[EdgeWithScore] = Nil,
- tailCursor: Array[Byte] = Array.empty,
- timestamp: Long = System.currentTimeMillis(),
- isFailure: Boolean = false)
-
-case class EdgeWithScore(edge: Edge, score: Double)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
deleted file mode 100644
index c2b86c2..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
+++ /dev/null
@@ -1,107 +0,0 @@
-package com.kakao.s2graph.core
-
-
-import com.kakao.s2graph.core.mysqls._
-
-//import com.kakao.s2graph.core.models._
-
-import com.kakao.s2graph.core.types._
-import play.api.libs.json.Json
-
-/**
- */
-case class Vertex(id: VertexId,
- ts: Long = System.currentTimeMillis(),
- props: Map[Int, InnerValLike] = Map.empty[Int, InnerValLike],
- op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty) extends GraphElement {
-
- val innerId = id.innerId
-
- def schemaVer = serviceColumn.schemaVersion
-
- def serviceColumn = ServiceColumn.findById(id.colId)
-
- def service = Service.findById(serviceColumn.serviceId)
-
- lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
-
- def defaultProps = Map(ColumnMeta.lastModifiedAtColumnSeq.toInt -> InnerVal.withLong(ts, schemaVer))
-
- // lazy val kvs = Graph.client.vertexSerializer(this).toKeyValues
-
- /** TODO: make this as configurable */
- override def serviceName = service.serviceName
-
- override def isAsync = false
-
- override def queueKey = Seq(ts.toString, serviceName).mkString("|")
-
- override def queuePartitionKey = id.innerId.toString
-
- def propsWithName = for {
- (seq, v) <- props
- meta <- ColumnMeta.findByIdAndSeq(id.colId, seq.toByte)
- } yield (meta.name -> v.toString)
-
- // /** only used by bulk loader */
- // def buildPuts(): List[Put] = {
- // // logger.error(s"put: $this => $rowKey")
- //// val put = new Put(rowKey.bytes)
- //// for ((q, v) <- qualifiersWithValues) {
- //// put.addColumn(vertexCf, q, ts, v)
- //// }
- //// List(put)
- // val kv = kvs.head
- // val put = new Put(kv.row)
- // kvs.map { kv =>
- // put.addColumn(kv.cf, kv.qualifier, kv.timestamp, kv.value)
- // }
- // List(put)
- // }
-
- def toEdgeVertex() = Vertex(SourceVertexId(id.colId, innerId), ts, props, op)
-
-
- override def hashCode() = {
- val hash = id.hashCode()
- // logger.debug(s"Vertex.hashCode: $this -> $hash")
- hash
- }
-
- override def equals(obj: Any) = {
- obj match {
- case otherVertex: Vertex =>
- val ret = id == otherVertex.id
- // logger.debug(s"Vertex.equals: $this, $obj => $ret")
- ret
- case _ => false
- }
- }
-
- def withProps(newProps: Map[Int, InnerValLike]) = Vertex(id, ts, newProps, op)
-
- def toLogString(): String = {
- val (serviceName, columnName) =
- if (!id.storeColId) ("", "")
- else (serviceColumn.service.serviceName, serviceColumn.columnName)
-
- if (propsWithName.nonEmpty)
- Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName, Json.toJson(propsWithName)).mkString("\t")
- else
- Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
- }
-}
-
-object Vertex {
-
- def toPropKey(labelId: Int): Int = Byte.MaxValue + labelId
-
- def toLabelId(propKey: Int): Int = propKey - Byte.MaxValue
-
- def isLabelId(propKey: Int): Boolean = propKey > Byte.MaxValue
-
- // val emptyVertex = Vertex(new CompositeId(CompositeId.defaultColId, CompositeId.defaultInnerId, false, true),
- // System.currentTimeMillis())
- def fromString(s: String): Option[Vertex] = Graph.toVertex(s)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Bucket.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Bucket.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Bucket.scala
deleted file mode 100644
index edc6147..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Bucket.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-/**
- * Created by shon on 8/5/15.
- */
-
-import scalikejdbc._
-
-object Bucket extends Model[Bucket] {
-
- val rangeDelimiter = "~"
- val INVALID_BUCKET_EXCEPTION = new RuntimeException("invalid bucket.")
-
- def apply(rs: WrappedResultSet): Bucket = {
- Bucket(rs.intOpt("id"),
- rs.int("experiment_id"),
- rs.string("modular"),
- rs.string("http_verb"),
- rs.string("api_path"),
- rs.string("request_body"),
- rs.int("timeout"),
- rs.string("impression_id"),
- rs.boolean("is_graph_query"),
- rs.boolean("is_empty"))
- }
-
- def finds(experimentId: Int)(implicit session: DBSession = AutoSession): List[Bucket] = {
- val cacheKey = "experimentId=" + experimentId
- withCaches(cacheKey) {
- sql"""select * from buckets where experiment_id = $experimentId"""
- .map { rs => Bucket(rs) }.list().apply()
- }
- }
-
- def toRange(str: String): Option[(Int, Int)] = {
- val range = str.split(rangeDelimiter)
- if (range.length == 2) Option(range.head.toInt, range.last.toInt)
- else None
- }
-
- def findByImpressionId(impressionId: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Bucket] = {
- val cacheKey = "impressionId=" + impressionId
- val sql = sql"""select * from buckets where impression_id=$impressionId"""
- .map { rs => Bucket(rs)}
- if (useCache) {
- withCache(cacheKey) {
- sql.single().apply()
- }
- } else {
- sql.single().apply()
- }
- }
-}
-
-case class Bucket(id: Option[Int],
- experimentId: Int,
- modular: String,
- httpVerb: String, apiPath: String,
- requestBody: String, timeout: Int, impressionId: String,
- isGraphQuery: Boolean = true,
- isEmpty: Boolean = false) {
-
- import Bucket._
-
- lazy val rangeOpt = toRange(modular)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ColumnMeta.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ColumnMeta.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ColumnMeta.scala
deleted file mode 100644
index 1b6d55f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/ColumnMeta.scala
+++ /dev/null
@@ -1,110 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-import play.api.libs.json.Json
-import scalikejdbc._
-
-object ColumnMeta extends Model[ColumnMeta] {
-
- val timeStampSeq = 0.toByte
- val countSeq = -1.toByte
- val lastModifiedAtColumnSeq = 0.toByte
- val lastModifiedAtColumn = ColumnMeta(Some(0), 0, "lastModifiedAt", lastModifiedAtColumnSeq, "long")
- val maxValue = Byte.MaxValue
-
- def apply(rs: WrappedResultSet): ColumnMeta = {
- ColumnMeta(Some(rs.int("id")), rs.int("column_id"), rs.string("name"), rs.byte("seq"), rs.string("data_type").toLowerCase())
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession) = {
- // val cacheKey = s"id=$id"
- val cacheKey = "id=" + id
- withCache(cacheKey) {
- sql"""select * from column_metas where id = ${id}""".map { rs => ColumnMeta(rs) }.single.apply
- }.get
- }
-
- def findAllByColumn(columnId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
- // val cacheKey = s"columnId=$columnId"
- val cacheKey = "columnId=" + columnId
- if (useCache) {
- withCaches(cacheKey)( sql"""select *from column_metas where column_id = ${columnId} order by seq ASC"""
- .map { rs => ColumnMeta(rs) }.list.apply())
- } else {
- sql"""select * from column_metas where column_id = ${columnId} order by seq ASC"""
- .map { rs => ColumnMeta(rs) }.list.apply()
- }
- }
-
- def findByName(columnId: Int, name: String)(implicit session: DBSession = AutoSession) = {
- // val cacheKey = s"columnId=$columnId:name=$name"
- val cacheKey = "columnId=" + columnId + ":name=" + name
- withCache(cacheKey)( sql"""select * from column_metas where column_id = ${columnId} and name = ${name}"""
- .map { rs => ColumnMeta(rs) }.single.apply())
- }
-
- def insert(columnId: Int, name: String, dataType: String)(implicit session: DBSession = AutoSession) = {
- val ls = findAllByColumn(columnId, false)
- val seq = ls.size + 1
- if (seq <= maxValue) {
- sql"""insert into column_metas(column_id, name, seq, data_type)
- select ${columnId}, ${name}, ${seq}, ${dataType}"""
- .updateAndReturnGeneratedKey.apply()
- }
- }
-
- def findOrInsert(columnId: Int, name: String, dataType: String)(implicit session: DBSession = AutoSession): ColumnMeta = {
- findByName(columnId, name) match {
- case Some(c) => c
- case None =>
- insert(columnId, name, dataType)
- expireCache(s"columnId=$columnId:name=$name")
- findByName(columnId, name).get
- }
- }
-
- def findByIdAndSeq(columnId: Int, seq: Byte, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
- val cacheKey = "columnId=" + columnId + ":seq=" + seq
- lazy val columnMetaOpt = sql"""
- select * from column_metas where column_id = ${columnId} and seq = ${seq}
- """.map { rs => ColumnMeta(rs) }.single.apply()
-
- if (useCache) withCache(cacheKey)(columnMetaOpt)
- else columnMetaOpt
- }
-
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val columnMeta = findById(id)
- val (columnId, name) = (columnMeta.columnId, columnMeta.name)
- sql"""delete from column_metas where id = ${id}""".execute.apply()
- val cacheKeys = List(s"id=$id", s"columnId=$columnId:name=$name", s"colunmId=$columnId")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- }
-
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from column_metas""".map { rs => ColumnMeta(rs) }.list().apply()
-
- putsToCache(ls.map { x =>
- val cacheKey = s"id=${x.id.get}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- val cacheKey = s"columnId=${x.columnId}:name=${x.name}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- val cacheKey = s"columnId=${x.columnId}:seq=${x.seq}"
- (cacheKey -> x)
- })
- putsToCaches(ls.groupBy(x => x.columnId).map { case (columnId, ls) =>
- val cacheKey = s"columnId=${columnId}"
- (cacheKey -> ls)
- }.toList)
- }
-}
-
-case class ColumnMeta(id: Option[Int], columnId: Int, name: String, seq: Byte, dataType: String) {
- lazy val toJson = Json.obj("name" -> name, "dataType" -> dataType)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
deleted file mode 100644
index 46b92ab..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-import com.kakao.s2graph.core.GraphUtil
-import com.kakao.s2graph.core.utils.logger
-import scalikejdbc._
-
-import scala.util.Random
-
-object Experiment extends Model[Experiment] {
- val impressionKey = "S2-Impression-Id"
-
- def apply(rs: WrappedResultSet): Experiment = {
- Experiment(rs.intOpt("id"),
- rs.int("service_id"),
- rs.string("name"),
- rs.string("description"),
- rs.string("experiment_type"),
- rs.int("total_modular"))
- }
-
- def finds(serviceId: Int)(implicit session: DBSession = AutoSession): List[Experiment] = {
- val cacheKey = "serviceId=" + serviceId
- withCaches(cacheKey) {
- sql"""select * from experiments where service_id = ${serviceId}"""
- .map { rs => Experiment(rs) }.list().apply()
- }
- }
-
- def findBy(serviceId: Int, name: String)(implicit session: DBSession = AutoSession): Option[Experiment] = {
- val cacheKey = "serviceId=" + serviceId + ":name=" + name
- withCache(cacheKey) {
- sql"""select * from experiments where service_id = ${serviceId} and name = ${name}"""
- .map { rs => Experiment(rs) }.single.apply
- }
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession): Option[Experiment] = {
- val cacheKey = "id=" + id
- withCache(cacheKey)(
- sql"""select * from experiments where id = ${id}"""
- .map { rs => Experiment(rs) }.single.apply
- )
- }
-}
-
-case class Experiment(id: Option[Int],
- serviceId: Int,
- name: String,
- description: String,
- experimentType: String,
- totalModular: Int) {
-
- def buckets = Bucket.finds(id.get)
-
- def rangeBuckets = for {
- bucket <- buckets
- range <- bucket.rangeOpt
- } yield range -> bucket
-
-
- def findBucket(uuid: String, impIdOpt: Option[String] = None): Option[Bucket] = {
- impIdOpt match {
- case Some(impId) => Bucket.findByImpressionId(impId)
- case None =>
- val seed = experimentType match {
- case "u" => (GraphUtil.murmur3(uuid) % totalModular) + 1
- case _ => Random.nextInt(totalModular) + 1
- }
- findBucket(seed)
- }
- }
-
- def findBucket(uuidMod: Int): Option[Bucket] = {
- rangeBuckets.find { case ((from, to), bucket) =>
- from <= uuidMod && uuidMod <= to
- }.map(_._2)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Label.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Label.scala
deleted file mode 100644
index 005a01e..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Label.scala
+++ /dev/null
@@ -1,372 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-import com.kakao.s2graph.core.GraphExceptions.ModelNotFoundException
-import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{GraphUtil, JSONParser, Management}
-import play.api.libs.json.Json
-import scalikejdbc._
-
-object Label extends Model[Label] {
-
- val maxHBaseTableNames = 2
-
- def apply(rs: WrappedResultSet): Label = {
- Label(Option(rs.int("id")), rs.string("label"),
- rs.int("src_service_id"), rs.string("src_column_name"), rs.string("src_column_type"),
- rs.int("tgt_service_id"), rs.string("tgt_column_name"), rs.string("tgt_column_type"),
- rs.boolean("is_directed"), rs.string("service_name"), rs.int("service_id"), rs.string("consistency_level"),
- rs.string("hbase_table_name"), rs.intOpt("hbase_table_ttl"), rs.string("schema_version"), rs.boolean("is_async"), rs.string("compressionAlgorithm"))
- }
-
- def deleteAll(label: Label)(implicit session: DBSession) = {
- val id = label.id
- LabelMeta.findAllByLabelId(id.get, false).foreach { x => LabelMeta.delete(x.id.get) }
- LabelIndex.findByLabelIdAll(id.get, false).foreach { x => LabelIndex.delete(x.id.get) }
- Label.delete(id.get)
- }
-
- def findByName(labelName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[Label] = {
- val cacheKey = "label=" + labelName
- lazy val labelOpt =
- sql"""
- select *
- from labels
- where label = ${labelName}""".map { rs => Label(rs) }.single.apply()
-
- if (useCache) withCache(cacheKey)(labelOpt)
- else labelOpt
- }
-
- def insert(label: String,
- srcServiceId: Int,
- srcColumnName: String,
- srcColumnType: String,
- tgtServiceId: Int,
- tgtColumnName: String,
- tgtColumnType: String,
- isDirected: Boolean,
- serviceName: String,
- serviceId: Int,
- consistencyLevel: String,
- hTableName: String,
- hTableTTL: Option[Int],
- schemaVersion: String,
- isAsync: Boolean,
- compressionAlgorithm: String)(implicit session: DBSession = AutoSession) = {
- sql"""
- insert into labels(label,
- src_service_id, src_column_name, src_column_type,
- tgt_service_id, tgt_column_name, tgt_column_type,
- is_directed, service_name, service_id, consistency_level, hbase_table_name, hbase_table_ttl, schema_version, is_async, compressionAlgorithm)
- values (${label},
- ${srcServiceId}, ${srcColumnName}, ${srcColumnType},
- ${tgtServiceId}, ${tgtColumnName}, ${tgtColumnType},
- ${isDirected}, ${serviceName}, ${serviceId}, ${consistencyLevel}, ${hTableName}, ${hTableTTL},
- ${schemaVersion}, ${isAsync}, ${compressionAlgorithm})
- """
- .updateAndReturnGeneratedKey.apply()
- }
-
- def findByIdOpt(id: Int)(implicit session: DBSession = AutoSession): Option[Label] = {
- val cacheKey = "id=" + id
- withCache(cacheKey)(
- sql"""
- select *
- from labels
- where id = ${id}"""
- .map { rs => Label(rs) }.single.apply())
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession): Label = {
- val cacheKey = "id=" + id
- withCache(cacheKey)(
- sql"""
- select *
- from labels
- where id = ${id}"""
- .map { rs => Label(rs) }.single.apply()).get
- }
-
- def findByTgtColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
- val cacheKey = "tgtColumnId=" + columnId
- val col = ServiceColumn.findById(columnId)
- withCaches(cacheKey)(
- sql"""
- select *
- from labels
- where tgt_column_name = ${col.columnName}
- and service_id = ${col.serviceId}
- """.map { rs => Label(rs) }.list().apply())
- }
-
- def findBySrcColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
- val cacheKey = "srcColumnId=" + columnId
- val col = ServiceColumn.findById(columnId)
- withCaches(cacheKey)(
- sql"""
- select *
- from labels
- where src_column_name = ${col.columnName}
- and service_id = ${col.serviceId}
- """.map { rs => Label(rs) }.list().apply())
- }
-
- def findBySrcServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
- val cacheKey = "srcServiceId=" + serviceId
- withCaches(cacheKey)(
- sql"""select * from labels where src_service_id = ${serviceId}""".map { rs => Label(rs) }.list().apply
- )
- }
-
- def findByTgtServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
- val cacheKey = "tgtServiceId=" + serviceId
- withCaches(cacheKey)(
- sql"""select * from labels where tgt_service_id = ${serviceId}""".map { rs => Label(rs) }.list().apply
- )
- }
-
- def insertAll(labelName: String, srcServiceName: String, srcColumnName: String, srcColumnType: String,
- tgtServiceName: String, tgtColumnName: String, tgtColumnType: String,
- isDirected: Boolean = true,
- serviceName: String,
- indices: Seq[Index],
- metaProps: Seq[Prop],
- consistencyLevel: String,
- hTableName: Option[String],
- hTableTTL: Option[Int],
- schemaVersion: String,
- isAsync: Boolean,
- compressionAlgorithm: String)(implicit session: DBSession = AutoSession): Label = {
-
- val srcServiceOpt = Service.findByName(srcServiceName, useCache = false)
- val tgtServiceOpt = Service.findByName(tgtServiceName, useCache = false)
- val serviceOpt = Service.findByName(serviceName, useCache = false)
- if (srcServiceOpt.isEmpty) throw new RuntimeException(s"source service $srcServiceName is not created.")
- if (tgtServiceOpt.isEmpty) throw new RuntimeException(s"target service $tgtServiceName is not created.")
- if (serviceOpt.isEmpty) throw new RuntimeException(s"service $serviceName is not created.")
-
- val newLabel = for {
- srcService <- srcServiceOpt
- tgtService <- tgtServiceOpt
- service <- serviceOpt
- } yield {
- val srcServiceId = srcService.id.get
- val tgtServiceId = tgtService.id.get
- val serviceId = service.id.get
-
- /** insert serviceColumn */
- val srcCol = ServiceColumn.findOrInsert(srcServiceId, srcColumnName, Some(srcColumnType), schemaVersion)
- val tgtCol = ServiceColumn.findOrInsert(tgtServiceId, tgtColumnName, Some(tgtColumnType), schemaVersion)
-
- if (srcCol.columnType != srcColumnType) throw new RuntimeException(s"source service column type not matched ${srcCol.columnType} != ${srcColumnType}")
- if (tgtCol.columnType != tgtColumnType) throw new RuntimeException(s"target service column type not matched ${tgtCol.columnType} != ${tgtColumnType}")
-
- /** create label */
- Label.findByName(labelName, useCache = false).getOrElse {
-
- val createdId = insert(labelName, srcServiceId, srcColumnName, srcColumnType,
- tgtServiceId, tgtColumnName, tgtColumnType, isDirected, serviceName, serviceId, consistencyLevel,
- hTableName.getOrElse(service.hTableName), hTableTTL.orElse(service.hTableTTL), schemaVersion, isAsync, compressionAlgorithm).toInt
-
- val labelMetaMap = metaProps.map { case Prop(propName, defaultValue, dataType) =>
- val labelMeta = LabelMeta.findOrInsert(createdId, propName, defaultValue, dataType)
- (propName -> labelMeta.seq)
- }.toMap ++ LabelMeta.reservedMetas.map (labelMeta => labelMeta.name -> labelMeta.seq).toMap
-
- if (indices.isEmpty) {
- // make default index with _PK, _timestamp, 0
- LabelIndex.findOrInsert(createdId, LabelIndex.DefaultName, LabelIndex.DefaultMetaSeqs.toList, "none")
- } else {
- indices.foreach { index =>
- val metaSeq = index.propNames.map { name => labelMetaMap(name) }
- LabelIndex.findOrInsert(createdId, index.name, metaSeq.toList, "none")
- }
- }
-
- val cacheKeys = List(s"id=$createdId", s"label=$labelName")
- val ret = findByName(labelName, useCache = false).get
- putsToCache(cacheKeys.map(k => k -> ret))
- ret
- }
- }
-
- newLabel.getOrElse(throw new RuntimeException("failed to create label"))
- }
-
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from labels""".map { rs => Label(rs) }.list().apply()
- putsToCache(ls.map { x =>
- val cacheKey = s"id=${x.id.get}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- val cacheKey = s"label=${x.label}"
- (cacheKey -> x)
- })
- }
-
- def updateName(oldName: String, newName: String)(implicit session: DBSession = AutoSession) = {
- logger.info(s"rename label: $oldName -> $newName")
- sql"""update labels set label = ${newName} where label = ${oldName}""".update.apply()
- }
-
- def updateHTableName(labelName: String, newHTableName: String)(implicit session: DBSession = AutoSession) = {
- logger.info(s"update HTable of label $labelName to $newHTableName")
- val cnt = sql"""update labels set hbase_table_name = $newHTableName where label = $labelName""".update().apply()
- val label = Label.findByName(labelName, useCache = false).get
-
- val cacheKeys = List(s"id=${label.id}", s"label=${label.label}")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- cnt
- }
-
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val label = findById(id)
- logger.info(s"delete label: $label")
- val cnt = sql"""delete from labels where id = ${label.id.get}""".update().apply()
- val cacheKeys = List(s"id=$id", s"label=${label.label}")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- cnt
- }
-}
-
-case class Label(id: Option[Int], label: String,
- srcServiceId: Int, srcColumnName: String, srcColumnType: String,
- tgtServiceId: Int, tgtColumnName: String, tgtColumnType: String,
- isDirected: Boolean = true, serviceName: String, serviceId: Int, consistencyLevel: String = "strong",
- hTableName: String, hTableTTL: Option[Int],
- schemaVersion: String, isAsync: Boolean = false,
- compressionAlgorithm: String) extends JSONParser {
- def metas = LabelMeta.findAllByLabelId(id.get)
-
- def metaSeqsToNames = metas.map(x => (x.seq, x.name)) toMap
-
- // lazy val firstHBaseTableName = hbaseTableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME)
- lazy val srcService = Service.findById(srcServiceId)
- lazy val tgtService = Service.findById(tgtServiceId)
- lazy val service = Service.findById(serviceId)
- /**
- * TODO
- * change this to apply hbase table from target serviceName
- */
- // lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.tableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME))
- // lazy val (hbaseZkAddr, hbaseTableName) = (Config.HBASE_ZOOKEEPER_QUORUM, hTableName.split(",").headOption.getOrElse(Config.HBASE_TABLE_NAME))
- // lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, hTableName.split(",").headOption.getOrElse(GraphConnection.getConfVal("hbase.table.name")))
- lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, hTableName.split(",").head)
-
- lazy val srcColumn = ServiceColumn.find(srcServiceId, srcColumnName).getOrElse(throw ModelNotFoundException("Source column not found"))
- lazy val tgtColumn = ServiceColumn.find(tgtServiceId, tgtColumnName).getOrElse(throw ModelNotFoundException("Target column not found"))
-
- lazy val direction = if (isDirected) "out" else "undirected"
- lazy val defaultIndex = LabelIndex.findByLabelIdAndSeq(id.get, LabelIndex.DefaultSeq)
-
- //TODO: Make sure this is correct
- lazy val indices = LabelIndex.findByLabelIdAll(id.get, useCache = true)
- lazy val indicesMap = indices.map(idx => (idx.seq, idx)) toMap
- lazy val indexSeqsMap = indices.map(idx => (idx.metaSeqs, idx)) toMap
- lazy val indexNameMap = indices.map(idx => (idx.name, idx)) toMap
- lazy val extraIndices = indices.filter(idx => defaultIndex.isDefined && idx.id.get != defaultIndex.get.id.get)
- // indices filterNot (_.id.get == defaultIndex.get.id.get)
- lazy val extraIndicesMap = extraIndices.map(idx => (idx.seq, idx)) toMap
-
- lazy val metaProps = LabelMeta.reservedMetas.map { m =>
- if (m == LabelMeta.to) m.copy(dataType = tgtColumnType)
- else if (m == LabelMeta.from) m.copy(dataType = srcColumnType)
- else m
- } ::: LabelMeta.findAllByLabelId(id.get, useCache = true)
-
- lazy val metaPropsInner = LabelMeta.reservedMetasInner.map { m =>
- if (m == LabelMeta.to) m.copy(dataType = tgtColumnType)
- else if (m == LabelMeta.from) m.copy(dataType = srcColumnType)
- else m
- } ::: LabelMeta.findAllByLabelId(id.get, useCache = true)
-
- lazy val metaPropsMap = metaProps.map(x => (x.seq, x)).toMap
- lazy val metaPropsInvMap = metaProps.map(x => (x.name, x)).toMap
- lazy val metaPropNames = metaProps.map(x => x.name)
- lazy val metaPropNamesMap = metaProps.map(x => (x.seq, x.name)) toMap
-
- /** this is used only by edgeToProps */
- lazy val metaPropsDefaultMap = (for {
- prop <- metaProps if LabelMeta.isValidSeq(prop.seq)
- jsValue <- innerValToJsValue(toInnerVal(prop.defaultValue, prop.dataType, schemaVersion), prop.dataType)
- } yield prop.name -> jsValue).toMap
-
- lazy val metaPropsDefaultMapInner = (for {
- prop <- metaPropsInner if LabelMeta.isValidSeq(prop.seq)
- jsValue <- innerValToJsValue(toInnerVal(prop.defaultValue, prop.dataType, schemaVersion), prop.dataType)
- } yield prop.name -> jsValue).toMap
-
- def srcColumnWithDir(dir: Int) = {
- if (dir == GraphUtil.directions("out")) srcColumn else tgtColumn
- }
-
- def tgtColumnWithDir(dir: Int) = {
- if (dir == GraphUtil.directions("out")) tgtColumn else srcColumn
- }
-
- def srcTgtColumn(dir: Int) =
- if (isDirected) {
- (srcColumnWithDir(dir), tgtColumnWithDir(dir))
- } else {
- if (dir == GraphUtil.directions("in")) {
- (tgtColumn, srcColumn)
- } else {
- (srcColumn, tgtColumn)
- }
- }
-
- def init() = {
- metas
- metaSeqsToNames
- service
- srcColumn
- tgtColumn
- defaultIndex
- indices
- metaProps
- }
-
- // def srcColumnInnerVal(jsValue: JsValue) = {
- // jsValueToInnerVal(jsValue, srcColumnType, version)
- // }
- // def tgtColumnInnerVal(jsValue: JsValue) = {
- // jsValueToInnerVal(jsValue, tgtColumnType, version)
- // }
-
- override def toString(): String = {
- val orderByKeys = LabelMeta.findAllByLabelId(id.get)
- super.toString() + orderByKeys.toString()
- }
-
- // def findLabelIndexSeq(scoring: List[(Byte, Double)]): Byte = {
- // if (scoring.isEmpty) LabelIndex.defaultSeq
- // else {
- // LabelIndex.findByLabelIdAndSeqs(id.get, scoring.map(_._1).sorted).map(_.seq).getOrElse(LabelIndex.defaultSeq)
- //
- //// LabelIndex.findByLabelIdAndSeqs(id.get, scoring.map(_._1).sorted).map(_.seq).getOrElse(LabelIndex.defaultSeq)
- // }
- // }
- lazy val toJson = Json.obj("labelName" -> label,
- "from" -> srcColumn.toJson, "to" -> tgtColumn.toJson,
- "isDirected" -> isDirected,
- "serviceName" -> serviceName,
- "consistencyLevel" -> consistencyLevel,
- "schemaVersion" -> schemaVersion,
- "isAsync" -> isAsync,
- "compressionAlgorithm" -> compressionAlgorithm,
- "defaultIndex" -> defaultIndex.map(x => x.toJson),
- "extraIndex" -> extraIndices.map(exIdx => exIdx.toJson),
- "metaProps" -> metaProps.filter { labelMeta => LabelMeta.isValidSeqForAdmin(labelMeta.seq) }.map(_.toJson)
- )
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelIndex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelIndex.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelIndex.scala
deleted file mode 100644
index 47f4a2a..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/LabelIndex.scala
+++ /dev/null
@@ -1,143 +0,0 @@
-package com.kakao.s2graph.core.mysqls
-
-/**
- * Created by shon on 6/3/15.
- */
-
-import play.api.libs.json.Json
-import scalikejdbc._
-
-object LabelIndex extends Model[LabelIndex] {
- val DefaultName = "_PK"
- val DefaultMetaSeqs = Seq(LabelMeta.timeStampSeq)
- val DefaultSeq = 1.toByte
- val MaxOrderSeq = 7
-
- def apply(rs: WrappedResultSet): LabelIndex = {
- LabelIndex(rs.intOpt("id"), rs.int("label_id"), rs.string("name"), rs.byte("seq"),
- rs.string("meta_seqs").split(",").filter(_ != "").map(s => s.toByte).toList match {
- case metaSeqsList => metaSeqsList
- },
- rs.string("formulars"))
- }
-
- def findById(id: Int)(implicit session: DBSession = AutoSession) = {
- val cacheKey = "id=" + id
- withCache(cacheKey) {
- sql"""select * from label_indices where id = ${id}""".map { rs => LabelIndex(rs) }.single.apply
- }.get
- }
-
- def findByLabelIdAll(labelId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession) = {
- val cacheKey = "labelId=" + labelId
- if (useCache) {
- withCaches(cacheKey)( sql"""
- select * from label_indices where label_id = ${labelId} and seq > 0 order by seq ASC
- """.map { rs => LabelIndex(rs) }.list.apply)
- } else {
- sql"""
- select * from label_indices where label_id = ${labelId} and seq > 0 order by seq ASC
- """.map { rs => LabelIndex(rs) }.list.apply
- }
- }
-
- def insert(labelId: Int, indexName: String, seq: Byte, metaSeqs: List[Byte], formulars: String)(implicit session: DBSession = AutoSession): Long = {
- sql"""
- insert into label_indices(label_id, name, seq, meta_seqs, formulars)
- values (${labelId}, ${indexName}, ${seq}, ${metaSeqs.mkString(",")}, ${formulars})
- """
- .updateAndReturnGeneratedKey.apply()
- }
-
- def findOrInsert(labelId: Int, indexName: String, metaSeqs: List[Byte], formulars: String)(implicit session: DBSession = AutoSession): LabelIndex = {
- findByLabelIdAndSeqs(labelId, metaSeqs) match {
- case Some(s) => s
- case None =>
- val orders = findByLabelIdAll(labelId, false)
- val seq = (orders.size + 1).toByte
- assert(seq <= MaxOrderSeq)
- val createdId = insert(labelId, indexName, seq, metaSeqs, formulars)
- val cacheKeys = List(s"labelId=$labelId:seq=$seq",
- s"labelId=$labelId:seqs=$metaSeqs", s"labelId=$labelId:seq=$seq", s"id=$createdId")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
-
- findByLabelIdAndSeq(labelId, seq).get
- }
- }
-
- def findByLabelIdAndSeqs(labelId: Int, seqs: List[Byte])(implicit session: DBSession = AutoSession): Option[LabelIndex] = {
- val cacheKey = "labelId=" + labelId + ":seqs=" + seqs.mkString(",")
- withCache(cacheKey) {
- sql"""
- select * from label_indices where label_id = ${labelId} and meta_seqs = ${seqs.mkString(",")}
- """.map { rs => LabelIndex(rs) }.single.apply
- }
- }
-
- def findByLabelIdAndSeq(labelId: Int, seq: Byte, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[LabelIndex] = {
- // val cacheKey = s"labelId=$labelId:seq=$seq"
- val cacheKey = "labelId=" + labelId + ":seq=" + seq
- if (useCache) {
- withCache(cacheKey)( sql"""
- select * from label_indices where label_id = ${labelId} and seq = ${seq}
- """.map { rs => LabelIndex(rs) }.single.apply)
- } else {
- sql"""
- select * from label_indices where label_id = ${labelId} and seq = ${seq}
- """.map { rs => LabelIndex(rs) }.single.apply
- }
- }
-
- def delete(id: Int)(implicit session: DBSession = AutoSession) = {
- val labelIndex = findById(id)
- val seqs = labelIndex.metaSeqs.mkString(",")
- val (labelId, seq) = (labelIndex.labelId, labelIndex.seq)
- sql"""delete from label_indices where id = ${id}""".execute.apply()
-
- val cacheKeys = List(s"id=$id", s"labelId=$labelId", s"labelId=$labelId:seq=$seq", s"labelId=$labelId:seqs=$seqs")
- cacheKeys.foreach { key =>
- expireCache(key)
- expireCaches(key)
- }
- }
-
- def findAll()(implicit session: DBSession = AutoSession) = {
- val ls = sql"""select * from label_indices""".map { rs => LabelIndex(rs) }.list.apply
- putsToCache(ls.map { x =>
- var cacheKey = s"id=${x.id.get}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- var cacheKey = s"labelId=${x.labelId}:seq=${x.seq}}"
- (cacheKey -> x)
- })
- putsToCache(ls.map { x =>
- var cacheKey = s"labelId=${x.labelId}:seqs=${x.metaSeqs.mkString(",")}"
- (cacheKey -> x)
- })
- putsToCaches(ls.groupBy(x => x.labelId).map { case (labelId, ls) =>
- val cacheKey = s"labelId=${labelId}"
- (cacheKey -> ls)
- }.toList)
- }
-}
-
-/**
- * formular
- * ex1): w1, w2, w3
- * ex2): 1.5 * w1^2 + 3.4 * (w1 * w2), w2, w1
- */
-
-case class LabelIndex(id: Option[Int], labelId: Int, name: String, seq: Byte, metaSeqs: Seq[Byte], formulars: String) {
- lazy val label = Label.findById(labelId)
- lazy val metas = label.metaPropsMap
- lazy val sortKeyTypes = metaSeqs.flatMap(metaSeq => label.metaPropsMap.get(metaSeq))
- lazy val propNames = sortKeyTypes.map { labelMeta => labelMeta.name }
- lazy val toJson = Json.obj(
- "name" -> name,
- "propNames" -> sortKeyTypes.map(x => x.name)
- )
-}
[07/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageHBase.scala b/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageHBase.scala
deleted file mode 100644
index a664de4..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v1/ExactStorageHBase.scala
+++ /dev/null
@@ -1,327 +0,0 @@
-package s2.counter.core.v1
-
-import com.kakao.s2graph.core.Graph
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.CellUtil
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.filter.{ColumnRangeFilter, FilterList}
-import org.apache.hadoop.hbase.util.Bytes
-import org.slf4j.LoggerFactory
-import s2.config.S2CounterConfig
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.counter.core._
-import s2.helper.{Management, WithHBase}
-import s2.models.Counter
-import s2.models.Counter.ItemType
-
-import scala.collection.JavaConversions._
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
-
-
-class ExactStorageHBase(config: Config) extends ExactStorage {
- import ExactStorageHBase._
-
- private val log = LoggerFactory.getLogger(getClass)
-
- lazy val s2config = new S2CounterConfig(config)
-
- private[counter] val withHBase = new WithHBase(config)
- private[counter] val hbaseManagement = new Management(config)
-
-
-
- private def getTableName(policy: Counter): String = {
- policy.hbaseTable.getOrElse(s2config.HBASE_TABLE_NAME)
- }
-
- override def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
- lazy val messageForLog = s"${policy.service}.${policy.action} $items $timeRange"
-
- val keys = {
- for {
- item <- items
- } yield {
- ExactKey(policy, item, checkItemType = true)
- }
- }
-
- val gets = {
- for {
- key <- keys
- } yield {
- val get = new Get(BytesUtilV1.toBytes(key))
- timeRange.map(t => intervalsMap(t._1.q)).distinct.foreach { cf =>
- get.addFamily(cf.toString.getBytes)
- }
- get.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ONE, {
- for {
- (from, to) <- timeRange
- } yield {
- new ColumnRangeFilter(
- BytesUtilV1.toBytes(from), true,
- BytesUtilV1.toBytes(to.copy(ts = to.ts + 1)), false)
- }
- }))
- }
- }
-
- // println(s"$messageForLog $gets")
- Future {
- withHBase(getTableName(policy)) { table =>
- for {
- (rst, key) <- table.get(gets).zip(keys) if !rst.isEmpty
- } yield {
- val qualifierWithCounts = {
- for {
- cell <- rst.listCells()
- eq = BytesUtilV1.toExactQualifier(CellUtil.cloneQualifier(cell))
- } yield {
- eq -> Bytes.toLong(CellUtil.cloneValue(cell))
- }
- }.toMap
- FetchedCounts(key, qualifierWithCounts)
- }
- } match {
- case Success(rst) => rst
- case Failure(ex) =>
- log.error(s"$ex: $messageForLog")
- Nil
- }
- }
- }
-
- override def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
- get(policy, items, timeRange).map { fetchedLs =>
- for {
- FetchedCounts(exactKey, qualifierWithCountMap) <- fetchedLs
- } yield {
- val intervalWithCountMap = qualifierWithCountMap
- .filter { case (eq, v) => eq.checkDimensionEquality(dimQuery) }
- .groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) }
- FetchedCountsGrouped(exactKey, intervalWithCountMap)
- }
- }
- }
-
- override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
- // increment mutation to hbase
- val increments = {
- for {
- (exactKey, values) <- counts
- inc = new Increment(BytesUtilV1.toBytes(exactKey))
- } yield {
- for {
- (eq, value) <- values
- } {
- inc.addColumn(intervalsMap.apply(eq.tq.q).toString.getBytes, BytesUtilV1.toBytes(eq), value)
- }
- // add column by dimension
- inc
- }
- }
-
- val results: Array[Object] = Array.fill(increments.size)(null)
-
- withHBase(getTableName(policy)) { table =>
- table.batch(increments, results)
- } match {
- case Failure(ex) =>
- log.error(s"${ex.getMessage}")
- case _ =>
- }
-
- assert(counts.length == results.length)
-
- for {
- ((exactKey, eqWithValue), result) <- counts.zip(results)
- } yield {
- val eqWithResult = result match {
- case r: Result =>
- for {
- (eq, value) <- eqWithValue
- } yield {
- val interval = eq.tq.q
- val cf = intervalsMap(interval)
- val result = Option(r.getColumnLatestCell(cf.toString.getBytes, BytesUtilV1.toBytes(eq))).map { cell =>
- Bytes.toLong(CellUtil.cloneValue(cell))
- }.getOrElse(-1l)
- eq -> result
- }
- case ex: Throwable =>
- log.error(s"${ex.getMessage}: $exactKey")
- Nil
- case _ =>
- log.error(s"result is null: $exactKey")
- Nil
- }
- (exactKey, eqWithResult.toMap)
- }
- }.toMap
-
- override def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
- withHBase(getTableName(policy)) { table =>
- table.delete {
- for {
- key <- keys
- } yield {
- new Delete(BytesUtilV1.toBytes(key))
- }
- }
- } match {
- case Failure(ex) =>
- log.error(ex.getMessage)
- case _ =>
- }
- }
-
- override def get(policy: Counter,
- queries: Seq[(ExactKeyTrait, Seq[ExactQualifier])])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
- lazy val messageForLog = s"${policy.service}.${policy.action} $queries"
-
- val gets = {
- for {
- (key, eqs) <- queries
- } yield {
- // println(s"$key $eqsGrouped")
- val get = new Get(BytesUtilV1.toBytes(key))
-
- for {
- eq <- eqs
- } {
- val cf = intervalsMap(eq.tq.q)
- get.addColumn(cf.toString.getBytes, BytesUtilV1.toBytes(eq))
- }
- get
- }
- }
-
- Future {
- withHBase(getTableName(policy)) { table =>
- for {
- (rst, key) <- table.get(gets).zip(queries.map(_._1)) if !rst.isEmpty
- } yield {
- val qualifierWithCounts = {
- for {
- cell <- rst.listCells()
- eq = BytesUtilV1.toExactQualifier(CellUtil.cloneQualifier(cell))
- } yield {
- eq -> Bytes.toLong(CellUtil.cloneValue(cell))
- }
- }.toMap
- FetchedCounts(key, qualifierWithCounts)
- }
- } match {
- case Success(rst) => rst.toSeq
- case Failure(ex) =>
- log.error(s"$ex: $messageForLog")
- Nil
- }
- }
- }
-
- override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
- val results: Array[Object] = Array.fill(keys.size)(null)
-
- val puts = keys.map { key =>
- val put = new Put(BytesUtilV1.toBytes(key))
- put.addColumn(blobCF, blobColumn, key.itemId.getBytes)
- }
-
- withHBase(getTableName(policy)) { table =>
- table.batch(puts, results)
- } match {
- case Failure(ex) =>
- log.error(s"${ex.getMessage}")
- case _ =>
- }
-
- for {
- (result, key) <- results.zip(keys)
- } yield {
- Option(result).map(_ => true).getOrElse {
- log.error(s"fail to insert blob value: $key")
- false
- }
- }
- }
-
- override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
- lazy val messageForLog = s"${policy.service}.${policy.action}.$blobId"
-
- policy.itemType match {
- case ItemType.BLOB =>
- withHBase(getTableName(policy)) { table =>
- val rowKey = BytesUtilV1.toBytes(ExactKey(policy.id, policy.version, policy.itemType, blobId))
- val get = new Get(rowKey)
- get.addColumn(blobCF, blobColumn)
- table.get(get)
- } match {
- case Success(result) =>
- Option(result).filter(!_.isEmpty).map { rst =>
- Bytes.toString(rst.getValue(blobCF, blobColumn))
- }
- case Failure(ex) =>
- throw ex
- }
- case _ =>
- log.warn(s"is not blob type counter. $messageForLog")
- throw new Exception(s"is not blob type counter. $messageForLog")
- }
- }
-
- override def prepare(policy: Counter): Unit = {
- // create hbase table
- policy.hbaseTable.foreach { table =>
-
- if (!hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)) {
- hbaseManagement.createTable(s2config.HBASE_ZOOKEEPER_QUORUM, table,
- ColumnFamily.values.map(_.toString).toList, 1)
- hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.SHORT.toString, policy.ttl)
- policy.dailyTtl.foreach { i =>
- hbaseManagement.setTTL(s2config.HBASE_ZOOKEEPER_QUORUM, table, ColumnFamily.LONG.toString, i * 24 * 60 * 60)
- }
- }
- }
- }
-
- override def destroy(policy: Counter): Unit = {
-
- }
-
- override def ready(policy: Counter): Boolean = {
- policy.hbaseTable.map { table =>
- hbaseManagement.tableExists(s2config.HBASE_ZOOKEEPER_QUORUM, table)
- }.getOrElse(true)
- }
-}
-
-object ExactStorageHBase {
- import TimedQualifier.IntervalUnit._
-
- object ColumnFamily extends Enumeration {
- type ColumnFamily = Value
-
- val SHORT = Value("s")
- val LONG = Value("l")
- }
-
- val blobCF = ColumnFamily.LONG.toString.getBytes
- val blobColumn = "b".getBytes
-
- val intervalsMap = Map(
- MINUTELY -> ColumnFamily.SHORT,
- HOURLY -> ColumnFamily.SHORT,
- DAILY -> ColumnFamily.LONG,
- MONTHLY -> ColumnFamily.LONG,
- TOTAL -> ColumnFamily.LONG
- )
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v1/RankingStorageRedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v1/RankingStorageRedis.scala b/s2counter_core/src/main/scala/s2/counter/core/v1/RankingStorageRedis.scala
deleted file mode 100644
index ea15a9c..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v1/RankingStorageRedis.scala
+++ /dev/null
@@ -1,189 +0,0 @@
-package s2.counter.core.v1
-
-import java.lang
-
-import com.typesafe.config.Config
-import org.slf4j.LoggerFactory
-import redis.clients.jedis.Pipeline
-import s2.counter.core.RankingCounter.RankingValueMap
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core.{RankingKey, RankingResult, RankingStorage}
-import s2.helper.WithRedis
-import s2.models.{Counter, CounterModel}
-
-import scala.collection.JavaConversions._
-import scala.util.{Failure, Success}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 22..
- */
-class RankingStorageRedis(config: Config) extends RankingStorage {
- private[counter] val log = LoggerFactory.getLogger(this.getClass)
- private[counter] val withRedis = new WithRedis(config)
-
- val counterModel = new CounterModel(config)
-
- val TOTAL = "_total_"
-
- /**
- * ex1)
- * dimension = "age.32"
- * ex2)
- * dimension = "age.gender.32.m"
- *
- */
- private def makeBucket(rankingKey: RankingKey): String = {
- val policyId = rankingKey.policyId
- val q = rankingKey.eq.tq.q
- val ts = rankingKey.eq.tq.ts
- val dimension = rankingKey.eq.dimension
- if (dimension.nonEmpty) {
- s"$policyId.$q.$ts.$dimension"
- }
- else {
- s"$policyId.$q.$ts"
- }
- }
-
- override def getTopK(rankingKey: RankingKey, k: Int): Option[RankingResult] = {
- val bucket = makeBucket(rankingKey)
- withRedis.doBlockWithKey(bucket) { jedis =>
- jedis.zrevrangeByScoreWithScores(bucket, "+inf", "-inf", 0, k + 1).toSeq.map(t => (t.getElement, t.getScore))
- } match {
- case Success(values) =>
- if (values.nonEmpty) {
-// println(values)
- Some(RankingResult(values.find(_._1 == TOTAL).map(_._2).getOrElse(-1d), values.filter(_._1 != TOTAL).take(k)))
- }
- else {
- None
- }
- case Failure(ex) =>
- log.error(s"fail to get top k($ex). $rankingKey")
- None
- }
- }
-
- private def getTTL(policyId: Int, intervalUnit: IntervalUnit.IntervalUnit): Option[Int] = {
- counterModel.findById(policyId).flatMap { policy =>
- intervalUnit match {
- case IntervalUnit.MINUTELY => Some(policy.ttl)
- case IntervalUnit.HOURLY => Some(policy.ttl)
- // default daily ttl 31 day
- case IntervalUnit.DAILY => Some(policy.dailyTtl.getOrElse(31) * 24 * 3600)
- case IntervalUnit.MONTHLY => policy.dailyTtl
- case IntervalUnit.TOTAL => policy.dailyTtl
- }
- }
- }
-
- override def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
- // update ranking by score
- val bucket = makeBucket(key)
- withRedis.doBlockWithKey(bucket) { jedis =>
- val pipeline = jedis.pipelined()
- updateItem(pipeline, bucket, key, value, k)
- pipeline.sync()
- } match {
- case Failure(ex) =>
- log.error(s"fail to update $key $value: $ex")
- case _ =>
- }
- }
-
- private def updateItem(pipeline: Pipeline, bucket: String, key: RankingKey, value: RankingValueMap, k: Int): Unit = {
- val topSeq = value.map { case (item, rv) =>
- // jedis client accept only java's double
- item -> rv.score.asInstanceOf[lang.Double]
- }.toSeq.sortBy(_._2).takeRight(k)
- pipeline.zadd(bucket, topSeq.toMap[String, lang.Double])
- pipeline.zincrby(bucket, value.mapValues(_.increment).values.sum, TOTAL)
- pipeline.zremrangeByRank(bucket, 0, -(k + 1))
- // if ttl defined, set expire time to bucket
- getTTL(key.policyId, key.eq.tq.q).foreach { ttl =>
- pipeline.expire(bucket, ttl)
- }
- }
-
- override def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
- values.map { case (key, value) =>
- (makeBucket(key), key, value)
- }.groupBy { case (bucket, key, value) =>
- withRedis.getBucketIdx(bucket)
- }.foreach { case (idx, seq) =>
- withRedis.doBlockWithIndex(idx) { jedis =>
- val pipeline = jedis.pipelined()
- for {
- (bucket, key, value) <- seq
- } {
- updateItem(pipeline, bucket, key, value, k)
- }
- pipeline.sync()
- } match {
- case Failure(ex) =>
- log.error(s"fail to update multi $idx: $ex")
- case _ =>
- }
- }
- }
-
- override def delete(key: RankingKey): Unit = {
- val bucket = makeBucket(key)
- withRedis.doBlockWithKey(bucket) { jedis =>
- jedis.del(bucket)
- } match {
- case Success(deleted) =>
- log.info(s"success to delete $key")
- case Failure(ex) =>
- log.error(s"fail to delete $key")
- }
- }
-
- override def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)] = {
- keys.map { key =>
- (makeBucket(key), key)
- }.groupBy { case (bucket, key) =>
- withRedis.getBucketIdx(bucket)
- }.toSeq.par.flatMap { case (idx, seq) =>
- withRedis.doBlockWithIndex(idx) { jedis =>
- val pipeline = jedis.pipelined()
- val keyWithRespLs = {
- for {
- (bucket, rankingKey) <- seq
- } yield {
- (rankingKey, pipeline.zrevrangeByScoreWithScores(bucket, "+inf", "-inf", 0, k + 1))
- }
- }
- pipeline.sync()
- for {
- (rankingKey, resp) <- keyWithRespLs
- } yield {
- (rankingKey, resp.get().toSeq.map { t => (t.getElement, t.getScore)})
- }
- } match {
- case Success(keyWithValues) =>
- for {
- (rankingKey, values) <- keyWithValues
- } yield {
- val result = RankingResult(values.find(_._1 == TOTAL).map(_._2).getOrElse(-1d), values.filter(_._1 != TOTAL).take(k))
- (rankingKey, result)
- }
- case Failure(ex) =>
- Nil
- }
- }
- }.seq
-
- override def prepare(policy: Counter): Unit = {
- // do nothing
- }
-
- override def destroy(policy: Counter): Unit = {
-
- }
-
- override def ready(policy: Counter): Boolean = {
- // always return true
- true
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v2/BytesUtilV2.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/BytesUtilV2.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/BytesUtilV2.scala
deleted file mode 100644
index f839221..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/BytesUtilV2.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-package s2.counter.core.v2
-
-import org.apache.hadoop.hbase.util._
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.models.Counter.ItemType
-import s2.util.Hashes
-
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 11..
- */
-object BytesUtilV2 extends BytesUtil {
- // ExactKey: [hash(1b)][version(1b)][policy(4b)][item(variable)]
- val BUCKET_BYTE_SIZE = Bytes.SIZEOF_BYTE
- val VERSION_BYTE_SIZE = Bytes.SIZEOF_BYTE
- val POLICY_ID_SIZE = Bytes.SIZEOF_INT
-
- val INTERVAL_SIZE = Bytes.SIZEOF_BYTE
- val TIMESTAMP_SIZE = Bytes.SIZEOF_LONG
- val TIMED_QUALIFIER_SIZE = INTERVAL_SIZE + TIMESTAMP_SIZE
-
- override def getRowKeyPrefix(id: Int): Array[Byte] = {
- Array(s2.counter.VERSION_2) ++ Bytes.toBytes(id)
- }
-
- override def toBytes(key: ExactKeyTrait): Array[Byte] = {
- val buff = new ArrayBuffer[Byte]
- // hash byte
- buff ++= Bytes.toBytes(Hashes.murmur3(key.itemKey)).take(BUCKET_BYTE_SIZE)
-
- // row key prefix
- // version + policy id
- buff ++= getRowKeyPrefix(key.policyId)
-
- buff ++= {
- key.itemType match {
- case ItemType.INT => Bytes.toBytes(key.itemKey.toInt)
- case ItemType.LONG => Bytes.toBytes(key.itemKey.toLong)
- case ItemType.STRING | ItemType.BLOB => Bytes.toBytes(key.itemKey)
- }
- }
- buff.toArray
- }
-
- override def toBytes(eq: ExactQualifier): Array[Byte] = {
- val len = eq.dimKeyValues.map { case (k, v) => k.length + 2 + v.length + 2 }.sum
- val pbr = new SimplePositionedMutableByteRange(len)
- for {
- v <- ExactQualifier.makeSortedDimension(eq.dimKeyValues)
- } {
- OrderedBytes.encodeString(pbr, v, Order.ASCENDING)
- }
- toBytes(eq.tq) ++ pbr.getBytes
- }
-
- override def toBytes(tq: TimedQualifier): Array[Byte] = {
- val pbr = new SimplePositionedMutableByteRange(INTERVAL_SIZE + 2 + TIMESTAMP_SIZE + 1)
- OrderedBytes.encodeString(pbr, tq.q.toString, Order.ASCENDING)
- OrderedBytes.encodeInt64(pbr, tq.ts, Order.DESCENDING)
- pbr.getBytes
- }
-
- private def decodeString(pbr: PositionedByteRange): Stream[String] = {
- if (pbr.getRemaining > 0) {
- Stream.cons(OrderedBytes.decodeString(pbr), decodeString(pbr))
- }
- else {
- Stream.empty
- }
- }
-
- override def toExactQualifier(bytes: Array[Byte]): ExactQualifier = {
- val pbr = new SimplePositionedByteRange(bytes)
- ExactQualifier(toTimedQualifier(pbr), {
- val seqStr = decodeString(pbr).toSeq
- val (keys, values) = seqStr.splitAt(seqStr.length / 2)
- keys.zip(values).toMap
- })
- }
-
- override def toTimedQualifier(bytes: Array[Byte]): TimedQualifier = {
- val pbr = new SimplePositionedByteRange(bytes)
- toTimedQualifier(pbr)
- }
-
- def toTimedQualifier(pbr: PositionedByteRange): TimedQualifier = {
- TimedQualifier(IntervalUnit.withName(OrderedBytes.decodeString(pbr)), OrderedBytes.decodeInt64(pbr))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
deleted file mode 100644
index 2b225c8..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
+++ /dev/null
@@ -1,346 +0,0 @@
-package s2.counter.core.v2
-
-import com.kakao.s2graph.core.mysqls.Label
-import com.typesafe.config.Config
-import org.apache.http.HttpStatus
-import org.slf4j.LoggerFactory
-import play.api.libs.json._
-import s2.config.S2CounterConfig
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.counter.core._
-import s2.models.Counter
-import s2.util.CartesianProduct
-
-import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 8. 19..
- */
-object ExactStorageGraph {
- case class RespGraph(success: Boolean, result: Long)
- implicit val respGraphFormat = Json.format[RespGraph]
-
- // using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
- private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
-}
-
-case class ExactStorageGraph(config: Config) extends ExactStorage {
- private val log = LoggerFactory.getLogger(this.getClass)
- private val s2config = new S2CounterConfig(config)
-
- private val SERVICE_NAME = "s2counter"
- private val COLUMN_NAME = "bucket"
- private val labelPostfix = "_counts"
-
- val s2graphUrl = s2config.GRAPH_URL
- val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
- val graphOp = new GraphOperation(config)
-
- import ExactStorageGraph._
-
- override def update(policy: Counter, counts: Seq[(ExactKeyTrait, ExactValueMap)]): Map[ExactKeyTrait, ExactValueMap] = {
- import scala.concurrent.ExecutionContext.Implicits.global
-
- val (keyWithEq, reqJsLs) = toIncrementCountRequests(policy, counts).unzip(x => ((x._1, x._2), x._3))
-
- val future = wsClient.url(s"$s2graphUrl/graphs/edges/incrementCount").post(Json.toJson(reqJsLs)).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val respSeq = resp.json.as[Seq[RespGraph]]
-
- val keyWithEqResult = {
- for {
- ((key, eq), RespGraph(success, result)) <- keyWithEq.zip(respSeq)
- } yield {
- (key, (eq, result))
- }
- }.groupBy(_._1).mapValues{ seq => seq.map(_._2).toMap }
- keyWithEqResult
- case _ =>
- throw new RuntimeException(s"update failed: $policy $counts")
- }
- }
- Await.result(future, 10 second)
- }
-
- def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
-
- }
-
- private def toIncrementCountRequests(policy: Counter,
- counts: Seq[(ExactKeyTrait, ExactValueMap)])
- : Seq[(ExactKeyTrait, ExactQualifier, JsValue)] = {
- val labelName = policy.action + labelPostfix
- val timestamp = System.currentTimeMillis()
- for {
- (exactKey, values) <- counts
- (eq, value) <- values
- } yield {
- val from = exactKey.itemKey
- val to = eq.dimension
- val json = Json.obj(
- "timestamp" -> timestamp,
- "operation" -> "incrementCount",
- "from" -> from,
- "to" -> to,
- "label" -> labelName,
- "props" -> Json.obj(
- "_count" -> value,
- "time_unit" -> eq.tq.q.toString,
- "time_value" -> eq.tq.ts
- )
- )
- (exactKey, eq, json)
- }
- }
-
- override def get(policy: Counter,
- items: Seq[String],
- timeRange: Seq[(TimedQualifier, TimedQualifier)],
- dimQuery: Map[String, Set[String]])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCountsGrouped]] = {
- val labelName = policy.action + labelPostfix
- val label = Label.findByName(labelName).get
-// val label = labelModel.findByName(labelName).get
-
- val ids = Json.toJson(items)
-
- val dimensions = {
- for {
- values <- CartesianProduct(dimQuery.values.map(ss => ss.toList).toList)
- } yield {
- dimQuery.keys.zip(values).toMap
- }
- }
-
- val stepJsLs = {
- for {
- (tqFrom, tqTo) <- timeRange
- dimension <- dimensions
- } yield {
- val eqFrom = ExactQualifier(tqFrom, dimension)
- val eqTo = ExactQualifier(tqTo, dimension)
- val intervalJs =
- s"""
- |{
- | "from": {
- | "_to": "${eqFrom.dimension}",
- | "time_unit": "${eqFrom.tq.q}",
- | "time_value": ${eqFrom.tq.ts}
- | },
- | "to": {
- | "_to": "${eqTo.dimension}",
- | "time_unit": "${eqTo.tq.q}",
- | "time_value": ${eqTo.tq.ts + 1}
- | }
- |}
- """.stripMargin
- val stepJs =
- s"""
- |{
- | "direction": "out",
- | "limit": -1,
- | "duplicate": "raw",
- | "label": "$labelName",
- | "interval": $intervalJs
- |}
- """.stripMargin
- stepJs
- }
- }
-
- val reqJsStr =
- s"""
- |{
- | "srcVertices": [
- | {"serviceName": "${policy.service}", "columnName": "${label.srcColumnName}", "ids": $ids}
- | ],
- | "steps": [
- | {
- | "step": [
- | ${stepJsLs.mkString(",")}
- | ]
- | }
- | ]
- |}
- """.stripMargin
-
- val reqJs = Json.parse(reqJsStr)
-// log.warn(s"query: ${reqJs.toString()}")
-
- wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(reqJs).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val respJs = resp.json
-// println(respJs)
- val keyWithValues = (respJs \ "results").as[Seq[JsValue]].map { result =>
-// println(s"result: $result")
- resultToExactKeyValues(policy, result)
- }.groupBy(_._1).mapValues(seq => seq.map(_._2).toMap.groupBy { case (eq, v) => (eq.tq.q, eq.dimKeyValues) })
- for {
- (k, v) <- keyWithValues.toSeq
- } yield {
- FetchedCountsGrouped(k, v)
- }
- case n: Int =>
- log.warn(s"getEdges status($n): $reqJsStr")
-// println(s"getEdges status($n): $reqJsStr")
- Nil
- }
- }
- }
-
- private def resultToExactKeyValues(policy: Counter, result: JsValue): (ExactKeyTrait, (ExactQualifier, Long)) = {
- val from = result \ "from" match {
- case s: JsString => s.as[String]
- case n: JsNumber => n.as[Long].toString
- case x: JsValue => throw new RuntimeException(s"$x's type must be string or number")
- }
- val dimension = (result \ "to").as[String]
- val props = result \ "props"
- val count = (props \ "_count").as[Long]
- val timeUnit = (props \ "time_unit").as[String]
- val timeValue = (props \ "time_value").as[Long]
- (ExactKey(policy, from, checkItemType = true), (ExactQualifier(TimedQualifier(timeUnit, timeValue), dimension), count))
- }
-
- private def getInner(policy: Counter, key: ExactKeyTrait, eqs: Seq[ExactQualifier])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
- val labelName = policy.action + labelPostfix
-
- Label.findByName(labelName) match {
- case Some(label) =>
-
- val src = Json.obj("serviceName" -> policy.service, "columnName" -> label.srcColumnName, "id" -> key.itemKey)
- val step = {
- val stepLs = {
- for {
- eq <- eqs
- } yield {
- val from = Json.obj("_to" -> eq.dimension, "time_unit" -> eq.tq.q.toString, "time_value" -> eq.tq.ts)
- val to = Json.obj("_to" -> eq.dimension, "time_unit" -> eq.tq.q.toString, "time_value" -> eq.tq.ts)
- val interval = Json.obj("from" -> from, "to" -> to)
- Json.obj("limit" -> 1, "label" -> labelName, "interval" -> interval)
- }
- }
- Json.obj("step" -> stepLs)
- }
- val query = Json.obj("srcVertices" -> Json.arr(src), "steps" -> Json.arr(step))
- // println(s"query: ${query.toString()}")
-
- wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(query).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val respJs = resp.json
- val keyWithValues = (respJs \ "results").as[Seq[JsValue]].map { result =>
- resultToExactKeyValues(policy, result)
- }.groupBy(_._1).mapValues(seq => seq.map(_._2).toMap)
- for {
- (key, eqWithValues) <- keyWithValues.toSeq
- } yield {
- FetchedCounts(key, eqWithValues)
- }
- case _ =>
- Nil
- }
- }
- case None =>
- Future.successful(Nil)
- }
- }
-
- // for query exact qualifier
- override def get(policy: Counter, queries: Seq[(ExactKeyTrait, Seq[ExactQualifier])])
- (implicit ex: ExecutionContext): Future[Seq[FetchedCounts]] = {
- val futures = {
- for {
- (key, eqs) <- queries
- } yield {
-// println(s"$key $eqs")
- getInner(policy, key, eqs)
- }
- }
- Future.sequence(futures).map(_.flatten)
- }
-
- override def getBlobValue(policy: Counter, blobId: String): Option[String] = {
- throw new RuntimeException("unsupported getBlobValue operation")
- }
-
- override def insertBlobValue(policy: Counter, keys: Seq[BlobExactKey]): Seq[Boolean] = {
- throw new RuntimeException("unsupported insertBlobValue operation")
- }
-
- private def existsLabel(policy: Counter, useCache: Boolean = true): Boolean = {
- val action = policy.action
- val counterLabelName = action + labelPostfix
-
- Label.findByName(counterLabelName, useCache).nonEmpty
- }
-
- override def prepare(policy: Counter): Unit = {
- val service = policy.service
- val action = policy.action
-
- val graphLabel = Label.findByName(action)
- if (graphLabel.isEmpty) {
- throw new Exception(s"label not found. $service.$action")
- }
-
- if (!existsLabel(policy, useCache = false)) {
- val label = Label.findByName(action, useCache = false).get
-
- val counterLabelName = action + labelPostfix
- val defaultJson =
- s"""
- |{
- | "label": "$counterLabelName",
- | "srcServiceName": "$service",
- | "srcColumnName": "${label.tgtColumnName}",
- | "srcColumnType": "${label.tgtColumnType}",
- | "tgtServiceName": "$SERVICE_NAME",
- | "tgtColumnName": "$COLUMN_NAME",
- | "tgtColumnType": "string",
- | "indices": [
- | {"name": "time", "propNames": ["_to", "time_unit", "time_value"]}
- | ],
- | "props": [
- | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
- | {"name": "time_value", "dataType": "long", "defaultValue": 0}
- | ],
- | "hTableName": "${policy.hbaseTable.get}"
- |}
- """.stripMargin
- val json = policy.dailyTtl.map(ttl => ttl * 24 * 60 * 60) match {
- case Some(ttl) =>
- Json.parse(defaultJson).as[JsObject] + ("hTableTTL" -> Json.toJson(ttl))
- case None =>
- Json.parse(defaultJson)
- }
-
- graphOp.createLabel(json)
- }
- }
-
- override def destroy(policy: Counter): Unit = {
- val action = policy.action
-
- if (existsLabel(policy, useCache = false)) {
- val counterLabelName = action + labelPostfix
-
- graphOp.deleteLabel(counterLabelName)
- }
- }
-
- override def ready(policy: Counter): Boolean = {
- existsLabel(policy)
- }
-
- // for range query
- override def get(policy: Counter, items: Seq[String], timeRange: Seq[(TimedQualifier, TimedQualifier)])
- (implicit ec: ExecutionContext): Future[Seq[FetchedCounts]] = {
- throw new NotImplementedError("Not implemented")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v2/GraphOperation.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/GraphOperation.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/GraphOperation.scala
deleted file mode 100644
index 0f43a52..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/GraphOperation.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package s2.counter.core.v2
-
-import com.typesafe.config.Config
-import org.apache.http.HttpStatus
-import org.slf4j.LoggerFactory
-import play.api.libs.json.{JsObject, JsValue, Json}
-import s2.config.S2CounterConfig
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 10..
- */
-class GraphOperation(config: Config) {
- // using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
- private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
- private val s2config = new S2CounterConfig(config)
- val s2graphUrl = s2config.GRAPH_URL
- private[counter] val log = LoggerFactory.getLogger(this.getClass)
-
- import scala.concurrent.ExecutionContext.Implicits.global
-
- def createLabel(json: JsValue): Boolean = {
- // fix counter label's schemaVersion
- val newJson = json.as[JsObject] ++ Json.obj("schemaVersion" -> "v2")
- val future = wsClient.url(s"$s2graphUrl/graphs/createLabel").post(newJson).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- true
- case _ =>
- throw new RuntimeException(s"failed createLabel. errCode: ${resp.status} body: ${resp.body} query: $json")
- }
- }
-
- Await.result(future, 10 second)
- }
-
- def deleteLabel(label: String): Boolean = {
- val future = wsClient.url(s"$s2graphUrl/graphs/deleteLabel/$label").put("").map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- true
- case _ =>
- throw new RuntimeException(s"failed deleteLabel. errCode: ${resp.status} body: ${resp.body}")
- }
- }
-
- Await.result(future, 10 second)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
deleted file mode 100644
index 18d7eda..0000000
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
+++ /dev/null
@@ -1,399 +0,0 @@
-package s2.counter.core.v2
-
-import com.kakao.s2graph.core.GraphUtil
-import com.kakao.s2graph.core.mysqls.Label
-import com.typesafe.config.Config
-import org.apache.commons.httpclient.HttpStatus
-import org.slf4j.LoggerFactory
-import play.api.libs.json.{JsObject, JsString, JsValue, Json}
-import s2.config.S2CounterConfig
-import s2.counter.core.RankingCounter.RankingValueMap
-import s2.counter.core.{RankingKey, RankingResult, RankingStorage}
-import s2.models.{Counter, CounterModel}
-import s2.util.{CollectionCache, CollectionCacheConfig}
-
-import scala.concurrent.duration._
-import scala.concurrent.{Await, Future}
-import scala.util.hashing.MurmurHash3
-
-/**
- * Created by shon on 7/28/15.
- */
-object RankingStorageGraph {
- // using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
- private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
-}
-
-class RankingStorageGraph(config: Config) extends RankingStorage {
- import RankingStorageGraph._
-
- private[counter] val log = LoggerFactory.getLogger(this.getClass)
- private val s2config = new S2CounterConfig(config)
-
- private val BUCKET_SHARD_COUNT = 53
- private val SERVICE_NAME = "s2counter"
- private val BUCKET_COLUMN_NAME = "bucket"
- private val counterModel = new CounterModel(config)
- private val labelPostfix = "_topK"
-
- val s2graphUrl = s2config.GRAPH_URL
- val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
-
- val prepareCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(10000, 600))
- val graphOp = new GraphOperation(config)
- import scala.concurrent.ExecutionContext.Implicits.global
-
- private def makeBucketKey(rankingKey: RankingKey): String = {
- val eq = rankingKey.eq
- val tq = eq.tq
- s"${tq.q}.${tq.ts}.${eq.dimension}"
- }
-
- // "", "age.32", "age.gender.32.M"
- private def makeBucketShardKey(shardIdx: Int, rankingKey: RankingKey): String = {
- s"$shardIdx.${makeBucketKey(rankingKey)}"
- }
-
- /**
- * indexProps: ["time_unit", "time_value", "score"]
- */
- override def getTopK(key: RankingKey, k: Int): Option[RankingResult] = {
- getTopK(Seq(key), k).headOption.map(_._2)
- }
-
- override def getTopK(keys: Seq[RankingKey], k: Int): Seq[(RankingKey, RankingResult)] = {
- val futures = for {
- key <- keys
- } yield {
- getEdges(key).map { edges =>
- key -> RankingResult(0d, toWithScoreLs(edges).take(k))
- }
- }
-
- Await.result(Future.sequence(futures), 10 seconds)
- }
-
- override def update(key: RankingKey, value: RankingValueMap, k: Int): Unit = {
- update(Seq((key, value)), k)
- }
-
- override def update(values: Seq[(RankingKey, RankingValueMap)], k: Int): Unit = {
- val futures = {
- for {
- (key, value) <- values
- } yield {
- // prepare dimension bucket edge
- checkAndPrepareDimensionBucket(key)
-
- val future = getEdges(key, "raw").flatMap { edges =>
- val prevRankingSeq = toWithScoreLs(edges)
- val prevRankingMap: Map[String, Double] = prevRankingSeq.groupBy(_._1).map(_._2.sortBy(-_._2).head)
- val currentRankingMap: Map[String, Double] = value.mapValues(_.score)
- val mergedRankingSeq = (prevRankingMap ++ currentRankingMap).toSeq.sortBy(-_._2).take(k)
- val mergedRankingMap = mergedRankingSeq.toMap
-
- val bucketRankingSeq = mergedRankingSeq.groupBy { case (itemId, score) =>
- // 0-index
- GraphUtil.transformHash(MurmurHash3.stringHash(itemId)) % BUCKET_SHARD_COUNT
- }.map { case (shardIdx, groupedRanking) =>
- shardIdx -> groupedRanking.filter { case (itemId, _) => currentRankingMap.contains(itemId) }
- }.toSeq
-
- insertBulk(key, bucketRankingSeq).flatMap { _ =>
- val duplicatedItems = prevRankingMap.filterKeys(s => currentRankingMap.contains(s))
- val cutoffItems = prevRankingMap.filterKeys(s => !mergedRankingMap.contains(s))
- val deleteItems = duplicatedItems ++ cutoffItems
-
- val keyWithEdgesLs = prevRankingSeq.map(_._1).zip(edges)
- val deleteEdges = keyWithEdgesLs.filter{ case (s, _) => deleteItems.contains(s) }.map(_._2)
-
- deleteAll(deleteEdges)
- }
- }
-
- future
- }
- }
-
- Await.result(Future.sequence(futures), 10 seconds)
- }
-
- private def toWithScoreLs(edges: List[JsValue]): List[(String, Double)] = {
- for {
- edgeJson <- edges
- to = (edgeJson \ "to").as[JsValue]
- score = (edgeJson \ "score").as[JsValue].toString().toDouble
- } yield {
- val toValue = to match {
- case s: JsString => s.as[String]
- case _ => to.toString()
- }
- toValue -> score
- }
- }
-
- private def insertBulk(key: RankingKey, newRankingSeq: Seq[(Int, Seq[(String, Double)])]): Future[Boolean] = {
- val labelName = counterModel.findById(key.policyId).get.action + labelPostfix
- val timestamp: Long = System.currentTimeMillis
- val payload = Json.toJson {
- for {
- (shardIdx, rankingSeq) <- newRankingSeq
- (itemId, score) <- rankingSeq
- } yield {
- val srcId = makeBucketShardKey(shardIdx, key)
- Json.obj(
- "timestamp" -> timestamp,
- "from" -> srcId,
- "to" -> itemId,
- "label" -> labelName,
- "props" -> Json.obj(
- "time_unit" -> key.eq.tq.q.toString,
- "time_value" -> key.eq.tq.ts,
- "date_time" -> key.eq.tq.dateTime,
- "score" -> score
- )
- )
- }
- }
-
- wsClient.url(s"$s2graphUrl/graphs/edges/insertBulk").post(payload).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- true
- case _ =>
- throw new RuntimeException(s"failed insertBulk. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
- }
- }
- }
-
- private def deleteAll(edges: List[JsValue]): Future[Boolean] = {
- // /graphs/edges/delete
- val futures = {
- for {
- groupedEdges <- edges.grouped(50)
- } yield {
- val payload = Json.toJson(groupedEdges)
- wsClient.url(s"$s2graphUrl/graphs/edges/delete").post(payload).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- true
- case _ =>
- log.error(s"failed delete. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
- false
- }
- }
- }
- }.toSeq
-
- Future.sequence(futures).map { seq =>
- seq.forall(x => x)
- }
- }
-
- /** select and delete */
- override def delete(key: RankingKey): Unit = {
- val future = getEdges(key).flatMap { edges =>
- deleteAll(edges)
- }
- Await.result(future, 10 second)
- }
-
- private def getEdges(key: RankingKey, duplicate: String="first"): Future[List[JsValue]] = {
- val labelName = counterModel.findById(key.policyId).get.action + labelPostfix
-
- val ids = {
- (0 until BUCKET_SHARD_COUNT).map { shardIdx =>
- s""""${makeBucketShardKey(shardIdx, key)}""""
- }
- }.mkString(",")
-
- val strJs =
- s"""
- |{
- | "srcVertices": [
- | {
- | "serviceName": "$SERVICE_NAME",
- | "columnName": "$BUCKET_COLUMN_NAME",
- | "ids": [$ids]
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "label": "$labelName",
- | "duplicate": "$duplicate",
- | "direction": "out",
- | "offset": 0,
- | "limit": -1,
- | "interval": {
- | "from": {"time_unit": "${key.eq.tq.q.toString}", "time_value": ${key.eq.tq.ts}},
- | "to": {"time_unit": "${key.eq.tq.q.toString}", "time_value": ${key.eq.tq.ts}}
- | },
- | "scoring": {"score": 1}
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
- log.debug(strJs)
-
- val payload = Json.parse(strJs)
- wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- (resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
- case _ =>
- throw new RuntimeException(s"failed getEdges. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
- }
- }
- }
-
- private def existsLabel(policy: Counter, useCache: Boolean = true): Boolean = {
- val action = policy.action
- val counterLabelName = action + labelPostfix
-
- Label.findByName(counterLabelName, useCache).nonEmpty
- }
-
- private def checkAndPrepareDimensionBucket(rankingKey: RankingKey): Boolean = {
- val dimension = rankingKey.eq.dimension
- val bucketKey = makeBucketKey(rankingKey)
- val labelName = "s2counter_topK_bucket"
-
- val prepared = prepareCache.withCache(s"$dimension:$bucketKey") {
- val checkReqJs = Json.arr(
- Json.obj(
- "label" -> labelName,
- "direction" -> "out",
- "from" -> dimension,
- "to" -> makeBucketShardKey(BUCKET_SHARD_COUNT - 1, rankingKey)
- )
- )
-
- val future = wsClient.url(s"$s2graphReadOnlyUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val checkRespJs = resp.json
- if (checkRespJs.as[Seq[JsValue]].nonEmpty) {
- true
- } else {
- false
- }
- case _ =>
- // throw exception
- throw new Exception(s"failed checkEdges. ${resp.body} ${resp.status}")
- }
- }.flatMap {
- case true => Future.successful(Some(true))
- case false =>
- val insertReqJsLs = {
- for {
- i <- 0 until BUCKET_SHARD_COUNT
- } yield {
- Json.obj(
- "timestamp" -> rankingKey.eq.tq.ts,
- "from" -> dimension,
- "to" -> makeBucketShardKey(i, rankingKey),
- "label" -> labelName,
- "props" -> Json.obj(
- "time_unit" -> rankingKey.eq.tq.q.toString,
- "date_time" -> rankingKey.eq.tq.dateTime
- )
- )
- }
- }
- wsClient.url(s"$s2graphUrl/graphs/edges/insert").post(Json.toJson(insertReqJsLs)).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- Some(true)
- case _ =>
- // throw exception
- throw new Exception(s"failed insertEdges. ${resp.body} ${resp.status}")
- }
- }
- }.recover {
- case e: Exception =>
- None
- }
-
- Await.result(future, 10 second)
- }
- prepared.getOrElse(false)
- }
-
- override def prepare(policy: Counter): Unit = {
- val service = policy.service
- val action = policy.action
-
- val graphLabel = {
- policy.rateActionId match {
- case Some(rateId) =>
- counterModel.findById(rateId, useCache = false).flatMap { ratePolicy =>
- Label.findByName(ratePolicy.action)
- }
- case None =>
- Label.findByName(action)
- }
- }
- if (graphLabel.isEmpty) {
- throw new Exception(s"label not found. $service.$action")
- }
-
- if (!existsLabel(policy, useCache = false)) {
- // find input label to specify target column
- val inputLabelName = policy.rateActionId.flatMap { id =>
- counterModel.findById(id, useCache = false).map(_.action)
- }.getOrElse(action)
- val label = graphLabel.get
-
- val counterLabelName = action + labelPostfix
- val defaultJson =
- s"""
- |{
- | "label": "$counterLabelName",
- | "srcServiceName": "$SERVICE_NAME",
- | "srcColumnName": "$BUCKET_COLUMN_NAME",
- | "srcColumnType": "string",
- | "tgtServiceName": "$service",
- | "tgtColumnName": "${label.tgtColumnName}",
- | "tgtColumnType": "${label.tgtColumnType}",
- | "indices": [
- | {"name": "time", "propNames": ["time_unit", "time_value", "score"]}
- | ],
- | "props": [
- | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
- | {"name": "time_value", "dataType": "long", "defaultValue": 0},
- | {"name": "date_time", "dataType": "long", "defaultValue": 0},
- | {"name": "score", "dataType": "float", "defaultValue": 0.0}
- | ],
- | "hTableName": "${policy.hbaseTable.get}"
- |}
- """.stripMargin
- val json = policy.dailyTtl.map(ttl => ttl * 24 * 60 * 60) match {
- case Some(ttl) =>
- Json.parse(defaultJson).as[JsObject] + ("hTableTTL" -> Json.toJson(ttl))
- case None =>
- Json.parse(defaultJson)
- }
-
- graphOp.createLabel(json)
- }
- }
-
- override def destroy(policy: Counter): Unit = {
- val action = policy.action
-
- if (existsLabel(policy, useCache = false)) {
- val counterLabelName = action + labelPostfix
-
- graphOp.deleteLabel(counterLabelName)
- }
- }
-
- override def ready(policy: Counter): Boolean = {
- existsLabel(policy)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/decay/DecayFormula.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/decay/DecayFormula.scala b/s2counter_core/src/main/scala/s2/counter/decay/DecayFormula.scala
deleted file mode 100644
index 66e1b93..0000000
--- a/s2counter_core/src/main/scala/s2/counter/decay/DecayFormula.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-package s2.counter.decay
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 26..
- */
-trait DecayFormula {
- def apply(value: Double, millis: Long): Double
- def apply(value: Double, seconds: Int): Double
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/decay/ExpDecayFormula.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/decay/ExpDecayFormula.scala b/s2counter_core/src/main/scala/s2/counter/decay/ExpDecayFormula.scala
deleted file mode 100644
index 6de9f69..0000000
--- a/s2counter_core/src/main/scala/s2/counter/decay/ExpDecayFormula.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-package s2.counter.decay
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 26..
- */
-case class ExpDecayFormula(halfLifeInMillis: Double) extends DecayFormula {
- val decayRate = - Math.log(2) / halfLifeInMillis
-
- override def apply(value: Double, millis: Long): Double = {
- value * Math.pow(Math.E, decayRate * millis)
- }
-
- override def apply(value: Double, seconds: Int): Double = {
- apply(value, seconds * 1000L)
- }
-}
-
-object ExpDecayFormula {
- @deprecated("do not use. just experimental", "0.14")
- def byWindowTime(windowInMillis: Long, pct: Double): ExpDecayFormula = {
- val halfLife = windowInMillis * Math.log(0.5) / Math.log(pct)
- ExpDecayFormula(halfLife)
- }
-
- def byMeanLifeTime(millis: Long): ExpDecayFormula = {
- ExpDecayFormula(millis * Math.log(2))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/counter/package.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/package.scala b/s2counter_core/src/main/scala/s2/counter/package.scala
deleted file mode 100644
index 40a9e41..0000000
--- a/s2counter_core/src/main/scala/s2/counter/package.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package s2
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 22..
- */
-package object counter {
- val VERSION_1: Byte = 1
- val VERSION_2: Byte = 2
-
- case class MethodNotSupportedException(message: String, cause: Throwable = null) extends Exception(message, cause)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/CounterAdmin.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/CounterAdmin.scala b/s2counter_core/src/main/scala/s2/helper/CounterAdmin.scala
deleted file mode 100644
index 3cf9181..0000000
--- a/s2counter_core/src/main/scala/s2/helper/CounterAdmin.scala
+++ /dev/null
@@ -1,114 +0,0 @@
-package s2.helper
-
-import com.kakao.s2graph.core.Graph
-import com.kakao.s2graph.core.mysqls.Label
-import com.typesafe.config.Config
-import play.api.libs.json.Json
-import s2.config.S2CounterConfig
-import s2.counter.core.v1.{ExactStorageAsyncHBase, RankingStorageRedis}
-import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
-import s2.counter.core.{ExactCounter, RankingCounter}
-import s2.models.{Counter, CounterModel}
-
-import scala.util.Try
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 11..
- */
-class CounterAdmin(config: Config) {
- val s2config = new S2CounterConfig(config)
- val counterModel = new CounterModel(config)
- val graphOp = new GraphOperation(config)
- val s2graph = new Graph(config)(scala.concurrent.ExecutionContext.global)
- val storageManagement = new com.kakao.s2graph.core.Management(s2graph)
-
- def setupCounterOnGraph(): Unit = {
- // create s2counter service
- val service = "s2counter"
- storageManagement.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"$service-${config.getString("phase")}", 1, None, "gz")
- // create bucket label
- val label = "s2counter_topK_bucket"
- if (Label.findByName(label, useCache = false).isEmpty) {
- val strJs =
- s"""
- |{
- | "label": "$label",
- | "srcServiceName": "s2counter",
- | "srcColumnName": "dimension",
- | "srcColumnType": "string",
- | "tgtServiceName": "s2counter",
- | "tgtColumnName": "bucket",
- | "tgtColumnType": "string",
- | "indices": [
- | {"name": "time", "propNames": ["time_unit", "date_time"]}
- | ],
- | "props": [
- | {"name": "time_unit", "dataType": "string", "defaultValue": ""},
- | {"name": "date_time", "dataType": "long", "defaultValue": 0}
- | ],
- | "hTableName": "s2counter_60",
- | "hTableTTL": 5184000
- |}
- """.stripMargin
- graphOp.createLabel(Json.parse(strJs))
- }
- }
-
- def createCounter(policy: Counter): Unit = {
- val newPolicy = policy.copy(hbaseTable = Some(makeHTableName(policy)))
- prepareStorage(newPolicy)
- counterModel.createServiceAction(newPolicy)
- }
-
- def deleteCounter(service: String, action: String): Option[Try[Unit]] = {
- for {
- policy <- counterModel.findByServiceAction(service, action, useCache = false)
- } yield {
- Try {
- exactCounter(policy).destroy(policy)
- if (policy.useRank) {
- rankingCounter(policy).destroy(policy)
- }
- counterModel.deleteServiceAction(policy)
- }
- }
- }
-
- def prepareStorage(policy: Counter): Unit = {
- if (policy.rateActionId.isEmpty) {
- // if defined rate action, do not use exact counter
- exactCounter(policy).prepare(policy)
- }
- if (policy.useRank) {
- rankingCounter(policy).prepare(policy)
- }
- }
-
- def prepareStorage(policy: Counter, version: Byte): Unit = {
- // this function to prepare storage by version parameter instead of policy.version
- prepareStorage(policy.copy(version = version))
- }
-
- private val exactCounterMap = Map(
- s2.counter.VERSION_1 -> new ExactCounter(config, new ExactStorageAsyncHBase(config)),
- s2.counter.VERSION_2 -> new ExactCounter(config, new ExactStorageGraph(config))
- )
- private val rankingCounterMap = Map(
- s2.counter.VERSION_1 -> new RankingCounter(config, new RankingStorageRedis(config)),
- s2.counter.VERSION_2 -> new RankingCounter(config, new RankingStorageGraph(config))
- )
-
- private val tablePrefixMap = Map (
- s2.counter.VERSION_1 -> "s2counter",
- s2.counter.VERSION_2 -> "s2counter_v2"
- )
-
- def exactCounter(version: Byte): ExactCounter = exactCounterMap(version)
- def exactCounter(policy: Counter): ExactCounter = exactCounter(policy.version)
- def rankingCounter(version: Byte): RankingCounter = rankingCounterMap(version)
- def rankingCounter(policy: Counter): RankingCounter = rankingCounter(policy.version)
-
- def makeHTableName(policy: Counter): String = {
- Seq(tablePrefixMap(policy.version), policy.service, policy.ttl) ++ policy.dailyTtl mkString "_"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/DistributedScanner.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/DistributedScanner.scala b/s2counter_core/src/main/scala/s2/helper/DistributedScanner.scala
deleted file mode 100644
index dcf4d03..0000000
--- a/s2counter_core/src/main/scala/s2/helper/DistributedScanner.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-package s2.helper
-
-import java.util
-import java.util.Comparator
-
-import com.google.common.primitives.SignedBytes
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 21..
- */
-
-object DistributedScanner {
- val BUCKET_BYTE_SIZE = Bytes.SIZEOF_BYTE
-
- def getRealRowKey(result: Result): Array[Byte] = {
- result.getRow.drop(BUCKET_BYTE_SIZE)
- }
-}
-
-class DistributedScanner(table: Table, scan: Scan) extends AbstractClientScanner {
- import DistributedScanner._
-
- private val BYTE_MAX = BigInt(256)
-
- private[helper] val scanners = {
- for {
- i <- 0 until BYTE_MAX.pow(BUCKET_BYTE_SIZE).toInt
- } yield {
- val bucketBytes: Array[Byte] = Bytes.toBytes(i).takeRight(BUCKET_BYTE_SIZE)
- val newScan = new Scan(scan).setStartRow(bucketBytes ++ scan.getStartRow).setStopRow(bucketBytes ++ scan.getStopRow)
- table.getScanner(newScan)
- }
- }
-
- val resultCache = new util.TreeMap[Result, java.util.Iterator[Result]](new Comparator[Result] {
- val comparator = SignedBytes.lexicographicalComparator()
- override def compare(o1: Result, o2: Result): Int = {
- comparator.compare(getRealRowKey(o1), getRealRowKey(o2))
- }
- })
-
- lazy val initialized = {
- val iterators = scanners.map(_.iterator()).filter(_.hasNext)
- iterators.foreach { it =>
- resultCache.put(it.next(), it)
- }
- iterators.nonEmpty
- }
-
- override def next(): Result = {
- if (initialized) {
- Option(resultCache.pollFirstEntry()).map { entry =>
- val it = entry.getValue
- if (it.hasNext) {
- // fill cache
- resultCache.put(it.next(), it)
- }
- entry.getKey
- }.orNull
- } else {
- null
- }
- }
-
- override def close(): Unit = {
- for {
- scanner <- scanners
- } {
- scanner.close()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/HashShardingJedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/HashShardingJedis.scala b/s2counter_core/src/main/scala/s2/helper/HashShardingJedis.scala
deleted file mode 100644
index 395486d..0000000
--- a/s2counter_core/src/main/scala/s2/helper/HashShardingJedis.scala
+++ /dev/null
@@ -1,156 +0,0 @@
-package s2.helper
-
-import com.typesafe.config.Config
-import org.slf4j.LoggerFactory
-import redis.clients.jedis.exceptions.JedisException
-import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
-import s2.config.S2CounterConfig
-import s2.util.Hashes
-
-/**
- * Created by jay on 14. 10. 31..
- */
-class HashShardingJedis(config: Config) {
- lazy val s2config = new S2CounterConfig(config)
-
- private val log = LoggerFactory.getLogger(getClass)
-
- val poolConfig = new JedisPoolConfig()
- poolConfig.setMaxTotal(150)
- poolConfig.setMaxIdle(50)
- poolConfig.setMaxWaitMillis(200)
-
- val jedisPools = s2config.REDIS_INSTANCES.map { case (host, port) =>
- new JedisPool(poolConfig, host, port)
- }
- val jedisPoolSize = jedisPools.size
-
- def getJedisPool(idx: Int): JedisPool = {
- if(idx >= jedisPoolSize)
- null
- else
- jedisPools(idx)
- }
-
- def getJedisPoolWithBucketname2(bucket: String): JedisPool = {
- val hashedValue = Hashes.murmur3(bucket)
- val idx = hashedValue % jedisPoolSize
- getJedisPool(idx)
- }
-
- def getJedisPoolWithBucketname(bucket: String): (JedisPool, JedisPool) = {
- val hashedValue = Hashes.murmur3(bucket)
- val idx = hashedValue % jedisPoolSize
- val secondaryIdx = if (jedisPoolSize <= 1) {
- throw new Exception("too small sharding pool <= 1")
- } else {
- val newIdx = (hashedValue / jedisPoolSize) % (jedisPoolSize -1)
- if(newIdx < idx) {
- newIdx
- } else {
- newIdx +1
- }
- }
- (getJedisPool(idx), getJedisPool(secondaryIdx))
- }
-
- def doBlockWithJedisInstace(f : Jedis => Any, fallBack : => Any, jedis : Jedis) = {
- try {
- f(jedis)
- }
- catch {
- case e:JedisException => {
- fallBack
- }
- }
- }
-
- def doBlockWithBucketName(f : Jedis => Any, fallBack : => Any, bucket : String) = {
-// Logger.debug(s"start jedis do block")
- //val (jedis_pool1, jedis_pool2) = getJedisPoolWithBucketname(bucket)
- val jedis_pool1= getJedisPoolWithBucketname2(bucket)
-// if(jedis_pool1 != null && jedis_pool2 != null) {
- if(jedis_pool1 != null) {
- var jedis1: Jedis = null
-// var jedis2: Jedis = null
- try {
- jedis1 = jedis_pool1.getResource()
-// jedis2 = jedis_pool2.getResource()
- log.info(s">> Jedis Pool Active Num : ${jedis_pool1.getNumActive}")
-
- /* val f1 = Future(f(jedis1))
- val f2 = Future(f(jedis2))
-
- val mixedFuture = Future.sequence(List(f1,f2)) */
-
- val r1 = f(jedis1)
- //val r2 = f(jedis2)
-
- r1
- }
- catch {
- case e:JedisException => {
-// Logger.debug(s"following exception catched")
-// Logger.debug(s"${e}")
- jedis_pool1.returnBrokenResource(jedis1)
-// jedis_pool2.returnBrokenResource(jedis2)
-
- jedis1 = null
-// jedis2 = null
- fallBack
- }
- }
- finally {
- if (jedis1 != null) jedis_pool1.returnResource(jedis1)
-// if (jedis2 != null) jedis_pool2.returnResource(jedis2)
- }
- }
- else{
-// Logger.debug(s"fallback executed")
- fallBack
- }
- }
-
- def doBlockWithKey[T](key: String)(f: Jedis => T)(fallBack: => T) = {
-// Logger.debug(s"start jedis do block")
- val (jedis_pool1, jedis_pool2) = getJedisPoolWithBucketname(key)
- if(jedis_pool1 != null && jedis_pool2 != null) {
- var jedis1: Jedis = null
- var jedis2: Jedis = null
- try {
- jedis1 = jedis_pool1.getResource()
- jedis2 = jedis_pool2.getResource()
-
- /* val f1 = Future(f(jedis1))
- val f2 = Future(f(jedis2))
-
- val mixedFuture = Future.sequence(List(f1,f2)) */
-
- val r1 = f(jedis1)
- //val r2 = f(jedis2)
-
- r1
- }
- catch {
- case e:JedisException => {
-// Logger.debug(s"following exception catched")
-// Logger.debug(s"${e}")
- jedis_pool1.returnBrokenResource(jedis1)
- jedis_pool2.returnBrokenResource(jedis2)
-
- jedis1 = null
- jedis2 = null
- fallBack
- }
- }
- finally {
- if (jedis1 != null) jedis_pool1.returnResource(jedis1)
- if (jedis2 != null) jedis_pool2.returnResource(jedis2)
- }
- }
- else{
-// Logger.debug(s"fallback executed")
- fallBack
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/Management.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/Management.scala b/s2counter_core/src/main/scala/s2/helper/Management.scala
deleted file mode 100644
index 266caba..0000000
--- a/s2counter_core/src/main/scala/s2/helper/Management.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-package s2.helper
-
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.client.{Admin, ConnectionFactory, Durability}
-import org.apache.hadoop.hbase.io.compress.Compression
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
-import org.apache.hadoop.hbase.regionserver.BloomType
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
-import org.slf4j.LoggerFactory
-import redis.clients.jedis.ScanParams
-
-import scala.collection.JavaConversions._
-import scala.util.Random
-
-/**
-* Created by hsleep(honeysleep@gmail.com) on 15. 3. 30..
-*/
-class Management(config: Config) {
- val withRedis = new HashShardingJedis(config)
-
- val log = LoggerFactory.getLogger(this.getClass)
-
- def describe(zkAddr: String, tableName: String) = {
- val admin = getAdmin(zkAddr)
- val table = admin.getTableDescriptor(TableName.valueOf(tableName))
-
- table.getColumnFamilies.foreach { cf =>
- println(s"columnFamily: ${cf.getNameAsString}")
- cf.getValues.foreach { case (k, v) =>
- println(s"${Bytes.toString(k.get())} ${Bytes.toString(v.get())}")
- }
- }
- }
-
- def setTTL(zkAddr: String, tableName: String, cfName: String, ttl: Int) = {
- val admin = getAdmin(zkAddr)
- val tableNameObj = TableName.valueOf(tableName)
- val table = admin.getTableDescriptor(tableNameObj)
-
- val cf = table.getFamily(cfName.getBytes)
- cf.setTimeToLive(ttl)
-
- admin.modifyColumn(tableNameObj, cf)
- }
-
- def getAdmin(zkAddr: String): Admin = {
- val conf = HBaseConfiguration.create()
- conf.set("hbase.zookeeper.quorum", zkAddr)
- val conn = ConnectionFactory.createConnection(conf)
- conn.getAdmin
- }
-
- def tableExists(zkAddr: String, tableName: String): Boolean = {
- getAdmin(zkAddr).tableExists(TableName.valueOf(tableName))
- }
-
- def createTable(zkAddr: String, tableName: String, cfs: List[String], regionMultiplier: Int) = {
- log.info(s"create table: $tableName on $zkAddr, $cfs, $regionMultiplier")
- val admin = getAdmin(zkAddr)
- val regionCount = admin.getClusterStatus.getServersSize * regionMultiplier
- try {
- val desc = new HTableDescriptor(TableName.valueOf(tableName))
- desc.setDurability(Durability.ASYNC_WAL)
- for (cf <- cfs) {
- val columnDesc = new HColumnDescriptor(cf)
- .setCompressionType(Compression.Algorithm.LZ4)
- .setBloomFilterType(BloomType.ROW)
- .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)
- .setMaxVersions(1)
- .setMinVersions(0)
- .setBlocksize(32768)
- .setBlockCacheEnabled(true)
- desc.addFamily(columnDesc)
- }
-
- if (regionCount <= 1) admin.createTable(desc)
- else admin.createTable(desc, getStartKey(regionCount), getEndKey(regionCount), regionCount)
- } catch {
- case e: Throwable =>
- log.error(s"$zkAddr, $tableName failed with $e", e)
- throw e
- }
- }
-
- // we only use murmur hash to distribute row key.
- private def getStartKey(regionCount: Int) = {
- Bytes.toBytes(Int.MaxValue / regionCount)
- }
-
- private def getEndKey(regionCount: Int) = {
- Bytes.toBytes(Int.MaxValue / regionCount * (regionCount - 1))
- }
-
- case class RedisScanIterator(scanParams: ScanParams = new ScanParams().count(100)) extends Iterator[String] {
- val nextCursorId: collection.mutable.Map[Int, String] = collection.mutable.Map.empty[Int, String]
- var innerIterator: Iterator[String] = _
-
- for {
- i <- 0 until withRedis.jedisPoolSize
- } {
- nextCursorId.put(i, "0")
- }
-
- def callScan(): Unit = {
- if (nextCursorId.nonEmpty) {
-// println(s"callScan: idx: $nextIdx, cursor: $nextCursorId")
- val idx = Random.shuffle(nextCursorId.keys).head
- val cursorId = nextCursorId(idx)
- val pool = withRedis.getJedisPool(idx)
- val conn = pool.getResource
- try {
- val result = conn.scan(cursorId, scanParams)
- result.getStringCursor match {
- case "0" =>
- log.debug(s"end scan: idx: $idx, cursor: $cursorId")
- nextCursorId.remove(idx)
- case x: String =>
- nextCursorId.put(idx, x)
- }
- innerIterator = result.getResult.toIterator
- } finally {
- pool.returnResource(conn)
- }
- }
- else {
- innerIterator = List.empty[String].toIterator
- }
- }
-
- // initialize
- callScan()
-
- override def hasNext: Boolean = {
- innerIterator.hasNext match {
- case true =>
- true
- case false =>
- callScan()
- innerIterator.hasNext
- }
- }
-
- override def next(): String = innerIterator.next()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/WithHBase.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/WithHBase.scala b/s2counter_core/src/main/scala/s2/helper/WithHBase.scala
deleted file mode 100644
index ae3ff1e..0000000
--- a/s2counter_core/src/main/scala/s2/helper/WithHBase.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-package s2.helper
-
-import com.stumbleupon.async.{Callback, Deferred}
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.client._
-import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
-import org.hbase.async.HBaseClient
-import org.slf4j.LoggerFactory
-import s2.config.S2CounterConfig
-
-import scala.concurrent.{Future, Promise}
-import scala.util.Try
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-class WithHBase(config: Config) {
- lazy val logger = LoggerFactory.getLogger(this.getClass)
- lazy val s2config = new S2CounterConfig(config)
-
- lazy val zkQuorum = s2config.HBASE_ZOOKEEPER_QUORUM
- lazy val defaultTableName = s2config.HBASE_TABLE_NAME
-
- logger.info(s"$zkQuorum, $defaultTableName")
-
- val hbaseConfig = HBaseConfiguration.create()
- s2config.getConfigMap("hbase").foreach { case (k, v) =>
- hbaseConfig.set(k, v)
- }
-
-// lazy val conn: HConnection = HConnectionManager.createConnection(hbaseConfig)
- lazy val conn: Connection = ConnectionFactory.createConnection(hbaseConfig)
-
- val writeBufferSize = 1024 * 1024 * 2 // 2MB
-
-// def apply[T](op: Table => T): Try[T] = {
-// Try {
-// val table = conn.getTable(TableName.valueOf(defaultTableName))
-// // do not keep failed operation in writer buffer
-// table.setWriteBufferSize(writeBufferSize)
-// try {
-// op(table)
-// } catch {
-// case e: Throwable =>
-// logger.error(s"Operation to table($defaultTableName) is failed: ${e.getMessage}")
-// throw e
-// } finally {
-// table.close()
-// }
-// }
-// }
-
- def apply[T](tableName: String)(op: Table => T): Try[T] = {
- Try {
- val table = conn.getTable(TableName.valueOf(tableName))
- // do not keep failed operation in writer buffer
- table.setWriteBufferSize(writeBufferSize)
- try {
- op(table)
- } catch {
- case ex: Exception =>
- logger.error(s"$ex: Operation to table($tableName) is failed")
- throw ex
- } finally {
- table.close()
- }
- }
- }
-}
-
-case class WithAsyncHBase(config: Config) {
- lazy val logger = LoggerFactory.getLogger(this.getClass)
- lazy val s2config = new S2CounterConfig(config)
-
- lazy val zkQuorum = s2config.HBASE_ZOOKEEPER_QUORUM
-
- val hbaseConfig = HBaseConfiguration.create()
- s2config.getConfigMap("hbase").foreach { case (k, v) =>
- hbaseConfig.set(k, v)
- }
-
-// lazy val conn: HConnection = HConnectionManager.createConnection(hbaseConfig)
- lazy val client: HBaseClient = new HBaseClient(zkQuorum)
-
- val writeBufferSize = 1024 * 1024 * 2 // 2MB
-
- def apply[T](op: HBaseClient => Deferred[T]): Future[T] = {
- val promise = Promise[T]()
-
- op(client).addCallback(new Callback[Unit, T] {
- def call(arg: T): Unit = {
- promise.success(arg)
- }
- }).addErrback(new Callback[Unit, Exception] {
- def call(ex: Exception): Unit = {
- promise.failure(ex)
- }
- })
- promise.future
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/helper/WithRedis.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/helper/WithRedis.scala b/s2counter_core/src/main/scala/s2/helper/WithRedis.scala
deleted file mode 100644
index 2046577..0000000
--- a/s2counter_core/src/main/scala/s2/helper/WithRedis.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-package s2.helper
-
-import com.typesafe.config.Config
-import org.slf4j.LoggerFactory
-import redis.clients.jedis.exceptions.JedisException
-import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
-import s2.config.S2CounterConfig
-import s2.util.Hashes
-
-import scala.util.Try
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-class WithRedis(config: Config) {
- lazy val s2config = new S2CounterConfig(config)
-
- private val log = LoggerFactory.getLogger(getClass)
-
- val poolConfig = new JedisPoolConfig()
- poolConfig.setMaxTotal(150)
- poolConfig.setMaxIdle(50)
- poolConfig.setMaxWaitMillis(200)
-
- val jedisPools = s2config.REDIS_INSTANCES.map { case (host, port) =>
- new JedisPool(poolConfig, host, port)
- }
-
- def getBucketIdx(key: String): Int = {
- Hashes.murmur3(key) % jedisPools.size
- }
-
- def doBlockWithIndex[T](idx: Int)(f: Jedis => T): Try[T] = {
- Try {
- val pool = jedisPools(idx)
-
- var jedis: Jedis = null
-
- try {
- jedis = pool.getResource
-
- f(jedis)
- }
- catch {
- case e: JedisException =>
- pool.returnBrokenResource(jedis)
-
- jedis = null
- throw e
- }
- finally {
- if (jedis != null) {
- pool.returnResource(jedis)
- }
- }
- }
- }
-
- def doBlockWithKey[T](key: String)(f: Jedis => T): Try[T] = {
- doBlockWithIndex(getBucketIdx(key))(f)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/models/CachedDBModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/models/CachedDBModel.scala b/s2counter_core/src/main/scala/s2/models/CachedDBModel.scala
deleted file mode 100644
index 5da0265..0000000
--- a/s2counter_core/src/main/scala/s2/models/CachedDBModel.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package s2.models
-
-import s2.util.{CollectionCache, CollectionCacheConfig}
-import scalikejdbc.AutoSession
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 5. 27..
- */
-trait CachedDBModel[T] {
- implicit val s = AutoSession
-
- val cacheConfig: CollectionCacheConfig
- lazy val cache = new CollectionCache[Option[T]](cacheConfig)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/models/Counter.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/models/Counter.scala b/s2counter_core/src/main/scala/s2/models/Counter.scala
deleted file mode 100644
index e26e071..0000000
--- a/s2counter_core/src/main/scala/s2/models/Counter.scala
+++ /dev/null
@@ -1,213 +0,0 @@
-package s2.models
-
-import com.typesafe.config.Config
-import s2.config.S2CounterConfig
-import s2.util.{CollectionCache, CollectionCacheConfig}
-import scalikejdbc._
-
-/**
-* Created by hsleep(honeysleep@gmail.com) on 15. 1. 30..
-*/
-case class Counter(id: Int, useFlag: Boolean, version: Byte, service: String, action: String,
- itemType: Counter.ItemType.ItemType, autoComb: Boolean, dimension: String,
- useProfile: Boolean, bucketImpId: Option[String],
- useRank: Boolean,
- ttl: Int, dailyTtl: Option[Int], hbaseTable: Option[String],
- intervalUnit: Option[String],
- rateActionId: Option[Int], rateBaseId: Option[Int], rateThreshold: Option[Int]) {
- val intervals: Array[String] = intervalUnit.map(s => s.split(',')).getOrElse(Array("t", "M", "d", "H"))
- val dimensionSp = if (dimension.isEmpty) Array.empty[String] else dimension.split(',').sorted
-
- val dimensionList: List[Array[String]] = {
- autoComb match {
- case true =>
- for {
- i <- (0 to math.min(4, dimensionSp.length)).toList
- combines <- dimensionSp.combinations(i)
- } yield {
- combines
- }
- case false =>
- dimensionSp isEmpty match {
- case true => List(Array())
- case false => dimensionSp.toList.map(sp => sp.split('.'))
- }
- }
- }
-
- val dimensionSet: Set[Set[String]] = {
- for {
- arr <- dimensionList
- } yield {
- arr.toSet
- }
- }.toSet
-
- val isRateCounter: Boolean = rateActionId.isDefined && rateBaseId.isDefined && rateActionId != rateBaseId
- val isTrendCounter: Boolean = rateActionId.isDefined && rateBaseId.isDefined && rateActionId == rateBaseId
-}
-
-object Counter extends SQLSyntaxSupport[Counter] {
- object ItemType extends Enumeration {
- type ItemType = Value
- val INT, LONG, STRING, BLOB = Value
- }
-
- def apply(c: SyntaxProvider[Counter])(rs: WrappedResultSet): Counter = apply(c.resultName)(rs)
- def apply(r: ResultName[Counter])(rs: WrappedResultSet): Counter = {
- lazy val itemType = Counter.ItemType(rs.int(r.itemType))
- Counter(rs.int(r.id), rs.boolean(r.useFlag), rs.byte(r.version), rs.string(r.service), rs.string(r.action),
- itemType, rs.boolean(r.autoComb), rs.string(r.dimension),
- rs.boolean(r.useProfile), rs.stringOpt(r.bucketImpId),
- rs.boolean(r.useRank),
- rs.int(r.ttl), rs.intOpt(r.dailyTtl), rs.stringOpt(r.hbaseTable), rs.stringOpt(r.intervalUnit),
- rs.intOpt(r.rateActionId), rs.intOpt(r.rateBaseId), rs.intOpt(r.rateThreshold))
- }
-
- def apply(useFlag: Boolean, version: Byte, service: String, action: String, itemType: Counter.ItemType.ItemType,
- autoComb: Boolean, dimension: String, useProfile: Boolean = false, bucketImpId: Option[String] = None,
- useRank: Boolean = false, ttl: Int = 259200, dailyTtl: Option[Int] = None,
- hbaseTable: Option[String] = None, intervalUnit: Option[String] = None,
- rateActionId: Option[Int] = None, rateBaseId: Option[Int] = None, rateThreshold: Option[Int] = None): Counter = {
- Counter(-1, useFlag, version, service, action, itemType, autoComb, dimension,
- useProfile, bucketImpId,
- useRank, ttl, dailyTtl, hbaseTable,
- intervalUnit, rateActionId, rateBaseId, rateThreshold)
- }
-}
-
-class CounterModel(config: Config) extends CachedDBModel[Counter] {
- private lazy val s2Config = new S2CounterConfig(config)
- // enable negative cache
- override val cacheConfig: CollectionCacheConfig =
- new CollectionCacheConfig(s2Config.CACHE_MAX_SIZE, s2Config.CACHE_TTL_SECONDS,
- negativeCache = true, s2Config.CACHE_NEGATIVE_TTL_SECONDS)
-
- val c = Counter.syntax("c")
- val r = c.result
-
- val multiCache = new CollectionCache[Seq[Counter]](cacheConfig)
-
- def findById(id: Int, useCache: Boolean = true): Option[Counter] = {
- lazy val sql = withSQL {
- selectFrom(Counter as c).where.eq(c.id, id).and.eq(c.useFlag, 1)
- }.map(Counter(c))
-
- if (useCache) {
- cache.withCache(s"_id:$id") {
- sql.single().apply()
- }
- } else {
- sql.single().apply()
- }
- }
-
- def findByServiceAction(service: String, action: String, useCache: Boolean = true): Option[Counter] = {
- lazy val sql = withSQL {
- selectFrom(Counter as c).where.eq(c.service, service).and.eq(c.action, action).and.eq(c.useFlag, 1)
- }.map(Counter(c))
-
- if (useCache) {
- cache.withCache(s"$service.$action") {
- sql.single().apply()
- }
- }
- else {
- sql.single().apply()
- }
- }
-
- def findByRateActionId(rateActionId: Int, useCache: Boolean = true): Seq[Counter] = {
- lazy val sql = withSQL {
- selectFrom(Counter as c).where.eq(c.rateActionId, rateActionId).and.ne(c.rateBaseId, rateActionId).and.eq(c.useFlag, 1)
- }.map(Counter(c))
-
- if (useCache) {
- multiCache.withCache(s"_rate_action_id.$rateActionId") {
- sql.list().apply()
- }
- } else {
- sql.list().apply()
- }
- }
-
- def findByRateBaseId(rateBaseId: Int, useCache: Boolean = true): Seq[Counter] = {
- lazy val sql = withSQL {
- selectFrom(Counter as c).where.eq(c.rateBaseId, rateBaseId).and.ne(c.rateActionId, rateBaseId).and.eq(c.useFlag, 1)
- }.map(Counter(c))
-
- if (useCache) {
- multiCache.withCache(s"_rate_base_id.$rateBaseId") {
- sql.list().apply()
- }
- } else {
- sql.list().apply()
- }
- }
-
- def findByTrendActionId(trendActionId: Int, useCache: Boolean = true): Seq[Counter] = {
- lazy val sql = withSQL {
- selectFrom(Counter as c).where.eq(c.rateActionId, trendActionId).and.eq(c.rateBaseId, trendActionId).and.eq(c.useFlag, 1)
- }.map(Counter(c))
-
- if (useCache) {
- multiCache.withCache(s"_trend_action_id.$trendActionId") {
- sql.list().apply()
- }
- } else {
- sql.list().apply()
- }
- }
-
- def createServiceAction(policy: Counter): Unit = {
- withSQL {
- val c = Counter.column
- insert.into(Counter).namedValues(
- c.useFlag -> policy.useFlag,
- c.version -> policy.version,
- c.service -> policy.service,
- c.action -> policy.action,
- c.itemType -> policy.itemType.id,
- c.autoComb -> policy.autoComb,
- c.dimension -> policy.dimension,
- c.useProfile -> policy.useProfile,
- c.bucketImpId -> policy.bucketImpId,
- c.useRank -> policy.useRank,
- c.ttl -> policy.ttl,
- c.dailyTtl -> policy.dailyTtl,
- c.hbaseTable -> policy.hbaseTable,
- c.intervalUnit -> policy.intervalUnit,
- c.rateActionId -> policy.rateActionId,
- c.rateBaseId -> policy.rateBaseId,
- c.rateThreshold -> policy.rateThreshold
- )
- }.update().apply()
- }
-
- def updateServiceAction(policy: Counter): Unit = {
- withSQL {
- val c = Counter.column
- update(Counter).set(
- c.autoComb -> policy.autoComb,
- c.dimension -> policy.dimension,
- c.useProfile -> policy.useProfile,
- c.bucketImpId -> policy.bucketImpId,
- c.useRank -> policy.useRank,
- c.intervalUnit -> policy.intervalUnit,
- c.rateActionId -> policy.rateActionId,
- c.rateBaseId -> policy.rateBaseId,
- c.rateThreshold -> policy.rateThreshold
- ).where.eq(c.id, policy.id)
- }.update().apply()
- }
-
- def deleteServiceAction(policy: Counter): Unit = {
- withSQL {
- val c = Counter.column
- update(Counter).set(
- c.action -> s"deleted_${System.currentTimeMillis()}_${policy.action}",
- c.useFlag -> false
- ).where.eq(c.id, policy.id)
- }.update().apply()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/models/DBModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/models/DBModel.scala b/s2counter_core/src/main/scala/s2/models/DBModel.scala
deleted file mode 100644
index 6cb34b9..0000000
--- a/s2counter_core/src/main/scala/s2/models/DBModel.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package s2.models
-
-import com.typesafe.config.Config
-import s2.config.S2CounterConfig
-import scalikejdbc._
-
-/**
- * Created by alec on 15. 3. 31..
- */
-object DBModel {
- private var initialized = false
-
- def initialize(config: Config): Unit = {
- if (!initialized) {
- this synchronized {
- if (!initialized) {
- val s2Config = new S2CounterConfig(config)
- Class.forName(s2Config.DB_DEFAULT_DRIVER)
- val settings = ConnectionPoolSettings(initialSize = 0, maxSize = 10, connectionTimeoutMillis = 5000L, validationQuery = "select 1;")
-
- ConnectionPool.singleton(s2Config.DB_DEFAULT_URL, s2Config.DB_DEFAULT_USER, s2Config.DB_DEFAULT_PASSWORD, settings)
- initialized = true
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/CartesianProduct.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/CartesianProduct.scala b/s2counter_core/src/main/scala/s2/util/CartesianProduct.scala
deleted file mode 100644
index 2077e3f..0000000
--- a/s2counter_core/src/main/scala/s2/util/CartesianProduct.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package s2.util
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-object CartesianProduct {
- def apply[T](xss: List[List[T]]): List[List[T]] = xss match {
- case Nil => List(Nil)
- case h :: t => for(xh <- h; xt <- apply(t)) yield xh :: xt
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_core/src/main/scala/s2/util/CollectionCache.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/util/CollectionCache.scala b/s2counter_core/src/main/scala/s2/util/CollectionCache.scala
deleted file mode 100644
index 122f87a..0000000
--- a/s2counter_core/src/main/scala/s2/util/CollectionCache.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package s2.util
-
-import java.net.InetAddress
-import java.util.concurrent.TimeUnit
-
-import com.google.common.cache.{Cache, CacheBuilder}
-import org.slf4j.LoggerFactory
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.language.{postfixOps, reflectiveCalls}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 7. 1..
- */
-case class CollectionCacheConfig(maxSize: Int, ttl: Int, negativeCache: Boolean = false, negativeTTL: Int = 600)
-
-class CollectionCache[C <: { def nonEmpty: Boolean; def isEmpty: Boolean } ](config: CollectionCacheConfig) {
- private val cache: Cache[String, C] = CacheBuilder.newBuilder()
- .expireAfterWrite(config.ttl, TimeUnit.SECONDS)
- .maximumSize(config.maxSize)
- .build[String, C]()
-
-// private lazy val cache = new SynchronizedLruMap[String, (C, Int)](config.maxSize)
- private lazy val className = this.getClass.getSimpleName
-
- private lazy val log = LoggerFactory.getLogger(this.getClass)
- val localHostname = InetAddress.getLocalHost.getHostName
-
- def size = cache.size
- val maxSize = config.maxSize
-
- // cache statistics
- def getStatsString: String = {
- s"$localHostname ${cache.stats().toString}"
- }
-
- def withCache(key: String)(op: => C): C = {
- Option(cache.getIfPresent(key)) match {
- case Some(r) => r
- case None =>
- val r = op
- if (r.nonEmpty || config.negativeCache) {
- cache.put(key, r)
- }
- r
- }
- }
-
- def withCacheAsync(key: String)(op: => Future[C])(implicit ec: ExecutionContext): Future[C] = {
- Option(cache.getIfPresent(key)) match {
- case Some(r) => Future.successful(r)
- case None =>
- op.map { r =>
- if (r.nonEmpty || config.negativeCache) {
- cache.put(key, r)
- }
- r
- }
- }
- }
-
- def purgeKey(key: String) = {
- cache.invalidate(key)
- }
-
- def contains(key: String): Boolean = {
- Option(cache.getIfPresent(key)).nonEmpty
- }
-}
[19/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
deleted file mode 100644
index 46ad15f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/tall/IndexEdgeSerializable.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.indexedge.tall
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
-import com.kakao.s2graph.core.types.VertexId
-import com.kakao.s2graph.core.{GraphUtil, IndexEdge}
-import org.apache.hadoop.hbase.util.Bytes
-
-
-class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] {
- import StorageSerializable._
-
- val label = indexEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = Serializable.edgeCf
-
- val idxPropsMap = indexEdge.orders.toMap
- val idxPropsBytes = propsToBytes(indexEdge.orders)
-
- override def toKeyValues: Seq[SKeyValue] = {
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
-
- val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
- // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
-
- val qualifier =
- if (indexEdge.degreeEdge) Array.empty[Byte]
- else
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => Bytes.add(idxPropsBytes, VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes)
- case Some(vId) => idxPropsBytes
- }
-
- /** TODO search usage of op byte. if there is no, then remove opByte */
- val rowBytes = Bytes.add(row, Array.fill(1)(GraphUtil.defaultOpByte), qualifier)
- // val qualifierBytes = Array.fill(1)(indexEdge.op)
- val qualifierBytes = Array.empty[Byte]
-
- val value =
- if (indexEdge.degreeEdge)
- Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.degreeSeq).innerVal.toString().toLong)
- else if (indexEdge.op == GraphUtil.operations("incrementCount"))
- Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.countSeq).innerVal.toString().toLong)
- else propsToKeyValues(indexEdge.metas.toSeq)
-
- val kv = SKeyValue(table, rowBytes, cf, qualifierBytes, value, indexEdge.version)
-
- // logger.debug(s"[Ser]: ${kv.row.toList}, ${kv.qualifier.toList}, ${kv.value.toList}")
- Seq(kv)
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
deleted file mode 100644
index f83dd1f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.indexedge.wide
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.StorageDeserializable._
-import com.kakao.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue, StorageDeserializable}
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.{GraphUtil, IndexEdge, QueryParam, Vertex}
-
-class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] {
- import StorageDeserializable._
-
- type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
- type ValueRaw = (Array[(Byte, InnerValLike)], Int)
-
- private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- // val degree = Bytes.toLong(kv.value)
- val degree = bytesToLongFunc(kv.value, 0)
- val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
- val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
- (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
- }
-
- private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- var qualifierLen = 0
- var pos = 0
- val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
- val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
- pos = endAt
- qualifierLen += endAt
- val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
- (HBaseType.defaultTgtVertexId, 0)
- } else {
- TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
- }
- qualifierLen += tgtVertexIdLen
- (props, endAt, tgtVertexId, tgtVertexIdLen)
- }
- val (op, opLen) =
- if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
- else (kv.qualifier(qualifierLen), 1)
-
- qualifierLen += opLen
-
- (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
- }
-
- private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
- val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
- (props, endAt)
- }
-
- private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
- (Array.empty[(Byte, InnerValLike)], 0)
- }
-
- override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[IndexEdge]): IndexEdge = {
- assert(_kvs.size == 1)
-
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-
- val kv = kvs.head
- val (srcVertexId, labelWithDir, labelIdxSeq, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.id, e.labelWithDir, e.labelIndexSeq, false, 0)
- }.getOrElse(parseRow(kv, version))
-
- val (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdInQualifier, _) =
- if (kv.qualifier.isEmpty) parseDegreeQualifier(kv, version)
- else parseQualifier(kv, version)
-
- val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
- // val countVal = Bytes.toLong(kv.value)
- val countVal = bytesToLongFunc(kv.value, 0)
- val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
- (dummyProps, 8)
- } else if (kv.qualifier.isEmpty) {
- parseDegreeValue(kv, version)
- } else {
- parseValue(kv, version)
- }
-
- val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${queryParam.label.id.get}, ${labelIdxSeq}"))
-
-
- // assert(kv.qualifier.nonEmpty && index.metaSeqs.size == idxPropsRaw.size)
-
- val idxProps = for {
- (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
- } yield {
- if (k == LabelMeta.degreeSeq) k -> v
- else seq -> v
- }
-
- val idxPropsMap = idxProps.toMap
- val tgtVertexId = if (tgtVertexIdInQualifier) {
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => tgtVertexIdRaw
- case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
- }
- } else tgtVertexIdRaw
-
- val _mergedProps = (idxProps ++ props).toMap
- val mergedProps =
- if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
- else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
-
- // logger.error(s"$mergedProps")
- // val ts = mergedProps(LabelMeta.timeStampSeq).toString().toLong
-
- val ts = kv.timestamp
- IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
-
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
deleted file mode 100644
index 716b6fb..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/indexedge/wide/IndexEdgeSerializable.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.indexedge.wide
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
-import com.kakao.s2graph.core.types.VertexId
-import com.kakao.s2graph.core.{GraphUtil, IndexEdge}
-import org.apache.hadoop.hbase.util.Bytes
-
-
-class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] {
- import StorageSerializable._
-
- val label = indexEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = Serializable.edgeCf
-
- val idxPropsMap = indexEdge.orders.toMap
- val idxPropsBytes = propsToBytes(indexEdge.orders)
-
- override def toKeyValues: Seq[SKeyValue] = {
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
-
- val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
- // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
- val tgtIdBytes = VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes
- val qualifier =
- if (indexEdge.degreeEdge) Array.empty[Byte]
- else {
- if (indexEdge.op == GraphUtil.operations("incrementCount")) {
- Bytes.add(idxPropsBytes, tgtIdBytes, Array.fill(1)(indexEdge.op))
- } else {
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => Bytes.add(idxPropsBytes, tgtIdBytes)
- case Some(vId) => idxPropsBytes
- }
- }
- }
-
-
- val value =
- if (indexEdge.degreeEdge)
- Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.degreeSeq).innerVal.toString().toLong)
- else if (indexEdge.op == GraphUtil.operations("incrementCount"))
- Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.countSeq).innerVal.toString().toLong)
- else propsToKeyValues(indexEdge.metas.toSeq)
-
- val kv = SKeyValue(table, row, cf, qualifier, value, indexEdge.version)
-
- Seq(kv)
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
deleted file mode 100644
index c97bed6..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.snapshotedge.tall
-
-import com.kakao.s2graph.core.mysqls.{LabelIndex, LabelMeta}
-import com.kakao.s2graph.core.storage.StorageDeserializable._
-import com.kakao.s2graph.core.storage.{CanSKeyValue, Deserializable, SKeyValue}
-import com.kakao.s2graph.core.types.{HBaseType, LabelWithDirection, SourceAndTargetVertexIdPair, SourceVertexId}
-import com.kakao.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-
-class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] {
-
- def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
- val statusCode = byte >> 4
- val op = byte & ((1 << 4) - 1)
- (statusCode.toByte, op.toByte)
- }
-
- override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
- assert(kvs.size == 1)
-
- val kv = kvs.head
- val schemaVer = queryParam.label.schemaVersion
- val cellVersion = kv.timestamp
- /** rowKey */
- def parseRowV3(kv: SKeyValue, version: String) = {
- var pos = 0
- val (srcIdAndTgtId, srcIdAndTgtIdLen) = SourceAndTargetVertexIdPair.fromBytes(kv.row, pos, kv.row.length, version)
- pos += srcIdAndTgtIdLen
- val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
- pos += 4
- val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
-
- val rowLen = srcIdAndTgtIdLen + 4 + 1
- (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isInverted, rowLen)
-
- }
- val (srcInnerId, tgtInnerId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
- }.getOrElse(parseRowV3(kv, schemaVer))
-
- val srcVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, srcInnerId)
- val tgtVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, tgtInnerId)
-
- val (props, op, ts, statusCode, _pendingEdgeOpt) = {
- var pos = 0
- val (statusCode, op) = statusCodeWithOp(kv.value(pos))
- pos += 1
- val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- val kvsMap = props.toMap
- val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
-
- pos = endAt
- val _pendingEdgeOpt =
- if (pos == kv.value.length) None
- else {
- val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
- pos += 1
- // val versionNum = Bytes.toLong(kv.value, pos, 8)
- // pos += 8
- val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- pos = endAt
- val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
-
- val pendingEdge =
- Edge(Vertex(srcVertexId, cellVersion),
- Vertex(tgtVertexId, cellVersion),
- labelWithDir, pendingEdgeOp,
- cellVersion, pendingEdgeProps.toMap,
- statusCode = pendingEdgeStatusCode, lockTs = lockTs)
- Option(pendingEdge)
- }
-
- (kvsMap, op, ts, statusCode, _pendingEdgeOpt)
- }
-
- SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
- labelWithDir, op, cellVersion, props, statusCode = statusCode,
- pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
deleted file mode 100644
index a507b90..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.snapshotedge.tall
-
-import com.kakao.s2graph.core.SnapshotEdge
-import com.kakao.s2graph.core.mysqls.LabelIndex
-import com.kakao.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
-import com.kakao.s2graph.core.types.SourceAndTargetVertexIdPair
-import org.apache.hadoop.hbase.util.Bytes
-
-
-class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] {
- import StorageSerializable._
-
- val label = snapshotEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = Serializable.edgeCf
-
- def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
- val byte = (((statusCode << 4) | op).toByte)
- Array.fill(1)(byte.toByte)
- }
- def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
- propsToKeyValuesWithTs(snapshotEdge.props.toList))
-
- override def toKeyValues: Seq[SKeyValue] = {
- val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes
- val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
-
- val row = Bytes.add(srcIdAndTgtIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
-
- val qualifier = Array.empty[Byte]
-
- val value = snapshotEdge.pendingEdgeOpt match {
- case None => valueBytes()
- case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
- val versionBytes = Array.empty[Byte]
- val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
-
- Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
- }
-
- val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
- Seq(kv)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
deleted file mode 100644
index 1174f50..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.snapshotedge.wide
-
-import com.kakao.s2graph.core.mysqls.{LabelIndex, LabelMeta}
-import com.kakao.s2graph.core.storage.StorageDeserializable._
-import com.kakao.s2graph.core.storage.{CanSKeyValue, Deserializable}
-import com.kakao.s2graph.core.types.TargetVertexId
-import com.kakao.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-
-class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] {
-
- def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
- val statusCode = byte >> 4
- val op = byte & ((1 << 4) - 1)
- (statusCode.toByte, op.toByte)
- }
-
- override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
- assert(kvs.size == 1)
-
- val kv = kvs.head
- val schemaVer = queryParam.label.schemaVersion
- val cellVersion = kv.timestamp
-
- val (srcVertexId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.id, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
- }.getOrElse(parseRow(kv, schemaVer))
-
- val (tgtVertexId, props, op, ts, statusCode, _pendingEdgeOpt) = {
- val (tgtVertexId, _) = TargetVertexId.fromBytes(kv.qualifier, 0, kv.qualifier.length, schemaVer)
- var pos = 0
- val (statusCode, op) = statusCodeWithOp(kv.value(pos))
- pos += 1
- val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- val kvsMap = props.toMap
- val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
-
- pos = endAt
- val _pendingEdgeOpt =
- if (pos == kv.value.length) None
- else {
- val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
- pos += 1
- // val versionNum = Bytes.toLong(kv.value, pos, 8)
- // pos += 8
- val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- pos = endAt
- val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
-
- val pendingEdge =
- Edge(Vertex(srcVertexId, cellVersion),
- Vertex(tgtVertexId, cellVersion),
- labelWithDir, pendingEdgeOp,
- cellVersion, pendingEdgeProps.toMap,
- statusCode = pendingEdgeStatusCode, lockTs = lockTs)
- Option(pendingEdge)
- }
-
- (tgtVertexId, kvsMap, op, ts, statusCode, _pendingEdgeOpt)
- }
-
- SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
- labelWithDir, op, cellVersion, props, statusCode = statusCode,
- pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
deleted file mode 100644
index e6074d9..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.snapshotedge.wide
-
-import com.kakao.s2graph.core.SnapshotEdge
-import com.kakao.s2graph.core.mysqls.LabelIndex
-import com.kakao.s2graph.core.storage.{SKeyValue, Serializable, StorageSerializable}
-import com.kakao.s2graph.core.types.VertexId
-import org.apache.hadoop.hbase.util.Bytes
-
-
-/**
- * this class serialize
- * @param snapshotEdge
- */
-class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] {
- import StorageSerializable._
-
- val label = snapshotEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = Serializable.edgeCf
-
- def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
- val byte = (((statusCode << 4) | op).toByte)
- Array.fill(1)(byte.toByte)
- }
- def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
- propsToKeyValuesWithTs(snapshotEdge.props.toList))
-
- override def toKeyValues: Seq[SKeyValue] = {
- val srcIdBytes = VertexId.toSourceVertexId(snapshotEdge.srcVertex.id).bytes
- val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
-
- val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
- val tgtIdBytes = VertexId.toTargetVertexId(snapshotEdge.tgtVertex.id).bytes
-
- val qualifier = tgtIdBytes
-
- val value = snapshotEdge.pendingEdgeOpt match {
- case None => valueBytes()
- case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
- val versionBytes = Array.empty[Byte]
- val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
- Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
- }
- val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
- Seq(kv)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
deleted file mode 100644
index e355401..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.vertex
-
-import com.kakao.s2graph.core.storage.{CanSKeyValue, Deserializable}
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLike, VertexId}
-import com.kakao.s2graph.core.{QueryParam, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-import scala.collection.mutable.ListBuffer
-
-class VertexDeserializable extends Deserializable[Vertex] {
- def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[Vertex]): Vertex = {
-
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-
- val kv = kvs.head
- val (vertexId, _) = VertexId.fromBytes(kv.row, 0, kv.row.length, version)
-
- var maxTs = Long.MinValue
- val propsMap = new collection.mutable.HashMap[Int, InnerValLike]
- val belongLabelIds = new ListBuffer[Int]
-
- for {
- kv <- kvs
- } {
- val propKey =
- if (kv.qualifier.length == 1) kv.qualifier.head.toInt
- else Bytes.toInt(kv.qualifier)
-
- val ts = kv.timestamp
- if (ts > maxTs) maxTs = ts
-
- if (Vertex.isLabelId(propKey)) {
- belongLabelIds += Vertex.toLabelId(propKey)
- } else {
- val v = kv.value
- val (value, _) = InnerVal.fromBytes(v, 0, v.length, version)
- propsMap += (propKey -> value)
- }
- }
- assert(maxTs != Long.MinValue)
- Vertex(vertexId, maxTs, propsMap.toMap, belongLabelIds = belongLabelIds)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexSerializable.scala
deleted file mode 100644
index 0c17592..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/serde/vertex/VertexSerializable.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package com.kakao.s2graph.core.storage.serde.vertex
-
-import com.kakao.s2graph.core.Vertex
-import com.kakao.s2graph.core.storage.{SKeyValue, Serializable}
-import org.apache.hadoop.hbase.util.Bytes
-
-
-case class VertexSerializable(vertex: Vertex) extends Serializable[Vertex] {
-
- val cf = Serializable.vertexCf
-
- override def toKeyValues: Seq[SKeyValue] = {
- val row = vertex.id.bytes
- val base = for ((k, v) <- vertex.props ++ vertex.defaultProps) yield Bytes.toBytes(k) -> v.bytes
- val belongsTo = vertex.belongLabelIds.map { labelId => Bytes.toBytes(Vertex.toPropKey(labelId)) -> Array.empty[Byte] }
- (base ++ belongsTo).map { case (qualifier, value) =>
- SKeyValue(vertex.hbaseTableName.getBytes, row, cf, qualifier, value, vertex.ts)
- } toSeq
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/HBaseType.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/HBaseType.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/HBaseType.scala
deleted file mode 100644
index 4b3b3db..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/HBaseType.scala
+++ /dev/null
@@ -1,163 +0,0 @@
-package com.kakao.s2graph.core.types
-
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * Created by shon on 6/6/15.
- */
-object HBaseType {
- val VERSION4 = "v4"
- val VERSION3 = "v3"
- val VERSION2 = "v2"
- val VERSION1 = "v1"
-// val DEFAULT_VERSION = VERSION2
- val DEFAULT_VERSION = VERSION3
- val EMPTY_SEQ_BYTE = Byte.MaxValue
- val DEFAULT_COL_ID = 0
- val bitsForDir = 2
- val maxBytes = Bytes.toBytes(Int.MaxValue)
- val toSeqByte = -5.toByte
- val defaultTgtVertexId = null
-}
-
-object HBaseDeserializable {
-
- import HBaseType._
-
- // 6 bits is used for index sequence so total index per label is limited to 2^6
- def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
- val byte = bytes(offset)
- val isInverted = if ((byte & 1) != 0) true else false
- val labelOrderSeq = byte >> 1
- (labelOrderSeq.toByte, isInverted)
- }
-
- def bytesToKeyValues(bytes: Array[Byte],
- offset: Int,
- length: Int,
- version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLike)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLike)](len)
- var i = 0
- while (i < len) {
- val k = bytes(pos)
- pos += 1
- val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- val ret = (kvs, pos)
- // logger.debug(s"bytesToProps: $ret")
- ret
- }
-
- def bytesToKeyValuesWithTs(bytes: Array[Byte],
- offset: Int,
- version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLikeWithTs)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLikeWithTs)](len)
- var i = 0
- while (i < len) {
- val k = bytes(pos)
- pos += 1
- val (v, numOfBytesUsed) = InnerValLikeWithTs.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- val ret = (kvs, pos)
- // logger.debug(s"bytesToProps: $ret")
- ret
- }
-
- def bytesToProps(bytes: Array[Byte],
- offset: Int,
- version: String = DEFAULT_VERSION): (Array[(Byte, InnerValLike)], Int) = {
- var pos = offset
- val len = bytes(pos)
- pos += 1
- val kvs = new Array[(Byte, InnerValLike)](len)
- var i = 0
- while (i < len) {
- val k = EMPTY_SEQ_BYTE
- val (v, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, 0, version)
- pos += numOfBytesUsed
- kvs(i) = (k -> v)
- i += 1
- }
- // logger.error(s"bytesToProps: $kvs")
- val ret = (kvs, pos)
-
- ret
- }
-}
-
-object HBaseSerializable {
- def propsToBytes(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, v.bytes)
- bytes
- }
-
- def propsToKeyValues(props: Seq[(Byte, InnerValLike)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
- bytes
- }
-
- def propsToKeyValuesWithTs(props: Seq[(Byte, InnerValLikeWithTs)]): Array[Byte] = {
- val len = props.length
- assert(len < Byte.MaxValue)
- var bytes = Array.fill(1)(len.toByte)
- for ((k, v) <- props) bytes = Bytes.add(bytes, Array.fill(1)(k), v.bytes)
- bytes
- }
-
- def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
- assert(labelOrderSeq < (1 << 6))
- val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
- Array.fill(1)(byte.toByte)
- }
-}
-
-trait HBaseSerializable {
- def bytes: Array[Byte]
-}
-
-trait HBaseDeserializable {
-
- import HBaseType._
-
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (HBaseSerializable, Int)
-
- // def fromBytesWithIndex(bytes: Array[Byte],
- // offset: Int,
- // len: Int,
- // version: String = DEFAULT_VERSION): (HBaseSerializable, Int)
- def notSupportedEx(version: String) = new RuntimeException(s"not supported version, $version")
-}
-
-trait HBaseDeserializableWithIsVertexId {
-
- import HBaseType._
-
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION,
- isVertexId: Boolean = false): (HBaseSerializable, Int)
-
- def notSupportedEx(version: String) = new RuntimeException(s"not supported version, $version")
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/InnerValLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/InnerValLike.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/InnerValLike.scala
deleted file mode 100644
index 8146f32..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/InnerValLike.scala
+++ /dev/null
@@ -1,246 +0,0 @@
-package com.kakao.s2graph.core.types
-
-import org.apache.hadoop.hbase.util._
-
-/**
- * Created by shon on 6/6/15.
- */
-object InnerVal extends HBaseDeserializableWithIsVertexId {
- import HBaseType._
-
- val order = Order.DESCENDING
- val stringLenOffset = 7.toByte
- val maxStringLen = Byte.MaxValue - stringLenOffset
- val maxMetaByte = Byte.MaxValue
- val minMetaByte = 0.toByte
-
- /** supported data type */
- val BLOB = "blob"
- val STRING = "string"
- val DOUBLE = "double"
- val FLOAT = "float"
- val LONG = "long"
- val INT = "integer"
- val SHORT = "short"
- val BYTE = "byte"
- val NUMERICS = List(DOUBLE, FLOAT, LONG, INT, SHORT, BYTE)
- val BOOLEAN = "boolean"
-
- def isNumericType(dataType: String): Boolean = {
- dataType match {
- case InnerVal.BYTE | InnerVal.SHORT | InnerVal.INT | InnerVal.LONG | InnerVal.FLOAT | InnerVal.DOUBLE => true
- case _ => false
- }
- }
- def toInnerDataType(dataType: String): String = {
- dataType match {
- case "blob" => BLOB
- case "string" | "str" | "s" => STRING
- case "double" | "d" | "float64" => DOUBLE
- case "float" | "f" | "float32" => FLOAT
- case "long" | "l" | "int64" | "integer64" => LONG
- case "int" | "integer" | "i" | "int32" | "integer32" => INT
- case "short" | "int16" | "integer16" => SHORT
- case "byte" | "b" | "tinyint" | "int8" | "integer8" => BYTE
- case "boolean" | "bool" => BOOLEAN
- case _ => throw new RuntimeException(s"can`t convert $dataType into InnerDataType")
- }
- }
-
- def numByteRange(num: BigDecimal) = {
-// val byteLen =
-// if (num.isValidByte | num.isValidChar) 1
-// else if (num.isValidShort) 2
-// else if (num.isValidInt) 4
-// else if (num.isValidLong) 8
-// else if (num.isValidFloat) 4
-// else 12
- val byteLen = 12
- // else throw new RuntimeException(s"wrong data $num")
- new SimplePositionedMutableByteRange(byteLen + 4)
- }
-
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String,
- isVertexId: Boolean): (InnerValLike, Int) = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal.fromBytes(bytes, offset, len, version, isVertexId)
- case VERSION1 => v1.InnerVal.fromBytes(bytes, offset, len, version, isVertexId)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withLong(l: Long, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(l))
- case VERSION1 => v1.InnerVal(Some(l), None, None)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withInt(i: Int, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(i))
- case VERSION1 => v1.InnerVal(Some(i.toLong), None, None)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withFloat(f: Float, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(f.toDouble))
- case VERSION1 => v1.InnerVal(Some(f.toLong), None, None)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withDouble(d: Double, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(BigDecimal(d))
- case VERSION1 => v1.InnerVal(Some(d.toLong), None, None)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withNumber(num: BigDecimal, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(num)
- case VERSION1 => v1.InnerVal(Some(num.toLong), None, None)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withBoolean(b: Boolean, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(b)
- case VERSION1 => v1.InnerVal(None, None, Some(b))
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withBlob(blob: Array[Byte], version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(blob)
- case _ => throw notSupportedEx(version)
- }
- }
-
- def withStr(s: String, version: String): InnerValLike = {
- version match {
- case VERSION2 | VERSION3 | VERSION4 => v2.InnerVal(s)
- case VERSION1 => v1.InnerVal(None, Some(s), None)
- case _ => throw notSupportedEx(version)
- }
- }
-
-// def withInnerVal(innerVal: InnerValLike, version: String): InnerValLike = {
-// val bytes = innerVal.bytes
-// version match {
-// case VERSION2 => v2.InnerVal.fromBytes(bytes, 0, bytes.length, version)._1
-// case VERSION1 => v1.InnerVal.fromBytes(bytes, 0, bytes.length, version)._1
-// case _ => throw notSupportedEx(version)
-// }
-// }
-
- /** nasty implementation for backward compatability */
- def convertVersion(innerVal: InnerValLike, dataType: String, toVersion: String): InnerValLike = {
- val ret = toVersion match {
- case VERSION2 | VERSION3 | VERSION4 =>
- if (innerVal.isInstanceOf[v1.InnerVal]) {
- val obj = innerVal.asInstanceOf[v1.InnerVal]
- obj.valueType match {
- case "long" => InnerVal.withLong(obj.longV.get, toVersion)
- case "string" => InnerVal.withStr(obj.strV.get, toVersion)
- case "boolean" => InnerVal.withBoolean(obj.boolV.get, toVersion)
- case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
- }
- } else {
- innerVal
- }
- case VERSION1 =>
- if (innerVal.isInstanceOf[v2.InnerVal]) {
- val obj = innerVal.asInstanceOf[v2.InnerVal]
- obj.value match {
- case str: String => InnerVal.withStr(str, toVersion)
- case b: Boolean => InnerVal.withBoolean(b, toVersion)
- case n: BigDecimal => InnerVal.withNumber(n, toVersion)
- case n: Long => InnerVal.withNumber(n, toVersion)
- case n: Double => InnerVal.withNumber(n, toVersion)
- case n: Int => InnerVal.withNumber(n, toVersion)
- case _ => throw notSupportedEx(s"v2 to v1: $obj -> $toVersion")
- }
- } else {
- innerVal
- }
- case _ => throw notSupportedEx(toVersion)
- }
-// logger.debug(s"convertVersion: $innerVal, $dataType, $toVersion, $ret, ${innerVal.bytes.toList}, ${ret.bytes.toList}")
- ret
- }
-
-}
-
-trait InnerValLike extends HBaseSerializable {
-
- val value: Any
-
- def compare(other: InnerValLike): Int
-
- def +(other: InnerValLike): InnerValLike
-
- def <(other: InnerValLike) = this.compare(other) < 0
-
- def <=(other: InnerValLike) = this.compare(other) <= 0
-
- def >(other: InnerValLike) = this.compare(other) > 0
-
- def >=(other: InnerValLike) = this.compare(other) >= 0
-
- override def toString(): String = value.toString
-
- override def hashCode(): Int = value.hashCode()
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case other: InnerValLike =>
- val ret = toString == obj.toString
-// logger.debug(s"InnerValLike.equals($this, $obj) => $ret")
- ret
- case _ => false
- }
- }
- def hashKey(dataType: String): Int
-
- def toIdString(): String
-
-}
-
-object InnerValLikeWithTs extends HBaseDeserializable {
- import HBaseType._
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (InnerValLikeWithTs, Int) = {
- val (innerVal, numOfBytesUsed) = InnerVal.fromBytes(bytes, offset, len, version)
- val ts = Bytes.toLong(bytes, offset + numOfBytesUsed)
- (InnerValLikeWithTs(innerVal, ts), numOfBytesUsed + 8)
- }
-
- def withLong(l: Long, ts: Long, version: String): InnerValLikeWithTs = {
- InnerValLikeWithTs(InnerVal.withLong(l, version), ts)
- }
-
- def withStr(s: String, ts: Long, version: String): InnerValLikeWithTs = {
- InnerValLikeWithTs(InnerVal.withStr(s, version), ts)
- }
-}
-
-case class InnerValLikeWithTs(innerVal: InnerValLike, ts: Long)
- extends HBaseSerializable {
-
- def bytes: Array[Byte] = {
- Bytes.add(innerVal.bytes, Bytes.toBytes(ts))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/LabelWithDirection.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/LabelWithDirection.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/LabelWithDirection.scala
deleted file mode 100644
index 6207ad7..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/LabelWithDirection.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-package com.kakao.s2graph.core.types
-
-import com.kakao.s2graph.core.GraphUtil
-import com.kakao.s2graph.core.utils.logger
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * Created by shon on 6/6/15.
- */
-object LabelWithDirection {
-
- import HBaseType._
-
- def apply(compositeInt: Int): LabelWithDirection = {
- // logger.debug(s"CompositeInt: $compositeInt")
-
- val dir = compositeInt & ((1 << bitsForDir) - 1)
- val labelId = compositeInt >> bitsForDir
- LabelWithDirection(labelId, dir)
- }
-
- def labelOrderSeqWithIsInverted(labelOrderSeq: Byte, isInverted: Boolean): Array[Byte] = {
- assert(labelOrderSeq < (1 << 6))
- val byte = labelOrderSeq << 1 | (if (isInverted) 1 else 0)
- Array.fill(1)(byte.toByte)
- }
-
- def bytesToLabelIndexSeqWithIsInverted(bytes: Array[Byte], offset: Int): (Byte, Boolean) = {
- val byte = bytes(offset)
- val isInverted = if ((byte & 1) != 0) true else false
- val labelOrderSeq = byte >> 1
- (labelOrderSeq.toByte, isInverted)
- }
-}
-
-case class LabelWithDirection(labelId: Int, dir: Int) extends HBaseSerializable {
-
- import HBaseType._
-
- assert(dir < (1 << bitsForDir))
- assert(labelId < (Int.MaxValue >> bitsForDir))
-
- lazy val labelBits = labelId << bitsForDir
-
- lazy val compositeInt = labelBits | dir
-
- def bytes = {
- Bytes.toBytes(compositeInt)
- }
-
- lazy val dirToggled = LabelWithDirection(labelId, GraphUtil.toggleDir(dir))
-
- def updateDir(newDir: Int) = LabelWithDirection(labelId, newDir)
-
- def isDirected = dir == 0 || dir == 1
-
- override def hashCode(): Int = compositeInt
-
- override def equals(other: Any): Boolean = {
- other match {
- case o: LabelWithDirection => hashCode == o.hashCode()
- case _ => false
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/VertexId.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/VertexId.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/VertexId.scala
deleted file mode 100644
index 13637bf..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/VertexId.scala
+++ /dev/null
@@ -1,145 +0,0 @@
-package com.kakao.s2graph.core.types
-
-import com.kakao.s2graph.core.GraphUtil
-import com.kakao.s2graph.core.types.HBaseType._
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * Created by shon on 6/10/15.
- */
-object VertexId extends HBaseDeserializable {
- import HBaseType._
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (VertexId, Int) = {
- /** since murmur hash is prepended, skip numOfBytes for murmur hash */
- var pos = offset + GraphUtil.bytesForMurMurHash
-
- val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
- pos += numOfBytesUsed
- val colId = Bytes.toInt(bytes, pos, 4)
- (VertexId(colId, innerId), GraphUtil.bytesForMurMurHash + numOfBytesUsed + 4)
- }
-
- def apply(colId: Int, innerId: InnerValLike): VertexId = new VertexId(colId, innerId)
-
- def toSourceVertexId(vid: VertexId) = {
- SourceVertexId(vid.colId, vid.innerId)
- }
-
- def toTargetVertexId(vid: VertexId) = {
- TargetVertexId(vid.colId, vid.innerId)
- }
-}
-
-class VertexId protected (val colId: Int, val innerId: InnerValLike) extends HBaseSerializable {
- val storeHash: Boolean = true
- val storeColId: Boolean = true
- lazy val hashBytes =
-// if (storeHash) Bytes.toBytes(GraphUtil.murmur3(innerId.toString))
- if (storeHash) Bytes.toBytes(GraphUtil.murmur3(innerId.toIdString()))
- else Array.empty[Byte]
-
- lazy val colIdBytes: Array[Byte] =
- if (storeColId) Bytes.toBytes(colId)
- else Array.empty[Byte]
-
- def bytes: Array[Byte] = Bytes.add(hashBytes, innerId.bytes, colIdBytes)
-
- override def toString(): String = {
- colId.toString() + "," + innerId.toString()
-// s"VertexId($colId, $innerId)"
- }
-
- override def hashCode(): Int = {
- val ret = if (storeColId) {
- colId * 31 + innerId.hashCode()
- } else {
- innerId.hashCode()
- }
-// logger.debug(s"VertexId.hashCode: $ret")
- ret
- }
- override def equals(obj: Any): Boolean = {
- val ret = obj match {
- case other: VertexId => colId == other.colId && innerId.toIdString() == other.innerId.toIdString()
- case _ => false
- }
-// logger.debug(s"VertexId.equals: $this, $obj => $ret")
- ret
- }
-
- def compareTo(other: VertexId): Int = {
- Bytes.compareTo(bytes, other.bytes)
- }
- def <(other: VertexId): Boolean = compareTo(other) < 0
- def <=(other: VertexId): Boolean = compareTo(other) <= 0
- def >(other: VertexId): Boolean = compareTo(other) > 0
- def >=(other: VertexId): Boolean = compareTo(other) >= 0
-}
-
-object SourceVertexId extends HBaseDeserializable {
- import HBaseType._
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (VertexId, Int) = {
- /** since murmur hash is prepended, skip numOfBytes for murmur hash */
- val pos = offset + GraphUtil.bytesForMurMurHash
- val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
-
- (SourceVertexId(DEFAULT_COL_ID, innerId), GraphUtil.bytesForMurMurHash + numOfBytesUsed)
- }
-
-}
-
-
-case class SourceVertexId(override val colId: Int,
- override val innerId: InnerValLike) extends VertexId(colId, innerId) {
- override val storeColId: Boolean = false
-}
-
-object TargetVertexId extends HBaseDeserializable {
- import HBaseType._
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (VertexId, Int) = {
- /** murmur has is not prepended so start from offset */
- val (innerId, numOfBytesUsed) = InnerVal.fromBytes(bytes, offset, len, version, isVertexId = true)
- (TargetVertexId(DEFAULT_COL_ID, innerId), numOfBytesUsed)
- }
-}
-
-case class TargetVertexId(override val colId: Int,
- override val innerId: InnerValLike)
- extends VertexId(colId, innerId) {
- override val storeColId: Boolean = false
- override val storeHash: Boolean = false
-
-}
-
-object SourceAndTargetVertexIdPair extends HBaseDeserializable {
- val delimiter = ":"
- import HBaseType._
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION): (SourceAndTargetVertexIdPair, Int) = {
- val pos = offset + GraphUtil.bytesForMurMurHash
- val (srcId, srcBytesLen) = InnerVal.fromBytes(bytes, pos, len, version, isVertexId = true)
- val (tgtId, tgtBytesLen) = InnerVal.fromBytes(bytes, pos + srcBytesLen, len, version, isVertexId = true)
- (SourceAndTargetVertexIdPair(srcId, tgtId), GraphUtil.bytesForMurMurHash + srcBytesLen + tgtBytesLen)
- }
-}
-
-case class SourceAndTargetVertexIdPair(val srcInnerId: InnerValLike, val tgtInnerId: InnerValLike) extends HBaseSerializable {
- val colId = DEFAULT_COL_ID
- import SourceAndTargetVertexIdPair._
- override def bytes: Array[Byte] = {
- val hashBytes = Bytes.toBytes(GraphUtil.murmur3(srcInnerId + delimiter + tgtInnerId))
- Bytes.add(hashBytes, srcInnerId.bytes, tgtInnerId.bytes)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/v1/InnerVal.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/v1/InnerVal.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/v1/InnerVal.scala
deleted file mode 100644
index 5111cbc..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/v1/InnerVal.scala
+++ /dev/null
@@ -1,226 +0,0 @@
-package com.kakao.s2graph.core.types.v1
-
-import com.kakao.s2graph.core.GraphExceptions.IllegalDataTypeException
-import com.kakao.s2graph.core.types._
-import org.apache.hadoop.hbase.util.Bytes
-
-/**
- * Created by shon on 6/6/15.
- */
-object InnerVal extends HBaseDeserializableWithIsVertexId {
- import HBaseType._
- // val defaultVal = new InnerVal(None, None, None)
- val stringLenOffset = 7.toByte
- val maxStringLen = Byte.MaxValue - stringLenOffset
- val maxMetaByte = Byte.MaxValue
- val minMetaByte = 0.toByte
- /**
- * first byte encoding rule.
- * 0 => default
- * 1 => long
- * 2 => int
- * 3 => short
- * 4 => byte
- * 5 => true
- * 6 => false
- * 7 ~ 127 => string len + 7
- */
- val metaByte = Map("default" -> 0, "long" -> 1, "int" -> 2, "short" -> 3,
- "byte" -> 4, "true" -> 5, "false" -> 6).map {
- case (k, v) => (k, v.toByte)
- }
- val metaByteRev = metaByte.map { case (k, v) => (v.toByte, k) } ++ metaByte.map { case (k, v) => ((-v).toByte, k) }
-
- def maxIdVal(dataType: String) = {
- dataType match {
- case "string" => InnerVal.withStr((0 until (Byte.MaxValue - stringLenOffset)).map("~").mkString)
- case "long" => InnerVal.withLong(Long.MaxValue)
- case "bool" => InnerVal.withBoolean(true)
- case _ => throw IllegalDataTypeException(dataType)
- }
- }
-
- def minIdVal(dataType: String) = {
- dataType match {
- case "string" => InnerVal.withStr("")
- case "long" => InnerVal.withLong(1)
- case "bool" => InnerVal.withBoolean(false)
- case _ => throw IllegalDataTypeException(dataType)
- }
- }
-
- def fromBytes(bytes: Array[Byte], offset: Int, len: Int, version: String = DEFAULT_VERSION, isVertexId: Boolean = false): (InnerVal, Int) = {
- var pos = offset
- //
- val header = bytes(pos)
- // logger.debug(s"${bytes(offset)}: ${bytes.toList.slice(pos, bytes.length)}")
- pos += 1
-
- var numOfBytesUsed = 0
- val (longV, strV, boolV) = metaByteRev.get(header) match {
- case Some(s) =>
- s match {
- case "default" =>
- (None, None, None)
- case "true" =>
- numOfBytesUsed = 0
- (None, None, Some(true))
- case "false" =>
- numOfBytesUsed = 0
- (None, None, Some(false))
- case "byte" =>
- numOfBytesUsed = 1
- val b = bytes(pos)
- val value = if (b >= 0) Byte.MaxValue - b else Byte.MinValue - b - 1
- (Some(value.toLong), None, None)
- case "short" =>
- numOfBytesUsed = 2
- val b = Bytes.toShort(bytes, pos, 2)
- val value = if (b >= 0) Short.MaxValue - b else Short.MinValue - b - 1
- (Some(value.toLong), None, None)
- case "int" =>
- numOfBytesUsed = 4
- val b = Bytes.toInt(bytes, pos, 4)
- val value = if (b >= 0) Int.MaxValue - b else Int.MinValue - b - 1
- (Some(value.toLong), None, None)
- case "long" =>
- numOfBytesUsed = 8
- val b = Bytes.toLong(bytes, pos, 8)
- val value = if (b >= 0) Long.MaxValue - b else Long.MinValue - b - 1
- (Some(value.toLong), None, None)
- }
- case _ => // string
- val strLen = header - stringLenOffset
- numOfBytesUsed = strLen
- (None, Some(Bytes.toString(bytes, pos, strLen)), None)
- }
-
- (InnerVal(longV, strV, boolV), numOfBytesUsed + 1)
- }
-
- def withLong(l: Long): InnerVal = {
- // if (l < 0) throw new IllegalDataRangeException("value shoudl be >= 0")
- InnerVal(Some(l), None, None)
- }
-
- def withStr(s: String): InnerVal = {
- InnerVal(None, Some(s), None)
- }
-
- def withBoolean(b: Boolean): InnerVal = {
- InnerVal(None, None, Some(b))
- }
-
- /**
- * In natural order
- * -129, -128 , -2, -1 < 0 < 1, 2, 127, 128
- *
- * In byte order
- * 0 < 1, 2, 127, 128 < -129, -128, -2, -1
- *
- */
- def transform(l: Long): (Byte, Array[Byte]) = {
- if (Byte.MinValue <= l && l <= Byte.MaxValue) {
- // val value = if (l < 0) l - Byte.MinValue else l + Byte.MinValue
- val key = if (l >= 0) metaByte("byte") else -metaByte("byte")
- val value = if (l >= 0) Byte.MaxValue - l else Byte.MinValue - l - 1
- val valueBytes = Array.fill(1)(value.toByte)
- (key.toByte, valueBytes)
- } else if (Short.MinValue <= l && l <= Short.MaxValue) {
- val key = if (l >= 0) metaByte("short") else -metaByte("short")
- val value = if (l >= 0) Short.MaxValue - l else Short.MinValue - l - 1
- val valueBytes = Bytes.toBytes(value.toShort)
- (key.toByte, valueBytes)
- } else if (Int.MinValue <= l && l <= Int.MaxValue) {
- val key = if (l >= 0) metaByte("int") else -metaByte("int")
- val value = if (l >= 0) Int.MaxValue - l else Int.MinValue - l - 1
- val valueBytes = Bytes.toBytes(value.toInt)
- (key.toByte, valueBytes)
- } else if (Long.MinValue <= l && l <= Long.MaxValue) {
- val key = if (l >= 0) metaByte("long") else -metaByte("long")
- val value = if (l >= 0) Long.MaxValue - l else Long.MinValue - l - 1
- val valueBytes = Bytes.toBytes(value.toLong)
- (key.toByte, valueBytes)
- } else {
- throw new Exception(s"InnerVal range is out: $l")
- }
- }
-}
-
-case class InnerVal(longV: Option[Long], strV: Option[String], boolV: Option[Boolean])
- extends HBaseSerializable with InnerValLike {
-
- import InnerVal._
-
- lazy val isDefault = longV.isEmpty && strV.isEmpty && boolV.isEmpty
- val value = (longV, strV, boolV) match {
- case (Some(l), None, None) => l
- case (None, Some(s), None) => s
- case (None, None, Some(b)) => b
- case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
- }
- def valueType = (longV, strV, boolV) match {
- case (Some(l), None, None) => "long"
- case (None, Some(s), None) => "string"
- case (None, None, Some(b)) => "boolean"
- case _ => throw new Exception(s"InnerVal should be [long/integeer/short/byte/string/boolean]")
- }
-
- def compare(other: InnerValLike): Int = {
- if (!other.isInstanceOf[InnerVal]) {
- throw new RuntimeException(s"compare between $this vs $other is not supported")
- } else {
-// (value, other.value) match {
-// case (v1: Long, v2: Long) => v1.compare(v2)
-// case (b1: Boolean, b2: Boolean) => b1.compare(b2)
-// case (s1: String, s2: String) => s1.compare(s2)
-// case _ => throw new Exception("Please check a type of the compare operands")
-// }
- Bytes.compareTo(bytes, other.bytes) * -1
- }
- }
-
- def +(other: InnerValLike) = {
- if (!other.isInstanceOf[InnerVal]) {
- throw new RuntimeException(s"+ between $this vs $other is not supported")
- } else {
- (value, other.value) match {
- case (v1: Long, v2: Long) => InnerVal.withLong(v1 + v2)
- case (b1: Boolean, b2: Boolean) => InnerVal.withBoolean(if (b2) !b1 else b1)
- case _ => throw new Exception("Please check a type of the incr operands")
- }
- }
- }
-
- def bytes = {
- val (meta, valBytes) = (longV, strV, boolV) match {
- case (None, None, None) =>
- (metaByte("default"), Array.empty[Byte])
- case (Some(l), None, None) =>
- transform(l)
- case (None, None, Some(b)) =>
- val meta = if (b) metaByte("true") else metaByte("false")
- (meta, Array.empty[Byte])
- case (None, Some(s), None) =>
- val sBytes = Bytes.toBytes(s)
- if (sBytes.length > maxStringLen) {
- throw new IllegalDataTypeException(s"string in innerVal maxSize is $maxStringLen, given ${sBytes.length}")
- }
- assert(sBytes.length <= maxStringLen)
- val meta = (stringLenOffset + sBytes.length).toByte
- (meta, sBytes)
- case _ => throw new IllegalDataTypeException("innerVal data type should be [long/string/bool]")
- }
- Bytes.add(Array.fill(1)(meta.toByte), valBytes)
- }
-
- override def toString(): String = {
- value.toString
- }
- override def hashKey(dataType: String): Int = {
- value.toString.hashCode()
- }
- override def toIdString(): String = {
- value.toString
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/types/v2/InnerVal.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/types/v2/InnerVal.scala b/s2core/src/main/scala/com/kakao/s2graph/core/types/v2/InnerVal.scala
deleted file mode 100644
index 630e53d..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/types/v2/InnerVal.scala
+++ /dev/null
@@ -1,162 +0,0 @@
-package com.kakao.s2graph.core.types.v2
-
-import com.kakao.s2graph.core.types._
-import org.apache.hadoop.hbase.util._
-
-/**
- * Created by shon on 6/6/15.
- */
-object InnerVal extends HBaseDeserializableWithIsVertexId {
-
- import HBaseType._
-
- val order = Order.DESCENDING
-
- def fromBytes(bytes: Array[Byte],
- offset: Int,
- len: Int,
- version: String = DEFAULT_VERSION,
- isVertexId: Boolean = false): (InnerVal, Int) = {
- val pbr = new SimplePositionedByteRange(bytes)
- pbr.setPosition(offset)
- val startPos = pbr.getPosition
- if (bytes(offset) == -1 | bytes(offset) == 0) {
- /** simple boolean */
- val boolean = order match {
- case Order.DESCENDING => bytes(offset) == 0
- case _ => bytes(offset) == -1
- }
- (InnerVal(boolean), 1)
- }
- else {
- if (OrderedBytes.isNumeric(pbr)) {
- val numeric = OrderedBytes.decodeNumericAsBigDecimal(pbr)
- if (isVertexId) (InnerVal(numeric.longValue()), pbr.getPosition - startPos)
- else (InnerVal(BigDecimal(numeric)), pbr.getPosition - startPos)
-// (InnerVal(numeric.doubleValue()), pbr.getPosition - startPos)
-// (InnerVal(BigDecimal(numeric)), pbr.getPosition - startPos)
- } else if (OrderedBytes.isText(pbr)) {
- val str = OrderedBytes.decodeString(pbr)
- (InnerVal(str), pbr.getPosition - startPos)
- } else if (OrderedBytes.isBlobVar(pbr)) {
- val blobVar = OrderedBytes.decodeBlobVar(pbr)
- (InnerVal(blobVar), pbr.getPosition - startPos)
- } else {
- throw new RuntimeException("!!")
- }
- }
- }
-}
-
-case class InnerVal(value: Any) extends HBaseSerializable with InnerValLike {
-
- import com.kakao.s2graph.core.types.InnerVal._
-
- def bytes: Array[Byte] = {
- val ret = value match {
- case b: Boolean =>
-
- /** since OrderedBytes header start from 0x05, it is safe to use -1, 0
- * for decreasing order (true, false) */
- // Bytes.toBytes(b)
- order match {
- case Order.DESCENDING => if (b) Array(0.toByte) else Array(-1.toByte)
- case _ => if (!b) Array(0.toByte) else Array(-1.toByte)
- }
- case d: Double =>
- val num = BigDecimal(d)
- val pbr = numByteRange(num)
- val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
- pbr.getBytes().take(len)
- case l: Long =>
- val num = BigDecimal(l)
- val pbr = numByteRange(num)
- val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
- pbr.getBytes().take(len)
- case i: Int =>
- val num = BigDecimal(i)
- val pbr = numByteRange(num)
- val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
- pbr.getBytes().take(len)
- case sh: Short =>
- val num = BigDecimal(sh)
- val pbr = numByteRange(num)
- val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
- pbr.getBytes().take(len)
- case b: Byte =>
- val num = BigDecimal(b)
- val pbr = numByteRange(num)
- val len = OrderedBytes.encodeNumeric(pbr, num.bigDecimal, order)
- pbr.getBytes().take(len)
-
-
- case b: BigDecimal =>
- val pbr = numByteRange(b)
- val len = OrderedBytes.encodeNumeric(pbr, b.bigDecimal, order)
- pbr.getBytes().take(len)
- case s: String =>
- val pbr = new SimplePositionedMutableByteRange(s.getBytes.length + 3)
- val len = OrderedBytes.encodeString(pbr, s, order)
- pbr.getBytes().take(len)
- case blob: Array[Byte] =>
- val len = OrderedBytes.blobVarEncodedLength(blob.length)
- val pbr = new SimplePositionedMutableByteRange(len)
- val totalLen = OrderedBytes.encodeBlobVar(pbr, blob, order)
- pbr.getBytes().take(totalLen)
- }
- // println(s"$value => ${ret.toList}, ${ret.length}")
- ret
- }
-
-//
- override def hashKey(dataType: String): Int = {
- if (value.isInstanceOf[String]) {
- // since we use dummy stringn value for degree edge.
- value.toString.hashCode()
- } else {
- dataType match {
- case BYTE => value.asInstanceOf[BigDecimal].bigDecimal.byteValue().hashCode()
- case FLOAT => value.asInstanceOf[BigDecimal].bigDecimal.floatValue().hashCode()
- case DOUBLE => value.asInstanceOf[BigDecimal].bigDecimal.doubleValue().hashCode()
- case LONG => value.asInstanceOf[BigDecimal].bigDecimal.longValue().hashCode()
- case INT => value.asInstanceOf[BigDecimal].bigDecimal.intValue().hashCode()
- case SHORT => value.asInstanceOf[BigDecimal].bigDecimal.shortValue().hashCode()
- case STRING => value.toString.hashCode
- case _ => throw new RuntimeException(s"NotSupportede type: $dataType")
- }
- }
- }
-
- def compare(other: InnerValLike): Int = {
- if (!other.isInstanceOf[InnerValLike])
- throw new RuntimeException(s"compare $this vs $other")
- Bytes.compareTo(bytes, other.bytes) * -1
- }
-
- def +(other: InnerValLike): InnerValLike = {
- if (!other.isInstanceOf[InnerValLike])
- throw new RuntimeException(s"+ $this, $other")
-
- (value, other.value) match {
- case (v1: BigDecimal, v2: BigDecimal) => new InnerVal(BigDecimal(v1.bigDecimal.add(v2.bigDecimal)))
- case _ => throw new RuntimeException("+ operation on inner val is for big decimal pair")
- }
- }
-
- //need to be removed ??
- override def toString(): String = {
-// value.toString()
- value match {
- case n: BigDecimal => n.bigDecimal.toPlainString
- case _ => value.toString
- }
- }
-
- override def toIdString(): String = {
- value match {
- case n: BigDecimal => n.bigDecimal.longValue().toString()
- case _ => value.toString
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
deleted file mode 100644
index 6777c28..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-package com.kakao.s2graph.core.utils
-
-import java.util.concurrent.TimeUnit
-
-import com.google.common.cache.CacheBuilder
-import com.stumbleupon.async.Deferred
-import com.typesafe.config.Config
-
-import scala.concurrent.ExecutionContext
-
-class DeferCache[R](config: Config)(implicit ex: ExecutionContext) {
-
- import com.kakao.s2graph.core.utils.Extensions.DeferOps
-
- type Value = (Long, Deferred[R])
-
- private val maxSize = config.getInt("future.cache.max.size")
- private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
- private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
-
- private val futureCache = CacheBuilder.newBuilder()
- .initialCapacity(maxSize)
- .concurrencyLevel(Runtime.getRuntime.availableProcessors())
- .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
- .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
- .maximumSize(maxSize).build[java.lang.Long, (Long, Deferred[R])]()
-
-
- def asMap() = futureCache.asMap()
-
- def getIfPresent(cacheKey: Long): Value = futureCache.getIfPresent(cacheKey)
-
- private def checkAndExpire(cacheKey: Long,
- cachedAt: Long,
- cacheTTL: Long,
- oldDefer: Deferred[R])(op: => Deferred[R]): Deferred[R] = {
- if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
- // future is too old. so need to expire and fetch new data from storage.
- futureCache.asMap().remove(cacheKey)
-
- val newPromise = new Deferred[R]()
- val now = System.currentTimeMillis()
-
- futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
- case null =>
- // only one thread succeed to come here concurrently
- // initiate fetch to storage then add callback on complete to finish promise.
- op withCallback { value =>
- newPromise.callback(value)
- value
- }
- newPromise
- case (cachedAt, oldDefer) => oldDefer
- }
- } else {
- // future is not to old so reuse it.
- oldDefer
- }
- }
- def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Deferred[R]): Deferred[R] = {
- val cacheVal = futureCache.getIfPresent(cacheKey)
- cacheVal match {
- case null =>
- val promise = new Deferred[R]()
- val now = System.currentTimeMillis()
- val (cachedAt, defer) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
- case null =>
- op.withCallback { value =>
- promise.callback(value)
- value
- }
- (now, promise)
- case oldVal => oldVal
- }
- checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
-
- case (cachedAt, defer) =>
- checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
deleted file mode 100644
index eea9a79..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-package com.kakao.s2graph.core.utils
-
-import com.stumbleupon.async.{Callback, Deferred}
-import com.typesafe.config.Config
-import scala.concurrent.{ExecutionContext, Future, Promise}
-
-object Extensions {
-
-
- def retryOnSuccess[T](maxRetryNum: Int, n: Int = 1)(fn: => Future[T])(shouldStop: T => Boolean)(implicit ex: ExecutionContext): Future[T] = n match {
- case i if n <= maxRetryNum =>
- fn.flatMap { result =>
- if (!shouldStop(result)) {
- logger.info(s"retryOnSuccess $n")
- retryOnSuccess(maxRetryNum, n + 1)(fn)(shouldStop)
- } else {
- Future.successful(result)
- }
- }
- case _ => fn
- }
-
- def retryOnFailure[T](maxRetryNum: Int, n: Int = 1)(fn: => Future[T])(fallback: => T)(implicit ex: ExecutionContext): Future[T] = n match {
- case i if n <= maxRetryNum =>
- fn recoverWith { case t: Throwable =>
- logger.info(s"retryOnFailure $n $t")
- retryOnFailure(maxRetryNum, n + 1)(fn)(fallback)
- }
- case _ =>
- Future.successful(fallback)
- }
-
-
- implicit class DeferOps[T](d: Deferred[T])(implicit ex: ExecutionContext) {
-
- def withCallback[R](op: T => R): Deferred[R] = {
- d.addCallback(new Callback[R, T] {
- override def call(arg: T): R = op(arg)
- })
- }
-
- def recoverWith(op: Exception => T): Deferred[T] = {
- d.addErrback(new Callback[Deferred[T], Exception] {
- override def call(e: Exception): Deferred[T] = Deferred.fromResult(op(e))
- })
- }
-
-
- def toFuture: Future[T] = {
- val promise = Promise[T]
-
- d.addBoth(new Callback[Unit, T] {
- def call(arg: T) = arg match {
- case e: Exception => promise.failure(e)
- case _ => promise.success(arg)
- }
- })
-
- promise.future
- }
-
- def toFutureWith(fallback: => T): Future[T] = {
- toFuture recoverWith { case t: Throwable => Future.successful(fallback) }
- }
-
- }
-
- implicit class ConfigOps(config: Config) {
- def getBooleanWithFallback(key: String, defaultValue: Boolean): Boolean =
- if (config.hasPath(key)) config.getBoolean(key) else defaultValue
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
deleted file mode 100644
index 17d9e8f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-package com.kakao.s2graph.core.utils
-
-import java.util.concurrent.TimeUnit
-
-import com.google.common.cache.CacheBuilder
-import com.typesafe.config.Config
-
-import scala.concurrent.{Promise, Future, ExecutionContext}
-
-
-class FutureCache[R](config: Config)(implicit ex: ExecutionContext) {
-
- type Value = (Long, Future[R])
-
- private val maxSize = config.getInt("future.cache.max.size")
- private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
- private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
-
- private val futureCache = CacheBuilder.newBuilder()
- .initialCapacity(maxSize)
- .concurrencyLevel(Runtime.getRuntime.availableProcessors())
- .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
- .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
- .maximumSize(maxSize).build[java.lang.Long, (Long, Promise[R])]()
-
-
- def asMap() = futureCache.asMap()
-
- def getIfPresent(cacheKey: Long): Value = {
- val (cachedAt, promise) = futureCache.getIfPresent(cacheKey)
- (cachedAt, promise.future)
- }
-
- private def checkAndExpire(cacheKey: Long,
- cachedAt: Long,
- cacheTTL: Long,
- oldFuture: Future[R])(op: => Future[R]): Future[R] = {
- if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
- // future is too old. so need to expire and fetch new data from storage.
- futureCache.asMap().remove(cacheKey)
-
- val newPromise = Promise[R]
- val now = System.currentTimeMillis()
-
- futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
- case null =>
- // only one thread succeed to come here concurrently
- // initiate fetch to storage then add callback on complete to finish promise.
- op.onSuccess { case value =>
- newPromise.success(value)
- value
- }
- newPromise.future
- case (cachedAt, oldPromise) => oldPromise.future
- }
- } else {
- // future is not to old so reuse it.
- oldFuture
- }
- }
- def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Future[R]): Future[R] = {
- val cacheVal = futureCache.getIfPresent(cacheKey)
- cacheVal match {
- case null =>
- val promise = Promise[R]
- val now = System.currentTimeMillis()
- val (cachedAt, cachedPromise) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
- case null =>
- op.onSuccess { case value =>
- promise.success(value)
- value
- }
- (now, promise)
- case oldVal => oldVal
- }
- checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
-
- case (cachedAt, cachedPromise) =>
- checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
deleted file mode 100644
index d281017..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-package com.kakao.s2graph.core.utils
-
-import play.api.libs.json.JsValue
-import org.slf4j.LoggerFactory
-
-import scala.language.{higherKinds, implicitConversions}
-
-object logger {
-
- trait Loggable[T] {
- def toLogMessage(msg: T): String
- }
-
- object Loggable {
- implicit val stringLoggable = new Loggable[String] {
- def toLogMessage(msg: String) = msg
- }
-
- implicit def numericLoggable[T: Numeric] = new Loggable[T] {
- def toLogMessage(msg: T) = msg.toString
- }
-
- implicit val jsonLoggable = new Loggable[JsValue] {
- def toLogMessage(msg: JsValue) = msg.toString()
- }
-
- implicit val booleanLoggable = new Loggable[Boolean] {
- def toLogMessage(msg: Boolean) = msg.toString()
- }
- }
-
- private val logger = LoggerFactory.getLogger("application")
- private val errorLogger = LoggerFactory.getLogger("error")
-
- def info[T: Loggable](msg: => T) = logger.info(implicitly[Loggable[T]].toLogMessage(msg))
-
- def debug[T: Loggable](msg: => T) = logger.debug(implicitly[Loggable[T]].toLogMessage(msg))
-
- def error[T: Loggable](msg: => T, exception: => Throwable) = errorLogger.error(implicitly[Loggable[T]].toLogMessage(msg), exception)
-
- def error[T: Loggable](msg: => T) = errorLogger.error(implicitly[Loggable[T]].toLogMessage(msg))
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2core/src/main/scala/com/kakao/s2graph/core/utils/SafeUpdateCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/SafeUpdateCache.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/SafeUpdateCache.scala
deleted file mode 100644
index d383b47..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/SafeUpdateCache.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-package com.kakao.s2graph.core.utils
-
-import java.util.concurrent.atomic.AtomicBoolean
-
-import com.google.common.cache.CacheBuilder
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
-
-object SafeUpdateCache {
-
- case class CacheKey(key: String)
-
-}
-
-class SafeUpdateCache[T](prefix: String, maxSize: Int, ttl: Int)(implicit executionContext: ExecutionContext) {
-
- import SafeUpdateCache._
-
- implicit class StringOps(key: String) {
- def toCacheKey = new CacheKey(prefix + ":" + key)
- }
-
- def toTs() = (System.currentTimeMillis() / 1000).toInt
-
- private val cache = CacheBuilder.newBuilder().maximumSize(maxSize).build[CacheKey, (T, Int, AtomicBoolean)]()
-
- def put(key: String, value: T) = cache.put(key.toCacheKey, (value, toTs, new AtomicBoolean(false)))
-
- def invalidate(key: String) = cache.invalidate(key.toCacheKey)
-
- def withCache(key: String)(op: => T): T = {
- val cacheKey = key.toCacheKey
- val cachedValWithTs = cache.getIfPresent(cacheKey)
-
- if (cachedValWithTs == null) {
- // fetch and update cache.
- val newValue = op
- cache.put(cacheKey, (newValue, toTs(), new AtomicBoolean(false)))
- newValue
- } else {
- val (cachedVal, updatedAt, isUpdating) = cachedValWithTs
- if (toTs() < updatedAt + ttl) cachedVal // in cache TTL
- else {
- val running = isUpdating.getAndSet(true)
- if (running) cachedVal
- else {
- Future(op)(executionContext) onComplete {
- case Failure(ex) =>
- cache.put(cacheKey, (cachedVal, toTs(), new AtomicBoolean(false))) // keep old value
- logger.error(s"withCache update failed: $cacheKey")
- case Success(newValue) =>
- cache.put(cacheKey, (newValue, toTs(), new AtomicBoolean(false))) // update new value
- logger.info(s"withCache update success: $cacheKey")
- }
- cachedVal
- }
- }
- }
- }
-}
-
[05/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/EraseDailyCounter.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/EraseDailyCounter.scala b/s2counter_loader/src/main/scala/s2/counter/EraseDailyCounter.scala
deleted file mode 100644
index 71784fa..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/EraseDailyCounter.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-package s2.counter
-
-import java.text.SimpleDateFormat
-
-import kafka.producer.KeyedMessage
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import play.api.libs.json.Json
-import s2.config.{S2ConfigFactory, StreamingConfig}
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.counter.core._
-import s2.counter.core.v1.ExactStorageHBase
-import s2.counter.core.v2.ExactStorageGraph
-import s2.models.{Counter, CounterModel, DBModel}
-import s2.spark.{SparkApp, WithKafka}
-
-import scala.collection.mutable
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 7. 1..
- */
-object EraseDailyCounter extends SparkApp with WithKafka {
- import scala.concurrent.ExecutionContext.Implicits.global
-
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- def valueToEtlItem(policy: Counter, key: ExactKeyTrait, values: ExactValueMap): Seq[CounterEtlItem] = {
- if (values.nonEmpty) {
- for {
- (eq, value) <- filter(values.toList)
- } yield {
- CounterEtlItem(eq.tq.ts, policy.service, policy.action, key.itemKey, Json.toJson(eq.dimKeyValues), Json.toJson(Map("value" -> -value)))
- }
- } else {
- Nil
- }
- }
-
- def filter(values: List[(ExactQualifier, Long)]): List[(ExactQualifier, Long)] = {
- val sorted = values.sortBy(_._1.dimKeyValues.size).reverse
- val (eq, value) = sorted.head
- val dimKeys = eq.dimKeyValues.toSeq
- val flat = {
- for {
- i <- 0 to dimKeys.length
- comb <- dimKeys.combinations(i)
- } yield {
- ExactQualifier(eq.tq, comb.toMap) -> value
- }
- }.toMap
-
-// println("flat >>>", flat)
-
- val valuesMap = values.toMap
- val remain = (valuesMap ++ flat.map { case (k, v) =>
- k -> (valuesMap(k) - v)
- }).filter(_._2 > 0).toList
-
-// println("remain >>>", remain)
-
- if (remain.isEmpty) {
- List((eq, value))
- } else {
- (eq, value) :: filter(remain)
- }
- }
-
- def produce(policy: Counter, exactRdd: RDD[(ExactKeyTrait, ExactValueMap)]): Unit = {
- exactRdd.mapPartitions { part =>
- for {
- (key, values) <- part
- item <- valueToEtlItem(policy, key, values)
- } yield {
- item
- }
- }.foreachPartition { part =>
- val m = MutableHashMap.empty[Int, mutable.MutableList[CounterEtlItem]]
- part.foreach { item =>
- val k = getPartKey(item.item, 20)
- val values = m.getOrElse(k, mutable.MutableList.empty[CounterEtlItem])
- values += item
- m.update(k, values)
- }
- m.foreach { case (k, v) =>
- v.map(_.toKafkaMessage).grouped(1000).foreach { grouped =>
-// println(grouped)
- producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_COUNTER, null, k, grouped.mkString("\n")))
- }
- }
- }
- }
-
- def rddToExactRdd(policy: Counter, date: String, rdd: RDD[String]): RDD[(ExactKeyTrait, ExactValueMap)] = {
- val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
- val fromTs = dateFormat.parse(date).getTime
- val toTs = fromTs + 23 * 60 * 60 * 1000
-
- rdd.mapPartitions { part =>
- val exactCounter = policy.version match {
- case VERSION_1 => new ExactCounter(S2ConfigFactory.config, new ExactStorageHBase(S2ConfigFactory.config))
- case VERSION_2 => new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
- }
-
- for {
- line <- part
- FetchedCounts(exactKey, qualifierWithCountMap) <- exactCounter.getCount(policy, line, Array(TimedQualifier.IntervalUnit.DAILY), fromTs, toTs)
- } yield {
- (exactKey, qualifierWithCountMap)
- }
- }
- }
-
- lazy val className = getClass.getName.stripSuffix("$")
-
- override def run(): Unit = {
- validateArgument("service", "action", "date", "file", "op")
- DBModel.initialize(S2ConfigFactory.config)
-
- val (service, action, date, file, op) = (args(0), args(1), args(2), args(3), args(4))
- val conf = sparkConf(s"$className: $service.$action")
-
- val ctx = new SparkContext(conf)
-
- val rdd = ctx.textFile(file, 20)
-
- val counterModel = new CounterModel(S2ConfigFactory.config)
-
- val policy = counterModel.findByServiceAction(service, action).get
- val exactRdd = rddToExactRdd(policy, date, rdd)
- produce(policy, exactRdd)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlFunctions.scala b/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlFunctions.scala
deleted file mode 100644
index cefce65..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlFunctions.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-package s2.counter.core
-
-import com.kakao.s2graph.core.{Edge, Graph, GraphUtil}
-import org.apache.spark.Logging
-import play.api.libs.json._
-import s2.config.{S2ConfigFactory, StreamingConfig}
-import s2.models.CounterModel
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 3. 17..
- */
-object CounterEtlFunctions extends Logging {
- lazy val filterOps = Seq("insert", "insertBulk", "update", "increment").map(op => GraphUtil.operations(op))
- lazy val preFetchSize = StreamingConfig.PROFILE_PREFETCH_SIZE
- lazy val config = S2ConfigFactory.config
- lazy val counterModel = new CounterModel(config)
-
- def logToEdge(line: String): Option[Edge] = {
- for {
- elem <- Graph.toGraphElement(line) if elem.isInstanceOf[Edge]
- edge <- Some(elem.asInstanceOf[Edge]).filter { x =>
- filterOps.contains(x.op)
- }
- } yield {
- edge
- }
- }
-
- def parseEdgeFormat(line: String): Option[CounterEtlItem] = {
- /**
- * 1427082276804 insert edge 19073318 52453027_93524145648511699 story_user_ch_doc_view {"doc_type" : "l", "channel_subscribing" : "y", "view_from" : "feed"}
- */
- for {
- elem <- Graph.toGraphElement(line) if elem.isInstanceOf[Edge]
- edge <- Some(elem.asInstanceOf[Edge]).filter { x =>
- filterOps.contains(x.op)
- }
- } yield {
- val label = edge.label
- val labelName = label.label
- val tgtService = label.tgtColumn.service.serviceName
- val tgtId = edge.tgtVertex.innerId.toString()
- val srcId = edge.srcVertex.innerId.toString()
-
- // make empty property if no exist edge property
- val dimension = Json.parse(Some(GraphUtil.split(line)).filter(_.length >= 7).map(_(6)).getOrElse("{}"))
- val bucketKeys = Seq("_from")
- val bucketKeyValues = {
- for {
- variable <- bucketKeys
- } yield {
- val jsValue = variable match {
- case "_from" => JsString(srcId)
- case s => dimension \ s
- }
- s"[[$variable]]" -> jsValue
- }
- }
- val property = Json.toJson(bucketKeyValues :+ ("value" -> JsString("1")) toMap)
-// val property = Json.toJson(Map("_from" -> srcId, "_to" -> tgtId, "value" -> "1"))
-
- CounterEtlItem(edge.ts, tgtService, labelName, tgtId, dimension, property)
- }
- }
-
- def parseEdgeFormat(lines: List[String]): List[CounterEtlItem] = {
- for {
- line <- lines
- item <- parseEdgeFormat(line)
- } yield {
- item
- }
- }
-
- def checkPolicyAndMergeDimension(service: String, action: String, items: List[CounterEtlItem]): List[CounterEtlItem] = {
- counterModel.findByServiceAction(service, action).map { policy =>
- if (policy.useProfile) {
- policy.bucketImpId match {
- case Some(_) => DimensionProps.mergeDimension(policy, items)
- case None => Nil
- }
- } else {
- items
- }
- }.getOrElse(Nil)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlItem.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlItem.scala b/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlItem.scala
deleted file mode 100644
index 1b0f3cd..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/core/CounterEtlItem.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-package s2.counter.core
-
-import org.slf4j.LoggerFactory
-import play.api.libs.json._
-import s2.util.UnitConverter
-
-import scala.util.{Failure, Success, Try}
-
-/**
-* Created by hsleep(honeysleep@gmail.com) on 15. 10. 6..
-*/
-case class CounterEtlItem(ts: Long, service: String, action: String, item: String, dimension: JsValue, property: JsValue, useProfile: Boolean = false) {
- def toKafkaMessage: String = {
- s"$ts\t$service\t$action\t$item\t${dimension.toString()}\t${property.toString()}"
- }
-
- lazy val value = {
- property \ "value" match {
- case JsNumber(n) => n.longValue()
- case JsString(s) => s.toLong
- case _: JsUndefined => 1L
- case _ => throw new Exception("wrong type")
- }
- }
-}
-
-object CounterEtlItem {
- val log = LoggerFactory.getLogger(this.getClass)
-
- def apply(line: String): Option[CounterEtlItem] = {
- Try {
- val Array(ts, service, action, item, dimension, property) = line.split('\t')
- CounterEtlItem(UnitConverter.toMillis(ts.toLong), service, action, item, Json.parse(dimension), Json.parse(property))
- } match {
- case Success(item) =>
- Some(item)
- case Failure(ex) =>
- log.error(">>> failed")
- log.error(s"${ex.toString}: $line")
- None
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
deleted file mode 100644
index a36b55f..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
+++ /dev/null
@@ -1,446 +0,0 @@
-package s2.counter.core
-
-import com.kakao.s2graph.core.GraphUtil
-import kafka.producer.KeyedMessage
-import org.apache.spark.rdd.RDD
-import org.apache.spark.{Accumulable, Logging}
-import play.api.libs.json.{JsString, JsNumber, JsValue, Json}
-import s2.config.{S2ConfigFactory, StreamingConfig}
-import s2.counter.TrxLog
-import s2.counter.core.ExactCounter.ExactValueMap
-import s2.counter.core.RankingCounter.RankingValueMap
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core.v2.{ExactStorageGraph, RankingStorageGraph}
-import s2.models.{Counter, DBModel, DefaultCounterModel}
-import s2.spark.WithKafka
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-import scala.util.Try
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 10. 6..
- */
-object CounterFunctions extends Logging with WithKafka {
- import scala.concurrent.ExecutionContext.Implicits.global
-
- private val K_MAX = 500
-
- val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
- val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
-
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- type HashMapAccumulable = Accumulable[MutableHashMap[String, Long], (String, Long)]
-
- val initialize = {
- logInfo("initialize CounterFunctions")
- DBModel.initialize(S2ConfigFactory.config)
- true
- }
-
- def getCountValue(policy: Counter, item: CounterEtlItem): ExactValueMap = {
- for {
- dimKeys <- policy.dimensionList
- dimValues <- getDimensionValues(item.dimension, dimKeys).toSeq
- eq <- ExactQualifier.getQualifiers(policy.intervals.map(IntervalUnit.withName), item.ts, dimKeys.zip(dimValues).toMap)
- } yield {
- eq -> item.value
- }
- }.toMap
-
- def getDimensionValues(dimension: JsValue, keys: Array[String]): Option[Array[String]] = {
- Try {
- for {
- k <- keys
- jsValue = dimension \ k
- } yield {
- jsValue match {
- case JsNumber(n) => n.toString()
- case JsString(s) => s
- case _ => throw new Exception()
- }
- }
- }.toOption
- }
-
- def exactMapper(item: CounterEtlItem): Option[(ExactKeyTrait, ExactValueMap)] = {
- DefaultCounterModel.findByServiceAction(item.service, item.action).map { policy =>
- (ExactKey(policy, item.item, checkItemType = true), getCountValue(policy, item))
- }
- }
-
- def rankingMapper(row: ItemRankingRow): Seq[(RankingKey, RankingValueMap)] = {
- // normal ranking
- for {
- (eq, rv) <- row.value
- } yield {
- (RankingKey(row.key.policyId, row.key.version, eq), Map(row.key.itemKey -> rv))
- }
- }.toSeq
-
- def logToRankValue(log: TrxLog): Option[(ExactKeyTrait, Map[ExactQualifier, RankingValue])] = {
- DefaultCounterModel.findById(log.policyId).map { policy =>
- val key = ExactKey(policy, log.item, checkItemType = false)
- val value = {
- for {
- result <- log.results
- } yield {
- ExactQualifier(TimedQualifier(result.interval, result.ts), result.dimension) -> RankingValue(result.result, result.value)
- }
- }.toMap
- key -> value
- }
- }
-
- def reduceValue[T, U](op: (U, U) => U, default: U)(m1: Map[T, U], m2: Map[T, U]): Map[T, U] = {
- m1 ++ m2.map { case (k, v) =>
- k -> op(m1.getOrElse(k, default), v)
- }
- }
-
- def makeExactRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[(ExactKeyTrait, ExactValueMap)] = {
- rdd.mapPartitions { part =>
- assert(initialize)
- for {
- (k, v) <- part
- line <- GraphUtil.parseString(v)
- item <- CounterEtlItem(line).toSeq
- ev <- exactMapper(item).toSeq
- } yield {
- ev
- }
- }.reduceByKey(reduceValue[ExactQualifier, Long](_ + _, 0L)(_, _), numPartitions)
- }
-
- def makeRankingRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
- val logRdd = makeTrxLogRdd(rdd, numPartitions)
- makeRankingRddFromTrxLog(logRdd, numPartitions)
- }
-
- def makeRankingRddFromTrxLog(rdd: RDD[TrxLog], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
- val itemRankingRdd = makeItemRankingRdd(rdd, numPartitions).cache()
- try {
- rankingCount(itemRankingRdd, numPartitions) union
- rateRankingCount(itemRankingRdd, numPartitions) union
- trendRankingCount(itemRankingRdd, numPartitions) coalesce numPartitions
- } finally {
- itemRankingRdd.unpersist(false)
- }
- }
-
- def makeTrxLogRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[TrxLog] = {
- rdd.mapPartitions { part =>
- assert(initialize)
- for {
- (k, v) <- part
- line <- GraphUtil.parseString(v)
- trxLog = Json.parse(line).as[TrxLog] if trxLog.success
- } yield {
- trxLog
- }
- }
- }
-
- def rankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
- rdd.mapPartitions { part =>
- for {
- row <- part
- rv <- rankingMapper(row)
- } yield {
- rv
- }
- }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
- }
-
- case class ItemRankingRow(key: ExactKeyTrait, value: Map[ExactQualifier, RankingValue])
-
- def makeItemRankingRdd(rdd: RDD[TrxLog], numPartitions: Int): RDD[ItemRankingRow] = {
- rdd.mapPartitions { part =>
- for {
- log <- part
- rv <- logToRankValue(log)
- } yield {
- rv
- }
- }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions).mapPartitions { part =>
- for {
- (key, value) <- part
- } yield {
- ItemRankingRow(key, value)
- }
- }
- }
-
- def mapTrendRankingValue(rows: Seq[ItemRankingRow]): Seq[(ExactKeyTrait, Map[ExactQualifier, RateRankingValue])] = {
- for {
- row <- rows
- trendPolicy <- DefaultCounterModel.findByTrendActionId(row.key.policyId)
- } yield {
- val key = ExactKey(trendPolicy, row.key.itemKey, checkItemType = false)
- val value = row.value.filter { case (eq, rv) =>
- // eq filter by rate policy dimension
- trendPolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
- }.map { case (eq, rv) =>
- eq -> RateRankingValue(rv.score, -1)
- }
- (key, value)
- }
- }
-
- def mapRateRankingValue(rows: Seq[ItemRankingRow]): Seq[(ExactKeyTrait, Map[ExactQualifier, RateRankingValue])] = {
- val actionPart = {
- for {
- row <- rows
- ratePolicy <- DefaultCounterModel.findByRateActionId(row.key.policyId)
- } yield {
- val key = ExactKey(ratePolicy, row.key.itemKey, checkItemType = false)
- val value = row.value.filter { case (eq, rv) =>
- // eq filter by rate policy dimension
- ratePolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
- }.map { case (eq, rv) =>
- eq -> RateRankingValue(rv.score, -1)
- }
- (key, value)
- }
- }
-
- val basePart = {
- for {
- row <- rows
- ratePolicy <- DefaultCounterModel.findByRateBaseId(row.key.policyId)
- } yield {
- val key = ExactKey(ratePolicy, row.key.itemKey, checkItemType = false)
- val value = row.value.filter { case (eq, rv) =>
- // eq filter by rate policy dimension
- ratePolicy.dimensionSet.exists { dimSet => dimSet == eq.dimKeyValues.keys }
- }.map { case (eq, rv) =>
- eq -> RateRankingValue(-1, rv.score)
- }
- (key, value)
- }
- }
-
- actionPart ++ basePart
- }
-
- def trendRankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
- rdd.mapPartitions { part =>
- mapTrendRankingValue(part.toSeq) toIterator
- }.reduceByKey(reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(_, _), numPartitions).mapPartitions { part =>
- val missingByPolicy = {
- for {
- (key, value) <- part.toSeq
- trendPolicy <- DefaultCounterModel.findById(key.policyId).toSeq
- actionId <- trendPolicy.rateActionId.toSeq
- actionPolicy <- DefaultCounterModel.findById(actionId).toSeq
- } yield {
- // filter total eq
- val missingQualifiersWithRRV = value.filterKeys { eq => eq.tq.q != IntervalUnit.TOTAL }
- (actionPolicy, key, missingQualifiersWithRRV)
- }
- }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
-
- val filled = {
- for {
- (policy, missing) <- missingByPolicy.toSeq
- keyWithPast = exactCounter.getPastCounts(policy, missing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
- (key, current) <- missing
- } yield {
- val past = keyWithPast.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
- val base = past.mapValues(l => RateRankingValue(-1, l))
-// log.warn(s"trend: $policy $key -> $current $base")
- key -> reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, base)
- }
- }
-
-// filled.foreach(println)
-
- {
- // filter by rate threshold
- for {
- (key, value) <- filled
- ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
- (eq, rrv) <- value if rrv.baseScore >= ratePolicy.rateThreshold.getOrElse(Int.MinValue)
- } yield {
- (RankingKey(key.policyId, key.version, eq), Map(key.itemKey -> rrv.rankingValue))
- }
- } toIterator
- }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
- }
-
- def rateRankingCount(rdd: RDD[ItemRankingRow], numPartitions: Int): RDD[(RankingKey, RankingValueMap)] = {
- rdd.mapPartitions { part =>
- mapRateRankingValue(part.toSeq) toIterator
- }.reduceByKey(reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(_, _), numPartitions).mapPartitions { part =>
- val seq = part.toSeq
-// seq.foreach(x => println(s"item ranking row>> $x"))
-
- // find and evaluate action value is -1
- val actionMissingByPolicy = {
- for {
- (key, value) <- seq if value.exists { case (eq, rrv) => rrv.actionScore == -1 }
- ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
- actionId <- ratePolicy.rateActionId.toSeq
- actionPolicy <- DefaultCounterModel.findById(actionId)
- } yield {
- (actionPolicy, key, value.filter { case (eq, rrv) => rrv.actionScore == -1 })
- }
- }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
-
- val actionFilled = {
- for {
- (actionPolicy, actionMissing) <- actionMissingByPolicy.toSeq
- keyWithRelated = exactCounter.getRelatedCounts(actionPolicy, actionMissing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
- (key, current) <- actionMissing
- } yield {
- val related = keyWithRelated.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
- val found = related.mapValues(l => RateRankingValue(l, -1))
- val filled = reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, found)
-// log.warn(s"action: $key -> $found $filled")
- key -> filled
- }
- }
-
-// actionFilled.foreach(x => println(s"action filled>> $x"))
-
- // find and evaluate base value is -1
- val baseMissingByPolicy = {
- for {
- (key, value) <- seq if value.exists { case (eq, rrv) => rrv.baseScore == -1 }
- ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
- baseId <- ratePolicy.rateBaseId.toSeq
- basePolicy <- DefaultCounterModel.findById(baseId)
- } yield {
- (basePolicy, key, value.filter { case (eq, rrv) => rrv.baseScore == -1 })
- }
- }.groupBy(_._1).mapValues(seq => seq.map(x => (x._2, x._3)))
-
- val baseFilled = {
- for {
- (basePolicy, baseMissing) <- baseMissingByPolicy.toSeq
- keyWithRelated = exactCounter.getRelatedCounts(basePolicy, baseMissing.map { case (k, v) => k.itemKey -> v.keys.toSeq })
- (key, current) <- baseMissing
- } yield {
- val related = keyWithRelated.getOrElse(key.itemKey, Map.empty[ExactQualifier, Long])
- val found = related.mapValues(l => RateRankingValue(-1, l))
- val filled = reduceValue(RateRankingValue.reduce, RateRankingValue(-1, -1))(current, found)
-// log.warn(s"base: $basePolicy $key -> $found $filled")
- key -> filled
- }
- }
-
-// baseFilled.foreach(x => println(s"base filled>> $x"))
-
- val alreadyFilled = {
- for {
- (key, value) <- seq if value.exists { case (eq, rrv) => rrv.actionScore != -1 && rrv.baseScore != -1 }
- } yield {
- key -> value.filter { case (eq, rrv) => rrv.actionScore != -1 && rrv.baseScore != -1 }
- }
- }
-
- val rtn = {
- // filter by rate threshold
- for {
- (key, value) <- actionFilled ++ baseFilled ++ alreadyFilled
- ratePolicy <- DefaultCounterModel.findById(key.policyId).toSeq
- (eq, rrv) <- value if rrv.baseScore >= ratePolicy.rateThreshold.getOrElse(Int.MinValue)
- } yield {
- (RankingKey(key.policyId, key.version, eq), Map(key.itemKey -> rrv.rankingValue))
- }
- }
- rtn.toIterator
- }.reduceByKey(reduceValue(RankingValue.reduce, RankingValue(0, 0))(_, _), numPartitions)
- }
-
- def insertBlobValue(keys: Seq[BlobExactKey], acc: HashMapAccumulable): Unit = {
- val keyByPolicy = {
- for {
- key <- keys
- policy <- DefaultCounterModel.findById(key.policyId)
- } yield {
- (policy, key)
- }
- }.groupBy(_._1).mapValues(values => values.map(_._2))
-
- for {
- (policy, allKeys) <- keyByPolicy
- keys <- allKeys.grouped(10)
- success <- exactCounter.insertBlobValue(policy, keys)
- } yield {
- success match {
- case true => acc += ("BLOB", 1)
- case false => acc += ("BLOBFailed", 1)
- }
- }
- }
-
- def updateExactCounter(counts: Seq[(ExactKeyTrait, ExactValueMap)], acc: HashMapAccumulable): Seq[TrxLog] = {
- val countsByPolicy = {
- for {
- (key, count) <- counts
- policy <- DefaultCounterModel.findById(key.policyId)
- } yield {
- (policy, (key, count))
- }
- }.groupBy { case (policy, _) => policy }.mapValues(values => values.map(_._2))
-
- for {
- (policy, allCounts) <- countsByPolicy
- counts <- allCounts.grouped(10)
- trxLog <- exactCounter.updateCount(policy, counts)
- } yield {
- trxLog.success match {
- case true => acc += (s"ExactV${policy.version}", 1)
- case false => acc += (s"ExactFailedV${policy.version}", 1)
- }
- trxLog
- }
- }.toSeq
-
- def exactCountFromEtl(rdd: RDD[CounterEtlItem], numPartitions: Int): RDD[(ExactKeyTrait, ExactValueMap)] = {
- rdd.mapPartitions { part =>
- for {
- item <- part
- ev <- exactMapper(item).toSeq
- } yield {
- ev
- }
- }.reduceByKey(reduceValue[ExactQualifier, Long](_ + _, 0L)(_, _), numPartitions)
- }
-
- def updateRankingCounter(values: TraversableOnce[(RankingKey, RankingValueMap)], acc: HashMapAccumulable): Unit = {
- assert(initialize)
- val valuesByPolicy = {
- for {
- (key, value) <- values.toSeq
- policy <- DefaultCounterModel.findById(key.policyId)
- if policy.useRank && rankingCounter.ready(policy) // update only rank counter enabled and ready
- } yield {
- (policy, (key, value))
- }
- }.groupBy { case (policy, _) => policy }.mapValues(values => values.map(_._2))
-
- for {
- (policy, allValues) <- valuesByPolicy
- groupedValues <- allValues.grouped(10)
- } {
- rankingCounter.update(groupedValues, K_MAX)
- acc += (s"RankingV${policy.version}", groupedValues.length)
- }
- }
-
- def produceTrxLog(trxLogs: TraversableOnce[TrxLog]): Unit = {
- for {
- trxLog <- trxLogs
- } {
- val topic = trxLog.success match {
- case true => StreamingConfig.KAFKA_TOPIC_COUNTER_TRX
- case false => StreamingConfig.KAFKA_TOPIC_COUNTER_FAIL
- }
- val msg = new KeyedMessage[String, String](topic, s"${trxLog.policyId}${trxLog.item}", Json.toJson(trxLog).toString())
- producer.send(msg)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/core/DimensionProps.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/DimensionProps.scala b/s2counter_loader/src/main/scala/s2/counter/core/DimensionProps.scala
deleted file mode 100644
index 98bc750..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/core/DimensionProps.scala
+++ /dev/null
@@ -1,154 +0,0 @@
-package s2.counter.core
-
-import com.kakao.s2graph.core.mysqls.{Bucket, Experiment, Service}
-import org.apache.commons.httpclient.HttpStatus
-import org.slf4j.LoggerFactory
-import play.api.libs.json._
-import s2.config.StreamingConfig
-import s2.models.Counter
-import s2.util.{CollectionCache, CollectionCacheConfig, RetryAsync}
-
-import scala.annotation.tailrec
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration._
-import scala.concurrent.{Await, Future}
-import scala.util.Try
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 10. 6..
- */
-object DimensionProps {
- // using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
- private val client = new play.api.libs.ws.ning.NingWSClient(builder.build)
- private val log = LoggerFactory.getLogger(this.getClass)
-
- private val retryCnt = 3
-
- val cacheConfig = CollectionCacheConfig(StreamingConfig.PROFILE_CACHE_MAX_SIZE,
- StreamingConfig.PROFILE_CACHE_TTL_SECONDS,
- negativeCache = true,
- 3600 // negative ttl 1 hour
- )
- val cache: CollectionCache[Option[JsValue]] = new CollectionCache[Option[JsValue]](cacheConfig)
-
- @tailrec
- private[counter] def makeRequestBody(requestBody: String, keyValues: List[(String, String)]): String = {
- keyValues match {
- case head :: tail =>
- makeRequestBody(requestBody.replace(head._1, head._2), tail)
- case Nil => requestBody
- }
- }
-
- private[counter] def query(bucket: Bucket, item: CounterEtlItem): Future[Option[JsValue]] = {
- val keyValues = (item.dimension.as[JsObject] ++ item.property.as[JsObject] fields)
- .filter { case (key, _) => key.startsWith("[[") && key.endsWith("]]") }
- .map { case (key, jsValue) =>
- val replacement = jsValue match {
- case JsString(s) => s
- case value => value.toString()
- }
- key -> replacement
- }.toList
-
- val cacheKey = s"${bucket.impressionId}=" + keyValues.flatMap(x => Seq(x._1, x._2)).mkString("_")
-
- cache.withCacheAsync(cacheKey) {
- val retryFuture = RetryAsync(retryCnt, withSleep = false) {
- val future = bucket.httpVerb.toUpperCase match {
- case "GET" =>
- client.url(bucket.apiPath).get()
- case "POST" =>
- val newBody = makeRequestBody(bucket.requestBody, keyValues)
- client.url(bucket.apiPath).post(Json.parse(newBody))
- }
-
- future.map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val json = Json.parse(resp.body)
- for {
- results <- (json \ "results").asOpt[Seq[JsValue]]
- result <- results.headOption
- props <- (result \ "props").asOpt[JsValue]
- } yield {
- props
- }
- case _ =>
- log.error(s"${resp.body}(${resp.status}}) item: $item")
- None
- }
- }
- }
-
- // if fail to retry
- retryFuture onFailure { case t => log.error(s"${t.getMessage} item: $item") }
-
- retryFuture
- }
- }
-
- private[counter] def query(service: Service, experiment: Experiment, item: CounterEtlItem): Future[Option[JsValue]] = {
- val keyValues = (item.dimension.as[JsObject] ++ item.property.as[JsObject] fields)
- .filter { case (key, _) => key.startsWith("[[") && key.endsWith("]]") }.toMap
-
- val cacheKey = s"${experiment.name}=" + keyValues.flatMap(x => Seq(x._1, x._2)).mkString("_")
-
- cache.withCacheAsync(cacheKey) {
- val retryFuture = RetryAsync(retryCnt, withSleep = false) {
- val url = s"${StreamingConfig.GRAPH_URL}/graphs/experiment/${service.accessToken}/${experiment.name}/0"
- val future = client.url(url).post(Json.toJson(keyValues))
-
- future.map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- val json = Json.parse(resp.body)
- for {
- results <- (json \ "results").asOpt[Seq[JsValue]]
- result <- results.headOption
- props <- (result \ "props").asOpt[JsValue]
- } yield {
- props
- }
- case _ =>
- log.error(s"${resp.body}(${resp.status}}) item: $item")
- None
- }
- }
- }
-
- // if fail to retry
- retryFuture onFailure { case t => log.error(s"${t.getMessage} item: $item") }
-
- retryFuture
- }
- }
-
- def mergeDimension(policy: Counter, items: List[CounterEtlItem]): List[CounterEtlItem] = {
- for {
- impId <- policy.bucketImpId
- bucket <- Bucket.findByImpressionId(impId)
- experiment <- Experiment.findById(bucket.experimentId)
- service <- Try { Service.findById(experiment.serviceId) }.toOption
- } yield {
- val futures = {
- for {
- item <- items
- } yield {
- query(service, experiment, item).map {
- case Some(jsValue) =>
- val newDimension = item.dimension.as[JsObject] ++ jsValue.as[JsObject]
- item.copy(dimension = newDimension)
- case None => item
- }
- }
- }
- Await.result(Future.sequence(futures), 10 seconds)
- }
- }.getOrElse(items)
-
- def getCacheStatsString: String = {
- cache.getStatsString
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/stream/EtlStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/stream/EtlStreaming.scala b/s2counter_loader/src/main/scala/s2/counter/stream/EtlStreaming.scala
deleted file mode 100644
index 03c42b5..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/stream/EtlStreaming.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-package s2.counter.stream
-
-import com.kakao.s2graph.core.{Graph, GraphUtil}
-import kafka.producer.KeyedMessage
-import kafka.serializer.StringDecoder
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
-import org.apache.spark.streaming.kafka.StreamHelper
-import s2.config.{S2ConfigFactory, S2CounterConfig, StreamingConfig}
-import s2.counter.core.{CounterEtlFunctions, CounterEtlItem, DimensionProps}
-import s2.models.{CounterModel, DBModel}
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.concurrent.ExecutionContext
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 10. 6..
- */
-object EtlStreaming extends SparkApp with WithKafka {
- lazy val config = S2ConfigFactory.config
- lazy val s2Config = new S2CounterConfig(config)
- lazy val counterModel = new CounterModel(config)
- lazy val className = getClass.getName.stripSuffix("$")
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- implicit val graphEx = ExecutionContext.Implicits.global
-
- val initialize = {
- println("streaming initialize")
-// Graph(config)
- DBModel.initialize(config)
- true
- }
-
- val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_ETL)
- val strInputTopics = inputTopics.mkString(",")
- val groupId = buildKafkaGroupId(strInputTopics, "etl_to_counter")
- val kafkaParam = Map(
- "group.id" -> groupId,
- "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
- "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
- "zookeeper.connection.timeout.ms" -> "10000"
- )
- val streamHelper = StreamHelper(kafkaParam)
-
- override def run(): Unit = {
- validateArgument("interval")
- val (intervalInSec) = seconds(args(0).toLong)
-
- val conf = sparkConf(s"$strInputTopics: $className")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- /**
- * read message from etl topic and join user profile from graph and then produce whole message to counter topic
- */
- val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
-
- // etl logic
- stream.foreachRDD { (rdd, ts) =>
- rdd.foreachPartitionWithOffsetRange { case (osr, part) =>
- assert(initialize)
-
- // convert to edge format
- val items = {
- for {
- (k, v) <- part
- line <- GraphUtil.parseString(v)
- item <- CounterEtlFunctions.parseEdgeFormat(line)
- } yield {
- acc += ("Edges", 1)
- item
- }
- }
-
- // join user profile
- val joinItems = items.toList.groupBy { e =>
- (e.service, e.action)
- }.flatMap { case ((service, action), v) =>
- CounterEtlFunctions.checkPolicyAndMergeDimension(service, action, v)
- }
-
- // group by kafka partition key and send to kafka
- val m = MutableHashMap.empty[Int, mutable.MutableList[CounterEtlItem]]
- joinItems.foreach { item =>
- if (item.useProfile) {
- acc += ("ETL", 1)
- }
- val k = getPartKey(item.item, 20)
- val values: mutable.MutableList[CounterEtlItem] = m.getOrElse(k, mutable.MutableList.empty[CounterEtlItem])
- values += item
- m.update(k, values)
- }
- m.foreach { case (k, v) =>
- v.map(_.toKafkaMessage).grouped(1000).foreach { grouped =>
- acc += ("Produce", grouped.size)
- producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_COUNTER, null, k, grouped.mkString("\n")))
- }
- }
-
- streamHelper.commitConsumerOffset(osr)
- }
-
- if (ts.milliseconds / 1000 % 60 == 0) {
- log.warn(DimensionProps.getCacheStatsString)
- }
- }
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/stream/ExactCounterStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/stream/ExactCounterStreaming.scala b/s2counter_loader/src/main/scala/s2/counter/stream/ExactCounterStreaming.scala
deleted file mode 100644
index 2b8ba21..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/stream/ExactCounterStreaming.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-package s2.counter.stream
-
-import kafka.serializer.StringDecoder
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
-import org.apache.spark.streaming.kafka.{HasOffsetRanges, StreamHelper}
-import s2.config.{S2ConfigFactory, S2CounterConfig, StreamingConfig}
-import s2.counter.core.CounterFunctions
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.language.postfixOps
-
-/**
- * Streaming job for counter topic
- * Created by hsleep(honeysleep@gmail.com) on 15. 1. 15..
- */
-object ExactCounterStreaming extends SparkApp with WithKafka {
- lazy val config = S2ConfigFactory.config
- lazy val s2Config = new S2CounterConfig(config)
- lazy val className = getClass.getName.stripSuffix("$")
-
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_COUNTER)
- val strInputTopics = inputTopics.mkString(",")
- val groupId = buildKafkaGroupId(strInputTopics, "counter_v2")
- val kafkaParam = Map(
-// "auto.offset.reset" -> "smallest",
- "group.id" -> groupId,
- "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
- "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
- "zookeeper.connection.timeout.ms" -> "10000"
- )
- val streamHelper = StreamHelper(kafkaParam)
-
- override def run() = {
- validateArgument("interval", "clear")
- val (intervalInSec, clear) = (seconds(args(0).toLong), args(1).toBoolean)
-
- if (clear) {
- streamHelper.kafkaHelper.consumerGroupCleanup()
- }
-
- val conf = sparkConf(s"$strInputTopics: $className")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- implicit val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- // make stream
- val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
- stream.foreachRDD { (rdd, ts) =>
- val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
-
- val exactRDD = CounterFunctions.makeExactRdd(rdd, offsets.length)
-
- // for at-least once semantic
- exactRDD.foreachPartitionWithIndex { (i, part) =>
- // update exact counter
- val trxLogs = CounterFunctions.updateExactCounter(part.toSeq, acc)
- CounterFunctions.produceTrxLog(trxLogs)
-
- // commit offset range
- streamHelper.commitConsumerOffset(offsets(i))
- }
- }
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/stream/GraphToETLStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/stream/GraphToETLStreaming.scala b/s2counter_loader/src/main/scala/s2/counter/stream/GraphToETLStreaming.scala
deleted file mode 100644
index 39654d3..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/stream/GraphToETLStreaming.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-package s2.counter.stream
-
-import com.kakao.s2graph.core.GraphUtil
-import kafka.producer.KeyedMessage
-import kafka.serializer.StringDecoder
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
-import s2.config.{S2ConfigFactory, S2CounterConfig, StreamingConfig}
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-/**
- * can be @deprecated
- * Created by hsleep(honeysleep@gmail.com) on 15. 3. 16..
- */
-object GraphToETLStreaming extends SparkApp with WithKafka {
- lazy val config = S2ConfigFactory.config
- lazy val s2Config = new S2CounterConfig(config)
- lazy val className = getClass.getName.stripSuffix("$")
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- override def run(): Unit = {
- validateArgument("interval", "topic")
- val (intervalInSec, topic) = (seconds(args(0).toLong), args(1))
-
- val groupId = buildKafkaGroupId(topic, "graph_to_etl")
- val kafkaParam = Map(
-// "auto.offset.reset" -> "smallest",
- "group.id" -> groupId,
- "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
- "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
- "zookeeper.connection.timeout.ms" -> "10000"
- )
-
- val conf = sparkConf(s"$topic: $className")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- val acc = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- /**
- * consume graphIn topic and produce messages to etl topic
- * two purpose
- * 1. partition by target vertex id
- * 2. expand kafka partition count
- */
- val stream = getStreamHelper(kafkaParam).createStream[String, String, StringDecoder, StringDecoder](ssc, topic.split(',').toSet)
- stream.foreachRDD { rdd =>
- rdd.foreachPartitionWithOffsetRange { case (osr, part) =>
- val m = MutableHashMap.empty[Int, mutable.MutableList[String]]
- for {
- (k, v) <- part
- line <- GraphUtil.parseString(v)
- } {
- try {
- val sp = GraphUtil.split(line)
- // get partition key by target vertex id
- val partKey = getPartKey(sp(4), 20)
- val values = m.getOrElse(partKey, mutable.MutableList.empty[String])
- values += line
- m.update(partKey, values)
- } catch {
- case ex: Throwable =>
- log.error(s"$ex: $line")
- }
- }
-
- m.foreach { case (k, v) =>
- v.grouped(1000).foreach { grouped =>
- producer.send(new KeyedMessage[String, String](StreamingConfig.KAFKA_TOPIC_ETL, null, k, grouped.mkString("\n")))
- }
- }
-
- getStreamHelper(kafkaParam).commitConsumerOffset(osr)
- }
- }
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/counter/stream/RankingCounterStreaming.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/stream/RankingCounterStreaming.scala b/s2counter_loader/src/main/scala/s2/counter/stream/RankingCounterStreaming.scala
deleted file mode 100644
index 4c0b927..0000000
--- a/s2counter_loader/src/main/scala/s2/counter/stream/RankingCounterStreaming.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-package s2.counter.stream
-
-import kafka.serializer.StringDecoder
-import org.apache.spark.streaming.Durations._
-import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
-import org.apache.spark.streaming.kafka.{HasOffsetRanges, StreamHelper}
-import s2.config.{S2ConfigFactory, S2CounterConfig, StreamingConfig}
-import s2.counter.core.CounterFunctions
-import s2.spark.{HashMapParam, SparkApp, WithKafka}
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 19..
- */
-object RankingCounterStreaming extends SparkApp with WithKafka {
- lazy val config = S2ConfigFactory.config
- lazy val s2Config = new S2CounterConfig(config)
- lazy val className = getClass.getName.stripSuffix("$")
-
- lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
-
- val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_COUNTER_TRX)
- val strInputTopics = inputTopics.mkString(",")
- val groupId = buildKafkaGroupId(strInputTopics, "ranking_v2")
- val kafkaParam = Map(
-// "auto.offset.reset" -> "smallest",
- "group.id" -> groupId,
- "metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
- "zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
- "zookeeper.connection.timeout.ms" -> "10000"
- )
- val streamHelper = StreamHelper(kafkaParam)
-
- override def run() = {
- validateArgument("interval", "clear")
- val (intervalInSec, clear) = (seconds(args(0).toLong), args(1).toBoolean)
-
- if (clear) {
- streamHelper.kafkaHelper.consumerGroupCleanup()
- }
-
- val conf = sparkConf(s"$strInputTopics: $className")
- val ssc = streamingContext(conf, intervalInSec)
- val sc = ssc.sparkContext
-
- implicit val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- // make stream
- val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
- stream.foreachRDD { (rdd, ts) =>
- // for at-least once semantic
- val nextRdd = {
- CounterFunctions.makeRankingRdd(rdd, sc.defaultParallelism).foreachPartition { part =>
- // update ranking counter
- CounterFunctions.updateRankingCounter(part, acc)
- }
- rdd
- }
-
- streamHelper.commitConsumerOffsets(nextRdd.asInstanceOf[HasOffsetRanges])
-// CounterFunctions.makeRankingRdd(rdd, offsets.length).foreachPartitionWithIndex { (i, part) =>
-// // update ranking counter
-// CounterFunctions.updateRankingCounter(part, acc)
-//
-// // commit offset range
-// streamHelper.commitConsumerOffset(offsets(i))
-// }
- }
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/main/scala/s2/models/DefaultCounterModel.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/models/DefaultCounterModel.scala b/s2counter_loader/src/main/scala/s2/models/DefaultCounterModel.scala
deleted file mode 100644
index 9cbe212..0000000
--- a/s2counter_loader/src/main/scala/s2/models/DefaultCounterModel.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package s2.models
-
-import s2.config.S2ConfigFactory
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 8..
- */
-case object DefaultCounterModel extends CounterModel(S2ConfigFactory.config)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
new file mode 100644
index 0000000..14e335e
--- /dev/null
+++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
@@ -0,0 +1,33 @@
+package org.apache.s2graph.counter.loader.core
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.{Matchers, BeforeAndAfterAll, FlatSpec}
+import s2.models.DBModel
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 7. 3..
+ */
+class CounterEtlFunctionsSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
+ override def beforeAll: Unit = {
+ DBModel.initialize(ConfigFactory.load())
+ }
+
+ "CounterEtlFunctions" should "parsing log" in {
+ val data =
+ """
+ |1435107139287 insert e aaPHfITGUU0B_150212123559509 abcd test_case {"cateid":"100110102","shopid":"1","brandid":""}
+ |1435106916136 insert e Tgc00-wtjp2B_140918153515441 efgh test_case {"cateid":"101104107","shopid":"2","brandid":""}
+ """.stripMargin.trim.split('\n')
+ val items = {
+ for {
+ line <- data
+ item <- CounterEtlFunctions.parseEdgeFormat(line)
+ } yield {
+ item.action should equal("test_case")
+ item
+ }
+ }
+
+ items should have size 2
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/DimensionPropsTest.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/DimensionPropsTest.scala b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/DimensionPropsTest.scala
new file mode 100644
index 0000000..c0f1db7
--- /dev/null
+++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/DimensionPropsTest.scala
@@ -0,0 +1,46 @@
+package org.apache.s2graph.counter.loader.core
+
+import org.scalatest.{FunSuite, Matchers}
+
+import scala.collection.mutable.ListBuffer
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 2015. 10. 7..
+ */
+class DimensionPropsTest extends FunSuite with Matchers {
+ test("makeRequestBody with Seq") {
+ val requestBody =
+ """
+ |{
+ | "_from" => [[_from]]
+ |}
+ """.stripMargin
+ val requestBodyExpected =
+ """
+ |{
+ | "_from" => 1
+ |}
+ """.stripMargin
+ val requestBodyResult = DimensionProps.makeRequestBody(requestBody, Seq(("[[_from]]", "1")).toList)
+
+ requestBodyResult shouldEqual requestBodyExpected
+ }
+
+ test("makeRequestBody with ListBuffer") {
+ val requestBody =
+ """
+ |{
+ | "_from" => [[_from]]
+ |}
+ """.stripMargin
+ val requestBodyExpected =
+ """
+ |{
+ | "_from" => 1
+ |}
+ """.stripMargin
+ val requestBodyResult = DimensionProps.makeRequestBody(requestBody, ListBuffer(("[[_from]]", "1")).toList)
+
+ requestBodyResult shouldEqual requestBodyExpected
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreamingSpec.scala b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreamingSpec.scala
new file mode 100644
index 0000000..b716f86
--- /dev/null
+++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/ExactCounterStreamingSpec.scala
@@ -0,0 +1,198 @@
+package org.apache.s2graph.counter.loader.stream
+
+import org.apache.s2graph.core.{Management, GraphUtil}
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter.loader.counter.core.{CounterEtlItem, CounterFunctions}
+import org.apache.s2graph.counter.loader.models.DefaultCounterModel
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.HashMapParam
+import org.apache.spark.streaming.{Seconds, StreamingContext}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import play.api.libs.json.Json
+import CounterFunctions.HashMapAccumulable
+import s2.counter.core.TimedQualifier.IntervalUnit
+import s2.counter.core._
+import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
+import s2.helper.CounterAdmin
+import s2.models.{Counter, DBModel}
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.util.{Failure, Success}
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 19..
+ */
+class ExactCounterStreamingSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
+ private val master = "local[2]"
+ private val appName = "exact_counter_streaming"
+ private val batchDuration = Seconds(1)
+
+ private var sc: SparkContext = _
+ private var ssc: StreamingContext = _
+
+ val admin = new CounterAdmin(S2ConfigFactory.config)
+ val graphOp = new GraphOperation(S2ConfigFactory.config)
+ val s2config = new S2CounterConfig(S2ConfigFactory.config)
+
+ val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
+ val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
+
+ val service = "test"
+ val action = "test_case"
+
+ override def beforeAll(): Unit = {
+ DBModel.initialize(S2ConfigFactory.config)
+
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ ssc = new StreamingContext(conf, batchDuration)
+
+ sc = ssc.sparkContext
+
+ // create test_case label
+ Management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
+ if (Label.findByName(action, useCache = false).isEmpty) {
+ val strJs =
+ s"""
+ |{
+ | "label": "$action",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "src",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "$action",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | ],
+ | "props": [
+ | ]
+ |}
+ """.stripMargin
+ graphOp.createLabel(Json.parse(strJs))
+ }
+
+ // action
+ admin.deleteCounter(service, action).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "is_shared,relationship", useRank = true))
+ }
+
+ override def afterAll(): Unit = {
+ admin.deleteCounter(service, action)
+ if (ssc != null) {
+ ssc.stop()
+ }
+ }
+
+ "ExactCounter" should "update" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+ val data =
+ s"""
+ |1434534565675 $service $action 70362200_94013572857366866 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
+ |1434534565675 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
+ |1434534566220 $service $action 51223360_94013140590929619 {"is_shared":"false","relationship":"FE"} {"userId":"312383","userIdType":"profile_id","value":"1"}
+ |1434534566508 $service $action 63808459_94013420047377826 {"is_shared":"false","relationship":"FE"} {"userId":"21968241","userIdType":"profile_id","value":"1"}
+ |1434534566210 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"6062217","userIdType":"profile_id","value":"1"}
+ |1434534566459 $service $action 49699692_94012186431261763 {"is_shared":"false","relationship":"FE"} {"userId":"67863471","userIdType":"profile_id","value":"1"}
+ |1434534565681 $service $action 64556827_94012311028641810 {"is_shared":"false","relationship":"FE"} {"userId":"19381218","userIdType":"profile_id","value":"1"}
+ |1434534565865 $service $action 41814266_94012477588942163 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
+ |1434534565865 $service $action 66697741_94007840665633458 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
+ |1434534566142 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"11917195","userIdType":"profile_id","value":"1"}
+ |1434534566077 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"37709890","userIdType":"profile_id","value":"1"}
+ |1434534565938 $service $action 40921487_94012905738975266 {"is_shared":"false","relationship":"FE"} {"userId":"59869223","userIdType":"profile_id","value":"1"}
+ |1434534566033 $service $action 64506628_93994707216829506 {"is_shared":"false","relationship":"FE"} {"userId":"50375575","userIdType":"profile_id","value":"1"}
+ |1434534566451 $service $action 40748868_94013448321919139 {"is_shared":"false","relationship":"FE"} {"userId":"12249539","userIdType":"profile_id","value":"1"}
+ |1434534566669 $service $action 64499956_94013227717457106 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
+ |1434534566669 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
+ |1434534566318 $service $action 64774665_94012837889027027 {"is_shared":"true","relationship":"F"} {"userId":"71557816","userIdType":"profile_id","value":"1"}
+ |1434534566274 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"57931860","userIdType":"profile_id","value":"1"}
+ |1434534566659 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"19990823","userIdType":"profile_id","value":"1"}
+ |1434534566250 $service $action 70670053_93719933175630611 {"is_shared":"true","relationship":"F"} {"userId":"68897412","userIdType":"profile_id","value":"1"}
+ |1434534566402 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"15541439","userIdType":"profile_id","value":"1"}
+ |1434534566122 $service $action 48890741_94013463616012786 {"is_shared":"false","relationship":"FE"} {"userId":"48040409","userIdType":"profile_id","value":"1"}
+ |1434534566055 $service $action 64509008_94002318232678546 {"is_shared":"true","relationship":"F"} {"userId":"46532039","userIdType":"profile_id","value":"1"}
+ |1434534565994 $service $action 66644368_94009163363033795 {"is_shared":"false","relationship":"FE"} {"userId":"4143147","userIdType":"profile_id","value":"1"}
+ |1434534566448 $service $action 64587644_93938555963733954 {"is_shared":"false","relationship":"FE"} {"userId":"689042","userIdType":"profile_id","value":"1"}
+ |1434534565935 $service $action 52812511_94012009551561315 {"is_shared":"false","relationship":"FE"} {"userId":"35509692","userIdType":"profile_id","value":"1"}
+ |1434534566544 $service $action 70452048_94008573197583762 {"is_shared":"false","relationship":"FE"} {"userId":"5172421","userIdType":"profile_id","value":"1"}
+ |1434534565929 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"33556498","userIdType":"profile_id","value":"1"}
+ |1434534566358 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"8987346","userIdType":"profile_id","value":"1"}
+ |1434534566057 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"35134964","userIdType":"profile_id","value":"1"}
+ |1434534566140 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"11900315","userIdType":"profile_id","value":"1"}
+ |1434534566158 $service $action 64639374_93888330176053635 {"is_shared":"true","relationship":"F"} {"userId":"49996643","userIdType":"profile_id","value":"1"}
+ |1434534566025 $service $action 67265128_94009084771192002 {"is_shared":"false","relationship":"FE"} {"userId":"37801480","userIdType":"profile_id","value":"1"}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+ val resultRdd = CounterFunctions.makeExactRdd(rdd, 2)
+ val result = resultRdd.collect().toMap
+
+ // result.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val parsed = {
+ for {
+ line <- GraphUtil.parseString(data)
+ item <- CounterEtlItem(line).toSeq
+ ev <- CounterFunctions.exactMapper(item).toSeq
+ } yield {
+ ev
+ }
+ }
+ val parsedResult = parsed.groupBy(_._1).mapValues(values => values.map(_._2).reduce(CounterFunctions.reduceValue[ExactQualifier, Long](_ + _, 0L)))
+
+ // parsedResult.foreach { case (k, v) =>
+ // println(k, v)
+ // }
+
+ result should not be empty
+ result should equal (parsedResult)
+
+ val itemId = "46889329_94013502934177075"
+ val key = ExactKey(DefaultCounterModel.findByServiceAction(service, action).get, itemId, checkItemType = true)
+ val value = result.get(key)
+
+ value should not be empty
+ value.get.get(ExactQualifier(TimedQualifier("t", 0), Map.empty[String, String])) should equal (Some(6L))
+
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (None)
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+ resultRdd.foreachPartition { part =>
+ CounterFunctions.updateExactCounter(part.toSeq, acc)
+ }
+
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map.empty[String, String]) -> Map(ExactQualifier(TimedQualifier("t", 0), "") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("is_shared" -> "false")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.false") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"))) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "relationship.FE") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("relationship" -> Set("FE"))) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("is_shared" -> "false", "relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.relationship.false.FE") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"), "relationship" -> Set("FE"))) should be (Some(expected))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreamingSpec.scala b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreamingSpec.scala
new file mode 100644
index 0000000..2a913e5
--- /dev/null
+++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/stream/RankingCounterStreamingSpec.scala
@@ -0,0 +1,451 @@
+package org.apache.s2graph.counter.loader.stream
+
+import org.apache.s2graph.core.Management
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.counter.loader.counter.core.CounterFunctions
+import org.apache.s2graph.counter.loader.models.DefaultCounterModel
+import org.apache.s2graph.spark.config.S2ConfigFactory
+import org.apache.s2graph.spark.spark.HashMapParam
+import org.apache.spark.streaming.{Seconds, StreamingContext}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import play.api.libs.json.Json
+import CounterFunctions.HashMapAccumulable
+import s2.counter.core.TimedQualifier.IntervalUnit
+import s2.counter.core._
+import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
+import s2.helper.CounterAdmin
+import s2.models.{Counter, DBModel}
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.util.{Failure, Success}
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 15. 6. 17..
+ */
+class RankingCounterStreamingSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
+ private val master = "local[2]"
+ private val appName = "ranking_counter_streaming"
+ private val batchDuration = Seconds(1)
+
+ private var sc: SparkContext = _
+ private var ssc: StreamingContext = _
+
+ val admin = new CounterAdmin(S2ConfigFactory.config)
+ val graphOp = new GraphOperation(S2ConfigFactory.config)
+ val s2config = new S2CounterConfig(S2ConfigFactory.config)
+
+ val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
+ val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
+
+ val service = "test"
+ val action = "test_case"
+ val action_base = "test_case_base"
+ val action_rate = "test_case_rate"
+ val action_rate_threshold = "test_case_rate_threshold"
+ val action_trend = "test_case_trend"
+
+ override def beforeAll(): Unit = {
+ DBModel.initialize(S2ConfigFactory.config)
+
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ ssc = new StreamingContext(conf, batchDuration)
+
+ sc = ssc.sparkContext
+
+ admin.setupCounterOnGraph()
+
+ // create test_case label
+ Management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
+ if (Label.findByName(action, useCache = false).isEmpty) {
+ val strJs =
+ s"""
+ |{
+ | "label": "$action",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "src",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "$action",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | ],
+ | "props": [
+ | ]
+ |}
+ """.stripMargin
+ graphOp.createLabel(Json.parse(strJs))
+ }
+ if (Label.findByName(action_base, useCache = false).isEmpty) {
+ val strJs =
+ s"""
+ |{
+ | "label": "$action_base",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "src",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "$action",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | ],
+ | "props": [
+ | ]
+ |}
+ """.stripMargin
+ graphOp.createLabel(Json.parse(strJs))
+ }
+
+ // action
+ admin.deleteCounter(service, action).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+
+ // action_base
+ admin.deleteCounter(service, action_base).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action_base, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
+ val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
+
+ // action_rate
+ admin.deleteCounter(service, action_rate).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action_rate, Counter.ItemType.STRING, autoComb = true, "gender,p1", useRank = true,
+ rateActionId = Some(policy.id), rateBaseId = Some(basePolicy.id)))
+
+ // action_rate_threshold
+ admin.deleteCounter(service, action_rate_threshold).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action_rate_threshold, Counter.ItemType.STRING, autoComb = true, "gender,p1", useRank = true,
+ rateActionId = Some(policy.id), rateBaseId = Some(basePolicy.id), rateThreshold = Some(3)))
+
+ // action_trend
+ admin.deleteCounter(service, action_trend).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action_trend, Counter.ItemType.STRING, autoComb = true, "p1", useRank = true,
+ rateActionId = Some(policy.id), rateBaseId = Some(policy.id)))
+ }
+
+ override def afterAll(): Unit = {
+ admin.deleteCounter(service, action)
+ admin.deleteCounter(service, action_base)
+ admin.deleteCounter(service, action_rate)
+ admin.deleteCounter(service, action_rate_threshold)
+ admin.deleteCounter(service, action_trend)
+ if (ssc != null) {
+ ssc.stop()
+ }
+ }
+
+ "RankingCounterStreaming" should "update" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+// val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
+
+ rankingCounter.ready(policy) should equal (true)
+ val data =
+ s"""
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":3}]}
+ |{"success":false,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
+ |{"success":true,"policyId":${policy.id},"item":"3","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"3","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
+ |{"success":true,"policyId":${policy.id},"item":"4","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val result = CounterFunctions.makeRankingRdd(rdd, 2).collect().toMap
+
+ // result.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ result should not be empty
+ val rankKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
+ result should contain (rankKey -> Map(
+ "1" -> RankingValue(3, 1),
+ "3" -> RankingValue(2, 2),
+ "4" -> RankingValue(1, 1)
+ ))
+
+ val key = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
+ val value = result.get(key)
+
+ value should not be empty
+ value.get.get("1").get should equal (RankingValue(3, 1))
+ value.get.get("2") shouldBe empty
+ value.get.get("3").get should equal (RankingValue(2, 2))
+
+ rankingCounter.ready(policy) should equal (true)
+
+ // delete, update and get
+ rankingCounter.delete(key)
+ Thread.sleep(1000)
+ CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
+ Thread.sleep(1000)
+ val rst = rankingCounter.getTopK(key)
+
+ rst should not be empty
+// rst.get.totalScore should equal(4f)
+ rst.get.values should contain allOf(("3", 2d), ("4", 1d), ("1", 3d))
+ }
+
+// "rate by base" >> {
+// val data =
+// """
+// |{"success":true,"policyId":42,"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
+// """.stripMargin.trim
+// val rdd = sc.parallelize(Seq(("", data)))
+//
+// val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2).collect()
+// trxLogRdd.foreach { log =>
+// CounterFunctions.rateBaseRankingMapper(log) must not be empty
+// }
+//
+// true must_== true
+// }
+
+ it should "update rate ranking counter" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+ val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
+ val ratePolicy = DefaultCounterModel.findByServiceAction(service, action_rate).get
+
+ // update base policy
+ val eq = ExactQualifier(TimedQualifier("M", 1433084400000l), "")
+ val exactKey = ExactKey(basePolicy, "1", checkItemType = true)
+
+ // check base item count
+ exactCounter.updateCount(basePolicy, Seq(
+ (exactKey, Map(eq -> 2l))
+ ))
+ Thread.sleep(1000)
+
+ // direct get
+ val baseCount = exactCounter.getCount(basePolicy, "1", Seq(IntervalUnit.MONTHLY), 1433084400000l, 1433084400000l, Map.empty[String, Set[String]])
+ baseCount should not be empty
+ baseCount.get should equal (FetchedCountsGrouped(exactKey, Map(
+ (eq.tq.q, Map.empty[String, String]) -> Map(eq-> 2l)
+ )))
+
+ // related get
+ val relatedCount = exactCounter.getRelatedCounts(basePolicy, Seq("1" -> Seq(eq)))
+ relatedCount should not be empty
+ relatedCount.get("1") should not be empty
+ relatedCount.get("1").get should equal (Map(eq -> 2l))
+
+ val data =
+ s"""
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
+ |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":2,"result":4}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"p1.1","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"p1.1","ts":1433084400000,"value":2,"result":4}]}
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
+ trxLogRdd.count() should equal (data.trim.split('\n').length)
+
+ val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
+ itemRankingRdd.foreach(println)
+
+ val result = CounterFunctions.rateRankingCount(itemRankingRdd, 2).collect().toMap.filterKeys(key => key.policyId == ratePolicy.id)
+ result.foreach(println)
+ result should have size 3
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ // rate ranking
+ val key = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
+ val value = result.get(key)
+
+// println(key, value)
+
+ value should not be empty
+ value.get.get("1") should not be empty
+ value.get.get("1").get should equal (RankingValue(1, 0))
+ value.get.get("2").get should equal (RankingValue(0.25, 0))
+
+ val key2 = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), "p1.1"))
+ val value2 = result.get(key2)
+
+// println(key2, value2)
+
+ val values = value.map(v => (key, v)).toSeq ++ value2.map(v => (key2, v)).toSeq
+ println(s"values: $values")
+
+ // delete, update and get
+ rankingCounter.delete(key)
+ rankingCounter.delete(key2)
+ Thread.sleep(1000)
+ CounterFunctions.updateRankingCounter(values, acc)
+ // for update graph
+ Thread.sleep(1000)
+
+ val rst = rankingCounter.getTopK(key)
+ rst should not be empty
+ rst.get.values should equal (Seq(("1", 1d), ("2", 0.25d)))
+
+ val rst2 = rankingCounter.getTopK(key2)
+ rst2 should not be empty
+ rst2.get.values should equal (Seq(("2", 0.25d)))
+ }
+
+ it should "update rate ranking counter with threshold" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+ val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
+ val ratePolicy = DefaultCounterModel.findByServiceAction(service, action_rate_threshold).get
+
+ val data =
+ s"""
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
+ |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":2,"result":4}]}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
+ trxLogRdd.count() should equal (data.trim.split('\n').length)
+
+ val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
+ itemRankingRdd.foreach(println)
+
+ val result = CounterFunctions.rateRankingCount(itemRankingRdd, 2).collect().toMap.filterKeys(key => key.policyId == ratePolicy.id)
+ result.foreach(println)
+ result should have size 2
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ // rate ranking
+ val key = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
+ val value = result.get(key)
+
+ value should not be empty
+ value.get.get("1") should be (None)
+ value.get.get("2").get should equal (RankingValue(0.25, 0))
+
+ // delete, update and get
+ rankingCounter.delete(key)
+ Thread.sleep(1000)
+ CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
+ Thread.sleep(1000)
+ val rst = rankingCounter.getTopK(key)
+
+ rst should not be empty
+ rst.get.values should equal (Seq(("2", 0.25d)))
+ }
+
+ it should "update trend ranking counter" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+ val trendPolicy = DefaultCounterModel.findByServiceAction(service, action_trend).get
+
+ val exactKey1 = ExactKey(policy, "1", checkItemType = true)
+ val exactKey2 = ExactKey(policy, "2", checkItemType = true)
+ // update old key value
+ val tq1 = TimedQualifier("M", 1435676400000l)
+ val tq2 = TimedQualifier("M", 1427814000000l)
+ exactCounter.updateCount(policy, Seq(
+ exactKey1 -> Map(ExactQualifier(tq1.add(-1), "") -> 1l, ExactQualifier(tq2.add(-1), "") -> 92l)
+ ))
+ val eq1 = ExactQualifier(tq1, "")
+ val eq2 = ExactQualifier(tq2, "")
+
+ val oldCount = exactCounter.getPastCounts(policy, Seq("1" -> Seq(eq1, eq2), "2" -> Seq(eq1, eq1.copy(dimension = "gender.M"))))
+ oldCount should not be empty
+ oldCount.get("1").get should equal(Map(eq1 -> 1l, eq2 -> 92l))
+ oldCount.get("2") should be (None)
+
+ val data =
+ s"""
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":2}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1435676400000,"value":1,"result":1}]}
+ |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1427814000000,"value":1,"result":92}]}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
+ trxLogRdd.count() should equal (data.trim.split('\n').length)
+
+ val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
+ itemRankingRdd.foreach(println)
+
+ val result = CounterFunctions.trendRankingCount(itemRankingRdd, 2).collect().toMap
+ result.foreach(println)
+ // dimension gender.M is ignored, because gender is not defined dimension in trend policy.
+ result should have size 2
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+
+ // trend ranking
+ val key = RankingKey(trendPolicy.id, 2, ExactQualifier(TimedQualifier("M", 1435676400000L), ""))
+ val value = result.get(key)
+
+ value should not be empty
+ value.get.get("1").get should equal (RankingValue(2, 0))
+ value.get.get("2").get should equal (RankingValue(1, 0))
+
+ val key2 = RankingKey(trendPolicy.id, 2, ExactQualifier(TimedQualifier("M", 1427814000000L), ""))
+ val value2 = result.get(key2)
+
+ value2 should not be empty
+ value2.get.get("1").get should equal (RankingValue(1, 0))
+
+ // delete, update and get
+ rankingCounter.delete(key)
+ Thread.sleep(1000)
+ CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
+ Thread.sleep(1000)
+ val rst = rankingCounter.getTopK(key)
+
+ rst should not be empty
+ rst.get.values should equal (Seq("1" -> 2, "2" -> 1))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/s2/counter/core/CounterEtlFunctionsSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/s2/counter/core/CounterEtlFunctionsSpec.scala b/s2counter_loader/src/test/scala/s2/counter/core/CounterEtlFunctionsSpec.scala
deleted file mode 100644
index 520b30f..0000000
--- a/s2counter_loader/src/test/scala/s2/counter/core/CounterEtlFunctionsSpec.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package s2.counter.core
-
-import com.typesafe.config.ConfigFactory
-import org.scalatest.{Matchers, BeforeAndAfterAll, FlatSpec}
-import s2.models.DBModel
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 7. 3..
- */
-class CounterEtlFunctionsSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
- override def beforeAll: Unit = {
- DBModel.initialize(ConfigFactory.load())
- }
-
- "CounterEtlFunctions" should "parsing log" in {
- val data =
- """
- |1435107139287 insert e aaPHfITGUU0B_150212123559509 abcd test_case {"cateid":"100110102","shopid":"1","brandid":""}
- |1435106916136 insert e Tgc00-wtjp2B_140918153515441 efgh test_case {"cateid":"101104107","shopid":"2","brandid":""}
- """.stripMargin.trim.split('\n')
- val items = {
- for {
- line <- data
- item <- CounterEtlFunctions.parseEdgeFormat(line)
- } yield {
- item.action should equal("test_case")
- item
- }
- }
-
- items should have size 2
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/s2/counter/core/DimensionPropsTest.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/s2/counter/core/DimensionPropsTest.scala b/s2counter_loader/src/test/scala/s2/counter/core/DimensionPropsTest.scala
deleted file mode 100644
index b658d05..0000000
--- a/s2counter_loader/src/test/scala/s2/counter/core/DimensionPropsTest.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-package s2.counter.core
-
-import org.scalatest.{FunSuite, Matchers}
-
-import scala.collection.mutable.ListBuffer
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 10. 7..
- */
-class DimensionPropsTest extends FunSuite with Matchers {
- test("makeRequestBody with Seq") {
- val requestBody =
- """
- |{
- | "_from" => [[_from]]
- |}
- """.stripMargin
- val requestBodyExpected =
- """
- |{
- | "_from" => 1
- |}
- """.stripMargin
- val requestBodyResult = DimensionProps.makeRequestBody(requestBody, Seq(("[[_from]]", "1")).toList)
-
- requestBodyResult shouldEqual requestBodyExpected
- }
-
- test("makeRequestBody with ListBuffer") {
- val requestBody =
- """
- |{
- | "_from" => [[_from]]
- |}
- """.stripMargin
- val requestBodyExpected =
- """
- |{
- | "_from" => 1
- |}
- """.stripMargin
- val requestBodyResult = DimensionProps.makeRequestBody(requestBody, ListBuffer(("[[_from]]", "1")).toList)
-
- requestBodyResult shouldEqual requestBodyExpected
- }
-}
[04/27] incubator-s2graph git commit: [S2GRAPH-57]: Change package
names into org.apahce.s2graph.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala b/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
deleted file mode 100644
index 4970399..0000000
--- a/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
+++ /dev/null
@@ -1,196 +0,0 @@
-package s2.counter.stream
-
-import com.kakao.s2graph.core.GraphUtil
-import com.kakao.s2graph.core.mysqls.Label
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{SparkConf, SparkContext}
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-import play.api.libs.json.Json
-import s2.config.{S2ConfigFactory, S2CounterConfig}
-import s2.counter.core.CounterFunctions.HashMapAccumulable
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
-import s2.helper.CounterAdmin
-import s2.models.{Counter, DBModel, DefaultCounterModel}
-import s2.spark.HashMapParam
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.util.{Failure, Success}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 19..
- */
-class ExactCounterStreamingSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
- private val master = "local[2]"
- private val appName = "exact_counter_streaming"
- private val batchDuration = Seconds(1)
-
- private var sc: SparkContext = _
- private var ssc: StreamingContext = _
-
- val admin = new CounterAdmin(S2ConfigFactory.config)
- val graphOp = new GraphOperation(S2ConfigFactory.config)
- val s2config = new S2CounterConfig(S2ConfigFactory.config)
-
- val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
- val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
-
- val service = "test"
- val action = "test_case"
-
- override def beforeAll(): Unit = {
- DBModel.initialize(S2ConfigFactory.config)
-
- val conf = new SparkConf()
- .setMaster(master)
- .setAppName(appName)
-
- ssc = new StreamingContext(conf, batchDuration)
-
- sc = ssc.sparkContext
-
- // create test_case label
- com.kakao.s2graph.core.Management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
- if (Label.findByName(action, useCache = false).isEmpty) {
- val strJs =
- s"""
- |{
- | "label": "$action",
- | "srcServiceName": "$service",
- | "srcColumnName": "src",
- | "srcColumnType": "string",
- | "tgtServiceName": "$service",
- | "tgtColumnName": "$action",
- | "tgtColumnType": "string",
- | "indices": [
- | ],
- | "props": [
- | ]
- |}
- """.stripMargin
- graphOp.createLabel(Json.parse(strJs))
- }
-
- // action
- admin.deleteCounter(service, action).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "is_shared,relationship", useRank = true))
- }
-
- override def afterAll(): Unit = {
- admin.deleteCounter(service, action)
- if (ssc != null) {
- ssc.stop()
- }
- }
-
- "ExactCounter" should "update" in {
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
- val data =
- s"""
- |1434534565675 $service $action 70362200_94013572857366866 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
- |1434534565675 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
- |1434534566220 $service $action 51223360_94013140590929619 {"is_shared":"false","relationship":"FE"} {"userId":"312383","userIdType":"profile_id","value":"1"}
- |1434534566508 $service $action 63808459_94013420047377826 {"is_shared":"false","relationship":"FE"} {"userId":"21968241","userIdType":"profile_id","value":"1"}
- |1434534566210 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"6062217","userIdType":"profile_id","value":"1"}
- |1434534566459 $service $action 49699692_94012186431261763 {"is_shared":"false","relationship":"FE"} {"userId":"67863471","userIdType":"profile_id","value":"1"}
- |1434534565681 $service $action 64556827_94012311028641810 {"is_shared":"false","relationship":"FE"} {"userId":"19381218","userIdType":"profile_id","value":"1"}
- |1434534565865 $service $action 41814266_94012477588942163 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
- |1434534565865 $service $action 66697741_94007840665633458 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
- |1434534566142 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"11917195","userIdType":"profile_id","value":"1"}
- |1434534566077 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"37709890","userIdType":"profile_id","value":"1"}
- |1434534565938 $service $action 40921487_94012905738975266 {"is_shared":"false","relationship":"FE"} {"userId":"59869223","userIdType":"profile_id","value":"1"}
- |1434534566033 $service $action 64506628_93994707216829506 {"is_shared":"false","relationship":"FE"} {"userId":"50375575","userIdType":"profile_id","value":"1"}
- |1434534566451 $service $action 40748868_94013448321919139 {"is_shared":"false","relationship":"FE"} {"userId":"12249539","userIdType":"profile_id","value":"1"}
- |1434534566669 $service $action 64499956_94013227717457106 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
- |1434534566669 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
- |1434534566318 $service $action 64774665_94012837889027027 {"is_shared":"true","relationship":"F"} {"userId":"71557816","userIdType":"profile_id","value":"1"}
- |1434534566274 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"57931860","userIdType":"profile_id","value":"1"}
- |1434534566659 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"19990823","userIdType":"profile_id","value":"1"}
- |1434534566250 $service $action 70670053_93719933175630611 {"is_shared":"true","relationship":"F"} {"userId":"68897412","userIdType":"profile_id","value":"1"}
- |1434534566402 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"15541439","userIdType":"profile_id","value":"1"}
- |1434534566122 $service $action 48890741_94013463616012786 {"is_shared":"false","relationship":"FE"} {"userId":"48040409","userIdType":"profile_id","value":"1"}
- |1434534566055 $service $action 64509008_94002318232678546 {"is_shared":"true","relationship":"F"} {"userId":"46532039","userIdType":"profile_id","value":"1"}
- |1434534565994 $service $action 66644368_94009163363033795 {"is_shared":"false","relationship":"FE"} {"userId":"4143147","userIdType":"profile_id","value":"1"}
- |1434534566448 $service $action 64587644_93938555963733954 {"is_shared":"false","relationship":"FE"} {"userId":"689042","userIdType":"profile_id","value":"1"}
- |1434534565935 $service $action 52812511_94012009551561315 {"is_shared":"false","relationship":"FE"} {"userId":"35509692","userIdType":"profile_id","value":"1"}
- |1434534566544 $service $action 70452048_94008573197583762 {"is_shared":"false","relationship":"FE"} {"userId":"5172421","userIdType":"profile_id","value":"1"}
- |1434534565929 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"33556498","userIdType":"profile_id","value":"1"}
- |1434534566358 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"8987346","userIdType":"profile_id","value":"1"}
- |1434534566057 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"35134964","userIdType":"profile_id","value":"1"}
- |1434534566140 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"11900315","userIdType":"profile_id","value":"1"}
- |1434534566158 $service $action 64639374_93888330176053635 {"is_shared":"true","relationship":"F"} {"userId":"49996643","userIdType":"profile_id","value":"1"}
- |1434534566025 $service $action 67265128_94009084771192002 {"is_shared":"false","relationship":"FE"} {"userId":"37801480","userIdType":"profile_id","value":"1"}
- """.stripMargin.trim
- // println(data)
- val rdd = sc.parallelize(Seq(("", data)))
-
- // rdd.foreachPartition { part =>
- // part.foreach(println)
- // }
- val resultRdd = CounterFunctions.makeExactRdd(rdd, 2)
- val result = resultRdd.collect().toMap
-
- // result.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val parsed = {
- for {
- line <- GraphUtil.parseString(data)
- item <- CounterEtlItem(line).toSeq
- ev <- CounterFunctions.exactMapper(item).toSeq
- } yield {
- ev
- }
- }
- val parsedResult = parsed.groupBy(_._1).mapValues(values => values.map(_._2).reduce(CounterFunctions.reduceValue[ExactQualifier, Long](_ + _, 0L)))
-
- // parsedResult.foreach { case (k, v) =>
- // println(k, v)
- // }
-
- result should not be empty
- result should equal (parsedResult)
-
- val itemId = "46889329_94013502934177075"
- val key = ExactKey(DefaultCounterModel.findByServiceAction(service, action).get, itemId, checkItemType = true)
- val value = result.get(key)
-
- value should not be empty
- value.get.get(ExactQualifier(TimedQualifier("t", 0), Map.empty[String, String])) should equal (Some(6L))
-
- exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (None)
-
- val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
- resultRdd.foreachPartition { part =>
- CounterFunctions.updateExactCounter(part.toSeq, acc)
- }
-
- Option(FetchedCountsGrouped(key, Map(
- (IntervalUnit.TOTAL, Map.empty[String, String]) -> Map(ExactQualifier(TimedQualifier("t", 0), "") -> 6l)
- ))).foreach { expected =>
- exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (Some(expected))
- }
- Option(FetchedCountsGrouped(key, Map(
- (IntervalUnit.TOTAL, Map("is_shared" -> "false")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.false") -> 6l)
- ))).foreach { expected =>
- exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"))) should be (Some(expected))
- }
- Option(FetchedCountsGrouped(key, Map(
- (IntervalUnit.TOTAL, Map("relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "relationship.FE") -> 6l)
- ))).foreach { expected =>
- exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("relationship" -> Set("FE"))) should be (Some(expected))
- }
- Option(FetchedCountsGrouped(key, Map(
- (IntervalUnit.TOTAL, Map("is_shared" -> "false", "relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.relationship.false.FE") -> 6l)
- ))).foreach { expected =>
- exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"), "relationship" -> Set("FE"))) should be (Some(expected))
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2counter_loader/src/test/scala/s2/counter/stream/RankingCounterStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/s2/counter/stream/RankingCounterStreamingSpec.scala b/s2counter_loader/src/test/scala/s2/counter/stream/RankingCounterStreamingSpec.scala
deleted file mode 100644
index 434673d..0000000
--- a/s2counter_loader/src/test/scala/s2/counter/stream/RankingCounterStreamingSpec.scala
+++ /dev/null
@@ -1,448 +0,0 @@
-package s2.counter.stream
-
-import com.kakao.s2graph.core.mysqls.Label
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{SparkConf, SparkContext}
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-import play.api.libs.json.Json
-import s2.config.{S2ConfigFactory, S2CounterConfig}
-import s2.counter.core.CounterFunctions.HashMapAccumulable
-import s2.counter.core.TimedQualifier.IntervalUnit
-import s2.counter.core._
-import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
-import s2.helper.CounterAdmin
-import s2.models.{Counter, DBModel, DefaultCounterModel}
-import s2.spark.HashMapParam
-
-import scala.collection.mutable.{HashMap => MutableHashMap}
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.util.{Failure, Success}
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 6. 17..
- */
-class RankingCounterStreamingSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
- private val master = "local[2]"
- private val appName = "ranking_counter_streaming"
- private val batchDuration = Seconds(1)
-
- private var sc: SparkContext = _
- private var ssc: StreamingContext = _
-
- val admin = new CounterAdmin(S2ConfigFactory.config)
- val graphOp = new GraphOperation(S2ConfigFactory.config)
- val s2config = new S2CounterConfig(S2ConfigFactory.config)
-
- val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
- val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
-
- val service = "test"
- val action = "test_case"
- val action_base = "test_case_base"
- val action_rate = "test_case_rate"
- val action_rate_threshold = "test_case_rate_threshold"
- val action_trend = "test_case_trend"
-
- override def beforeAll(): Unit = {
- DBModel.initialize(S2ConfigFactory.config)
-
- val conf = new SparkConf()
- .setMaster(master)
- .setAppName(appName)
-
- ssc = new StreamingContext(conf, batchDuration)
-
- sc = ssc.sparkContext
-
- admin.setupCounterOnGraph()
-
- // create test_case label
- com.kakao.s2graph.core.Management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
- if (Label.findByName(action, useCache = false).isEmpty) {
- val strJs =
- s"""
- |{
- | "label": "$action",
- | "srcServiceName": "$service",
- | "srcColumnName": "src",
- | "srcColumnType": "string",
- | "tgtServiceName": "$service",
- | "tgtColumnName": "$action",
- | "tgtColumnType": "string",
- | "indices": [
- | ],
- | "props": [
- | ]
- |}
- """.stripMargin
- graphOp.createLabel(Json.parse(strJs))
- }
- if (Label.findByName(action_base, useCache = false).isEmpty) {
- val strJs =
- s"""
- |{
- | "label": "$action_base",
- | "srcServiceName": "$service",
- | "srcColumnName": "src",
- | "srcColumnType": "string",
- | "tgtServiceName": "$service",
- | "tgtColumnName": "$action",
- | "tgtColumnType": "string",
- | "indices": [
- | ],
- | "props": [
- | ]
- |}
- """.stripMargin
- graphOp.createLabel(Json.parse(strJs))
- }
-
- // action
- admin.deleteCounter(service, action).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
-
- // action_base
- admin.deleteCounter(service, action_base).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action_base, Counter.ItemType.STRING, autoComb = true, "", useRank = true))
- val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
-
- // action_rate
- admin.deleteCounter(service, action_rate).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action_rate, Counter.ItemType.STRING, autoComb = true, "gender,p1", useRank = true,
- rateActionId = Some(policy.id), rateBaseId = Some(basePolicy.id)))
-
- // action_rate_threshold
- admin.deleteCounter(service, action_rate_threshold).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action_rate_threshold, Counter.ItemType.STRING, autoComb = true, "gender,p1", useRank = true,
- rateActionId = Some(policy.id), rateBaseId = Some(basePolicy.id), rateThreshold = Some(3)))
-
- // action_trend
- admin.deleteCounter(service, action_trend).foreach {
- case Success(v) =>
- case Failure(ex) =>
- println(s"$ex")
- }
- admin.createCounter(Counter(useFlag = true, 2, service, action_trend, Counter.ItemType.STRING, autoComb = true, "p1", useRank = true,
- rateActionId = Some(policy.id), rateBaseId = Some(policy.id)))
- }
-
- override def afterAll(): Unit = {
- admin.deleteCounter(service, action)
- admin.deleteCounter(service, action_base)
- admin.deleteCounter(service, action_rate)
- admin.deleteCounter(service, action_rate_threshold)
- admin.deleteCounter(service, action_trend)
- if (ssc != null) {
- ssc.stop()
- }
- }
-
- "RankingCounterStreaming" should "update" in {
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
-// val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
-
- rankingCounter.ready(policy) should equal (true)
- val data =
- s"""
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":3}]}
- |{"success":false,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
- |{"success":true,"policyId":${policy.id},"item":"3","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"3","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
- |{"success":true,"policyId":${policy.id},"item":"4","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- """.stripMargin.trim
- // println(data)
- val rdd = sc.parallelize(Seq(("", data)))
-
- // rdd.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val result = CounterFunctions.makeRankingRdd(rdd, 2).collect().toMap
-
- // result.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- result should not be empty
- val rankKey = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
- result should contain (rankKey -> Map(
- "1" -> RankingValue(3, 1),
- "3" -> RankingValue(2, 2),
- "4" -> RankingValue(1, 1)
- ))
-
- val key = RankingKey(policy.id, policy.version, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
- val value = result.get(key)
-
- value should not be empty
- value.get.get("1").get should equal (RankingValue(3, 1))
- value.get.get("2") shouldBe empty
- value.get.get("3").get should equal (RankingValue(2, 2))
-
- rankingCounter.ready(policy) should equal (true)
-
- // delete, update and get
- rankingCounter.delete(key)
- Thread.sleep(1000)
- CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
- Thread.sleep(1000)
- val rst = rankingCounter.getTopK(key)
-
- rst should not be empty
-// rst.get.totalScore should equal(4f)
- rst.get.values should contain allOf(("3", 2d), ("4", 1d), ("1", 3d))
- }
-
-// "rate by base" >> {
-// val data =
-// """
-// |{"success":true,"policyId":42,"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
-// """.stripMargin.trim
-// val rdd = sc.parallelize(Seq(("", data)))
-//
-// val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2).collect()
-// trxLogRdd.foreach { log =>
-// CounterFunctions.rateBaseRankingMapper(log) must not be empty
-// }
-//
-// true must_== true
-// }
-
- it should "update rate ranking counter" in {
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
- val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
- val ratePolicy = DefaultCounterModel.findByServiceAction(service, action_rate).get
-
- // update base policy
- val eq = ExactQualifier(TimedQualifier("M", 1433084400000l), "")
- val exactKey = ExactKey(basePolicy, "1", checkItemType = true)
-
- // check base item count
- exactCounter.updateCount(basePolicy, Seq(
- (exactKey, Map(eq -> 2l))
- ))
- Thread.sleep(1000)
-
- // direct get
- val baseCount = exactCounter.getCount(basePolicy, "1", Seq(IntervalUnit.MONTHLY), 1433084400000l, 1433084400000l, Map.empty[String, Set[String]])
- baseCount should not be empty
- baseCount.get should equal (FetchedCountsGrouped(exactKey, Map(
- (eq.tq.q, Map.empty[String, String]) -> Map(eq-> 2l)
- )))
-
- // related get
- val relatedCount = exactCounter.getRelatedCounts(basePolicy, Seq("1" -> Seq(eq)))
- relatedCount should not be empty
- relatedCount.get("1") should not be empty
- relatedCount.get("1").get should equal (Map(eq -> 2l))
-
- val data =
- s"""
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
- |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":2,"result":4}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"p1.1","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"p1.1","ts":1433084400000,"value":2,"result":4}]}
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
- """.stripMargin.trim
- // println(data)
- val rdd = sc.parallelize(Seq(("", data)))
-
- // rdd.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
- trxLogRdd.count() should equal (data.trim.split('\n').length)
-
- val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
- itemRankingRdd.foreach(println)
-
- val result = CounterFunctions.rateRankingCount(itemRankingRdd, 2).collect().toMap.filterKeys(key => key.policyId == ratePolicy.id)
- result.foreach(println)
- result should have size 3
-
- val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- // rate ranking
- val key = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
- val value = result.get(key)
-
-// println(key, value)
-
- value should not be empty
- value.get.get("1") should not be empty
- value.get.get("1").get should equal (RankingValue(1, 0))
- value.get.get("2").get should equal (RankingValue(0.25, 0))
-
- val key2 = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), "p1.1"))
- val value2 = result.get(key2)
-
-// println(key2, value2)
-
- val values = value.map(v => (key, v)).toSeq ++ value2.map(v => (key2, v)).toSeq
- println(s"values: $values")
-
- // delete, update and get
- rankingCounter.delete(key)
- rankingCounter.delete(key2)
- Thread.sleep(1000)
- CounterFunctions.updateRankingCounter(values, acc)
- // for update graph
- Thread.sleep(1000)
-
- val rst = rankingCounter.getTopK(key)
- rst should not be empty
- rst.get.values should equal (Seq(("1", 1d), ("2", 0.25d)))
-
- val rst2 = rankingCounter.getTopK(key2)
- rst2 should not be empty
- rst2.get.values should equal (Seq(("2", 0.25d)))
- }
-
- it should "update rate ranking counter with threshold" in {
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
- val basePolicy = DefaultCounterModel.findByServiceAction(service, action_base).get
- val ratePolicy = DefaultCounterModel.findByServiceAction(service, action_rate_threshold).get
-
- val data =
- s"""
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":2}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1433084400000,"value":2,"result":4}]}
- |{"success":true,"policyId":${basePolicy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1433084400000,"value":2,"result":4}]}
- """.stripMargin.trim
- // println(data)
- val rdd = sc.parallelize(Seq(("", data)))
-
- // rdd.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
- trxLogRdd.count() should equal (data.trim.split('\n').length)
-
- val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
- itemRankingRdd.foreach(println)
-
- val result = CounterFunctions.rateRankingCount(itemRankingRdd, 2).collect().toMap.filterKeys(key => key.policyId == ratePolicy.id)
- result.foreach(println)
- result should have size 2
-
- val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- // rate ranking
- val key = RankingKey(ratePolicy.id, 2, ExactQualifier(TimedQualifier("M", 1433084400000L), ""))
- val value = result.get(key)
-
- value should not be empty
- value.get.get("1") should be (None)
- value.get.get("2").get should equal (RankingValue(0.25, 0))
-
- // delete, update and get
- rankingCounter.delete(key)
- Thread.sleep(1000)
- CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
- Thread.sleep(1000)
- val rst = rankingCounter.getTopK(key)
-
- rst should not be empty
- rst.get.values should equal (Seq(("2", 0.25d)))
- }
-
- it should "update trend ranking counter" in {
- val policy = DefaultCounterModel.findByServiceAction(service, action).get
- val trendPolicy = DefaultCounterModel.findByServiceAction(service, action_trend).get
-
- val exactKey1 = ExactKey(policy, "1", checkItemType = true)
- val exactKey2 = ExactKey(policy, "2", checkItemType = true)
- // update old key value
- val tq1 = TimedQualifier("M", 1435676400000l)
- val tq2 = TimedQualifier("M", 1427814000000l)
- exactCounter.updateCount(policy, Seq(
- exactKey1 -> Map(ExactQualifier(tq1.add(-1), "") -> 1l, ExactQualifier(tq2.add(-1), "") -> 92l)
- ))
- val eq1 = ExactQualifier(tq1, "")
- val eq2 = ExactQualifier(tq2, "")
-
- val oldCount = exactCounter.getPastCounts(policy, Seq("1" -> Seq(eq1, eq2), "2" -> Seq(eq1, eq1.copy(dimension = "gender.M"))))
- oldCount should not be empty
- oldCount.get("1").get should equal(Map(eq1 -> 1l, eq2 -> 92l))
- oldCount.get("2") should be (None)
-
- val data =
- s"""
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":2}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"","ts":1435676400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"2","results":[{"interval":"M","dimension":"gender.M","ts":1435676400000,"value":1,"result":1}]}
- |{"success":true,"policyId":${policy.id},"item":"1","results":[{"interval":"M","dimension":"","ts":1427814000000,"value":1,"result":92}]}
- """.stripMargin.trim
- // println(data)
- val rdd = sc.parallelize(Seq(("", data)))
-
- // rdd.foreachPartition { part =>
- // part.foreach(println)
- // }
-
- val trxLogRdd = CounterFunctions.makeTrxLogRdd(rdd, 2)
- trxLogRdd.count() should equal (data.trim.split('\n').length)
-
- val itemRankingRdd = CounterFunctions.makeItemRankingRdd(trxLogRdd, 2)
- itemRankingRdd.foreach(println)
-
- val result = CounterFunctions.trendRankingCount(itemRankingRdd, 2).collect().toMap
- result.foreach(println)
- // dimension gender.M is ignored, because gender is not defined dimension in trend policy.
- result should have size 2
-
- val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
-
- // trend ranking
- val key = RankingKey(trendPolicy.id, 2, ExactQualifier(TimedQualifier("M", 1435676400000L), ""))
- val value = result.get(key)
-
- value should not be empty
- value.get.get("1").get should equal (RankingValue(2, 0))
- value.get.get("2").get should equal (RankingValue(1, 0))
-
- val key2 = RankingKey(trendPolicy.id, 2, ExactQualifier(TimedQualifier("M", 1427814000000L), ""))
- val value2 = result.get(key2)
-
- value2 should not be empty
- value2.get.get("1").get should equal (RankingValue(1, 0))
-
- // delete, update and get
- rankingCounter.delete(key)
- Thread.sleep(1000)
- CounterFunctions.updateRankingCounter(Seq((key, value.get)), acc)
- Thread.sleep(1000)
- val rst = rankingCounter.getTopK(key)
-
- rst should not be empty
- rst.get.values should equal (Seq("1" -> 2, "2" -> 1))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_netty/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/resources/application.conf b/s2rest_netty/src/main/resources/application.conf
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_netty/src/main/scala/Server.scala
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala
deleted file mode 100644
index 16477b1..0000000
--- a/s2rest_netty/src/main/scala/Server.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-package com.kakao.s2graph.rest.netty
-
-import java.util.Map.Entry
-import java.util.concurrent.Executors
-import java.util.function.Consumer
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.Experiment
-import com.kakao.s2graph.core.rest.RestHandler.HandlerResult
-import com.kakao.s2graph.core.rest._
-import com.kakao.s2graph.core.utils.Extensions._
-import com.kakao.s2graph.core.utils.logger
-import com.typesafe.config.ConfigFactory
-import io.netty.bootstrap.ServerBootstrap
-import io.netty.buffer.{ByteBuf, Unpooled}
-import io.netty.channel._
-import io.netty.channel.nio.NioEventLoopGroup
-import io.netty.channel.socket.SocketChannel
-import io.netty.channel.socket.nio.NioServerSocketChannel
-import io.netty.handler.codec.http.HttpHeaders._
-import io.netty.handler.codec.http._
-import io.netty.handler.logging.{LogLevel, LoggingHandler}
-import io.netty.util.CharsetUtil
-import play.api.libs.json._
-
-import scala.collection.mutable
-import scala.concurrent.ExecutionContext
-import scala.io.Source
-import scala.util.{Failure, Success, Try}
-
-class S2RestHandler(s2rest: RestHandler)(implicit ec: ExecutionContext) extends SimpleChannelInboundHandler[FullHttpRequest] with JSONParser {
- val ApplicationJson = "application/json"
-
- val Ok = HttpResponseStatus.OK
- val CloseOpt = Option(ChannelFutureListener.CLOSE)
- val BadRequest = HttpResponseStatus.BAD_REQUEST
- val BadGateway = HttpResponseStatus.BAD_GATEWAY
- val NotFound = HttpResponseStatus.NOT_FOUND
- val InternalServerError = HttpResponseStatus.INTERNAL_SERVER_ERROR
-
- def badRoute(ctx: ChannelHandlerContext) =
- simpleResponse(ctx, BadGateway, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
-
- def simpleResponse(ctx: ChannelHandlerContext,
- httpResponseStatus: HttpResponseStatus,
- byteBufOpt: Option[ByteBuf] = None,
- headers: Seq[(String, String)] = Nil,
- channelFutureListenerOpt: Option[ChannelFutureListener] = None): Unit = {
-
- val res: FullHttpResponse = byteBufOpt match {
- case None => new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus)
- case Some(byteBuf) =>
- new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus, byteBuf)
- }
-
- headers.foreach { case (k, v) => res.headers().set(k, v) }
- val channelFuture = ctx.writeAndFlush(res)
-
- channelFutureListenerOpt match {
- case None =>
- case Some(listener) => channelFuture.addListener(listener)
- }
- }
-
- def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, requestBody: JsValue, result: HandlerResult, startedAt: Long) = {
- var closeOpt = CloseOpt
- var headers = mutable.ArrayBuilder.make[(String, String)]
-
- headers += (Names.CONTENT_TYPE -> ApplicationJson)
- result.headers.foreach(headers += _)
-
- if (HttpHeaders.isKeepAlive(req)) {
- headers += (Names.CONNECTION -> HttpHeaders.Values.KEEP_ALIVE)
- closeOpt = None
- }
-
- result.body onComplete {
- case Success(json) =>
- val duration = System.currentTimeMillis() - startedAt
-
- val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(json)} ${requestBody}"
- logger.info(log)
-
- val buf: ByteBuf = Unpooled.copiedBuffer(json.toString, CharsetUtil.UTF_8)
-
- headers += (Names.CONTENT_LENGTH -> buf.readableBytes().toString)
-
- simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result())
- case Failure(ex) => ex match {
- case e: BadQueryException =>
- logger.error(s"{$requestBody}, ${e.getMessage}", e)
- val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8)
- simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
- case e: Exception =>
- logger.error(s"${requestBody}, ${e.getMessage}", e)
- val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8)
- simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
- }
- }
- }
-
- override def channelRead0(ctx: ChannelHandlerContext, req: FullHttpRequest): Unit = {
- val uri = req.getUri
- val startedAt = System.currentTimeMillis()
-
- req.getMethod match {
- case HttpMethod.GET =>
- uri match {
- case "/health_check.html" =>
- if (NettyServer.isHealthy) {
- val healthCheckMsg = Unpooled.copiedBuffer(NettyServer.deployInfo, CharsetUtil.UTF_8)
- simpleResponse(ctx, Ok, byteBufOpt = Option(healthCheckMsg), channelFutureListenerOpt = CloseOpt)
- } else {
- simpleResponse(ctx, NotFound, channelFutureListenerOpt = CloseOpt)
- }
-
- case s if s.startsWith("/graphs/getEdge/") =>
- // src, tgt, label, dir
- val Array(srcId, tgtId, labelName, direction) = s.split("/").takeRight(4)
- val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
- val result = s2rest.checkEdges(params)
- toResponse(ctx, req, params, result, startedAt)
- case _ => badRoute(ctx)
- }
-
- case HttpMethod.PUT =>
- if (uri.startsWith("/health_check/")) {
- val newHealthCheck = uri.split("/").last.toBoolean
- NettyServer.isHealthy = newHealthCheck
- val newHealthCheckMsg = Unpooled.copiedBuffer(NettyServer.isHealthy.toString, CharsetUtil.UTF_8)
- simpleResponse(ctx, Ok, byteBufOpt = Option(newHealthCheckMsg), channelFutureListenerOpt = CloseOpt)
- } else badRoute(ctx)
-
- case HttpMethod.POST =>
- val body = req.content.toString(CharsetUtil.UTF_8)
-
- val result = s2rest.doPost(uri, body, Option(req.headers().get(Experiment.impressionKey)))
- toResponse(ctx, req, Json.parse(body), result, startedAt)
-
- case _ =>
- simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
- }
- }
-
- override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
- cause.printStackTrace()
- logger.error(s"exception on query.", cause)
- simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
- }
-}
-
-// Simple http server
-object NettyServer extends App {
- /** should be same with Boostrap.onStart on play */
-
- val numOfThread = Runtime.getRuntime.availableProcessors()
- val threadPool = Executors.newFixedThreadPool(numOfThread)
- val ec = ExecutionContext.fromExecutor(threadPool)
-
- val config = ConfigFactory.load()
- val port = Try(config.getInt("http.port")).recover { case _ => 9000 }.get
-
- // init s2graph with config
- val s2graph = new Graph(config)(ec)
- val rest = new RestHandler(s2graph)(ec)
-
- val deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
- var isHealthy = config.getBooleanWithFallback("app.health.on", true)
-
- logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
-
- // Configure the server.
- val bossGroup: EventLoopGroup = new NioEventLoopGroup(1)
- val workerGroup: EventLoopGroup = new NioEventLoopGroup()
-
- try {
- val b: ServerBootstrap = new ServerBootstrap()
- b.option(ChannelOption.SO_BACKLOG, Int.box(2048))
-
- b.group(bossGroup, workerGroup).channel(classOf[NioServerSocketChannel])
- .handler(new LoggingHandler(LogLevel.INFO))
- .childHandler(new ChannelInitializer[SocketChannel] {
- override def initChannel(ch: SocketChannel) {
- val p = ch.pipeline()
- p.addLast(new HttpServerCodec())
- p.addLast(new HttpObjectAggregator(65536))
- p.addLast(new S2RestHandler(rest)(ec))
- }
- })
-
- logger.info(s"Listening for HTTP on /0.0.0.0:$port")
- val ch: Channel = b.bind(port).sync().channel()
- ch.closeFuture().sync()
-
- } finally {
- bossGroup.shutdownGracefully()
- workerGroup.shutdownGracefully()
- s2graph.shutdown()
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_netty/src/main/scala/org/apache/s2graph/rest/netty/Server.scala
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/scala/org/apache/s2graph/rest/netty/Server.scala b/s2rest_netty/src/main/scala/org/apache/s2graph/rest/netty/Server.scala
new file mode 100644
index 0000000..6669c9c
--- /dev/null
+++ b/s2rest_netty/src/main/scala/org/apache/s2graph/rest/netty/Server.scala
@@ -0,0 +1,200 @@
+package org.apache.s2graph.rest.netty
+
+import java.util.concurrent.Executors
+
+import com.typesafe.config.ConfigFactory
+import io.netty.bootstrap.ServerBootstrap
+import io.netty.buffer.{ByteBuf, Unpooled}
+import io.netty.channel._
+import io.netty.channel.nio.NioEventLoopGroup
+import io.netty.channel.socket.SocketChannel
+import io.netty.channel.socket.nio.NioServerSocketChannel
+import io.netty.handler.codec.http.HttpHeaders._
+import io.netty.handler.codec.http._
+import io.netty.handler.logging.{LogLevel, LoggingHandler}
+import io.netty.util.CharsetUtil
+import org.apache.s2graph.core.GraphExceptions.BadQueryException
+import org.apache.s2graph.core.mysqls.Experiment
+import org.apache.s2graph.core.rest.RestHandler
+import org.apache.s2graph.core.rest.RestHandler.HandlerResult
+import org.apache.s2graph.core.utils.Extensions._
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{Graph, JSONParser, PostProcess}
+import play.api.libs.json._
+
+import scala.collection.mutable
+import scala.concurrent.ExecutionContext
+import scala.io.Source
+import scala.util.{Failure, Success, Try}
+
+class S2RestHandler(s2rest: RestHandler)(implicit ec: ExecutionContext) extends SimpleChannelInboundHandler[FullHttpRequest] with JSONParser {
+ val ApplicationJson = "application/json"
+
+ val Ok = HttpResponseStatus.OK
+ val CloseOpt = Option(ChannelFutureListener.CLOSE)
+ val BadRequest = HttpResponseStatus.BAD_REQUEST
+ val BadGateway = HttpResponseStatus.BAD_GATEWAY
+ val NotFound = HttpResponseStatus.NOT_FOUND
+ val InternalServerError = HttpResponseStatus.INTERNAL_SERVER_ERROR
+
+ def badRoute(ctx: ChannelHandlerContext) =
+ simpleResponse(ctx, BadGateway, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
+
+ def simpleResponse(ctx: ChannelHandlerContext,
+ httpResponseStatus: HttpResponseStatus,
+ byteBufOpt: Option[ByteBuf] = None,
+ headers: Seq[(String, String)] = Nil,
+ channelFutureListenerOpt: Option[ChannelFutureListener] = None): Unit = {
+
+ val res: FullHttpResponse = byteBufOpt match {
+ case None => new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus)
+ case Some(byteBuf) =>
+ new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus, byteBuf)
+ }
+
+ headers.foreach { case (k, v) => res.headers().set(k, v) }
+ val channelFuture = ctx.writeAndFlush(res)
+
+ channelFutureListenerOpt match {
+ case None =>
+ case Some(listener) => channelFuture.addListener(listener)
+ }
+ }
+
+ def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, requestBody: JsValue, result: HandlerResult, startedAt: Long) = {
+ var closeOpt = CloseOpt
+ var headers = mutable.ArrayBuilder.make[(String, String)]
+
+ headers += (Names.CONTENT_TYPE -> ApplicationJson)
+ result.headers.foreach(headers += _)
+
+ if (HttpHeaders.isKeepAlive(req)) {
+ headers += (Names.CONNECTION -> HttpHeaders.Values.KEEP_ALIVE)
+ closeOpt = None
+ }
+
+ result.body onComplete {
+ case Success(json) =>
+ val duration = System.currentTimeMillis() - startedAt
+
+ val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(json)} ${requestBody}"
+ logger.info(log)
+
+ val buf: ByteBuf = Unpooled.copiedBuffer(json.toString, CharsetUtil.UTF_8)
+
+ headers += (Names.CONTENT_LENGTH -> buf.readableBytes().toString)
+
+ simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result())
+ case Failure(ex) => ex match {
+ case e: BadQueryException =>
+ logger.error(s"{$requestBody}, ${e.getMessage}", e)
+ val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8)
+ simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
+ case e: Exception =>
+ logger.error(s"${requestBody}, ${e.getMessage}", e)
+ val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8)
+ simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
+ }
+ }
+ }
+
+ override def channelRead0(ctx: ChannelHandlerContext, req: FullHttpRequest): Unit = {
+ val uri = req.getUri
+ val startedAt = System.currentTimeMillis()
+
+ req.getMethod match {
+ case HttpMethod.GET =>
+ uri match {
+ case "/health_check.html" =>
+ if (NettyServer.isHealthy) {
+ val healthCheckMsg = Unpooled.copiedBuffer(NettyServer.deployInfo, CharsetUtil.UTF_8)
+ simpleResponse(ctx, Ok, byteBufOpt = Option(healthCheckMsg), channelFutureListenerOpt = CloseOpt)
+ } else {
+ simpleResponse(ctx, NotFound, channelFutureListenerOpt = CloseOpt)
+ }
+
+ case s if s.startsWith("/graphs/getEdge/") =>
+ // src, tgt, label, dir
+ val Array(srcId, tgtId, labelName, direction) = s.split("/").takeRight(4)
+ val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId))
+ val result = s2rest.checkEdges(params)
+ toResponse(ctx, req, params, result, startedAt)
+ case _ => badRoute(ctx)
+ }
+
+ case HttpMethod.PUT =>
+ if (uri.startsWith("/health_check/")) {
+ val newHealthCheck = uri.split("/").last.toBoolean
+ NettyServer.isHealthy = newHealthCheck
+ val newHealthCheckMsg = Unpooled.copiedBuffer(NettyServer.isHealthy.toString, CharsetUtil.UTF_8)
+ simpleResponse(ctx, Ok, byteBufOpt = Option(newHealthCheckMsg), channelFutureListenerOpt = CloseOpt)
+ } else badRoute(ctx)
+
+ case HttpMethod.POST =>
+ val body = req.content.toString(CharsetUtil.UTF_8)
+
+ val result = s2rest.doPost(uri, body, Option(req.headers().get(Experiment.impressionKey)))
+ toResponse(ctx, req, Json.parse(body), result, startedAt)
+
+ case _ =>
+ simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
+ }
+ }
+
+ override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
+ cause.printStackTrace()
+ logger.error(s"exception on query.", cause)
+ simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
+ }
+}
+
+// Simple http server
+object NettyServer extends App {
+ /** should be same with Boostrap.onStart on play */
+
+ val numOfThread = Runtime.getRuntime.availableProcessors()
+ val threadPool = Executors.newFixedThreadPool(numOfThread)
+ val ec = ExecutionContext.fromExecutor(threadPool)
+
+ val config = ConfigFactory.load()
+ val port = Try(config.getInt("http.port")).recover { case _ => 9000 }.get
+
+ // init s2graph with config
+ val s2graph = new Graph(config)(ec)
+ val rest = new RestHandler(s2graph)(ec)
+
+ val deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
+ var isHealthy = config.getBooleanWithFallback("app.health.on", true)
+
+ logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
+
+ // Configure the server.
+ val bossGroup: EventLoopGroup = new NioEventLoopGroup(1)
+ val workerGroup: EventLoopGroup = new NioEventLoopGroup()
+
+ try {
+ val b: ServerBootstrap = new ServerBootstrap()
+ b.option(ChannelOption.SO_BACKLOG, Int.box(2048))
+
+ b.group(bossGroup, workerGroup).channel(classOf[NioServerSocketChannel])
+ .handler(new LoggingHandler(LogLevel.INFO))
+ .childHandler(new ChannelInitializer[SocketChannel] {
+ override def initChannel(ch: SocketChannel) {
+ val p = ch.pipeline()
+ p.addLast(new HttpServerCodec())
+ p.addLast(new HttpObjectAggregator(65536))
+ p.addLast(new S2RestHandler(rest)(ec))
+ }
+ })
+
+ logger.info(s"Listening for HTTP on /0.0.0.0:$port")
+ val ch: Channel = b.bind(port).sync().channel()
+ ch.closeFuture().sync()
+
+ } finally {
+ bossGroup.shutdownGracefully()
+ workerGroup.shutdownGracefully()
+ s2graph.shutdown()
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/Bootstrap.scala b/s2rest_play/app/Bootstrap.scala
deleted file mode 100644
index 6ce3ac4..0000000
--- a/s2rest_play/app/Bootstrap.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-package com.kakao.s2graph.rest
-
-import java.util.concurrent.Executors
-
-import actors.QueueActor
-import com.kakao.s2graph.core.rest._
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{Management, ExceptionHandler, Graph}
-import config.Config
-import controllers.{ApplicationController}
-import play.api.Application
-import play.api.mvc.{WithFilters, _}
-import play.filters.gzip.GzipFilter
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.io.Source
-import scala.util.Try
-
-object Global extends WithFilters(new GzipFilter()) {
- var s2graph: Graph = _
- var storageManagement: Management = _
- var s2parser: RequestParser = _
- var s2rest: RestHandler = _
-
- // Application entry point
- override def onStart(app: Application) {
- ApplicationController.isHealthy = false
-
- val numOfThread = Runtime.getRuntime.availableProcessors()
- val threadPool = Executors.newFixedThreadPool(numOfThread)
- val ec = ExecutionContext.fromExecutor(threadPool)
-
- val config = Config.conf.underlying
-
- // init s2graph with config
- s2graph = new Graph(config)(ec)
- storageManagement = new Management(s2graph)
- s2parser = new RequestParser(s2graph.config) // merged config
- s2rest = new RestHandler(s2graph)(ec)
-
- QueueActor.init(s2graph)
-
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
- ExceptionHandler.apply(config)
- }
-
- val defaultHealthOn = Config.conf.getBoolean("app.health.on").getOrElse(true)
- ApplicationController.deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get
-
- ApplicationController.isHealthy = defaultHealthOn
- logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}")
- }
-
- override def onStop(app: Application) {
- QueueActor.shutdown()
-
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
- ExceptionHandler.shutdown()
- }
-
- /**
- * shutdown hbase client for flush buffers.
- */
- s2graph.shutdown()
- }
-
- override def onError(request: RequestHeader, ex: Throwable): Future[Result] = {
- logger.error(s"onError => ip:${request.remoteAddress}, request:${request}", ex)
- Future.successful(Results.InternalServerError)
- }
-
- override def onHandlerNotFound(request: RequestHeader): Future[Result] = {
- logger.error(s"onHandlerNotFound => ip:${request.remoteAddress}, request:${request}")
- Future.successful(Results.NotFound)
- }
-
- override def onBadRequest(request: RequestHeader, error: String): Future[Result] = {
- logger.error(s"onBadRequest => ip:${request.remoteAddress}, request:$request, error:$error")
- Future.successful(Results.BadRequest(error))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/actors/QueueActor.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/actors/QueueActor.scala b/s2rest_play/app/actors/QueueActor.scala
deleted file mode 100644
index 74bc65d..0000000
--- a/s2rest_play/app/actors/QueueActor.scala
+++ /dev/null
@@ -1,92 +0,0 @@
-package actors
-
-import java.util.concurrent.TimeUnit
-
-import actors.Protocol.FlushAll
-import akka.actor._
-import com.kakao.s2graph.core.ExceptionHandler._
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.utils.logger
-import config.Config
-import play.api.Play.current
-import play.api.libs.concurrent.Akka
-
-import scala.collection.mutable
-import scala.concurrent.duration.Duration
-
-/**
- * Created by shon on 9/2/15.
- */
-object Protocol {
-
- case object Flush
-
- case object FlushAll
-
-}
-
-object QueueActor {
- /** we are throttling down here so fixed number of actor to constant */
- var router: ActorRef = _
-
- // Akka.system.actorOf(props(), name = "queueActor")
- def init(s2: Graph) = {
- router = Akka.system.actorOf(props(s2))
- }
-
- def shutdown() = {
- router ! FlushAll
- Akka.system.shutdown()
- Thread.sleep(Config.ASYNC_HBASE_CLIENT_FLUSH_INTERVAL * 2)
- }
-
- def props(s2: Graph): Props = Props(classOf[QueueActor], s2)
-}
-
-class QueueActor(s2: Graph) extends Actor with ActorLogging {
-
- import Protocol._
-
- implicit val ec = context.system.dispatcher
- // logger.error(s"QueueActor: $self")
- val queue = mutable.Queue.empty[GraphElement]
- var queueSize = 0L
- val maxQueueSize = Config.LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE
- val timeUnitInMillis = 10
- val rateLimitTimeStep = 1000 / timeUnitInMillis
- val rateLimit = Config.LOCAL_QUEUE_ACTOR_RATE_LIMIT / rateLimitTimeStep
-
-
- context.system.scheduler.schedule(Duration.Zero, Duration(timeUnitInMillis, TimeUnit.MILLISECONDS), self, Flush)
-
- override def receive: Receive = {
- case element: GraphElement =>
-
- if (queueSize > maxQueueSize) {
- ExceptionHandler.enqueue(toKafkaMessage(Config.KAFKA_FAIL_TOPIC, element, None))
- } else {
- queueSize += 1L
- queue.enqueue(element)
- }
-
- case Flush =>
- val elementsToFlush =
- if (queue.size < rateLimit) queue.dequeueAll(_ => true)
- else (0 until rateLimit).map(_ => queue.dequeue())
-
- val flushSize = elementsToFlush.size
-
- queueSize -= elementsToFlush.length
- s2.mutateElements(elementsToFlush)
-
- if (flushSize > 0) {
- logger.info(s"flush: $flushSize, $queueSize")
- }
-
- case FlushAll =>
- s2.mutateElements(queue)
- context.stop(self)
-
- case _ => logger.error("unknown protocol")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/config/Config.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/config/Config.scala b/s2rest_play/app/config/Config.scala
deleted file mode 100644
index 98b87c5..0000000
--- a/s2rest_play/app/config/Config.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-package config
-
-import play.api.Play
-
-object Config {
- // HBASE
- lazy val HBASE_ZOOKEEPER_QUORUM = conf.getString("hbase.zookeeper.quorum").getOrElse("localhost")
-
-
- // HBASE CLIENT
- lazy val ASYNC_HBASE_CLIENT_FLUSH_INTERVAL = conf.getInt("async.hbase.client.flush.interval").getOrElse(1000).toShort
- lazy val RPC_TIMEOUT = conf.getInt("hbase.client.operation.timeout").getOrElse(1000)
- lazy val MAX_ATTEMPT = conf.getInt("hbase.client.operation.maxAttempt").getOrElse(3)
-
- // PHASE
- lazy val PHASE = conf.getString("phase").getOrElse("dev")
- lazy val conf = Play.current.configuration
-
- // CACHE
- lazy val CACHE_TTL_SECONDS = conf.getInt("cache.ttl.seconds").getOrElse(600)
- lazy val CACHE_MAX_SIZE = conf.getInt("cache.max.size").getOrElse(10000)
-
- //KAFKA
- lazy val KAFKA_METADATA_BROKER_LIST = conf.getString("kafka.metadata.broker.list").getOrElse("localhost")
- lazy val KAFKA_PRODUCER_POOL_SIZE = conf.getInt("kafka.producer.pool.size").getOrElse(0)
- lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}"
- lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async"
- lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"
-
- // is query or write
- lazy val IS_QUERY_SERVER = conf.getBoolean("is.query.server").getOrElse(true)
- lazy val IS_WRITE_SERVER = conf.getBoolean("is.write.server").getOrElse(true)
-
-
- // query limit per step
- lazy val QUERY_HARD_LIMIT = conf.getInt("query.hard.limit").getOrElse(300)
-
- // local queue actor
- lazy val LOCAL_QUEUE_ACTOR_MAX_QUEUE_SIZE = conf.getInt("local.queue.actor.max.queue.size").getOrElse(10000)
- lazy val LOCAL_QUEUE_ACTOR_RATE_LIMIT = conf.getInt("local.queue.actor.rate.limit").getOrElse(1000)
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/config/CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/config/CounterConfig.scala b/s2rest_play/app/config/CounterConfig.scala
deleted file mode 100644
index 2569d55..0000000
--- a/s2rest_play/app/config/CounterConfig.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package config
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 15. 9. 3..
- */
-object CounterConfig {
- // kafka
- lazy val KAFKA_TOPIC_COUNTER = s"s2counter-${Config.PHASE}"
- lazy val KAFKA_TOPIC_COUNTER_TRX = s"s2counter-trx-${Config.PHASE}"
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/AdminController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/AdminController.scala b/s2rest_play/app/controllers/AdminController.scala
deleted file mode 100644
index bb73c40..0000000
--- a/s2rest_play/app/controllers/AdminController.scala
+++ /dev/null
@@ -1,424 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.rest.RequestParser
-import com.kakao.s2graph.core.utils.logger
-import play.api.mvc
-import play.api.mvc.{Action, Controller}
-import play.api.libs.json._
-import play.api.libs.functional.syntax._
-
-import scala.util.{Failure, Success, Try}
-
-object AdminController extends Controller {
-
- import ApplicationController._
- private val management: Management = com.kakao.s2graph.rest.Global.storageManagement
- private val requestParser: RequestParser = com.kakao.s2graph.rest.Global.s2parser
-
- /**
- * admin message formatter
- * @tparam T
- */
- trait AdminMessageFormatter[T] {
- def toJson(msg: T): JsValue
- }
-
- object AdminMessageFormatter {
- implicit def jsValueToJson[T <: JsValue] = new AdminMessageFormatter[T] {
- def toJson(js: T) = js
- }
-
- implicit val stringToJson = new AdminMessageFormatter[String] {
- def toJson(js: String) = Json.obj("message" -> js)
- }
- }
-
- def format[T: AdminMessageFormatter](f: JsValue => play.mvc.Result)(message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- f(formatter.toJson(message))
- }
-
- /**
- * ok response
- * @param message
- * @tparam T
- * @return
- */
- def ok[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- Ok(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- /**
- * bad request response
- * @param message
- * @tparam T
- * @return
- */
- def bad[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- BadRequest(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- /**
- * not found response
- * @param message
- * @tparam T
- * @return
- */
- def notFound[T: AdminMessageFormatter](message: T) = {
- val formatter = implicitly[AdminMessageFormatter[T]]
- NotFound(formatter.toJson(message)).as(applicationJsonHeader)
- }
-
- private[AdminController] def tryResponse[T, R: AdminMessageFormatter](res: Try[T])(callback: T => R): mvc.Result = res match {
- case Success(m) =>
- val ret = callback(m)
- logger.info(ret.toString)
- ok(ret)
- case Failure(error) =>
- logger.error(error.getMessage, error)
- error match {
- case JsResultException(e) => bad(JsError.toFlatJson(e))
- case _ => bad(error.getMessage)
- }
- }
-
- def optionResponse[T, R: AdminMessageFormatter](res: Option[T])(callback: T => R): mvc.Result = res match {
- case Some(m) => ok(callback(m))
- case None => notFound("not found")
- }
-
- /**
- * load all model cache
- * @return
- */
- def loadCache() = Action { request =>
- val startTs = System.currentTimeMillis()
-
- if (!ApplicationController.isHealthy) {
- loadCacheInner()
- }
-
- ok(s"${System.currentTimeMillis() - startTs}")
- }
-
- def loadCacheInner() = {
- Service.findAll()
- ServiceColumn.findAll()
- Label.findAll()
- LabelMeta.findAll()
- LabelIndex.findAll()
- ColumnMeta.findAll()
- }
-
- /**
- * read
- */
-
- /**
- * get service info
- * @param serviceName
- * @return
- */
- def getService(serviceName: String) = Action { request =>
- val serviceOpt = Management.findService(serviceName)
- optionResponse(serviceOpt)(_.toJson)
- }
-
- /**
- * get label info
- * @param labelName
- * @return
- */
- def getLabel(labelName: String) = Action { request =>
- val labelOpt = Management.findLabel(labelName)
- optionResponse(labelOpt)(_.toJson)
- }
-
- /**
- * get all labels of service
- * @param serviceName
- * @return
- */
- def getLabels(serviceName: String) = Action { request =>
- Service.findByName(serviceName) match {
- case None => notFound(s"Service $serviceName not found")
- case Some(service) =>
- val src = Label.findBySrcServiceId(service.id.get)
- val tgt = Label.findByTgtServiceId(service.id.get)
-
- ok(Json.obj("from" -> src.map(_.toJson), "to" -> tgt.map(_.toJson)))
- }
- }
-
- /**
- * get service columns
- * @param serviceName
- * @param columnName
- * @return
- */
- def getServiceColumn(serviceName: String, columnName: String) = Action { request =>
- val serviceColumnOpt = for {
- service <- Service.findByName(serviceName)
- serviceColumn <- ServiceColumn.find(service.id.get, columnName, useCache = false)
- } yield serviceColumn
-
- optionResponse(serviceColumnOpt)(_.toJson)
- }
-
- /**
- * create
- */
-
- /**
- * create service
- * @return
- */
- def createService() = Action(parse.json) { request =>
- val serviceTry = createServiceInner(request.body)
- tryResponse(serviceTry)(_.toJson)
- }
-
-
- def createServiceInner(jsValue: JsValue) = {
- val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = requestParser.toServiceElements(jsValue)
- management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
- }
-
- /**
- * create label
- * @return
- */
- def createLabel() = Action(parse.json) { request =>
- val ret = createLabelInner(request.body)
- tryResponse(ret)(_.toJson)
- }
-
-
- def createLabelInner(json: JsValue) = for {
- labelArgs <- requestParser.toLabelElements(json)
- label <- (management.createLabel _).tupled(labelArgs)
- } yield label
-
- /**
- * add index
- * @return
- */
- def addIndex() = Action(parse.json) { request =>
- val ret = addIndexInner(request.body)
- tryResponse(ret)(_.label + " is updated")
- }
-
- def addIndexInner(json: JsValue) = for {
- (labelName, indices) <- requestParser.toIndexElements(json)
- label <- Management.addIndex(labelName, indices)
- } yield label
-
- /**
- * create service column
- * @return
- */
- def createServiceColumn() = Action(parse.json) { request =>
- val serviceColumnTry = createServiceColumnInner(request.body)
- tryResponse(serviceColumnTry) { (columns: Seq[ColumnMeta]) => Json.obj("metas" -> columns.map(_.toJson)) }
- }
-
- def createServiceColumnInner(jsValue: JsValue) = for {
- (serviceName, columnName, columnType, props) <- requestParser.toServiceColumnElements(jsValue)
- serviceColumn <- Management.createServiceColumn(serviceName, columnName, columnType, props)
- } yield serviceColumn
-
- /**
- * delete
- */
-
- /**
- * delete label
- * @param labelName
- * @return
- */
- def deleteLabel(labelName: String) = Action { request =>
- val deleteLabelTry = deleteLabelInner(labelName)
- tryResponse(deleteLabelTry)(labelName => labelName + " is deleted")
- }
-
- def deleteLabelInner(labelName: String) = Management.deleteLabel(labelName)
-
- /**
- * delete servieColumn
- * @param serviceName
- * @param columnName
- * @return
- */
- def deleteServiceColumn(serviceName: String, columnName: String) = Action { request =>
- val serviceColumnTry = deleteServiceColumnInner(serviceName, columnName)
- tryResponse(serviceColumnTry)(columnName => columnName + " is deleted")
- }
-
- def deleteServiceColumnInner(serviceName: String, columnName: String) =
- Management.deleteColumn(serviceName, columnName)
-
- /**
- * update
- */
-
- /**
- * add Prop to label
- * @param labelName
- * @return
- */
- def addProp(labelName: String) = Action(parse.json) { request =>
- val labelMetaTry = addPropInner(labelName, request.body)
- tryResponse(labelMetaTry)(_.toJson)
- }
-
- def addPropInner(labelName: String, js: JsValue) = for {
- prop <- requestParser.toPropElements(js)
- labelMeta <- Management.addProp(labelName, prop)
- } yield labelMeta
-
- /**
- * add prop to serviceColumn
- * @param serviceName
- * @param columnName
- * @return
- */
- def addServiceColumnProp(serviceName: String, columnName: String) = Action(parse.json) { request =>
- addServiceColumnPropInner(serviceName, columnName)(request.body) match {
- case None => bad(s"can`t find service with $serviceName or can`t find serviceColumn with $columnName")
- case Some(m) => Ok(m.toJson).as(applicationJsonHeader)
- }
- }
-
- def addServiceColumnPropInner(serviceName: String, columnName: String)(js: JsValue) = {
- for {
- service <- Service.findByName(serviceName)
- serviceColumn <- ServiceColumn.find(service.id.get, columnName)
- prop <- requestParser.toPropElements(js).toOption
- } yield {
- ColumnMeta.findOrInsert(serviceColumn.id.get, prop.name, prop.defaultValue)
- }
- }
-
- /**
- * add props to serviceColumn
- * @param serviecName
- * @param columnName
- * @return
- */
- def addServiceColumnProps(serviecName: String, columnName: String) = Action(parse.json) { request =>
- val jsObjs = request.body.asOpt[List[JsObject]].getOrElse(List.empty[JsObject])
- val newProps = for {
- js <- jsObjs
- newProp <- addServiceColumnPropInner(serviecName, columnName)(js)
- } yield newProp
- ok(s"${newProps.size} is added.")
- }
-
- /**
- * copy label
- * @param oldLabelName
- * @param newLabelName
- * @return
- */
- def copyLabel(oldLabelName: String, newLabelName: String) = Action { request =>
- val copyTry = management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
- tryResponse(copyTry)(_.label + "created")
- }
-
- /**
- * rename label
- * @param oldLabelName
- * @param newLabelName
- * @return
- */
- def renameLabel(oldLabelName: String, newLabelName: String) = Action { request =>
- Label.findByName(oldLabelName) match {
- case None => NotFound.as(applicationJsonHeader)
- case Some(label) =>
- Management.updateLabelName(oldLabelName, newLabelName)
- ok(s"Label was updated")
- }
- }
-
- /**
- * update HTable for a label
- * @param labelName
- * @param newHTableName
- * @return
- */
- def updateHTable(labelName: String, newHTableName: String) = Action { request =>
- val updateTry = Management.updateHTable(labelName, newHTableName)
- tryResponse(updateTry)(_.toString + " label(s) updated.")
- }
-
-
- case class HTableParams(cluster: String, hTableName: String,
- preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
-
- override def toString(): String = {
- s"""HtableParams
- |-- cluster : $cluster
- |-- hTableName : $hTableName
- |-- preSplitSize : $preSplitSize
- |-- hTableTTL : $hTableTTL
- |-- compressionAlgorithm : $compressionAlgorithm
- |""".stripMargin
- }
- }
-
- implicit object HTableParamsJsonConverter extends Format[HTableParams] {
- def reads(json: JsValue): JsResult[HTableParams] = (
- (__ \ "cluster").read[String] and
- (__ \ "hTableName").read[String] and
- (__ \ "preSplitSize").read[Int] and
- (__ \ "hTableTTL").readNullable[Int] and
- (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
-
- def writes(o: HTableParams): JsValue = Json.obj(
- "cluster" -> o.cluster,
- "hTableName" -> o.hTableName,
- "preSplitSize" -> o.preSplitSize,
- "hTableTTL" -> o.hTableTTL,
- "compressionAlgorithm" -> o.compressionAlgorithm
- )
- }
-
- implicit object JsErrorJsonWriter extends Writes[JsError] {
- def writes(o: JsError): JsValue = Json.obj(
- "errors" -> JsArray(
- o.errors.map {
- case (path, validationErrors) => Json.obj(
- "path" -> Json.toJson(path.toString()),
- "validationErrors" -> JsArray(validationErrors.map(validationError => Json.obj(
- "message" -> JsString(validationError.message),
- "args" -> JsArray(validationError.args.map(_ match {
- case x: Int => JsNumber(x)
- case x => JsString(x.toString)
- }))
- )))
- )
- }
- )
- )
- }
-
- def createHTable() = Action { request =>
-
- // Management.createTable(cluster, hTableName, List("e", "v"), preSplitSize, hTableTTL, compressionAlgorithm)
- request.body.asJson.map(_.validate[HTableParams] match {
- case JsSuccess(hTableParams, _) => {
- management.createTable(hTableParams.cluster, hTableParams.hTableName, List("e", "v"),
- hTableParams.preSplitSize, hTableParams.hTableTTL,
- hTableParams.compressionAlgorithm.getOrElse(Management.DefaultCompressionAlgorithm))
- logger.info(hTableParams.toString())
- ok(s"HTable was created.")
- }
- case err@JsError(_) => bad(Json.toJson(err))
- }).getOrElse(bad("Invalid Json."))
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b6fe32fc/s2rest_play/app/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ApplicationController.scala b/s2rest_play/app/controllers/ApplicationController.scala
deleted file mode 100644
index 5f54edd..0000000
--- a/s2rest_play/app/controllers/ApplicationController.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-package controllers
-
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
-import com.kakao.s2graph.core.PostProcess
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.iteratee.Enumerator
-import play.api.libs.json.{JsString, JsValue, Json}
-import play.api.mvc._
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object ApplicationController extends Controller {
-
- var isHealthy = true
- var deployInfo = ""
- val applicationJsonHeader = "application/json"
-
- val jsonParser: BodyParser[JsValue] = controllers.s2parse.json
-
- val jsonText: BodyParser[String] = controllers.s2parse.jsonText
-
- private def badQueryExceptionResults(ex: Exception) =
- Future.successful(BadRequest(PostProcess.badRequestResults(ex)).as(applicationJsonHeader))
-
- private def errorResults =
- Future.successful(Ok(PostProcess.emptyResults).as(applicationJsonHeader))
-
- def requestFallback(body: String): PartialFunction[Throwable, Future[Result]] = {
- case e: BadQueryException =>
- logger.error(s"{$body}, ${e.getMessage}", e)
- badQueryExceptionResults(e)
- case e: Exception =>
- logger.error(s"${body}, ${e.getMessage}", e)
- errorResults
- }
-
- def updateHealthCheck(isHealthy: Boolean) = Action { request =>
- this.isHealthy = isHealthy
- Ok(this.isHealthy + "\n")
- }
-
- def healthCheck() = withHeader(parse.anyContent) { request =>
- if (isHealthy) Ok(deployInfo)
- else NotFound
- }
-
- def jsonResponse(json: JsValue, headers: (String, String)*) =
- if (ApplicationController.isHealthy) {
- Ok(json).as(applicationJsonHeader).withHeaders(headers: _*)
- } else {
- Result(
- header = ResponseHeader(OK),
- body = Enumerator(json.toString.getBytes()),
- connection = HttpConnection.Close
- ).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*)
- }
-
- def toLogMessage[A](request: Request[A], result: Result)(startedAt: Long): String = {
- val duration = System.currentTimeMillis() - startedAt
- val isQueryRequest = result.header.headers.contains("result_size")
- val resultSize = result.header.headers.getOrElse("result_size", "-1")
-
- try {
- val body = request.body match {
- case AnyContentAsJson(jsValue) => jsValue match {
- case JsString(str) => str
- case _ => jsValue.toString
- }
- case AnyContentAsEmpty => ""
- case _ => request.body.toString
- }
-
- val str =
- if (isQueryRequest)
- s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
- else
- s"${request.method} ${request.uri} took ${duration} ms ${result.header.status} ${resultSize} ${body}"
-
- str
- } finally {
- /* pass */
- }
- }
-
- def withHeaderAsync[A](bodyParser: BodyParser[A])(block: Request[A] => Future[Result])(implicit ex: ExecutionContext) =
- Action.async(bodyParser) { request =>
- val startedAt = System.currentTimeMillis()
- block(request).map { r =>
- logger.info(toLogMessage(request, r)(startedAt))
- r
- }
- }
-
- def withHeader[A](bodyParser: BodyParser[A])(block: Request[A] => Result) =
- Action(bodyParser) { request: Request[A] =>
- val startedAt = System.currentTimeMillis()
- val r = block(request)
- logger.info(toLogMessage(request, r)(startedAt))
- r
- }
-}