You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2016/02/29 07:15:06 UTC
[1/4] incubator-s2graph git commit: [S2GRAPH-17]: Remove unnecessary
abstraction layer, Storage.
Repository: incubator-s2graph
Updated Branches:
refs/heads/master 36d5485bd -> e207f676f
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
new file mode 100644
index 0000000..17d9e8f
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/FutureCache.scala
@@ -0,0 +1,82 @@
+package com.kakao.s2graph.core.utils
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.CacheBuilder
+import com.typesafe.config.Config
+
+import scala.concurrent.{Promise, Future, ExecutionContext}
+
+
+class FutureCache[R](config: Config)(implicit ex: ExecutionContext) {
+
+ type Value = (Long, Future[R])
+
+ private val maxSize = config.getInt("future.cache.max.size")
+ private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+ private val futureCache = CacheBuilder.newBuilder()
+ .initialCapacity(maxSize)
+ .concurrencyLevel(Runtime.getRuntime.availableProcessors())
+ .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
+ .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
+ .maximumSize(maxSize).build[java.lang.Long, (Long, Promise[R])]()
+
+
+ def asMap() = futureCache.asMap()
+
+ def getIfPresent(cacheKey: Long): Value = {
+ val (cachedAt, promise) = futureCache.getIfPresent(cacheKey)
+ (cachedAt, promise.future)
+ }
+
+ private def checkAndExpire(cacheKey: Long,
+ cachedAt: Long,
+ cacheTTL: Long,
+ oldFuture: Future[R])(op: => Future[R]): Future[R] = {
+ if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
+ // future is too old. so need to expire and fetch new data from storage.
+ futureCache.asMap().remove(cacheKey)
+
+ val newPromise = Promise[R]
+ val now = System.currentTimeMillis()
+
+ futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
+ case null =>
+ // only one thread succeed to come here concurrently
+ // initiate fetch to storage then add callback on complete to finish promise.
+ op.onSuccess { case value =>
+ newPromise.success(value)
+ value
+ }
+ newPromise.future
+ case (cachedAt, oldPromise) => oldPromise.future
+ }
+ } else {
+ // future is not to old so reuse it.
+ oldFuture
+ }
+ }
+ def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Future[R]): Future[R] = {
+ val cacheVal = futureCache.getIfPresent(cacheKey)
+ cacheVal match {
+ case null =>
+ val promise = Promise[R]
+ val now = System.currentTimeMillis()
+ val (cachedAt, cachedPromise) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
+ case null =>
+ op.onSuccess { case value =>
+ promise.success(value)
+ value
+ }
+ (now, promise)
+ case oldVal => oldVal
+ }
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
+
+ case (cachedAt, cachedPromise) =>
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, cachedPromise.future)(op)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
index 438b97d..1c09778 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -173,7 +173,7 @@ class CrudTest extends IntegrateCommon {
TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
})
- TestUtil.mutateEdgesSync(bulkEdges: _*)
+ TestUtil.insertEdgesSync(bulkEdges: _*)
for {
label <- Label.findByName(labelName)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
index 751a7a7..ae9e514 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -2,7 +2,7 @@ package com.kakao.s2graph.core.Integrate
import com.kakao.s2graph.core._
import com.kakao.s2graph.core.mysqls.Label
-import com.kakao.s2graph.core.rest.RequestParser
+import com.kakao.s2graph.core.rest.{RequestParser, RestHandler}
import com.kakao.s2graph.core.utils.logger
import com.typesafe.config._
import org.scalatest._
@@ -33,8 +33,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
}
/**
- * Make Service, Label, Vertex for integrate test
- */
+ * Make Service, Label, Vertex for integrate test
+ */
def initTestData() = {
println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
Management.deleteService(testServiceName)
@@ -82,8 +82,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
/**
- * Test Helpers
- */
+ * Test Helpers
+ */
object TestUtil {
implicit def ec = scala.concurrent.ExecutionContext.global
@@ -111,22 +111,18 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
def getEdgesSync(queryJson: JsValue): JsValue = {
logger.info(Json.prettyPrint(queryJson))
-
- val ret = graph.getEdges(parser.toQuery(queryJson))
- val result = Await.result(ret, HttpRequestWaitingTime)
- val jsResult = PostProcess.toSimpleVertexArrJson(result)
-
- jsResult
+ val restHandler = new RestHandler(graph)
+ Await.result(restHandler.getEdgesAsync(queryJson)(PostProcess.toSimpleVertexArrJson), HttpRequestWaitingTime)
}
- def mutateEdgesSync(bulkEdges: String*) = {
+ def insertEdgesSync(bulkEdges: String*) = {
val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
val jsResult = Await.result(req, HttpRequestWaitingTime)
jsResult
}
- def mutateEdgesAsync(bulkEdges: String*) = {
+ def insertEdgesAsync(bulkEdges: String*) = {
val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
req
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
index 0d2d82e..0b26608 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
@@ -1,14 +1,17 @@
package com.kakao.s2graph.core.Integrate
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import com.kakao.s2graph.core.utils.logger
import org.scalatest.BeforeAndAfterEach
-import play.api.libs.json._
+import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
+
+import scala.util.{Success, Try}
class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
import TestUtil._
val insert = "insert"
- val delete = "delete"
val e = "e"
val weight = "weight"
val is_hidden = "is_hidden"
@@ -132,7 +135,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
val result = getEdgesSync(queryGroupBy(0, Seq("weight")))
(result \ "size").as[Int] should be(2)
- val weights = (result \\ "groupBy").map { js =>
+ val weights = (result \ "results" \\ "groupBy").map { js =>
(js \ "weight").as[Int]
}
weights should contain(30)
@@ -163,11 +166,10 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
(result \ "results").as[List[JsValue]].size should be(2)
result = getEdgesSync(queryTransform(0, "[[\"weight\"]]"))
- (result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted)
+ (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "weight").map(_.toString).sorted)
result = getEdgesSync(queryTransform(0, "[[\"_from\"]]"))
- val results = (result \ "results").as[JsValue]
- (result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted)
+ (result \ "results" \\ "to").map(_.toString).sorted should be((result \ "results" \\ "from").map(_.toString).sorted)
}
test("index") {
@@ -217,6 +219,57 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
// }
// }
+
+
+ test("duration") {
+ def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj(
+ "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100,
+ "duration" -> Json.obj("from" -> from, "to" -> to)))
+
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ Json.obj("srcVertices" -> $from, "steps" -> $steps)
+ }
+
+ // get all
+ var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+ // inclusive, exclusive
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+ insertEdgesSync(bulkEdges: _*)
+
+ // duration test after udpate
+ // get all
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+
+ // inclusive, exclusive
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ }
+
+
test("return tree") {
def queryParents(id: Long) = Json.parse(
s"""
@@ -246,7 +299,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
val src = 100
val tgt = 200
- mutateEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
+ insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
val result = TestUtil.getEdgesSync(queryParents(src))
val parents = (result \ "results").as[Seq[JsValue]]
@@ -259,54 +312,55 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
- test("pagination and _to") {
- def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit,
- "_to": $to
- }
- ]]
- }
- """)
-
- val src = System.currentTimeMillis().toInt
-
- val bulkEdges = Seq(
- toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
- toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
- toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
- toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
- )
- mutateEdgesSync(bulkEdges: _*)
-
- var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
- var edges = (result \ "results").as[List[JsValue]]
-
- edges.size should be(2)
- (edges(0) \ "to").as[Long] should be(4)
- (edges(1) \ "to").as[Long] should be(3)
-
- result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
-
- edges = (result \ "results").as[List[JsValue]]
- edges.size should be(2)
- (edges(0) \ "to").as[Long] should be(3)
- (edges(1) \ "to").as[Long] should be(2)
+// test("pagination and _to") {
+// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
+// s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": $offset,
+// "limit": $limit,
+// "_to": $to
+// }
+// ]]
+// }
+// """)
+//
+// val src = System.currentTimeMillis().toInt
+//
+// val bulkEdges = Seq(
+// toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+// toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+// toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
+// toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
+// )
+// insertEdgesSync(bulkEdges: _*)
+//
+// var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
+// var edges = (result \ "results").as[List[JsValue]]
+//
+// edges.size should be(2)
+// (edges(0) \ "to").as[Long] should be(4)
+// (edges(1) \ "to").as[Long] should be(3)
+//
+// result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
+//
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size should be(2)
+// (edges(0) \ "to").as[Long] should be(3)
+// (edges(1) \ "to").as[Long] should be(2)
+//
+// result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size should be(1)
+// }
- result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
- edges = (result \ "results").as[List[JsValue]]
- edges.size should be(1)
- }
test("order by") {
def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj(
"srcVertices" -> Json.arr(
@@ -351,7 +405,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
)
- mutateEdgesSync(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
// get edges
val edges = getEdgesSync(queryScore(0, Map("weight" -> 1)))
@@ -368,39 +422,6 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
edgesTo.reverse should be(ascOrderByTo)
}
- test("query with '_to' option after delete") {
- val from = 90210
- val to = 90211
- val inserts = Seq(toEdge(1, insert, e, from, to, testLabelName))
- mutateEdgesSync(inserts: _*)
-
- val deletes = Seq(toEdge(2, delete, e, from, to, testLabelName))
- mutateEdgesSync(deletes: _*)
-
- def queryWithTo = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "id": $from
- }],
- "steps": [
- {
- "step": [{
- "label": "$testLabelName",
- "direction": "out",
- "offset": 0,
- "limit": 10,
- "_to": $to
- }]
- }
- ]
- }
- """)
- val result = getEdgesSync(queryWithTo)
- (result \ "results").as[List[JsValue]].size should be(0)
-
- }
test("query with sampling") {
def queryWithSampling(id: Int, sample: Int) = Json.parse(
@@ -502,32 +523,324 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(ts, insert, e, 322, 3322, testLabelName)
)
- mutateEdgesSync(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
- var result = getEdgesSync(queryWithSampling(testId, sampleSize))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(scala.math.min(sampleSize, bulkEdges.size))
+ val result1 = getEdgesSync(queryWithSampling(testId, sampleSize))
+ (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
- result = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
+ val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
+ (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
- result = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
+ val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
+ (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
+ }
+ test("test query with filterOut query") {
+ def queryWithFilterOut(id1: String, id2: String) = Json.parse(
+ s"""{
+ | "limit": 10,
+ | "filterOut": {
+ | "srcVertices": [{
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }],
+ | "steps": [{
+ | "step": [{
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }]
+ | }]
+ | },
+ | "srcVertices": [{
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }],
+ | "steps": [{
+ | "step": [{
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }]
+ | }]
+ |}
+ """.stripMargin
+ )
+
+ val testId1 = "-23"
+ val testId2 = "-25"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 111, testLabelName, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 333, testLabelName, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 555, testLabelName, Json.obj(weight -> 1))
+ )
+ logger.debug(s"${bulkEdges.mkString("\n")}")
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithFilterOut(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(1)
+ (results(0) \ "to").toString should be("555")
+ }
- result = getEdgesSync(queryWithSampling(testId, 0))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(0) // edges in testLabelName2 = 3
- result = getEdgesSync(queryWithSampling(testId, 10))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(3) // edges in testLabelName2 = 3
+ /** note that this merge two different label result into one */
+ test("weighted union") {
+ def queryWithWeightedUnion(id1: String, id2: String) = Json.parse(
+ s"""
+ |{
+ | "limit": 10,
+ | "weights": [
+ | 10,
+ | 1
+ | ],
+ | "groupBy": ["weight"],
+ | "queries": [
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName2",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
- result = getEdgesSync(queryWithSampling(testId, -1))
- println(Json.toJson(result))
- (result \ "results").as[List[JsValue]].size should be(3) // edges in testLabelName2 = 3
+ val testId1 = "1"
+ val testId2 = "2"
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 444, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 666, testLabelName2, Json.obj(weight -> 1))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithWeightedUnion(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(2)
+ (results(0) \ "scoreSum").as[Float] should be(30.0)
+ (results(0) \ "agg").as[List[JsValue]].size should be(3)
+ (results(1) \ "scoreSum").as[Float] should be(3.0)
+ (results(1) \ "agg").as[List[JsValue]].size should be(3)
+ }
+
+ test("weighted union with options") {
+ def queryWithWeightedUnionWithOptions(id1: String, id2: String) = Json.parse(
+ s"""
+ |{
+ | "limit": 10,
+ | "weights": [
+ | 10,
+ | 1
+ | ],
+ | "groupBy": ["to"],
+ | "select": ["to", "weight"],
+ | "filterOut": {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | "queries": [
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id1
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | },
+ | {
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id2
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName2",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 5
+ | }
+ | ]
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
+
+ val testId1 = "-192848"
+ val testId2 = "-193849"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId1, 111, testLabelName, Json.obj(weight -> 10)),
+ toEdge(2, insert, e, testId1, 222, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3, insert, e, testId1, 333, testLabelName, Json.obj(weight -> 10)),
+ toEdge(4, insert, e, testId2, 111, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(5, insert, e, testId2, 333, testLabelName2, Json.obj(weight -> 1)),
+ toEdge(6, insert, e, testId2, 555, testLabelName2, Json.obj(weight -> 1))
+ )
+
+ insertEdgesSync(bulkEdges: _*)
+
+ val rs = getEdgesSync(queryWithWeightedUnionWithOptions(testId1, testId2))
+ logger.debug(Json.prettyPrint(rs))
+ val results = (rs \ "results").as[List[JsValue]]
+ results.size should be(1)
+
+ }
+
+ test("scoreThreshold") {
+ def queryWithScoreThreshold(id: String, scoreThreshold: Int) = Json.parse(
+ s"""{
+ | "limit": 10,
+ | "scoreThreshold": $scoreThreshold,
+ | "groupBy": ["to"],
+ | "srcVertices": [
+ | {
+ | "serviceName": "$testServiceName",
+ | "columnName": "$testColumnName",
+ | "id": $id
+ | }
+ | ],
+ | "steps": [
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | },
+ | {
+ | "step": [
+ | {
+ | "label": "$testLabelName",
+ | "direction": "out",
+ | "offset": 0,
+ | "limit": 10
+ | }
+ | ]
+ | }
+ | ]
+ |}
+ """.stripMargin
+ )
+
+ val testId = "-23903"
+
+ val bulkEdges = Seq(
+ toEdge(1, insert, e, testId, 101, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, testId, 102, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, testId, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 102, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 101, 104, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 102, 103, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 102, 104, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, 103, 105, testLabelName, Json.obj(weight -> 10))
+ )
+ // expected: 104 -> 2, 103 -> 2, 102 -> 1,, 105 -> 1
+ insertEdgesSync(bulkEdges: _*)
+
+ var rs = getEdgesSync(queryWithScoreThreshold(testId, 2))
+ logger.debug(Json.prettyPrint(rs))
+ var results = (rs \ "results").as[List[JsValue]]
+ results.size should be(2)
+
+ rs = getEdgesSync(queryWithScoreThreshold(testId, 1))
+ logger.debug(Json.prettyPrint(rs))
+
+ results = (rs \ "results").as[List[JsValue]]
+ results.size should be(4)
}
def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse(
@@ -548,6 +861,23 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
}
""")
+ def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj(
+ "limit" -> limit,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+
+
// called by each test, each
override def beforeEach = initTestData()
@@ -555,7 +885,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
override def initTestData(): Unit = {
super.initTestData()
- mutateEdgesSync(
+ insertEdgesSync(
toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
index bdcdb9e..aae108e 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
@@ -14,7 +14,7 @@ class StrongLabelDeleteTest extends IntegrateCommon {
import TestUtil._
test("Strong consistency select") {
- mutateEdgesSync(bulkEdges(): _*)
+ insertEdgesSync(bulkEdges(): _*)
var result = getEdgesSync(query(0))
(result \ "results").as[List[JsValue]].size should be(2)
@@ -56,7 +56,7 @@ class StrongLabelDeleteTest extends IntegrateCommon {
println(result)
(result \ "results").as[List[JsValue]].size should be(0)
- mutateEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
println(result)
@@ -69,7 +69,8 @@ class StrongLabelDeleteTest extends IntegrateCommon {
val ret = for {
i <- 0 until testNum
} yield {
- val src = System.currentTimeMillis()
+ val src = (i + 1) * 10000
+// val src = System.currentTimeMillis()
val (ret, last) = testInner(i, src)
ret should be(true)
@@ -136,7 +137,7 @@ class StrongLabelDeleteTest extends IntegrateCommon {
val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
// val allRequests = insertRequests ++ deleteRequests
val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
- mutateEdgesAsync(bulkRequests: _*)
+ insertEdgesAsync(bulkRequests: _*)
}
Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
@@ -175,7 +176,7 @@ class StrongLabelDeleteTest extends IntegrateCommon {
}
val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
- mutateEdgesAsync(bulkRequests: _*)
+ insertEdgesAsync(bulkRequests: _*)
}
Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
@@ -199,7 +200,7 @@ class StrongLabelDeleteTest extends IntegrateCommon {
val labelName = testLabelName2
val maxTgtId = 10
val batchSize = 10
- val testNum = 3
+ val testNum = 100
val numOfBatch = 10
def testInner(startTs: Long, src: Long) = {
@@ -217,13 +218,13 @@ class StrongLabelDeleteTest extends IntegrateCommon {
val op = if (Random.nextDouble() < 0.5) "delete" else "update"
lastOps(tgt) = op
- Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
+ Seq(currentTs, op, "e", src, tgt, labelName, "{}").mkString("\t")
}
allRequests.foreach(println(_))
val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequests =>
- mutateEdgesAsync(bulkRequests: _*)
+ insertEdgesAsync(bulkRequests: _*)
}
Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
index 1e2d836..b80d9c7 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
@@ -76,7 +76,7 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
println(result)
(result \ "results").as[List[JsValue]].size should be(0)
- mutateEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
result = getEdgesSync(query(20, "in", testTgtColumnName))
(result \ "results").as[List[JsValue]].size should be(3)
@@ -90,7 +90,7 @@ class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
override def initTestData(): Unit = {
super.initTestData()
- mutateEdgesSync(bulkEdges(): _*)
+ insertEdgesSync(bulkEdges(): _*)
}
object WeakLabelDeleteHelper {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/ManagementTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/ManagementTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/ManagementTest.scala
deleted file mode 100644
index e5b3e51..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/ManagementTest.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-//package com.kakao.s2graph.core
-//import com.kakao.s2graph.core.Graph
-//import com.kakao.s2graph.core.models.{HLabel, HService, HServiceColumn, HBaseModel}
-//import com.typesafe.config.ConfigFactory
-//import org.scalatest.{FunSuite, Matchers}
-//import play.api.libs.json.{JsString, JsBoolean, JsNumber, Json}
-//
-//import scala.concurrent.ExecutionContext
-//
-///**
-// * Created by shon on 5/15/15.
-// */
-//class ManagementTest extends FunSuite with Matchers {
-// val labelName = "test_label"
-// val serviceName = "test_service"
-// val columnName = "test_column"
-// val columnType = "long"
-// val indexProps = Seq("weight" -> JsNumber(5), "is_hidden" -> JsBoolean(true))
-// val props = Seq("is_blocked" -> JsBoolean(true), "category" -> JsString("sports"))
-// val consistencyLevel = "weak"
-// val hTableName = Some("testHTable")
-// val hTableTTL = Some(86000)
-// val preSplitSize = 10
-// val zkQuorum = "localhost"
-//
-// val config = ConfigFactory.parseString(s"hbase.zookeeper.quorum=$zkQuorum")
-// Graph(config)(ExecutionContext.Implicits.global)
-// HBaseModel(zkQuorum)
-// val current = System.currentTimeMillis()
-// val serviceNames = (0 until 2).map { i => s"$serviceName-${current + i}" }
-// val labelNames = (0 until 2).map { i => s"$labelName-${current + i}" }
-//
-//// def runTC[T <: HBaseModel](prevSeq: Long, prevSize: Int, prefix: String)(testSize: Int)(createF: String => T)(fetchF: String => Option[T])(deleteF: String => Boolean) = {
-//// var lastSeq = prevSeq
-//// val createds = collection.mutable.Map.empty[String, T]
-//// val names = (0 until testSize) map { i => s"$prefix-${current + i}"}
-////
-//// val rets = for {
-//// name <- names
-//// } yield {
-//// val created = createF(name)
-//// val testSeq = created.id.get > lastSeq
-//// lastSeq = created.id.get
-//// createds += (name -> created)
-//// val fetched = fetchF(name)
-//// fetched.isDefined && created == fetched.get && testSeq
-//// }
-////
-//// val deletes = for {
-//// name <- names
-//// } yield {
-//// deleteF(name)
-//// }
-////
-//// (rets ++ deletes).forall(_)
-//// }
-// test("test create service") {
-//
-// var prevSeq = Management.getSequence("HService")
-// val prevSize = HService.findAllServices().size
-// val createds = collection.mutable.Map.empty[String, HService]
-//
-// val rets = for {
-// serviceName <- serviceNames
-// } yield {
-// val service = Management.createService(serviceName, zkQuorum, hTableName.get, preSplitSize, hTableTTL)
-// val testSeq = service.id.get > prevSeq
-// prevSeq = service.id.get
-// createds += (service.serviceName -> service)
-// val other = Management.findService(service.serviceName)
-// other.isDefined && service == other.get && testSeq
-// }
-//
-// val deletes = for {
-// serviceName <- serviceNames
-// } yield {
-// Management.deleteService(serviceName)
-// }
-// (rets ++ deletes).forall(_)
-//
-// HService.findAllServices().size == prevSize
-// }
-// test("test create label") {
-// val service = Management.createService(serviceName, zkQuorum, hTableName.get, preSplitSize, hTableTTL)
-// var prevSeq = Management.getSequence("HLabel")
-// val prevSize = HLabel.findAll(useCache = false)
-// val createds = collection.mutable.Map.empty[String, HLabel]
-//
-// val rets = for {
-// lName <- labelNames
-// } yield {
-// val label = Management.createLabel(lName, serviceName, columnName, columnType,
-// serviceName, columnName, columnType,
-// true, serviceName, indexProps, props,
-// consistencyLevel, hTableName, hTableTTL
-// )
-// val testSeq = label.id.get > prevSeq
-// prevSeq = label.id.get
-//
-// createds += (label.label -> label)
-// val other = Management.findLabel(label.label)
-// other.isDefined && label == other.get && testSeq
-// }
-// println(HLabel.findAll(useCache = false))
-// val deletes = for {
-// lName <- labelNames
-// } yield {
-// Management.deleteLabel(lName)
-// }
-// (rets ++ deletes).forall(_)
-// HLabel.findAll(useCache = false).size == prevSize
-// }
-// test("test update label") {
-// HLabel.updateLabel(labelName, Seq("is_blocked" -> JsBoolean(false)))
-// for {
-// label <- HLabel.findByName(labelName, useCache = false)
-// } yield {
-// println(label)
-// }
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
index b431cc2..febad3d 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
@@ -12,7 +12,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
// dummy data for dummy edge
initTests()
-
+
import HBaseType.{VERSION1, VERSION2}
val ts = System.currentTimeMillis()
@@ -167,6 +167,58 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
}
}
+ test("replace reserved") {
+ val ts = 0
+ import com.kakao.s2graph.core.rest.TemplateHelper._
+
+ calculate(ts, 1, "hour") should be(hour + ts)
+ calculate(ts, 1, "day") should be(day + ts)
+
+ calculate(ts + 10, 1, "HOUR") should be(hour + ts + 10)
+ calculate(ts + 10, 1, "DAY") should be(day + ts + 10)
+
+ val body = """{
+ "day": ${1day},
+ "hour": ${1hour},
+ "-day": "${-10 day}",
+ "-hour": ${-10 hour},
+ "now": "${now}"
+ }
+ """
+
+ val parsed = replaceVariable(ts, body)
+ val json = Json.parse(parsed)
+
+ (json \ "day").as[Long] should be (1 * day + ts)
+ (json \ "hour").as[Long] should be (1 * hour + ts)
+
+ (json \ "-day").as[Long] should be (-10 * day + ts)
+ (json \ "-hour").as[Long] should be (-10 * hour + ts)
+
+ (json \ "now").as[Long] should be (ts)
+
+ val otherBody = """{
+ "nextday": "${next_day}",
+ "3dayago": "${next_day - 3 day}",
+ "nexthour": "${next_hour}"
+ }"""
+
+ val currentTs = System.currentTimeMillis()
+ val expectedDayTs = currentTs / day * day + day
+ val expectedHourTs = currentTs / hour * hour + hour
+ val threeDayAgo = expectedDayTs - 3 * day
+ val currentTsLs = (1 until 1000).map(currentTs + _)
+
+ currentTsLs.foreach { ts =>
+ val parsed = replaceVariable(ts, otherBody)
+ val json = Json.parse(parsed)
+
+ (json \ "nextday").as[Long] should be(expectedDayTs)
+ (json \ "nexthour").as[Long] should be(expectedHourTs)
+ (json \ "3dayago").as[Long] should be(threeDayAgo)
+ }
+ }
+
// test("time decay") {
// val ts = System.currentTimeMillis()
//
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
index 38b5b99..128f2d7 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
@@ -1,53 +1,53 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.Graph
-import com.typesafe.config.ConfigFactory
-
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.GetRequest
-import org.scalatest.{FunSuite, Matchers}
-
-import scala.concurrent.ExecutionContext
-
-class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
-
- val dummyRequests = {
- for {
- id <- 0 until 1000
- } yield {
- new GetRequest("a", Bytes.toBytes(id))
- }
- }
-
- implicit val ec = ExecutionContext.Implicits.global
- val config = ConfigFactory.load()
- val graph = new Graph(config)
-
- val qb = new AsynchbaseQueryBuilder(graph.storage.asInstanceOf[AsynchbaseStorage])
-
- test("test toCacheKeyBytes") {
- val startedAt = System.nanoTime()
-
- for {
- i <- dummyRequests.indices
- x = qb.toCacheKeyBytes(dummyRequests(i))
- } {
- for {
- j <- dummyRequests.indices if i != j
- y = qb.toCacheKeyBytes(dummyRequests(j))
- } {
- x should not equal y
- }
- }
-
- dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
- val xHash = qb.toCacheKeyBytes(x)
- val yHash = qb.toCacheKeyBytes(y)
- // println(xHash, yHash)
- xHash should be(yHash)
- }
- val duration = System.nanoTime() - startedAt
-
- println(s">> bytes: $duration")
- }
-}
+//package com.kakao.s2graph.core.storage.hbase
+//
+//import com.kakao.s2graph.core.Graph
+//import com.typesafe.config.ConfigFactory
+//
+//import org.apache.hadoop.hbase.util.Bytes
+//import org.hbase.async.GetRequest
+//import org.scalatest.{FunSuite, Matchers}
+//
+//import scala.concurrent.ExecutionContext
+//
+//class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
+//
+// val dummyRequests = {
+// for {
+// id <- 0 until 1000
+// } yield {
+// new GetRequest("a", Bytes.toBytes(id))
+// }
+// }
+//
+// implicit val ec = ExecutionContext.Implicits.global
+// val config = ConfigFactory.load()
+// val graph = new Graph(config)
+//
+// val qb = new AsynchbaseQueryBuilder(graph.storage.asInstanceOf[AsynchbaseStorage])
+//
+// test("test toCacheKeyBytes") {
+// val startedAt = System.nanoTime()
+//
+// for {
+// i <- dummyRequests.indices
+// x = qb.toCacheKeyBytes(dummyRequests(i))
+// } {
+// for {
+// j <- dummyRequests.indices if i != j
+// y = qb.toCacheKeyBytes(dummyRequests(j))
+// } {
+// x should not equal y
+// }
+// }
+//
+// dummyRequests.zip(dummyRequests).foreach { case (x, y) =>
+// val xHash = qb.toCacheKeyBytes(x)
+// val yHash = qb.toCacheKeyBytes(y)
+// // println(xHash, yHash)
+// xHash should be(yHash)
+// }
+// val duration = System.nanoTime() - startedAt
+//
+// println(s">> bytes: $duration")
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
index 32e3d0c..a36b55f 100644
--- a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
+++ b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
@@ -128,7 +128,7 @@ object CounterFunctions extends Logging with WithKafka {
itemRankingRdd.unpersist(false)
}
}
-
+
def makeTrxLogRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[TrxLog] = {
rdd.mapPartitions { part =>
assert(initialize)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala b/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
new file mode 100644
index 0000000..4970399
--- /dev/null
+++ b/s2counter_loader/src/test/scala/s2/counter/stream/ExactCounterStreamingSpec.scala
@@ -0,0 +1,196 @@
+package s2.counter.stream
+
+import com.kakao.s2graph.core.GraphUtil
+import com.kakao.s2graph.core.mysqls.Label
+import org.apache.spark.streaming.{Seconds, StreamingContext}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import play.api.libs.json.Json
+import s2.config.{S2ConfigFactory, S2CounterConfig}
+import s2.counter.core.CounterFunctions.HashMapAccumulable
+import s2.counter.core.TimedQualifier.IntervalUnit
+import s2.counter.core._
+import s2.counter.core.v2.{ExactStorageGraph, GraphOperation, RankingStorageGraph}
+import s2.helper.CounterAdmin
+import s2.models.{Counter, DBModel, DefaultCounterModel}
+import s2.spark.HashMapParam
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.util.{Failure, Success}
+
+/**
+ * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 19..
+ */
+class ExactCounterStreamingSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
+ private val master = "local[2]"
+ private val appName = "exact_counter_streaming"
+ private val batchDuration = Seconds(1)
+
+ private var sc: SparkContext = _
+ private var ssc: StreamingContext = _
+
+ val admin = new CounterAdmin(S2ConfigFactory.config)
+ val graphOp = new GraphOperation(S2ConfigFactory.config)
+ val s2config = new S2CounterConfig(S2ConfigFactory.config)
+
+ val exactCounter = new ExactCounter(S2ConfigFactory.config, new ExactStorageGraph(S2ConfigFactory.config))
+ val rankingCounter = new RankingCounter(S2ConfigFactory.config, new RankingStorageGraph(S2ConfigFactory.config))
+
+ val service = "test"
+ val action = "test_case"
+
+ override def beforeAll(): Unit = {
+ DBModel.initialize(S2ConfigFactory.config)
+
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ ssc = new StreamingContext(conf, batchDuration)
+
+ sc = ssc.sparkContext
+
+ // create test_case label
+ com.kakao.s2graph.core.Management.createService(service, s2config.HBASE_ZOOKEEPER_QUORUM, s"${service}_dev", 1, None, "gz")
+ if (Label.findByName(action, useCache = false).isEmpty) {
+ val strJs =
+ s"""
+ |{
+ | "label": "$action",
+ | "srcServiceName": "$service",
+ | "srcColumnName": "src",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "$service",
+ | "tgtColumnName": "$action",
+ | "tgtColumnType": "string",
+ | "indices": [
+ | ],
+ | "props": [
+ | ]
+ |}
+ """.stripMargin
+ graphOp.createLabel(Json.parse(strJs))
+ }
+
+ // action
+ admin.deleteCounter(service, action).foreach {
+ case Success(v) =>
+ case Failure(ex) =>
+ println(s"$ex")
+ }
+ admin.createCounter(Counter(useFlag = true, 2, service, action, Counter.ItemType.STRING, autoComb = true, "is_shared,relationship", useRank = true))
+ }
+
+ override def afterAll(): Unit = {
+ admin.deleteCounter(service, action)
+ if (ssc != null) {
+ ssc.stop()
+ }
+ }
+
+ "ExactCounter" should "update" in {
+ val policy = DefaultCounterModel.findByServiceAction(service, action).get
+ val data =
+ s"""
+ |1434534565675 $service $action 70362200_94013572857366866 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
+ |1434534565675 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"48255079","userIdType":"profile_id","value":"1"}
+ |1434534566220 $service $action 51223360_94013140590929619 {"is_shared":"false","relationship":"FE"} {"userId":"312383","userIdType":"profile_id","value":"1"}
+ |1434534566508 $service $action 63808459_94013420047377826 {"is_shared":"false","relationship":"FE"} {"userId":"21968241","userIdType":"profile_id","value":"1"}
+ |1434534566210 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"6062217","userIdType":"profile_id","value":"1"}
+ |1434534566459 $service $action 49699692_94012186431261763 {"is_shared":"false","relationship":"FE"} {"userId":"67863471","userIdType":"profile_id","value":"1"}
+ |1434534565681 $service $action 64556827_94012311028641810 {"is_shared":"false","relationship":"FE"} {"userId":"19381218","userIdType":"profile_id","value":"1"}
+ |1434534565865 $service $action 41814266_94012477588942163 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
+ |1434534565865 $service $action 66697741_94007840665633458 {"is_shared":"false","relationship":"FE"} {"userId":"19268547","userIdType":"profile_id","value":"1"}
+ |1434534566142 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"11917195","userIdType":"profile_id","value":"1"}
+ |1434534566077 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"37709890","userIdType":"profile_id","value":"1"}
+ |1434534565938 $service $action 40921487_94012905738975266 {"is_shared":"false","relationship":"FE"} {"userId":"59869223","userIdType":"profile_id","value":"1"}
+ |1434534566033 $service $action 64506628_93994707216829506 {"is_shared":"false","relationship":"FE"} {"userId":"50375575","userIdType":"profile_id","value":"1"}
+ |1434534566451 $service $action 40748868_94013448321919139 {"is_shared":"false","relationship":"FE"} {"userId":"12249539","userIdType":"profile_id","value":"1"}
+ |1434534566669 $service $action 64499956_94013227717457106 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
+ |1434534566669 $service $action 66444074_94012737377133826 {"is_shared":"false","relationship":"FE"} {"userId":"25167419","userIdType":"profile_id","value":"1"}
+ |1434534566318 $service $action 64774665_94012837889027027 {"is_shared":"true","relationship":"F"} {"userId":"71557816","userIdType":"profile_id","value":"1"}
+ |1434534566274 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"57931860","userIdType":"profile_id","value":"1"}
+ |1434534566659 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"19990823","userIdType":"profile_id","value":"1"}
+ |1434534566250 $service $action 70670053_93719933175630611 {"is_shared":"true","relationship":"F"} {"userId":"68897412","userIdType":"profile_id","value":"1"}
+ |1434534566402 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"15541439","userIdType":"profile_id","value":"1"}
+ |1434534566122 $service $action 48890741_94013463616012786 {"is_shared":"false","relationship":"FE"} {"userId":"48040409","userIdType":"profile_id","value":"1"}
+ |1434534566055 $service $action 64509008_94002318232678546 {"is_shared":"true","relationship":"F"} {"userId":"46532039","userIdType":"profile_id","value":"1"}
+ |1434534565994 $service $action 66644368_94009163363033795 {"is_shared":"false","relationship":"FE"} {"userId":"4143147","userIdType":"profile_id","value":"1"}
+ |1434534566448 $service $action 64587644_93938555963733954 {"is_shared":"false","relationship":"FE"} {"userId":"689042","userIdType":"profile_id","value":"1"}
+ |1434534565935 $service $action 52812511_94012009551561315 {"is_shared":"false","relationship":"FE"} {"userId":"35509692","userIdType":"profile_id","value":"1"}
+ |1434534566544 $service $action 70452048_94008573197583762 {"is_shared":"false","relationship":"FE"} {"userId":"5172421","userIdType":"profile_id","value":"1"}
+ |1434534565929 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"33556498","userIdType":"profile_id","value":"1"}
+ |1434534566358 $service $action 46889329_94013502934177075 {"is_shared":"false","relationship":"FE"} {"userId":"8987346","userIdType":"profile_id","value":"1"}
+ |1434534566057 $service $action 67075480_94008509166933763 {"is_shared":"false","relationship":"FE"} {"userId":"35134964","userIdType":"profile_id","value":"1"}
+ |1434534566140 $service $action 54547023_94013384964278435 {"is_shared":"false","relationship":"FE"} {"userId":"11900315","userIdType":"profile_id","value":"1"}
+ |1434534566158 $service $action 64639374_93888330176053635 {"is_shared":"true","relationship":"F"} {"userId":"49996643","userIdType":"profile_id","value":"1"}
+ |1434534566025 $service $action 67265128_94009084771192002 {"is_shared":"false","relationship":"FE"} {"userId":"37801480","userIdType":"profile_id","value":"1"}
+ """.stripMargin.trim
+ // println(data)
+ val rdd = sc.parallelize(Seq(("", data)))
+
+ // rdd.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+ val resultRdd = CounterFunctions.makeExactRdd(rdd, 2)
+ val result = resultRdd.collect().toMap
+
+ // result.foreachPartition { part =>
+ // part.foreach(println)
+ // }
+
+ val parsed = {
+ for {
+ line <- GraphUtil.parseString(data)
+ item <- CounterEtlItem(line).toSeq
+ ev <- CounterFunctions.exactMapper(item).toSeq
+ } yield {
+ ev
+ }
+ }
+ val parsedResult = parsed.groupBy(_._1).mapValues(values => values.map(_._2).reduce(CounterFunctions.reduceValue[ExactQualifier, Long](_ + _, 0L)))
+
+ // parsedResult.foreach { case (k, v) =>
+ // println(k, v)
+ // }
+
+ result should not be empty
+ result should equal (parsedResult)
+
+ val itemId = "46889329_94013502934177075"
+ val key = ExactKey(DefaultCounterModel.findByServiceAction(service, action).get, itemId, checkItemType = true)
+ val value = result.get(key)
+
+ value should not be empty
+ value.get.get(ExactQualifier(TimedQualifier("t", 0), Map.empty[String, String])) should equal (Some(6L))
+
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (None)
+
+ val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
+ resultRdd.foreachPartition { part =>
+ CounterFunctions.updateExactCounter(part.toSeq, acc)
+ }
+
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map.empty[String, String]) -> Map(ExactQualifier(TimedQualifier("t", 0), "") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map.empty[String, Set[String]]) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("is_shared" -> "false")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.false") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"))) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "relationship.FE") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("relationship" -> Set("FE"))) should be (Some(expected))
+ }
+ Option(FetchedCountsGrouped(key, Map(
+ (IntervalUnit.TOTAL, Map("is_shared" -> "false", "relationship" -> "FE")) -> Map(ExactQualifier(TimedQualifier("t", 0), "is_shared.relationship.false.FE") -> 6l)
+ ))).foreach { expected =>
+ exactCounter.getCount(policy, itemId, Seq(IntervalUnit.TOTAL), 0, 0, Map("is_shared" -> Set("false"), "relationship" -> Set("FE"))) should be (Some(expected))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_netty/build.sbt b/s2rest_netty/build.sbt
new file mode 100644
index 0000000..666fd80
--- /dev/null
+++ b/s2rest_netty/build.sbt
@@ -0,0 +1,7 @@
+name := "s2rest_netty"
+
+enablePlugins(JavaAppPackaging)
+
+libraryDependencies ++= Seq(
+ "io.netty" % "netty-all" % "4.0.33.Final"
+)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/conf/logger.xml
----------------------------------------------------------------------
diff --git a/s2rest_netty/conf/logger.xml b/s2rest_netty/conf/logger.xml
new file mode 100644
index 0000000..2d767c2
--- /dev/null
+++ b/s2rest_netty/conf/logger.xml
@@ -0,0 +1,83 @@
+<configuration>
+
+ <conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel"/>
+
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/application.log</file>
+
+ <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>logs/application.%i.log</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+
+ <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>500MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ </appender>
+
+
+ <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/error.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>logs/error.log.%d.%i</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>500MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>3</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%coloredLevel %logger{15} - %message%n%xException%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="ACTOR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/actor.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>logs/actor.log.%d.%i</fileNamePattern>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>200MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ <maxHistory>7</maxHistory>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="play" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="application" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="error" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="ERROR"/>
+ </logger>
+
+ <logger name="actor" level="INFO">
+ <appender-ref ref="ACTOR"/>
+ </logger>
+
+ <logger name="akka" level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/conf/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/conf/reference.conf b/s2rest_netty/conf/reference.conf
new file mode 100644
index 0000000..a992e5d
--- /dev/null
+++ b/s2rest_netty/conf/reference.conf
@@ -0,0 +1,131 @@
+# This is the main configuration file for the application.
+# ~~~~~
+
+# Secret key
+# ~~~~~
+# The secret key is used to secure cryptographics functions.
+#
+# This must be changed for production, but we recommend not changing it in this file.
+#
+# See http://www.playframework.com/documentation/latest/ApplicationSecret for more details.
+application.secret="/`==g^yr2DNnZGK_L^rguLZeR`60uLOVgY@OhyTv:maatl:Tl>9or/d1xME3b/Pi"
+
+# The application languages
+# ~~~~~
+application.langs="en"
+
+# Global object class
+# ~~~~~
+# Define the Global object class for this application.
+# Default to Global in the root package.
+# application.global=Global
+
+# Router
+# ~~~~~
+# Define the Router object to use for this application.
+# This router will be looked up first when the application is starting up,
+# so make sure this is the entry point.
+# Furthermore, it's assumed your route file is named properly.
+# So for an application router like `my.application.Router`,
+# you may need to define a router file `conf/my.application.routes`.
+# Default to Routes in the root package (and conf/routes)
+# application.router=my.application.Routes
+
+# Database configuration
+# ~~~~~
+# You can declare as many datasources as you want.
+# By convention, the default datasource is named `default`
+#
+# db.default.driver=org.h2.Driver
+# db.default.url="jdbc:h2:mem:play"
+# db.default.user=sa
+# db.default.password=""
+
+# Evolutions
+# ~~~~~
+# You can disable evolutions if needed
+# evolutionplugin=disabled
+
+# Logger
+# ~~~~~
+# You can also configure logback (http://logback.qos.ch/),
+# by providing an application-logger.xml file in the conf directory.
+
+# Root logger:
+logger.root=ERROR
+
+# Logger used by the framework:
+logger.play=INFO
+
+# Logger provided to your application:
+logger.application=DEBUG
+
+# APP PHASE
+phase=dev
+host=localhost
+
+# DB
+s2graph.models.table.name="models-dev"
+hbase.zookeeper.quorum=${host}
+db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
+# Query server
+is.query.server=true
+is.write.server=true
+query.hard.limit=100000
+
+# Local Cache
+cache.ttl.seconds=60
+cache.max.size=100000
+
+# HBASE
+#hbase.client.operation.timeout=1000
+#async.hbase.client.flush.interval=100
+hbase.table.compression.algorithm="gz"
+
+# Asynchbase
+hbase.client.retries.number=100
+hbase.rpcs.buffered_flush_interval=100
+hbase.rpc.timeout=0
+#hbase.nsre.high_watermark=1000000
+#hbase.timer.tick=5
+#hbase.timer.ticks_per_wheel=5
+
+# Kafka
+kafka.metadata.broker.list=${host}
+kafka.producer.pool.size=0
+
+# HTTP
+parsers.text.maxLength=512K
+parsers.json.maxLength=512K
+trustxforwarded=false
+
+# Local Queue Actor
+local.queue.actor.max.queue.size=100000
+local.queue.actor.rate.limit=1000000
+
+# local retry number
+max.retry.number=100
+max.back.off=50
+delete.all.fetch.size=10000
+hbase.fail.prob=-1.0
+
+# max allowd edges for deleteAll is multiply of above two configuration.
+
+# set global obejct package, TODO: remove global
+application.global=com.kakao.s2graph.rest.Global
+
+akka {
+ loggers = ["akka.event.slf4j.Slf4jLogger"]
+ loglevel = "DEBUG"
+}
+
+
+# Future cache.
+future.cache.max.size=100000
+future.cache.expire.after.write=10000
+future.cache.expire.after.access=5000
+
+
+# Counter
+redis.instances = [${host}]
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/resources/application.conf b/s2rest_netty/src/main/resources/application.conf
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf
new file mode 100644
index 0000000..a992e5d
--- /dev/null
+++ b/s2rest_netty/src/main/resources/reference.conf
@@ -0,0 +1,131 @@
+# This is the main configuration file for the application.
+# ~~~~~
+
+# Secret key
+# ~~~~~
+# The secret key is used to secure cryptographics functions.
+#
+# This must be changed for production, but we recommend not changing it in this file.
+#
+# See http://www.playframework.com/documentation/latest/ApplicationSecret for more details.
+application.secret="/`==g^yr2DNnZGK_L^rguLZeR`60uLOVgY@OhyTv:maatl:Tl>9or/d1xME3b/Pi"
+
+# The application languages
+# ~~~~~
+application.langs="en"
+
+# Global object class
+# ~~~~~
+# Define the Global object class for this application.
+# Default to Global in the root package.
+# application.global=Global
+
+# Router
+# ~~~~~
+# Define the Router object to use for this application.
+# This router will be looked up first when the application is starting up,
+# so make sure this is the entry point.
+# Furthermore, it's assumed your route file is named properly.
+# So for an application router like `my.application.Router`,
+# you may need to define a router file `conf/my.application.routes`.
+# Default to Routes in the root package (and conf/routes)
+# application.router=my.application.Routes
+
+# Database configuration
+# ~~~~~
+# You can declare as many datasources as you want.
+# By convention, the default datasource is named `default`
+#
+# db.default.driver=org.h2.Driver
+# db.default.url="jdbc:h2:mem:play"
+# db.default.user=sa
+# db.default.password=""
+
+# Evolutions
+# ~~~~~
+# You can disable evolutions if needed
+# evolutionplugin=disabled
+
+# Logger
+# ~~~~~
+# You can also configure logback (http://logback.qos.ch/),
+# by providing an application-logger.xml file in the conf directory.
+
+# Root logger:
+logger.root=ERROR
+
+# Logger used by the framework:
+logger.play=INFO
+
+# Logger provided to your application:
+logger.application=DEBUG
+
+# APP PHASE
+phase=dev
+host=localhost
+
+# DB
+s2graph.models.table.name="models-dev"
+hbase.zookeeper.quorum=${host}
+db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
+# Query server
+is.query.server=true
+is.write.server=true
+query.hard.limit=100000
+
+# Local Cache
+cache.ttl.seconds=60
+cache.max.size=100000
+
+# HBASE
+#hbase.client.operation.timeout=1000
+#async.hbase.client.flush.interval=100
+hbase.table.compression.algorithm="gz"
+
+# Asynchbase
+hbase.client.retries.number=100
+hbase.rpcs.buffered_flush_interval=100
+hbase.rpc.timeout=0
+#hbase.nsre.high_watermark=1000000
+#hbase.timer.tick=5
+#hbase.timer.ticks_per_wheel=5
+
+# Kafka
+kafka.metadata.broker.list=${host}
+kafka.producer.pool.size=0
+
+# HTTP
+parsers.text.maxLength=512K
+parsers.json.maxLength=512K
+trustxforwarded=false
+
+# Local Queue Actor
+local.queue.actor.max.queue.size=100000
+local.queue.actor.rate.limit=1000000
+
+# local retry number
+max.retry.number=100
+max.back.off=50
+delete.all.fetch.size=10000
+hbase.fail.prob=-1.0
+
+# max allowd edges for deleteAll is multiply of above two configuration.
+
+# set global obejct package, TODO: remove global
+application.global=com.kakao.s2graph.rest.Global
+
+akka {
+ loggers = ["akka.event.slf4j.Slf4jLogger"]
+ loglevel = "DEBUG"
+}
+
+
+# Future cache.
+future.cache.max.size=100000
+future.cache.expire.after.write=10000
+future.cache.expire.after.access=5000
+
+
+# Counter
+redis.instances = [${host}]
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_netty/src/main/scala/Server.scala
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala
index 7290b8a..16477b1 100644
--- a/s2rest_netty/src/main/scala/Server.scala
+++ b/s2rest_netty/src/main/scala/Server.scala
@@ -1,9 +1,12 @@
package com.kakao.s2graph.rest.netty
+import java.util.Map.Entry
import java.util.concurrent.Executors
+import java.util.function.Consumer
import com.kakao.s2graph.core.GraphExceptions.BadQueryException
import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.Experiment
import com.kakao.s2graph.core.rest.RestHandler.HandlerResult
import com.kakao.s2graph.core.rest._
import com.kakao.s2graph.core.utils.Extensions._
@@ -88,11 +91,11 @@ class S2RestHandler(s2rest: RestHandler)(implicit ec: ExecutionContext) extends
case e: BadQueryException =>
logger.error(s"{$requestBody}, ${e.getMessage}", e)
val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8)
- simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result())
+ simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
case e: Exception =>
logger.error(s"${requestBody}, ${e.getMessage}", e)
val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8)
- simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result())
+ simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = CloseOpt, headers = headers.result())
}
}
}
@@ -130,11 +133,10 @@ class S2RestHandler(s2rest: RestHandler)(implicit ec: ExecutionContext) extends
} else badRoute(ctx)
case HttpMethod.POST =>
- val jsonString = req.content.toString(CharsetUtil.UTF_8)
- val jsQuery = Json.parse(jsonString)
+ val body = req.content.toString(CharsetUtil.UTF_8)
- val result = s2rest.doPost(uri, jsQuery)
- toResponse(ctx, req, jsQuery, result, startedAt)
+ val result = s2rest.doPost(uri, body, Option(req.headers().get(Experiment.impressionKey)))
+ toResponse(ctx, req, Json.parse(body), result, startedAt)
case _ =>
simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_play/app/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/Bootstrap.scala b/s2rest_play/app/Bootstrap.scala
index 8005f79..6ce3ac4 100644
--- a/s2rest_play/app/Bootstrap.scala
+++ b/s2rest_play/app/Bootstrap.scala
@@ -7,7 +7,7 @@ import com.kakao.s2graph.core.rest._
import com.kakao.s2graph.core.utils.logger
import com.kakao.s2graph.core.{Management, ExceptionHandler, Graph}
import config.Config
-import controllers.{AdminController, ApplicationController}
+import controllers.{ApplicationController}
import play.api.Application
import play.api.mvc.{WithFilters, _}
import play.filters.gzip.GzipFilter
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_play/app/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/EdgeController.scala b/s2rest_play/app/controllers/EdgeController.scala
index 2b3b32e..a8e1a41 100644
--- a/s2rest_play/app/controllers/EdgeController.scala
+++ b/s2rest_play/app/controllers/EdgeController.scala
@@ -1,11 +1,9 @@
package controllers
import actors.QueueActor
-import com.kakao.s2graph.core.GraphExceptions.BadQueryException
import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{LabelMeta, Label}
+import com.kakao.s2graph.core.mysqls.{Label}
import com.kakao.s2graph.core.rest.RequestParser
-import com.kakao.s2graph.core.types.LabelWithDirection
import com.kakao.s2graph.core.utils.logger
import config.Config
import org.apache.kafka.clients.producer.ProducerRecord
@@ -14,7 +12,6 @@ import play.api.mvc.{Controller, Result}
import scala.collection.Seq
import scala.concurrent.Future
-import scala.util.{Failure, Success}
object EdgeController extends Controller {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
index bf24ed7..a387ba5 100644
--- a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
@@ -4,43 +4,43 @@ import play.api.libs.json.JsNumber
import play.api.test.{FakeApplication, PlaySpecification, WithApplication}
import play.libs.Json
-class JsonBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
- "to json" should {
- implicit val app = FakeApplication()
-
- "json benchmark" in new WithApplication(app) {
+class JsonBenchmarkSpec extends BenchmarkCommon {
+ "to json" >> {
+ "json benchmark" >> {
duration("map to json") {
- (0 to 100) foreach { n =>
+ (0 to 10) foreach { n =>
val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
Json.toJson(numberMaps)
}
}
duration("directMakeJson") {
- (0 to 100) foreach { n =>
+ (0 to 10) foreach { n =>
var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
+ (0 to 10).foreach { n =>
jsObj += (n.toString -> JsNumber(n * n))
}
}
}
duration("map to json 2") {
- (0 to 500) foreach { n =>
- val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ (0 to 50) foreach { n =>
+ val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap
Json.toJson(numberMaps)
}
}
duration("directMakeJson 2") {
- (0 to 500) foreach { n =>
+ (0 to 50) foreach { n =>
var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
+ (0 to 10).foreach { n =>
jsObj += (n.toString -> JsNumber(n * n))
}
}
}
+ true
}
+ true
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
index d38ff5e..d2d3624 100644
--- a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
@@ -8,10 +8,7 @@ import play.api.{Application => PlayApplication}
import scala.util.Random
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 9..
- */
-class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
+class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
"OrderingUtilBenchmarkSpec" should {
"performance MultiOrdering any" >> {
@@ -24,7 +21,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -35,7 +32,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}.map { x => x._1 }
val sorted2 = duration("MultiOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
}
seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
@@ -45,7 +42,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
"performance MultiOrdering double" >> {
- val tupLs = (0 until 500) map { i =>
+ val tupLs = (0 until 50) map { i =>
Random.nextDouble() -> Random.nextDouble()
}
@@ -54,13 +51,13 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
duration("MultiOrdering double") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
}
}
duration("TupleOrdering double") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -71,7 +68,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
"performance MultiOrdering jsvalue" >> {
- val tupLs = (0 until 500) map { i =>
+ val tupLs = (0 until 50) map { i =>
Random.nextDouble() -> Random.nextLong()
}
@@ -80,7 +77,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -91,7 +88,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon with PlaySpecification {
}
val sorted2 = duration("MultiOrdering jsvalue") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
}
seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
[2/4] incubator-s2graph git commit: [S2GRAPH-17]: Remove unnecessary
abstraction layer, Storage.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
deleted file mode 100644
index 337ed3f..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
+++ /dev/null
@@ -1,255 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import java.util
-import java.util.concurrent.TimeUnit
-import com.google.common.cache.CacheBuilder
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.QueryBuilder
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.{Extensions, logger}
-import com.stumbleupon.async.Deferred
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.GetRequest
-import scala.annotation.tailrec
-import scala.collection.JavaConversions._
-import scala.collection.{Map, Seq}
-import scala.util.Random
-import scala.concurrent.{ExecutionContext, Future}
-
-class AsynchbaseQueryBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionContext)
- extends QueryBuilder[GetRequest, Deferred[QueryRequestWithResult]](storage) {
-
- import Extensions.DeferOps
-
- val maxSize = storage.config.getInt("future.cache.max.size")
- val expreAfterWrite = storage.config.getInt("future.cache.expire.after.write")
- val expreAfterAccess = storage.config.getInt("future.cache.expire.after.access")
-
- val futureCache = CacheBuilder.newBuilder()
-// .recordStats()
- .initialCapacity(maxSize)
- .concurrencyLevel(Runtime.getRuntime.availableProcessors())
- .expireAfterWrite(expreAfterWrite, TimeUnit.MILLISECONDS)
- .expireAfterAccess(expreAfterAccess, TimeUnit.MILLISECONDS)
-// .weakKeys()
- .maximumSize(maxSize).build[java.lang.Long, (Long, Deferred[QueryRequestWithResult])]()
-
- // val scheduleTime = 60L * 60
-// val scheduleTime = 60
-// val scheduler = Executors.newScheduledThreadPool(1)
-//
-// scheduler.scheduleAtFixedRate(new Runnable(){
-// override def run() = {
-// logger.info(s"[FutureCache]: ${futureCache.stats()}")
-// }
-// }, scheduleTime, scheduleTime, TimeUnit.SECONDS)
-
- override def buildRequest(queryRequest: QueryRequest): GetRequest = {
- val srcVertex = queryRequest.vertex
- // val tgtVertexOpt = queryRequest.tgtVertexOpt
- val edgeCf = HSerializable.edgeCf
-
- val queryParam = queryRequest.queryParam
- val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
- val label = queryParam.label
- val labelWithDir = queryParam.labelWithDir
- val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
- val (srcInnerId, tgtInnerId) = tgtVertexIdOpt match {
- case Some(tgtVertexId) => // _to is given.
- /** we use toSnapshotEdge so dont need to swap src, tgt */
- val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- val tgt = InnerVal.convertVersion(tgtVertexId, tgtColumn.columnType, label.schemaVersion)
- (src, tgt)
- case None =>
- val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
- (src, src)
- }
-
- val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
- val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
- val currentTs = System.currentTimeMillis()
- val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs)).toMap
- val edge = Edge(srcV, tgtV, labelWithDir, propsWithTs = propsWithTs)
-
- val get = if (tgtVertexIdOpt.isDefined) {
- val snapshotEdge = edge.toSnapshotEdge
- val kv = storage.snapshotEdgeSerializer(snapshotEdge).toKeyValues.head
- new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
- } else {
- val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == queryParam.labelOrderSeq)
- assert(indexedEdgeOpt.isDefined)
-
- val indexedEdge = indexedEdgeOpt.get
- val kv = storage.indexEdgeSerializer(indexedEdge).toKeyValues.head
- val table = label.hbaseTableName.getBytes
- val rowKey = kv.row
- val cf = edgeCf
- new GetRequest(table, rowKey, cf)
- }
-
- val (minTs, maxTs) = queryParam.duration.getOrElse((0L, Long.MaxValue))
-
- get.maxVersions(1)
- get.setFailfast(true)
- get.setMaxResultsPerColumnFamily(queryParam.limit)
- get.setRowOffsetPerColumnFamily(queryParam.offset)
- get.setMinTimestamp(minTs)
- get.setMaxTimestamp(maxTs)
- get.setTimeout(queryParam.rpcTimeoutInMillis)
-
- if (queryParam.columnRangeFilter != null) get.setFilter(queryParam.columnRangeFilter)
-
- get
- }
-
- override def getEdge(srcVertex: Vertex, tgtVertex: Vertex, queryParam: QueryParam, isInnerCall: Boolean): Deferred[QueryRequestWithResult] = {
- //TODO:
- val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(tgtVertex.innerId))
- val q = Query.toQuery(Seq(srcVertex), _queryParam)
- val queryRequest = QueryRequest(q, 0, srcVertex, _queryParam)
- fetch(queryRequest, 1.0, isInnerCall = true, parentEdges = Nil)
- }
-
- override def fetch(queryRequest: QueryRequest,
- prevStepScore: Double,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): Deferred[QueryRequestWithResult] = {
- @tailrec
- def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (range < sampleNumber || set.size == sampleNumber) set
- else randomInt(sampleNumber, range, set + Random.nextInt(range))
- }
-
- def sample(edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
- if (edges.size <= n){
- edges
- }else{
- val plainEdges = if (queryRequest.queryParam.offset == 0) {
- edges.tail
- } else edges
-
- val randoms = randomInt(n, plainEdges.size)
- var samples = List.empty[EdgeWithScore]
- var idx = 0
- plainEdges.foreach { e =>
- if (randoms.contains(idx)) samples = e :: samples
- idx += 1
- }
- samples
- }
-
- }
- def normalize(edgeWithScores: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
- val sum = edgeWithScores.foldLeft(0.0) { case (acc, cur) => acc + cur.score }
- edgeWithScores.map { edgeWithScore =>
- edgeWithScore.copy(score = edgeWithScore.score / sum)
- }
- }
- def fetchInner(request: GetRequest) = {
- storage.client.get(request) withCallback { kvs =>
- val edgeWithScores = storage.toEdges(kvs.toSeq, queryRequest.queryParam, prevStepScore, isInnerCall, parentEdges)
-
- val normalized =
- if (queryRequest.queryParam.shouldNormalize) normalize(edgeWithScores)
- else edgeWithScores
-
- val resultEdgesWithScores =
- if (queryRequest.queryParam.sample >= 0 ) sample(normalized, queryRequest.queryParam.sample)
- else normalized
-
- QueryRequestWithResult(queryRequest, QueryResult(resultEdgesWithScores))
- } recoverWith { ex =>
- logger.error(s"fetchQueryParam failed. fallback return.", ex)
- QueryRequestWithResult(queryRequest, QueryResult(isFailure = true))
- }
- }
- def checkAndExpire(request: GetRequest,
- cacheKey: Long,
- cacheTTL: Long,
- cachedAt: Long,
- defer: Deferred[QueryRequestWithResult]): Deferred[QueryRequestWithResult] = {
- if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
- // future is too old. so need to expire and fetch new data from storage.
- futureCache.asMap().remove(cacheKey)
- val newPromise = new Deferred[QueryRequestWithResult]()
- futureCache.asMap().putIfAbsent(cacheKey, (System.currentTimeMillis(), newPromise)) match {
- case null =>
- // only one thread succeed to come here concurrently
- // initiate fetch to storage then add callback on complete to finish promise.
- fetchInner(request) withCallback { queryRequestWithResult =>
- newPromise.callback(queryRequestWithResult)
- queryRequestWithResult
- }
- newPromise
- case (cachedAt, oldDefer) => oldDefer
- }
- } else {
- // future is not to old so reuse it.
- defer
- }
- }
-
- val queryParam = queryRequest.queryParam
- val cacheTTL = queryParam.cacheTTLInMillis
- val request = buildRequest(queryRequest)
-
- if (cacheTTL <= 0) fetchInner(request)
- else {
- val cacheKeyBytes = Bytes.add(queryRequest.query.cacheKeyBytes, toCacheKeyBytes(request))
- val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
-
- val cacheVal = futureCache.getIfPresent(cacheKey)
- cacheVal match {
- case null =>
- // here there is no promise set up for this cacheKey so we need to set promise on future cache.
- val promise = new Deferred[QueryRequestWithResult]()
- val now = System.currentTimeMillis()
- val (cachedAt, defer) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
- case null =>
- fetchInner(request) withCallback { queryRequestWithResult =>
- promise.callback(queryRequestWithResult)
- queryRequestWithResult
- }
- (now, promise)
- case oldVal => oldVal
- }
- checkAndExpire(request, cacheKey, cacheTTL, cachedAt, defer)
- case (cachedAt, defer) =>
- checkAndExpire(request, cacheKey, cacheTTL, cachedAt, defer)
- }
- }
-
- }
-
-
- override def toCacheKeyBytes(getRequest: GetRequest): Array[Byte] = {
- var bytes = getRequest.key()
- Option(getRequest.family()).foreach(family => bytes = Bytes.add(bytes, family))
- Option(getRequest.qualifiers()).foreach {
- qualifiers =>
- qualifiers.filter(q => Option(q).isDefined).foreach {
- qualifier =>
- bytes = Bytes.add(bytes, qualifier)
- }
- }
- // if (getRequest.family() != null) bytes = Bytes.add(bytes, getRequest.family())
- // if (getRequest.qualifiers() != null) getRequest.qualifiers().filter(_ != null).foreach(q => bytes = Bytes.add(bytes, q))
- bytes
- }
-
-
- override def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]] = {
- val defers: Seq[Deferred[QueryRequestWithResult]] = for {
- (queryRequest, prevStepScore) <- queryRequestWithScoreLs
- parentEdges <- prevStepEdges.get(queryRequest.vertex.id)
- } yield fetch(queryRequest, prevStepScore, isInnerCall = false, parentEdges)
-
- val grouped: Deferred[util.ArrayList[QueryRequestWithResult]] = Deferred.group(defers)
- grouped withCallback {
- queryResults: util.ArrayList[QueryRequestWithResult] =>
- queryResults.toIndexedSeq
- } toFuture
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index a83aacd..7c05aed 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -1,17 +1,13 @@
package com.kakao.s2graph.core.storage.hbase
-import java.util
-import com.google.common.cache.Cache
-import com.kakao.s2graph.core.ExceptionHandler.{KafkaMessage, Key, Val}
-import com.kakao.s2graph.core.GraphExceptions.FetchTimeoutException
import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{Label, LabelMeta}
-import com.kakao.s2graph.core.storage.Storage
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.storage._
import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.{Extensions, logger}
+import com.kakao.s2graph.core.utils.{FutureCache, DeferCache, Extensions, logger}
import com.stumbleupon.async.Deferred
-import com.typesafe.config.Config
+import com.typesafe.config.{ConfigFactory, Config}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client.{ConnectionFactory, Durability}
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
@@ -19,20 +15,20 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
import org.apache.hadoop.hbase.regionserver.BloomType
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
-import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.hadoop.security.UserGroupInformation
import org.hbase.async._
import scala.collection.JavaConversions._
-import scala.collection.Seq
+import scala.collection.{Map, Seq}
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future, duration}
-import scala.util.Random
import scala.util.hashing.MurmurHash3
+import java.util
+
object AsynchbaseStorage {
- val vertexCf = HSerializable.vertexCf
- val edgeCf = HSerializable.edgeCf
+ val vertexCf = Serializable.vertexCf
+ val edgeCf = Serializable.edgeCf
val emptyKVs = new util.ArrayList[KeyValue]()
@@ -63,757 +59,245 @@ object AsynchbaseStorage {
}
}
-class AsynchbaseStorage(override val config: Config, vertexCache: Cache[Integer, Option[Vertex]])
- (implicit ec: ExecutionContext) extends Storage(config) {
-
- import AsynchbaseStorage._
- // import Extensions.FutureOps
+class AsynchbaseStorage(override val config: Config)(implicit ec: ExecutionContext)
+ extends Storage[Deferred[QueryRequestWithResult]](config) {
import Extensions.DeferOps
+ /**
+ * Asynchbase client setup.
+ * note that we need two client, one for bulk(withWait=false) and another for withWait=true
+ */
+ val configWithFlush = config.withFallback(ConfigFactory.parseMap(Map("hbase.rpcs.buffered_flush_interval" -> "0")))
val client = AsynchbaseStorage.makeClient(config)
- val queryBuilder = new AsynchbaseQueryBuilder(this)(ec)
- val mutationBuilder = new AsynchbaseMutationBuilder(this)(ec)
-
- // val cacheOpt = Option(cache)
- val cacheOpt = None
- val vertexCacheOpt = Option(vertexCache)
private val clientWithFlush = AsynchbaseStorage.makeClient(config, "hbase.rpcs.buffered_flush_interval" -> "0")
private val clients = Seq(client, clientWithFlush)
-
private val clientFlushInterval = config.getInt("hbase.rpcs.buffered_flush_interval").toString().toShort
- val MaxRetryNum = config.getInt("max.retry.number")
- val MaxBackOff = config.getInt("max.back.off")
- val DeleteAllFetchSize = config.getInt("delete.all.fetch.size")
- val FailProb = config.getDouble("hbase.fail.prob")
- val LockExpireDuration = config.getInt("lock.expire.time")
-
- /**
- * Serializer/Deserializer
- */
- def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge) = new SnapshotEdgeSerializable(snapshotEdge)
-
- def indexEdgeSerializer(indexedEdge: IndexEdge) = new IndexEdgeSerializable(indexedEdge)
-
- def vertexSerializer(vertex: Vertex) = new VertexSerializable(vertex)
-
- val snapshotEdgeDeserializer = new SnapshotEdgeDeserializable
- val indexEdgeDeserializer = new IndexEdgeDeserializable
- val vertexDeserializer = new VertexDeserializable
-
- def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = queryBuilder.getEdges(q)
-
- def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = {
- val futures = for {
- (srcVertex, tgtVertex, queryParam) <- params
- } yield queryBuilder.getEdge(srcVertex, tgtVertex, queryParam, false).toFuture
-
- Future.sequence(futures)
- }
-
- def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
- def fromResult(queryParam: QueryParam,
- kvs: Seq[org.hbase.async.KeyValue],
- version: String): Option[Vertex] = {
-
- if (kvs.isEmpty) None
- else {
- val newKVs = kvs
- Option(vertexDeserializer.fromKeyValues(queryParam, newKVs, version, None))
- }
- }
-
- val futures = vertices.map { vertex =>
- val kvs = vertexSerializer(vertex).toKeyValues
- val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, vertexCf)
- // get.setTimeout(this.singleGetTimeout.toShort)
- get.setFailfast(true)
- get.maxVersions(1)
-
- val cacheKey = MurmurHash3.stringHash(get.toString)
- val cacheVal = vertexCache.getIfPresent(cacheKey)
- if (cacheVal == null)
- client.get(get).toFutureWith(emptyKVs).map { kvs =>
- fromResult(QueryParam.Empty, kvs, vertex.serviceColumn.schemaVersion)
- }
+ private val emptyKeyValues = new util.ArrayList[KeyValue]()
+ private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
- else Future.successful(cacheVal)
- }
+ /** Future Cache to squash request */
+ private val futureCache = new DeferCache[QueryResult](config)(ec)
- Future.sequence(futures).map { result => result.toList.flatten }
- }
+ /** Simple Vertex Cache */
+ private val vertexCache = new FutureCache[Seq[SKeyValue]](config)(ec)
- def mutateEdge(edge: Edge, withWait: Boolean): Future[Boolean] = {
- val edgeFuture =
- if (edge.op == GraphUtil.operations("deleteAll")) {
- deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.label), edge.labelWithDir.dir, edge.ts)
- } else {
- val strongConsistency = edge.label.consistencyLevel == "strong"
- if (edge.op == GraphUtil.operations("delete") && !strongConsistency) {
- val zkQuorum = edge.label.hbaseZkAddr
- val (_, edgeUpdate) = Edge.buildDeleteBulk(None, edge)
- val mutations =
- mutationBuilder.indexedEdgeMutations(edgeUpdate) ++
- mutationBuilder.snapshotEdgeMutations(edgeUpdate) ++
- mutationBuilder.increments(edgeUpdate)
- writeAsyncSimple(zkQuorum, mutations, withWait)
- } else {
- mutateEdgesInner(Seq(edge), strongConsistency, withWait)(Edge.buildOperation)
+ /**
+ * fire rpcs into proper hbase cluster using client and
+ * return true on all mutation success. otherwise return false.
+ */
+ override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
+ if (kvs.isEmpty) Future.successful(true)
+ else {
+ val _client = client(withWait)
+ val futures = kvs.map { kv =>
+ val _defer = kv.operation match {
+ case SKeyValue.Put => _client.put(new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp))
+ case SKeyValue.Delete =>
+ if (kv.qualifier == null) _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.timestamp))
+ else _client.delete(new DeleteRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.timestamp))
+ case SKeyValue.Increment =>
+ _client.atomicIncrement(new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, Bytes.toLong(kv.value)))
}
- }
-
- val vertexFuture = writeAsyncSimple(edge.label.hbaseZkAddr,
- mutationBuilder.buildVertexPutsAsync(edge), withWait)
+ val future = _defer.withCallback { ret => true }.recoverWith { ex =>
+ logger.error(s"mutation failed. $kv", ex)
+ false
+ }.toFuture
- Future.sequence(Seq(edgeFuture, vertexFuture)).map(_.forall(identity))
- }
-
- override def mutateEdges(_edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
- val grouped = _edges.groupBy { edge => (edge.label, edge.srcVertex.innerId, edge.tgtVertex.innerId) } toSeq
-
- val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
- val (deleteAllEdges, edges) = edgeGroup.partition(_.op == GraphUtil.operations("deleteAll"))
-
- // DeleteAll first
- val deleteAllFutures = deleteAllEdges.map { edge =>
- deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.label), edge.labelWithDir.dir, edge.ts)
- }
-
- // After deleteAll, process others
- lazy val mutateEdgeFutures = edges.toList match {
- case head :: tail =>
- val strongConsistency = edges.head.label.consistencyLevel == "strong"
- if (strongConsistency) {
- val edgeFuture = mutateEdgesInner(edges, strongConsistency, withWait)(Edge.buildOperation)
-
- //TODO: decide what we will do on failure on vertex put
- val puts = mutationBuilder.buildVertexPutsAsync(head)
- val vertexFuture = writeAsyncSimple(head.label.hbaseZkAddr, puts, withWait)
- Seq(edgeFuture, vertexFuture)
- } else {
- edges.map { edge => mutateEdge(edge, withWait = withWait) }
- }
- case Nil => Nil
+ if (withWait) future else Future.successful(true)
}
- val composed = for {
- deleteRet <- Future.sequence(deleteAllFutures)
- mutateRet <- Future.sequence(mutateEdgeFutures)
- } yield deleteRet ++ mutateRet
-
- composed.map(_.forall(identity))
+ Future.sequence(futures).map(_.forall(identity))
}
-
- Future.sequence(mutateEdges)
}
- def mutateVertex(vertex: Vertex, withWait: Boolean): Future[Boolean] = {
- if (vertex.op == GraphUtil.operations("delete")) {
- writeAsyncSimple(vertex.hbaseZkAddr, mutationBuilder.buildDeleteAsync(vertex), withWait)
- } else if (vertex.op == GraphUtil.operations("deleteAll")) {
- logger.info(s"deleteAll for vertex is truncated. $vertex")
- Future.successful(true) // Ignore withWait parameter, because deleteAll operation may takes long time
- } else {
- writeAsyncSimple(vertex.hbaseZkAddr, mutationBuilder.buildPutsAll(vertex), withWait)
- }
- }
- def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = {
- val _client = if (withWait) clientWithFlush else client
- val defers: Seq[Deferred[(Boolean, Long)]] = for {
- edge <- edges
- } yield {
- val edgeWithIndex = edge.edgesWithIndex.head
- val countWithTs = edge.propsWithTs(LabelMeta.countSeq)
- val countVal = countWithTs.innerVal.toString().toLong
- val incr = mutationBuilder.buildIncrementsCountAsync(edgeWithIndex, countVal).head
- val request = incr.asInstanceOf[AtomicIncrementRequest]
- val defer = _client.bufferAtomicIncrement(request) withCallback { resultCount: java.lang.Long =>
- (true, resultCount.longValue())
- } recoverWith { ex =>
- logger.error(s"mutation failed. $request", ex)
- (false, -1L)
- }
- if (withWait) defer
- else Deferred.fromResult((true, -1L))
+ override def fetchSnapshotEdgeKeyValues(hbaseRpc: AnyRef): Future[Seq[SKeyValue]] = {
+ val defer = fetchKeyValuesInner(hbaseRpc)
+ defer.toFuture.map { kvsArr =>
+ kvsArr.map { kv =>
+ implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
+ } toSeq
}
-
- val grouped: Deferred[util.ArrayList[(Boolean, Long)]] = Deferred.groupInOrder(defers)
- grouped.toFuture.map(_.toSeq)
}
- private def writeAsyncSimpleRetry(zkQuorum: String, elementRpcs: Seq[HBaseRpc], withWait: Boolean): Future[Boolean] = {
- def compute = writeAsyncSimple(zkQuorum, elementRpcs, withWait).flatMap { ret =>
- if (ret) Future.successful(ret)
- else throw FetchTimeoutException("writeAsyncWithWaitRetrySimple")
- }
- Extensions.retryOnFailure(MaxRetryNum) {
- compute
- } {
- logger.error(s"writeAsyncWithWaitRetrySimple: $elementRpcs")
- false
- }
+ /**
+ * since HBase natively provide CheckAndSet on storage level, implementation becomes simple.
+ * @param rpc: key value that is need to be stored on storage.
+ * @param expectedOpt: last valid value for rpc's KeyValue.value from fetching.
+ * @return return true if expected value matches and our rpc is successfully applied, otherwise false.
+ * note that when some other thread modified same cell and have different value on this KeyValue,
+ * then HBase atomically return false.
+ */
+ override def writeLock(rpc: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean] = {
+ val put = new PutRequest(rpc.table, rpc.row, rpc.cf, rpc.qualifier, rpc.value, rpc.timestamp)
+ val expected = expectedOpt.map(_.value).getOrElse(Array.empty)
+ client(withWait = true).compareAndSet(put, expected).withCallback(ret => ret.booleanValue()).toFuture
}
- private def writeToStorage(_client: HBaseClient, rpc: HBaseRpc): Deferred[Boolean] = {
- // logger.debug(s"$rpc")
- val defer = rpc match {
- case d: DeleteRequest => _client.delete(d)
- case p: PutRequest => _client.put(p)
- case i: AtomicIncrementRequest => _client.bufferAtomicIncrement(i)
- }
- defer withCallback { ret => true } recoverWith { ex =>
- logger.error(s"mutation failed. $rpc", ex)
- false
- }
- }
- private def writeAsyncSimple(zkQuorum: String, elementRpcs: Seq[HBaseRpc], withWait: Boolean): Future[Boolean] = {
- val _client = if (withWait) clientWithFlush else client
- if (elementRpcs.isEmpty) {
- Future.successful(true)
- } else {
- val defers = elementRpcs.map { rpc => writeToStorage(_client, rpc) }
- if (withWait)
- Deferred.group(defers).toFuture map { arr => arr.forall(identity) }
- else
- Future.successful(true)
- }
- }
+ /**
+ * given queryRequest, build storage specific RPC Request.
+ * In HBase case, we either build Scanner or GetRequest.
+ *
+ * IndexEdge layer:
+ * Tall schema(v4): use scanner.
+ * Wide schema(label's schema version in v1, v2, v3): use GetRequest with columnRangeFilter
+ * when query is given with itnerval option.
+ * SnapshotEdge layer:
+ * Tall schema(v3, v4): use GetRequest without column filter.
+ * Wide schema(label's schema version in v1, v2): use GetRequest with columnRangeFilter.
+ * Vertex layer:
+ * all version: use GetRequest without column filter.
+ * @param queryRequest
+ * @return Scanner or GetRequest with proper setup with StartKey, EndKey, RangeFilter.
+ */
+ override def buildRequest(queryRequest: QueryRequest): AnyRef = {
+ import Serializable._
+ val queryParam = queryRequest.queryParam
+ val label = queryParam.label
+ val edge = toRequestEdge(queryRequest)
- private def writeAsync(zkQuorum: String, elementRpcs: Seq[Seq[HBaseRpc]], withWait: Boolean): Future[Seq[Boolean]] = {
- val _client = if (withWait) clientWithFlush else client
- if (elementRpcs.isEmpty) {
- Future.successful(Seq.empty[Boolean])
+ val kv = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
+ val snapshotEdge = edge.toSnapshotEdge
+ snapshotEdgeSerializer(snapshotEdge).toKeyValues.head
+ // new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
} else {
- val futures = elementRpcs.map { rpcs =>
- val defers = rpcs.map { rpc => writeToStorage(_client, rpc) }
- if (withWait)
- Deferred.group(defers).toFuture map { arr => arr.forall(identity) }
- else
- Future.successful(true)
- }
- if (withWait)
- Future.sequence(futures)
- else
- Future.successful(elementRpcs.map(_ => true))
- }
- }
-
- private def fetchSnapshotEdge(edge: Edge): Future[(QueryParam, Option[Edge], Option[KeyValue])] = {
- val labelWithDir = edge.labelWithDir
- val queryParam = QueryParam(labelWithDir)
- val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
- val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
- val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
-
- client.get(queryBuilder.buildRequest(queryRequest)) withCallback { kvs =>
- val (edgeOpt, kvOpt) =
- if (kvs.isEmpty()) (None, None)
- else {
- val _edgeOpt = toEdges(kvs, queryParam, 1.0, isInnerCall = true, parentEdges = Nil).headOption.map(_.edge)
- val _kvOpt = kvs.headOption
- (_edgeOpt, _kvOpt)
- }
- (queryParam, edgeOpt, kvOpt)
- } recoverWith { ex =>
- logger.error(s"fetchQueryParam failed. fallback return.", ex)
- throw new FetchTimeoutException(s"${edge.toLogString}")
- } toFuture
- }
-
-
- case class PartialFailureException(edge: Edge, statusCode: Byte, faileReason: String) extends Exception
-
- def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge) = {
- val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}").mkString("\n")
- logger.debug(msg)
- }
-
- def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge, edgeMutate: EdgeMutate) = {
- val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}",
- s"${edgeMutate.toLogString}").mkString("\n")
- logger.debug(msg)
- }
+ val indexedEdgeOpt = edge.edgesWithIndex.find(e => e.labelIndexSeq == queryParam.labelOrderSeq)
+ assert(indexedEdgeOpt.isDefined)
- private def buildLockEdge(snapshotEdgeOpt: Option[Edge], edge: Edge, kvOpt: Option[KeyValue]) = {
- val currentTs = System.currentTimeMillis()
- val lockTs = snapshotEdgeOpt match {
- case None => Option(currentTs)
- case Some(snapshotEdge) =>
- snapshotEdge.pendingEdgeOpt match {
- case None => Option(currentTs)
- case Some(pendingEdge) => pendingEdge.lockTs
- }
+ val indexedEdge = indexedEdgeOpt.get
+ indexEdgeSerializer(indexedEdge).toKeyValues.head
}
- val newVersion = kvOpt.map(_.timestamp()).getOrElse(edge.ts) + 1
- // snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
- val pendingEdge = edge.copy(version = newVersion, statusCode = 1, lockTs = lockTs)
- val base = snapshotEdgeOpt match {
- case None =>
- // no one ever mutated on this snapshotEdge.
- edge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
- case Some(snapshotEdge) =>
- // there is at least one mutation have been succeed.
- snapshotEdgeOpt.get.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
- }
- base.copy(version = newVersion, statusCode = 1, lockTs = None)
- }
- private def buildReleaseLockEdge(snapshotEdgeOpt: Option[Edge], lockEdge: SnapshotEdge,
- edgeMutate: EdgeMutate) = {
- val newVersion = lockEdge.version + 1
- val base = edgeMutate.newSnapshotEdge match {
- case None =>
- // shouldReplace false
- assert(snapshotEdgeOpt.isDefined)
- snapshotEdgeOpt.get.toSnapshotEdge
- case Some(newSnapshotEdge) => newSnapshotEdge
- }
- base.copy(version = newVersion, statusCode = 0, pendingEdgeOpt = None)
- }
+ val (minTs, maxTs) = queryParam.duration.getOrElse((0L, Long.MaxValue))
+ val get =
+ if (queryParam.tgtVertexInnerIdOpt.isDefined) new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf, kv.qualifier)
+ else new GetRequest(label.hbaseTableName.getBytes, kv.row, edgeCf)
- def mutate(predicate: Boolean,
- edge: Edge,
- statusCode: Byte,
- _edgeMutate: EdgeMutate): Future[Boolean] = {
- if (!predicate) throw new PartialFailureException(edge, 1, "predicate failed.")
+ get.maxVersions(1)
+ get.setFailfast(true)
+ get.setMaxResultsPerColumnFamily(queryParam.limit)
+ get.setRowOffsetPerColumnFamily(queryParam.offset)
+ get.setMinTimestamp(minTs)
+ get.setMaxTimestamp(maxTs)
+ get.setTimeout(queryParam.rpcTimeoutInMillis)
- if (statusCode >= 2) {
- logger.debug(s"skip mutate: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 1, s"$p")
- else
- writeAsyncSimple(edge.label.hbaseZkAddr, mutationBuilder.indexedEdgeMutations(_edgeMutate), withWait = true).map { ret =>
- if (ret) {
- debug(ret, "mutate", edge.toSnapshotEdge, _edgeMutate)
- } else {
- throw new PartialFailureException(edge, 1, "hbase fail.")
- }
- true
- }
- }
- }
+ if (queryParam.columnRangeFilter != null) get.setFilter(queryParam.columnRangeFilter)
- def increment(predicate: Boolean,
- edge: Edge,
- statusCode: Byte, _edgeMutate: EdgeMutate): Future[Boolean] = {
- if (!predicate) throw new PartialFailureException(edge, 2, "predicate failed.")
- if (statusCode >= 3) {
- logger.debug(s"skip increment: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 2, s"$p")
- else
- writeAsyncSimple(edge.label.hbaseZkAddr, mutationBuilder.increments(_edgeMutate), withWait = true).map { ret =>
- if (ret) {
- debug(ret, "increment", edge.toSnapshotEdge, _edgeMutate)
- } else {
- throw new PartialFailureException(edge, 2, "hbase fail.")
- }
- true
- }
- }
+ get
}
- def acquireLock(statusCode: Byte, edge: Edge,
- lockEdge: SnapshotEdge, oldBytes: Array[Byte]): Future[Boolean] =
- if (statusCode >= 1) {
- logger.debug(s"skip acquireLock: [$statusCode]\n${edge.toLogString}")
- Future.successful(true)
- } else {
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 0, s"$p")
- else {
- val lockEdgePut = toPutRequest(lockEdge)
- client.compareAndSet(lockEdgePut, oldBytes).toFuture.recoverWith {
- case ex: Exception =>
- logger.error(s"AcquireLock RPC Failed.")
- throw new PartialFailureException(edge, 0, "AcquireLock RPC Failed")
- }.map { ret =>
- if (ret) {
- val log = Seq(
- "\n",
- "=" * 50,
- s"[Success]: acquireLock",
- s"[RequestEdge]: ${edge.toLogString}",
- s"[LockEdge]: ${lockEdge.toLogString()}",
- s"[PendingEdge]: ${lockEdge.pendingEdgeOpt.map(_.toLogString).getOrElse("")}",
- "=" * 50, "\n").mkString("\n")
-
- logger.debug(log)
- // debug(ret, "acquireLock", edge.toSnapshotEdge)
- } else {
- throw new PartialFailureException(edge, 0, "hbase fail.")
- }
- true
- }
- }
- }
+ /**
+ * we are using future cache to squash requests into same key on storage.
+ *
+ * @param queryRequest
+ * @param prevStepScore
+ * @param isInnerCall
+ * @param parentEdges
+ * @return we use Deferred here since it has much better performrance compared to scala.concurrent.Future.
+ * seems like map, flatMap on scala.concurrent.Future is slower than Deferred's addCallback
+ */
+ override def fetch(queryRequest: QueryRequest,
+ prevStepScore: Double,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): Deferred[QueryRequestWithResult] = {
+
+ def fetchInner(hbaseRpc: AnyRef): Deferred[QueryResult] = {
+ fetchKeyValuesInner(hbaseRpc).withCallback { kvs =>
+ val edgeWithScores = toEdges(kvs, queryRequest.queryParam, prevStepScore, isInnerCall, parentEdges)
+ val resultEdgesWithScores = if (queryRequest.queryParam.sample >= 0) {
+ sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
+ } else edgeWithScores
+ QueryResult(resultEdgesWithScores)
+ // QueryRequestWithResult(queryRequest, QueryResult(resultEdgesWithScores, tailCursor = kvs.lastOption.map(_.key).getOrElse(Array.empty)))
- def releaseLock(predicate: Boolean,
- edge: Edge,
- lockEdge: SnapshotEdge,
- releaseLockEdge: SnapshotEdge,
- _edgeMutate: EdgeMutate,
- oldBytes: Array[Byte]): Future[Boolean] = {
- if (!predicate) {
- throw new PartialFailureException(edge, 3, "predicate failed.")
- }
- val p = Random.nextDouble()
- if (p < FailProb) throw new PartialFailureException(edge, 3, s"$p")
- else {
- val releaseLockEdgePut = toPutRequest(releaseLockEdge)
- val lockEdgePut = toPutRequest(lockEdge)
-
- client.compareAndSet(releaseLockEdgePut, lockEdgePut.value()).toFuture.recoverWith {
- case ex: Exception =>
- logger.error(s"ReleaseLock RPC Failed.")
- throw new PartialFailureException(edge, 3, "ReleaseLock RPC Failed")
- }.map { ret =>
- if (ret) {
- debug(ret, "releaseLock", edge.toSnapshotEdge)
- } else {
- val msg = Seq("\nFATAL ERROR\n",
- "=" * 50,
- oldBytes.toList,
- lockEdgePut.value.toList,
- releaseLockEdgePut.value().toList,
- "=" * 50,
- "\n"
- )
- logger.error(msg.mkString("\n"))
- // error(ret, "releaseLock", edge.toSnapshotEdge)
- throw new PartialFailureException(edge, 3, "hbase fail.")
- }
- true
+ } recoverWith { ex =>
+ logger.error(s"fetchInner failed. fallback return. $hbaseRpc}", ex)
+ QueryResult(isFailure = true)
+ // QueryRequestWithResult(queryRequest, QueryResult(isFailure = true))
}
}
- // }
- }
-
- private def toPutRequest(snapshotEdge: SnapshotEdge): PutRequest = {
- mutationBuilder.buildPutAsync(snapshotEdge).head.asInstanceOf[PutRequest]
- }
+ val queryParam = queryRequest.queryParam
+ val cacheTTL = queryParam.cacheTTLInMillis
+ val request = buildRequest(queryRequest)
- private def commitUpdate(edge: Edge,
- statusCode: Byte)(snapshotEdgeOpt: Option[Edge],
- kvOpt: Option[KeyValue],
- edgeUpdate: EdgeMutate): Future[Boolean] = {
- val label = edge.label
- def oldBytes = kvOpt.map(_.value()).getOrElse(Array.empty)
- // def oldBytes = snapshotEdgeOpt.map { e =>
- // snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head.value
- // }.getOrElse(Array.empty)
- def process(lockEdge: SnapshotEdge,
- releaseLockEdge: SnapshotEdge,
- _edgeMutate: EdgeMutate,
- statusCode: Byte): Future[Boolean] = {
-
- for {
- locked <- acquireLock(statusCode, edge, lockEdge, oldBytes)
- mutated <- mutate(locked, edge, statusCode, _edgeMutate)
- incremented <- increment(mutated, edge, statusCode, _edgeMutate)
- released <- releaseLock(incremented, edge, lockEdge, releaseLockEdge, _edgeMutate, oldBytes)
- } yield {
- released
+ val defer =
+ if (cacheTTL <= 0) fetchInner(request)
+ else {
+ val cacheKeyBytes = Bytes.add(queryRequest.query.cacheKeyBytes, toCacheKeyBytes(request))
+ val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
+ futureCache.getOrElseUpdate(cacheKey, cacheTTL)(fetchInner(request))
}
- }
-
-
- val lockEdge = buildLockEdge(snapshotEdgeOpt, edge, kvOpt)
- val releaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, edgeUpdate)
- snapshotEdgeOpt match {
- case None =>
- // no one ever did success on acquire lock.
- process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
- case Some(snapshotEdge) =>
- // someone did success on acquire lock at least one.
- snapshotEdge.pendingEdgeOpt match {
- case None =>
- // not locked
- process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
- case Some(pendingEdge) =>
- def isLockExpired = pendingEdge.lockTs.get + LockExpireDuration < System.currentTimeMillis()
- if (isLockExpired) {
- val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
- val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(pendingEdge))
- val newLockEdge = buildLockEdge(snapshotEdgeOpt, pendingEdge, kvOpt)
- val _newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, newLockEdge, newEdgeUpdate)
-
- // set lock ts as current ts
- val newPendingEdgeOpt = _newReleaseLockEdge.pendingEdgeOpt.map(_.copy(lockTs = Option(System.currentTimeMillis())))
- val newReleaseLockEdge = _newReleaseLockEdge.copy(pendingEdgeOpt = newPendingEdgeOpt)
-
- process(newLockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode = 0).flatMap { ret =>
- val log = s"[Success]: Resolving expired pending edge.\n${pendingEdge.toLogString}"
- throw new PartialFailureException(edge, 0, log)
- }
- } else {
- // locked
- if (pendingEdge.ts == edge.ts && statusCode > 0) {
- // self locked
- val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
- val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(edge))
- val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, newEdgeUpdate)
-
- /** lockEdge will be ignored */
- process(lockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode)
- } else {
- throw new PartialFailureException(edge, statusCode, s"others[${pendingEdge.ts}] is mutating. me[${edge.ts}]")
- }
- }
- }
- }
+ defer withCallback { queryResult => QueryRequestWithResult(queryRequest, queryResult)}
}
- private def mutateEdgesInner(edges: Seq[Edge],
- checkConsistency: Boolean,
- withWait: Boolean)(f: (Option[Edge], Seq[Edge]) => (Edge, EdgeMutate)): Future[Boolean] = {
- if (!checkConsistency) {
- val zkQuorum = edges.head.label.hbaseZkAddr
- val futures = edges.map { edge =>
- val (_, edgeUpdate) = f(None, Seq(edge))
- val mutations =
- mutationBuilder.indexedEdgeMutations(edgeUpdate) ++
- mutationBuilder.snapshotEdgeMutations(edgeUpdate) ++
- mutationBuilder.increments(edgeUpdate)
- writeAsyncSimple(zkQuorum, mutations, withWait)
- }
- Future.sequence(futures).map { rets => rets.forall(identity) }
- } else {
- def commit(_edges: Seq[Edge], statusCode: Byte): Future[Boolean] = {
-
- fetchSnapshotEdge(_edges.head) flatMap { case (queryParam, snapshotEdgeOpt, kvOpt) =>
-
- val (newEdge, edgeUpdate) = f(snapshotEdgeOpt, _edges)
- //shouldReplace false.
- if (edgeUpdate.newSnapshotEdge.isEmpty && statusCode <= 0) {
- logger.debug(s"${newEdge.toLogString} drop.")
- Future.successful(true)
- } else {
- commitUpdate(newEdge, statusCode)(snapshotEdgeOpt, kvOpt, edgeUpdate).map { ret =>
- if (ret) {
- logger.info(s"[Success] commit: \n${_edges.map(_.toLogString).mkString("\n")}")
- } else {
- throw new PartialFailureException(newEdge, 3, "commit failed.")
- }
- true
- }
- }
- }
- }
- def retry(tryNum: Int)(edges: Seq[Edge], statusCode: Byte)(fn: (Seq[Edge], Byte) => Future[Boolean]): Future[Boolean] = {
- if (tryNum >= MaxRetryNum) {
- edges.foreach { edge =>
- logger.error(s"commit failed after $MaxRetryNum\n${edge.toLogString}")
- ExceptionHandler.enqueue(ExceptionHandler.toKafkaMessage(element = edge))
- }
- Future.successful(false)
- } else {
- val future = fn(edges, statusCode)
- future.onSuccess {
- case success =>
- logger.debug(s"Finished. [$tryNum]\n${edges.head.toLogString}\n")
- }
- future recoverWith {
- case FetchTimeoutException(retryEdge) =>
- logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}")
- retry(tryNum + 1)(edges, statusCode)(fn)
-
- case PartialFailureException(retryEdge, failedStatusCode, faileReason) =>
- val status = failedStatusCode match {
- case 0 => "AcquireLock failed."
- case 1 => "Mutation failed."
- case 2 => "Increment failed."
- case 3 => "ReleaseLock failed."
- case 4 => "Unknown"
- }
-
- Thread.sleep(Random.nextInt(MaxBackOff))
- logger.info(s"[Try: $tryNum], [Status: $status] partial fail.\n${retryEdge.toLogString}\nFailReason: ${faileReason}")
- retry(tryNum + 1)(Seq(retryEdge), failedStatusCode)(fn)
- case ex: Exception =>
- logger.error("Unknown exception", ex)
- Future.successful(false)
- }
- }
- }
- retry(1)(edges, 0)(commit)
- }
- }
+ override def fetches(queryRequestWithScoreLs: scala.Seq[(QueryRequest, Double)],
+ prevStepEdges: Predef.Map[VertexId, scala.Seq[EdgeWithScore]]): Future[scala.Seq[QueryRequestWithResult]] = {
+ val defers: Seq[Deferred[QueryRequestWithResult]] = for {
+ (queryRequest, prevStepScore) <- queryRequestWithScoreLs
+ parentEdges <- prevStepEdges.get(queryRequest.vertex.id)
+ } yield fetch(queryRequest, prevStepScore, isInnerCall = false, parentEdges)
- def mutateLog(snapshotEdgeOpt: Option[Edge], edges: Seq[Edge],
- newEdge: Edge, edgeMutate: EdgeMutate) = {
- Seq("----------------------------------------------",
- s"SnapshotEdge: ${snapshotEdgeOpt.map(_.toLogString)}",
- s"requestEdges: ${edges.map(_.toLogString).mkString("\n")}",
- s"newEdge: ${newEdge.toLogString}",
- s"mutation: \n${edgeMutate.toLogString}",
- "----------------------------------------------").mkString("\n")
+ val grouped: Deferred[util.ArrayList[QueryRequestWithResult]] = Deferred.group(defers)
+ grouped withCallback {
+ queryResults: util.ArrayList[QueryRequestWithResult] =>
+ queryResults.toIndexedSeq
+ } toFuture
}
- private def deleteAllFetchedEdgesAsyncOld(queryRequest: QueryRequest,
- queryResult: QueryResult,
- requestTs: Long,
- retryNum: Int): Future[Boolean] = {
- val queryParam = queryRequest.queryParam
- val zkQuorum = queryParam.label.hbaseZkAddr
- val futures = for {
- edgeWithScore <- queryResult.edgeWithScoreLs
- (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
- } yield {
- /** reverted direction */
- val reversedIndexedEdgesMutations = edge.duplicateEdge.edgesWithIndex.flatMap { indexedEdge =>
- mutationBuilder.buildDeletesAsync(indexedEdge) ++ mutationBuilder.buildIncrementsAsync(indexedEdge, -1L)
- }
- val reversedSnapshotEdgeMutations = mutationBuilder.buildDeleteAsync(edge.toSnapshotEdge)
- val forwardIndexedEdgeMutations = edge.edgesWithIndex.flatMap { indexedEdge =>
- mutationBuilder.buildDeletesAsync(indexedEdge) ++ mutationBuilder.buildIncrementsAsync(indexedEdge, -1L)
- }
- val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
- writeAsyncSimple(zkQuorum, mutations, withWait = true)
- }
- Future.sequence(futures).map { rets => rets.forall(identity) }
- }
+ def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]] = fetchSnapshotEdgeKeyValues(request)
- private def buildEdgesToDelete(queryRequestWithResultLs: QueryRequestWithResult, requestTs: Long): QueryResult = {
- val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs).get
- val edgeWithScoreLs = queryResult.edgeWithScoreLs.filter { edgeWithScore =>
- (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
- }.map { edgeWithScore =>
- val label = queryRequest.queryParam.label
- val newPropsWithTs = edgeWithScore.edge.propsWithTs ++
- Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(requestTs, requestTs, label.schemaVersion))
- val copiedEdge = edgeWithScore.edge.copy(op = GraphUtil.operations("delete"), version = requestTs,
- propsWithTs = newPropsWithTs)
- edgeWithScore.copy(edge = copiedEdge)
- }
- queryResult.copy(edgeWithScoreLs = edgeWithScoreLs)
- }
- private def deleteAllFetchedEdgesLs(queryRequestWithResultLs: Seq[QueryRequestWithResult], requestTs: Long): Future[(Boolean, Boolean)] = {
- val queryResultLs = queryRequestWithResultLs.map(_.queryResult)
- queryResultLs.foreach { queryResult =>
- if (queryResult.isFailure) throw new RuntimeException("fetched result is fallback.")
- }
-
- val futures = for {
- queryRequestWithResult <- queryRequestWithResultLs
- (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
- deleteQueryResult = buildEdgesToDelete(queryRequestWithResult, requestTs)
- if deleteQueryResult.edgeWithScoreLs.nonEmpty
+ /**
+ * when withWait is given, we use client with flushInterval set to 0.
+ * if we are not using this, then we are adding extra wait time as much as flushInterval in worst case.
+ *
+ * @param edges
+ * @param withWait
+ * @return
+ */
+ override def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]] = {
+ val _client = client(withWait)
+ val defers: Seq[Deferred[(Boolean, Long)]] = for {
+ edge <- edges
} yield {
- val label = queryRequest.queryParam.label
- label.schemaVersion match {
- case HBaseType.VERSION3 if label.consistencyLevel == "strong" =>
-
- /**
- * read: snapshotEdge on queryResult = O(N)
- * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
- */
- mutateEdges(deleteQueryResult.edgeWithScoreLs.map(_.edge), withWait = true).map(_.forall(identity))
- case _ =>
-
- /**
- * read: x
- * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
- */
- deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
+ val edgeWithIndex = edge.edgesWithIndex.head
+ val countWithTs = edge.propsWithTs(LabelMeta.countSeq)
+ val countVal = countWithTs.innerVal.toString().toLong
+ val incr = buildIncrementsCountAsync(edgeWithIndex, countVal).head
+ val request = incr.asInstanceOf[AtomicIncrementRequest]
+ _client.bufferAtomicIncrement(request) withCallback { resultCount: java.lang.Long =>
+ (true, resultCount.longValue())
+ } recoverWith { ex =>
+ logger.error(s"mutation failed. $request", ex)
+ (false, -1L)
+ }
}
- }
- if (futures.isEmpty) {
- // all deleted.
- Future.successful(true -> true)
- } else {
- Future.sequence(futures).map { rets => false -> rets.forall(identity) }
- }
- }
-
- def fetchAndDeleteAll(query: Query, requestTs: Long): Future[(Boolean, Boolean)] = {
- val future = for {
- queryRequestWithResultLs <- getEdges(query)
- (allDeleted, ret) <- deleteAllFetchedEdgesLs(queryRequestWithResultLs, requestTs)
- } yield {
- (allDeleted, ret)
- }
- Extensions.retryOnFailure(MaxRetryNum) {
- future
- } {
- logger.error(s"fetch and deleteAll failed.")
- (true, false)
- }
+ val grouped: Deferred[util.ArrayList[(Boolean, Long)]] = Deferred.groupInOrder(defers)
+ grouped.toFuture.map(_.toSeq)
}
- def deleteAllAdjacentEdges(srcVertices: Seq[Vertex],
- labels: Seq[Label],
- dir: Int,
- ts: Long): Future[Boolean] = {
-
- def enqueueLogMessage() = {
- val kafkaMessages = for {
- vertice <- srcVertices
- id = vertice.innerId.toIdString()
- label <- labels
- } yield {
- val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", GraphUtil.fromOp(dir.toByte)).mkString("\t")
- val topic = ExceptionHandler.failTopic
- val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
- kafkaMsg
- }
-
- ExceptionHandler.enqueues(kafkaMessages)
- }
- val requestTs = ts
- val queryParams = for {
- label <- labels
- } yield {
- val labelWithDir = LabelWithDirection(label.id.get, dir)
- QueryParam(labelWithDir).limit(0, DeleteAllFetchSize).duplicatePolicy(Option(Query.DuplicatePolicy.Raw))
- }
-
- val step = Step(queryParams.toList)
- val q = Query(srcVertices, Vector(step))
-
- // Extensions.retryOnSuccessWithBackoff(MaxRetryNum, Random.nextInt(MaxBackOff) + 1) {
- val retryFuture = Extensions.retryOnSuccess(MaxRetryNum) {
- fetchAndDeleteAll(q, requestTs)
- } { case (allDeleted, deleteSuccess) =>
- allDeleted && deleteSuccess
- }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
-
- retryFuture onFailure {
- case ex =>
- logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
- enqueueLogMessage()
- }
-
- retryFuture
- }
-
- def flush(): Unit = clients.foreach { client =>
+ override def flush(): Unit = clients.foreach { client =>
val timeout = Duration((clientFlushInterval + 10) * 20, duration.MILLISECONDS)
Await.result(client.flush().toFuture, timeout)
}
- def createTable(zkAddr: String,
- tableName: String,
- cfs: List[String],
- regionMultiplier: Int,
- ttl: Option[Int],
- compressionAlgorithm: String): Unit = {
+ override def createTable(zkAddr: String,
+ tableName: String,
+ cfs: List[String],
+ regionMultiplier: Int,
+ ttl: Option[Int],
+ compressionAlgorithm: String): Unit = {
logger.info(s"create table: $tableName on $zkAddr, $cfs, $regionMultiplier, $compressionAlgorithm")
val admin = getAdmin(zkAddr)
val regionCount = admin.getClusterStatus.getServersSize * regionMultiplier
@@ -847,6 +331,77 @@ class AsynchbaseStorage(override val config: Config, vertexCache: Cache[Integer,
}
}
+
+ /** Asynchbase implementation override default getVertices to use future Cache */
+ override def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
+ def fromResult(queryParam: QueryParam,
+ kvs: Seq[SKeyValue],
+ version: String): Option[Vertex] = {
+
+ if (kvs.isEmpty) None
+ else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
+ }
+
+ val futures = vertices.map { vertex =>
+ val kvs = vertexSerializer(vertex).toKeyValues
+ val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
+ // get.setTimeout(this.singleGetTimeout.toShort)
+ get.setFailfast(true)
+ get.maxVersions(1)
+
+ val cacheKey = MurmurHash3.stringHash(get.toString)
+ vertexCache.getOrElseUpdate(cacheKey, cacheTTL = 10000)(fetchVertexKeyValues(get)).map { kvs =>
+ fromResult(QueryParam.Empty, kvs, vertex.serviceColumn.schemaVersion)
+ }
+ }
+
+ Future.sequence(futures).map { result => result.toList.flatten }
+ }
+
+
+
+
+
+ /**
+ * Private Methods which is specific to Asynchbase implementation.
+ */
+ private def fetchKeyValuesInner(rpc: AnyRef): Deferred[util.ArrayList[KeyValue]] = {
+ rpc match {
+ case getRequest: GetRequest => client.get(getRequest)
+ case scanner: Scanner =>
+ scanner.nextRows().withCallback { kvsLs =>
+ val ls = new util.ArrayList[KeyValue]
+ if (kvsLs == null) {
+
+ } else {
+ kvsLs.foreach { kvs =>
+ if (kvs != null) kvs.foreach { kv => ls.add(kv) }
+ else {
+
+ }
+ }
+ }
+ scanner.close()
+ ls
+ }.recoverWith { ex =>
+ logger.error(s"fetchKeyValuesInner failed.", ex)
+ scanner.close()
+ emptyKeyValues
+ }
+ case _ => Deferred.fromError(new RuntimeException(s"fetchKeyValues failed. $rpc"))
+ }
+ }
+
+ private def toCacheKeyBytes(hbaseRpc: AnyRef): Array[Byte] = {
+ hbaseRpc match {
+ case getRequest: GetRequest => getRequest.key()
+ case scanner: Scanner => scanner.getCurrentKey()
+ case _ =>
+ logger.error(s"toCacheKeyBytes failed. not supported class type. $hbaseRpc")
+ Array.empty[Byte]
+ }
+ }
+
private def getSecureClusterAdmin(zkAddr: String) = {
val jaas = config.getString("java.security.auth.login.config")
val krb5Conf = config.getString("java.security.krb5.conf")
@@ -855,6 +410,7 @@ class AsynchbaseStorage(override val config: Config, vertexCache: Cache[Integer,
val keytab = config.getString("keytab")
+
System.setProperty("java.security.auth.login.config", jaas)
System.setProperty("java.security.krb5.conf", krb5Conf)
// System.setProperty("sun.security.krb5.debug", "true")
@@ -922,5 +478,4 @@ class AsynchbaseStorage(override val config: Config, vertexCache: Cache[Integer,
private def getEndKey(regionCount: Int): Array[Byte] = {
Bytes.toBytes((Int.MaxValue / regionCount * (regionCount - 1)))
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HDeserializable.scala
deleted file mode 100644
index 25fe642..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HDeserializable.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.storage.{SKeyValue, StorageDeserializable}
-import com.kakao.s2graph.core.types.{LabelWithDirection, SourceVertexId, VertexId}
-import org.apache.hadoop.hbase.util.Bytes
-
-
-trait HDeserializable[E] extends StorageDeserializable[E] {
- import StorageDeserializable._
-
- type RowKeyRaw = (VertexId, LabelWithDirection, Byte, Boolean, Int)
-
- /** version 1 and version 2 share same code for parsing row key part */
- def parseRow(kv: SKeyValue, version: String): RowKeyRaw = {
- var pos = 0
- val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
- pos += srcIdLen
- val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
- pos += 4
- val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
-
- val rowLen = srcIdLen + 4 + 1
- (srcVertexId, labelWithDir, labelIdxSeq, isInverted, rowLen)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HSerializable.scala
deleted file mode 100644
index cc21cba..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/HSerializable.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.storage.StorageSerializable
-
-object HSerializable {
- val vertexCf = "v".getBytes()
- val edgeCf = "e".getBytes()
-}
-
-trait HSerializable[E] extends StorageSerializable[E]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeDeserializable.scala
deleted file mode 100644
index 64ac1cf..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeDeserializable.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.{CanSKeyValue, StorageDeserializable, SKeyValue}
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.utils.logger
-import org.apache.hadoop.hbase.util.Bytes
-
-class IndexEdgeDeserializable extends HDeserializable[IndexEdge] {
-
- import StorageDeserializable._
-
- type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
- type ValueRaw = (Array[(Byte, InnerValLike)], Int)
-
- private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- val degree = Bytes.toLong(kv.value)
- val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
- val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
- (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
- }
-
- private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
- var qualifierLen = 0
- var pos = 0
- val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
- val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
- pos = endAt
- qualifierLen += endAt
- val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
- (HBaseType.defaultTgtVertexId, 0)
- } else {
- TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
- }
- qualifierLen += tgtVertexIdLen
- (props, endAt, tgtVertexId, tgtVertexIdLen)
- }
- val (op, opLen) =
- if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
- else (kv.qualifier(qualifierLen), 1)
-
- qualifierLen += opLen
-
- (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
- }
-
- private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
- val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
- (props, endAt)
- }
-
- private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
- (Array.empty[(Byte, InnerValLike)], 0)
- }
-
-
-
- /** version 1 and version 2 is same logic */
- override def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[IndexEdge] = None): IndexEdge = {
- assert(_kvs.size == 1)
-
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-
- val kv = kvs.head
- val (srcVertexId, labelWithDir, labelIdxSeq, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.id, e.labelWithDir, e.labelIndexSeq, false, 0)
- }.getOrElse(parseRow(kv, version))
-
- val (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdInQualifier, _) =
- if (kv.qualifier.isEmpty) parseDegreeQualifier(kv, version)
- else parseQualifier(kv, version)
-
- val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
- val countVal = Bytes.toLong(kv.value)
- val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
- (dummyProps, 8)
- } else if (kv.qualifier.isEmpty) {
- parseDegreeValue(kv, version)
- } else {
- parseValue(kv, version)
- }
-
- val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException("invalid index seq"))
-
-
- // assert(kv.qualifier.nonEmpty && index.metaSeqs.size == idxPropsRaw.size)
-
- val idxProps = for {
- (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
- } yield {
- if (k == LabelMeta.degreeSeq) k -> v
- else seq -> v
- }
-
- val idxPropsMap = idxProps.toMap
- val tgtVertexId = if (tgtVertexIdInQualifier) {
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => tgtVertexIdRaw
- case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
- }
- } else tgtVertexIdRaw
-
- val _mergedProps = (idxProps ++ props).toMap
- val mergedProps =
- if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
- else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
-
-// logger.error(s"$mergedProps")
-// val ts = mergedProps(LabelMeta.timeStampSeq).toString().toLong
-
- val ts = kv.timestamp
- IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeSerializable.scala
deleted file mode 100644
index ceba4b9..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/IndexEdgeSerializable.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.mysqls.LabelMeta
-import com.kakao.s2graph.core.storage.{StorageSerializable, SKeyValue}
-import com.kakao.s2graph.core.types.VertexId
-import com.kakao.s2graph.core.utils.logger
-import com.kakao.s2graph.core.{GraphUtil, IndexEdge}
-import org.apache.hadoop.hbase.util.Bytes
-
-case class IndexEdgeSerializable(indexEdge: IndexEdge) extends HSerializable[IndexEdge] {
-
- import StorageSerializable._
-
- val label = indexEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = HSerializable.edgeCf
-
- val idxPropsMap = indexEdge.orders.toMap
- val idxPropsBytes = propsToBytes(indexEdge.orders)
-
- /** version 1 and version 2 share same code for serialize row key part */
- override def toKeyValues: Seq[SKeyValue] = {
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
-
- val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
-// logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
- val tgtIdBytes = VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes
- val qualifier =
- if (indexEdge.op == GraphUtil.operations("incrementCount")) {
- Bytes.add(idxPropsBytes, tgtIdBytes, Array.fill(1)(indexEdge.op))
- } else {
- idxPropsMap.get(LabelMeta.toSeq) match {
- case None => Bytes.add(idxPropsBytes, tgtIdBytes)
- case Some(vId) => idxPropsBytes
- }
- }
-
- val value = propsToKeyValues(indexEdge.metas.toSeq)
- val kv = SKeyValue(table, row, cf, qualifier, value, indexEdge.version)
-
- Seq(kv)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeDeserializable.scala
deleted file mode 100644
index 97f63e4..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeDeserializable.scala
+++ /dev/null
@@ -1,140 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.mysqls.{LabelIndex, LabelMeta}
-import com.kakao.s2graph.core.storage.{CanSKeyValue, SKeyValue, StorageDeserializable}
-import com.kakao.s2graph.core.types._
-import com.kakao.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-
-class SnapshotEdgeDeserializable extends HDeserializable[SnapshotEdge] {
-
- import StorageDeserializable._
-
- override def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
- queryParam.label.schemaVersion match {
- case HBaseType.VERSION3 => fromKeyValuesInnerV3(queryParam, _kvs, version, cacheElementOpt)
- case _ => fromKeyValuesInner(queryParam, _kvs, version, cacheElementOpt)
- }
- }
-
- def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
- val statusCode = byte >> 4
- val op = byte & ((1 << 4) - 1)
- (statusCode.toByte, op.toByte)
- }
-
- private def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
- assert(kvs.size == 1)
-
- val kv = kvs.head
- val schemaVer = queryParam.label.schemaVersion
- val cellVersion = kv.timestamp
-
- val (srcVertexId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.id, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
- }.getOrElse(parseRow(kv, schemaVer))
-
- val (tgtVertexId, props, op, ts, statusCode, _pendingEdgeOpt) = {
- val (tgtVertexId, _) = TargetVertexId.fromBytes(kv.qualifier, 0, kv.qualifier.length, schemaVer)
- var pos = 0
- val (statusCode, op) = statusCodeWithOp(kv.value(pos))
- pos += 1
- val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- val kvsMap = props.toMap
- val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
-
- pos = endAt
- val _pendingEdgeOpt =
- if (pos == kv.value.length) None
- else {
- val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
- pos += 1
- // val versionNum = Bytes.toLong(kv.value, pos, 8)
- // pos += 8
- val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- pos = endAt
- val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
-
- val pendingEdge =
- Edge(Vertex(srcVertexId, cellVersion),
- Vertex(tgtVertexId, cellVersion),
- labelWithDir, pendingEdgeOp,
- cellVersion, pendingEdgeProps.toMap,
- statusCode = pendingEdgeStatusCode, lockTs = lockTs)
- Option(pendingEdge)
- }
-
- (tgtVertexId, kvsMap, op, ts, statusCode, _pendingEdgeOpt)
- }
-
- SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
- labelWithDir, op, cellVersion, props, statusCode = statusCode,
- pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
- }
-
- private def fromKeyValuesInnerV3[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
- assert(kvs.size == 1)
-
- val kv = kvs.head
- val schemaVer = queryParam.label.schemaVersion
- val cellVersion = kv.timestamp
- /** rowKey */
- def parseRowV3(kv: SKeyValue, version: String) = {
- var pos = 0
- val (srcIdAndTgtId, srcIdAndTgtIdLen) = SourceAndTargetVertexIdPair.fromBytes(kv.row, pos, kv.row.length, version)
- pos += srcIdAndTgtIdLen
- val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
- pos += 4
- val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
-
- val rowLen = srcIdAndTgtIdLen + 4 + 1
- (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isInverted, rowLen)
-
- }
- val (srcInnerId, tgtInnerId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
- (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
- }.getOrElse(parseRowV3(kv, schemaVer))
-
- val srcVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, srcInnerId)
- val tgtVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, tgtInnerId)
-
- val (props, op, ts, statusCode, _pendingEdgeOpt) = {
- var pos = 0
- val (statusCode, op) = statusCodeWithOp(kv.value(pos))
- pos += 1
- val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- val kvsMap = props.toMap
- val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
-
- pos = endAt
- val _pendingEdgeOpt =
- if (pos == kv.value.length) None
- else {
- val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
- pos += 1
- // val versionNum = Bytes.toLong(kv.value, pos, 8)
- // pos += 8
- val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
- pos = endAt
- val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
-
- val pendingEdge =
- Edge(Vertex(srcVertexId, cellVersion),
- Vertex(tgtVertexId, cellVersion),
- labelWithDir, pendingEdgeOp,
- cellVersion, pendingEdgeProps.toMap,
- statusCode = pendingEdgeStatusCode, lockTs = lockTs)
- Option(pendingEdge)
- }
-
- (kvsMap, op, ts, statusCode, _pendingEdgeOpt)
- }
-
- SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
- labelWithDir, op, cellVersion, props, statusCode = statusCode,
- pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeSerializable.scala
deleted file mode 100644
index a9e77dc..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/SnapshotEdgeSerializable.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import java.util.UUID
-
-import com.kakao.s2graph.core.SnapshotEdge
-import com.kakao.s2graph.core.mysqls.LabelIndex
-import com.kakao.s2graph.core.storage.{StorageSerializable, SKeyValue}
-import com.kakao.s2graph.core.types.{HBaseType, SourceAndTargetVertexIdPair, VertexId}
-import com.kakao.s2graph.core.utils.logger
-import org.apache.hadoop.hbase.util.Bytes
-
-import scala.util.Random
-
-class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends HSerializable[SnapshotEdge] {
- import StorageSerializable._
-
- val label = snapshotEdge.label
- val table = label.hbaseTableName.getBytes()
- val cf = HSerializable.edgeCf
-
- def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
- val byte = (((statusCode << 4) | op).toByte)
- Array.fill(1)(byte.toByte)
- }
- def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
- propsToKeyValuesWithTs(snapshotEdge.props.toList))
-
- override def toKeyValues: Seq[SKeyValue] = {
- label.schemaVersion match {
- case HBaseType.VERSION3 => toKeyValuesInnerV3
- case _ => toKeyValuesInner
- }
- }
-
- private def toKeyValuesInner: Seq[SKeyValue] = {
- val srcIdBytes = VertexId.toSourceVertexId(snapshotEdge.srcVertex.id).bytes
- val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
-
- val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
- val tgtIdBytes = VertexId.toTargetVertexId(snapshotEdge.tgtVertex.id).bytes
-
- val qualifier = tgtIdBytes
-
- val value = snapshotEdge.pendingEdgeOpt match {
- case None => valueBytes()
- case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
- val versionBytes = Array.empty[Byte]
-// Bytes.toBytes(snapshotEdge.version)
- val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
-// Array.empty[Byte]
-// snapshotEdge.lockedAtOpt.map(lockedAt => Bytes.toBytes(lockedAt)).getOrElse(Array.empty[Byte])
- Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
- }
- val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
- Seq(kv)
- }
-
- private def toKeyValuesInnerV3: Seq[SKeyValue] = {
- val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes
- val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
-
- val row = Bytes.add(srcIdAndTgtIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
-
- val qualifier = Array.empty[Byte]
-
- val value = snapshotEdge.pendingEdgeOpt match {
- case None => valueBytes()
- case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
- val versionBytes = Array.empty[Byte]
-// Bytes.toBytes(snapshotEdge.version)
- val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
-// Array.empty[Byte]
-// snapshotEdge.lockedAtOpt.map(lockedAt => Bytes.toBytes(lockedAt)).getOrElse(Array.empty[Byte])
-// logger.error(s"ValueBytes: ${valueBytes().toList}")
-// logger.error(s"opBytes: ${opBytes.toList}")
-// logger.error(s"versionBytes: ${versionBytes.toList}")
-// logger.error(s"PropsBytes: ${propsBytes.toList}")
-// logger.error(s"LockBytes: ${lockBytes.toList}")
- Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
- }
-
- val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
- Seq(kv)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexDeserializable.scala
deleted file mode 100644
index 4c0ca19..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexDeserializable.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.storage.CanSKeyValue
-import com.kakao.s2graph.core.types.{InnerVal, InnerValLike, VertexId}
-import com.kakao.s2graph.core.{QueryParam, Vertex}
-import org.apache.hadoop.hbase.util.Bytes
-
-import scala.collection.mutable.ListBuffer
-
-class VertexDeserializable extends HDeserializable[Vertex] {
- def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam,
- _kvs: Seq[T],
- version: String,
- cacheElementOpt: Option[Vertex]): Vertex = {
-
- val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-
- val kv = kvs.head
- val (vertexId, _) = VertexId.fromBytes(kv.row, 0, kv.row.length, version)
-
- var maxTs = Long.MinValue
- val propsMap = new collection.mutable.HashMap[Int, InnerValLike]
- val belongLabelIds = new ListBuffer[Int]
-
- for {
- kv <- kvs
- } {
- val propKey =
- if (kv.qualifier.length == 1) kv.qualifier.head.toInt
- else Bytes.toInt(kv.qualifier)
-
- val ts = kv.timestamp
- if (ts > maxTs) maxTs = ts
-
- if (Vertex.isLabelId(propKey)) {
- belongLabelIds += Vertex.toLabelId(propKey)
- } else {
- val v = kv.value
- val (value, _) = InnerVal.fromBytes(v, 0, v.length, version)
- propsMap += (propKey -> value)
- }
- }
- assert(maxTs != Long.MinValue)
- Vertex(vertexId, maxTs, propsMap.toMap, belongLabelIds = belongLabelIds)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexSerializable.scala
deleted file mode 100644
index 370b844..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/VertexSerializable.scala
+++ /dev/null
@@ -1,19 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core.Vertex
-import com.kakao.s2graph.core.storage.SKeyValue
-import org.apache.hadoop.hbase.util.Bytes
-
-case class VertexSerializable(vertex: Vertex) extends HSerializable[Vertex] {
-
- val cf = HSerializable.vertexCf
-
- override def toKeyValues: Seq[SKeyValue] = {
- val row = vertex.id.bytes
- val base = for ((k, v) <- vertex.props ++ vertex.defaultProps) yield Bytes.toBytes(k) -> v.bytes
- val belongsTo = vertex.belongLabelIds.map { labelId => Bytes.toBytes(Vertex.toPropKey(labelId)) -> Array.empty[Byte] }
- (base ++ belongsTo).map { case (qualifier, value) =>
- SKeyValue(vertex.hbaseTableName.getBytes, row, cf, qualifier, value, vertex.ts)
- } toSeq
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
new file mode 100644
index 0000000..6777c28
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/DeferCache.scala
@@ -0,0 +1,82 @@
+package com.kakao.s2graph.core.utils
+
+import java.util.concurrent.TimeUnit
+
+import com.google.common.cache.CacheBuilder
+import com.stumbleupon.async.Deferred
+import com.typesafe.config.Config
+
+import scala.concurrent.ExecutionContext
+
+class DeferCache[R](config: Config)(implicit ex: ExecutionContext) {
+
+ import com.kakao.s2graph.core.utils.Extensions.DeferOps
+
+ type Value = (Long, Deferred[R])
+
+ private val maxSize = config.getInt("future.cache.max.size")
+ private val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ private val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+ private val futureCache = CacheBuilder.newBuilder()
+ .initialCapacity(maxSize)
+ .concurrencyLevel(Runtime.getRuntime.availableProcessors())
+ .expireAfterWrite(expireAfterWrite, TimeUnit.MILLISECONDS)
+ .expireAfterAccess(expireAfterAccess, TimeUnit.MILLISECONDS)
+ .maximumSize(maxSize).build[java.lang.Long, (Long, Deferred[R])]()
+
+
+ def asMap() = futureCache.asMap()
+
+ def getIfPresent(cacheKey: Long): Value = futureCache.getIfPresent(cacheKey)
+
+ private def checkAndExpire(cacheKey: Long,
+ cachedAt: Long,
+ cacheTTL: Long,
+ oldDefer: Deferred[R])(op: => Deferred[R]): Deferred[R] = {
+ if (System.currentTimeMillis() >= cachedAt + cacheTTL) {
+ // future is too old. so need to expire and fetch new data from storage.
+ futureCache.asMap().remove(cacheKey)
+
+ val newPromise = new Deferred[R]()
+ val now = System.currentTimeMillis()
+
+ futureCache.asMap().putIfAbsent(cacheKey, (now, newPromise)) match {
+ case null =>
+ // only one thread succeed to come here concurrently
+ // initiate fetch to storage then add callback on complete to finish promise.
+ op withCallback { value =>
+ newPromise.callback(value)
+ value
+ }
+ newPromise
+ case (cachedAt, oldDefer) => oldDefer
+ }
+ } else {
+ // future is not to old so reuse it.
+ oldDefer
+ }
+ }
+ def getOrElseUpdate(cacheKey: Long, cacheTTL: Long)(op: => Deferred[R]): Deferred[R] = {
+ val cacheVal = futureCache.getIfPresent(cacheKey)
+ cacheVal match {
+ case null =>
+ val promise = new Deferred[R]()
+ val now = System.currentTimeMillis()
+ val (cachedAt, defer) = futureCache.asMap().putIfAbsent(cacheKey, (now, promise)) match {
+ case null =>
+ op.withCallback { value =>
+ promise.callback(value)
+ value
+ }
+ (now, promise)
+ case oldVal => oldVal
+ }
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
+
+ case (cachedAt, defer) =>
+ checkAndExpire(cacheKey, cacheTTL, cachedAt, defer)(op)
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
index 4858f60..eea9a79 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala
@@ -2,7 +2,6 @@ package com.kakao.s2graph.core.utils
import com.stumbleupon.async.{Callback, Deferred}
import com.typesafe.config.Config
-
import scala.concurrent.{ExecutionContext, Future, Promise}
object Extensions {
@@ -70,5 +69,4 @@ object Extensions {
def getBooleanWithFallback(key: String, defaultValue: Boolean): Boolean =
if (config.hasPath(key)) config.getBoolean(key) else defaultValue
}
-
}
[4/4] incubator-s2graph git commit: [S2GRAPH-17]: Remove unnecessary
abstraction layer, Storage.
Posted by st...@apache.org.
[S2GRAPH-17]: Remove unnecessary abstraction layer, Storage.
1. refactor all common codes from AsynchbaseStorage to Storage trait.
2. little bit of documentation.
3. restructuring package(move serializer/deserializer from storage/hbase into storage).
4. extract future cache from AsynchbaseStorage into utils.
JIRA:
[S2GRAPH-17] https://issues.apache.org/jira/browse/S2GRAPH-17
Pull Request:
Closes #34
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e207f676
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e207f676
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e207f676
Branch: refs/heads/master
Commit: e207f676fe145690b027a8015fd25794ec47ff94
Parents: 36d5485
Author: DO YUNG YOON <st...@apache.org>
Authored: Mon Feb 29 15:14:07 2016 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Mon Feb 29 15:14:07 2016 +0900
----------------------------------------------------------------------
.gitignore | 2 +-
CHANGES | 2 +
README.md | 2 +-
dev_support/README.md | 45 +
dev_support/docker-compose.yml | 30 +
dev_support/graph_mysql/Dockerfile | 5 +
dev_support/graph_mysql/schema.sql | 218 ++++
docker-compose.yml | 30 -
loader/build.sbt | 7 +
.../main/scala/subscriber/GraphSubscriber.scala | 9 +-
.../main/scala/subscriber/TransferToHFile.scala | 19 +-
.../src/main/scala/subscriber/WalLogStat.scala | 94 ++
.../main/scala/subscriber/WalLogToHDFS.scala | 235 ++--
.../scala/subscriber/GraphSubscriberTest.scala | 8 +-
.../scala/subscriber/TransferToHFileTest.scala | 169 +++
project/plugins.sbt | 5 +-
s2core/build.sbt | 1 -
s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar | Bin 1305124 -> 1302410 bytes
s2core/migrate/mysql/schema.sql | 1 +
s2core/src/main/resources/create_model.hql | 1 -
s2core/src/main/resources/logback.xml | 13 +-
s2core/src/main/resources/reference.conf | 16 +-
.../scala/com/kakao/s2graph/core/Edge.scala | 1 +
.../scala/com/kakao/s2graph/core/Graph.scala | 21 +-
.../com/kakao/s2graph/core/Management.scala | 1 -
.../com/kakao/s2graph/core/PostProcess.scala | 33 +
.../com/kakao/s2graph/core/QueryParam.scala | 20 +-
.../com/kakao/s2graph/core/QueryRequest.scala | 12 -
.../com/kakao/s2graph/core/QueryResult.scala | 28 +-
.../scala/com/kakao/s2graph/core/Vertex.scala | 1 +
.../kakao/s2graph/core/mysqls/Experiment.scala | 2 +
.../kakao/s2graph/core/rest/RequestParser.scala | 43 +-
.../s2graph/core/storage/Deserializable.scala | 25 +
.../core/storage/IndexEdgeDeserializable.scala | 128 ++
.../core/storage/IndexEdgeSerializable.scala | 58 +
.../s2graph/core/storage/MutationBuilder.scala | 57 -
.../s2graph/core/storage/QueryBuilder.scala | 104 --
.../kakao/s2graph/core/storage/SKeyValue.scala | 22 +-
.../s2graph/core/storage/Serializable.scala | 10 +
.../storage/SnapshotEdgeDeserializable.scala | 142 +++
.../core/storage/SnapshotEdgeSerializable.scala | 76 ++
.../kakao/s2graph/core/storage/Storage.scala | 1137 +++++++++++++++++-
.../core/storage/StorageDeserializable.scala | 17 +-
.../core/storage/VertexDeserializable.scala | 46 +
.../core/storage/VertexSerializable.scala | 18 +
.../hbase/AsynchbaseMutationBuilder.scala | 119 --
.../storage/hbase/AsynchbaseQueryBuilder.scala | 255 ----
.../core/storage/hbase/AsynchbaseStorage.scala | 961 ++++-----------
.../core/storage/hbase/HDeserializable.scala | 25 -
.../core/storage/hbase/HSerializable.scala | 10 -
.../storage/hbase/IndexEdgeDeserializable.scala | 116 --
.../storage/hbase/IndexEdgeSerializable.scala | 45 -
.../hbase/SnapshotEdgeDeserializable.scala | 140 ---
.../hbase/SnapshotEdgeSerializable.scala | 91 --
.../storage/hbase/VertexDeserializable.scala | 47 -
.../core/storage/hbase/VertexSerializable.scala | 19 -
.../kakao/s2graph/core/utils/DeferCache.scala | 82 ++
.../kakao/s2graph/core/utils/Extentions.scala | 2 -
.../kakao/s2graph/core/utils/FutureCache.scala | 82 ++
.../kakao/s2graph/core/Integrate/CrudTest.scala | 2 +-
.../core/Integrate/IntegrateCommon.scala | 22 +-
.../s2graph/core/Integrate/QueryTest.scala | 546 +++++++--
.../core/Integrate/StrongLabelDeleteTest.scala | 17 +-
.../core/Integrate/WeakLabelDeleteTest.scala | 4 +-
.../com/kakao/s2graph/core/ManagementTest.scala | 121 --
.../s2graph/core/parsers/WhereParserTest.scala | 54 +-
.../hbase/AsynchbaseQueryBuilderTest.scala | 106 +-
.../s2/counter/core/CounterFunctions.scala | 2 +-
.../stream/ExactCounterStreamingSpec.scala | 196 +++
s2rest_netty/build.sbt | 7 +
s2rest_netty/conf/logger.xml | 83 ++
s2rest_netty/conf/reference.conf | 131 ++
.../src/main/resources/application.conf | 0
s2rest_netty/src/main/resources/reference.conf | 131 ++
s2rest_netty/src/main/scala/Server.scala | 14 +-
s2rest_play/app/Bootstrap.scala | 2 +-
.../app/controllers/EdgeController.scala | 5 +-
s2rest_play/build.sbt | 0
.../test/benchmark/JsonBenchmarkSpec.scala | 24 +-
.../benchmark/OrderingUtilBenchmarkSpec.scala | 21 +-
80 files changed, 4039 insertions(+), 2357 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 76f2bb8..1e21302 100644
--- a/.gitignore
+++ b/.gitignore
@@ -94,7 +94,7 @@ bin/
/db
.eclipse
/lib/
-/logs/
+logs/
/modules
/project/project
/project/target
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 1512043..97434d6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -63,6 +63,8 @@ Release 0.12.1 - unreleased
S2GRAPH-5: Add Apache RAT to valid LICENSE errors. (Committed by DOYUNG YOON).
+ S2GRAPH-17: Remove unnecessary abstraction layer, Storage. (Committed by DOYUNG YOON).
+
SUB TASKS
S2GRAPH-9: Provide rest server using netty. (Committed by daewon).
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 07a49f6..61430ee 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@
S2Graph comes with a Vagrantfile that lets you spin up a virtual environment for test and development purposes.
(On setting up S2Graph in your local environment directly, please refer to [Quick Start in Your Local Environment](https://steamshon.gitbooks.io/s2graph-book/content/getting_started.html).)
-You will need [VirtualBox](https://www.virtualbox.org/wiki/Downloads) and [Vagrant](http://docs.ansible.com/ansible/intro_installation.html) installed on your system.
+You will need [VirtualBox](https://www.virtualbox.org/wiki/Downloads) and [Vagrant](https://www.vagrantup.com/downloads.html) installed on your system.
With everything ready, let's get started by running the following commands:
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/dev_support/README.md
----------------------------------------------------------------------
diff --git a/dev_support/README.md b/dev_support/README.md
new file mode 100644
index 0000000..556a7d9
--- /dev/null
+++ b/dev_support/README.md
@@ -0,0 +1,45 @@
+# Run S2Graph using Docker
+
+1. Build a docker image of the s2graph in the project's root directory
+ - `sbt "project s2rest_play" docker:publishLocal`
+2. Run MySQL and HBase container first.
+ - change directory to dev-support. `cd dev-support`
+ - `docker-compose build`
+ - `docker-compose up -d graph_mysql` will run MySQL and HBase at same time.
+3. Run graph container
+ - `docker-compose up -d`
+
+> S2Graph should be connected with MySQL at initial state. Therefore you have to run MySQL and HBase before running it.
+
+## For OS X
+
+In OS X, the docker container is running on VirtualBox. In order to connect with HBase in the docker container from your local machine. You have to register the IP of the docker-machine into the `/etc/hosts` file.
+
+Within the `docker-compose.yml` file, I had supposed the name of docker-machine as `default`. So, in the `/etc/hosts` file, register the docker-machine name as `default`.
+
+```
+ex)
+192.168.99.100 default
+```
+
+# Run S2Graph on your local machine
+
+In order to develop and test S2Graph. You might be want to run S2Graph as `dev` mode on your local machine. In this case, the following commands are helpful.
+
+- Run only MySQL and HBase
+
+```
+# docker-compose up -d graph_mysql
+```
+
+- Run s2graph as 'dev' mode
+
+```
+# sbt "project s2rest_play" run -Dhost=default
+```
+
+- or run test cases
+
+```
+# sbt test -Dhost=default
+```
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/dev_support/docker-compose.yml
----------------------------------------------------------------------
diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml
new file mode 100644
index 0000000..9818251
--- /dev/null
+++ b/dev_support/docker-compose.yml
@@ -0,0 +1,30 @@
+graph:
+ image: s2rest_play:0.12.1-SNAPSHOT
+ container_name: graph
+ net: container:graph_hbase
+ links:
+ - graph_mysql
+ - graph_hbase
+
+graph_mysql:
+ build: graph_mysql
+ container_name: graph_mysql
+ environment:
+ MYSQL_ROOT_PASSWORD: graph
+ net: container:graph_hbase
+
+graph_hbase:
+ image: nerdammer/hbase:0.98.10.1
+ container_name: graph_hbase
+ hostname: "${DOCKER_MACHINE_NAME}"
+ ports:
+ - "3306:3306"
+ - "2181:2181"
+ - "60010:60010"
+ - "60000:60000"
+ - "60020:60020"
+ - "60030:60030"
+ - "9000:9000"
+ expose:
+ - "3306"
+ - "9000"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/dev_support/graph_mysql/Dockerfile
----------------------------------------------------------------------
diff --git a/dev_support/graph_mysql/Dockerfile b/dev_support/graph_mysql/Dockerfile
new file mode 100644
index 0000000..c6849f0
--- /dev/null
+++ b/dev_support/graph_mysql/Dockerfile
@@ -0,0 +1,5 @@
+FROM mysql
+
+MAINTAINER Jaesang Kim <ho...@gmail.com>
+
+ADD ./schema.sql /docker-entrypoint-initdb.d/
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/dev_support/graph_mysql/schema.sql
----------------------------------------------------------------------
diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql
new file mode 100644
index 0000000..2216896
--- /dev/null
+++ b/dev_support/graph_mysql/schema.sql
@@ -0,0 +1,218 @@
+CREATE DATABASE IF NOT EXISTS graph_dev;
+
+CREATE USER 'graph'@'%' IDENTIFIED BY 'graph';
+
+GRANT ALL PRIVILEGES ON graph_dev.* TO 'graph'@'%' identified by 'graph';
+
+flush privileges;
+
+use graph_dev;
+
+
+SET FOREIGN_KEY_CHECKS = 0;
+
+-- ----------------------------
+-- Table structure for `services`
+-- ----------------------------
+DROP TABLE IF EXISTS `services`;
+CREATE TABLE `services` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_name` varchar(64) NOT NULL,
+ `access_token` varchar(64) NOT NULL,
+ `cluster` varchar(255) NOT NULL,
+ `hbase_table_name` varchar(255) NOT NULL,
+ `pre_split_size` integer NOT NULL default 0,
+ `hbase_table_ttl` integer,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_service_name` (`service_name`),
+ INDEX `idx_access_token` (`access_token`),
+ INDEX `idx_cluster` (cluster(75))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+-- ----------------------------
+-- Table structure for `services_columns`
+-- ----------------------------
+DROP TABLE IF EXISTS `service_columns`;
+CREATE TABLE `service_columns` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_id` integer NOT NULL,
+ `column_name` varchar(64) NOT NULL,
+ `column_type` varchar(8) NOT NULL,
+ `schema_version` varchar(8) NOT NULL default 'v2',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE service_columns add FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `column_metas`
+-- ----------------------------
+DROP TABLE IF EXISTS `column_metas`;
+CREATE TABLE `column_metas` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `column_id` integer NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `seq` tinyint NOT NULL,
+ `data_type` varchar(8) NOT NULL DEFAULT 'string',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_column_id_name` (`column_id`, `name`),
+ INDEX `idx_column_id_seq` (`column_id`, `seq`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE column_metas ADD FOREIGN KEY(column_id) REFERENCES service_columns(id) ON DELETE CASCADE;
+
+-- ----------------------------
+-- Table structure for `labels`
+-- ----------------------------
+
+DROP TABLE IF EXISTS `labels`;
+CREATE TABLE `labels` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `label` varchar(64) NOT NULL,
+ `src_service_id` integer NOT NULL,
+ `src_column_name` varchar(64) NOT NULL,
+ `src_column_type` varchar(8) NOT NULL,
+ `tgt_service_id` integer NOT NULL,
+ `tgt_column_name` varchar(64) NOT NULL,
+ `tgt_column_type` varchar(8) NOT NULL,
+ `is_directed` tinyint NOT NULL DEFAULT 1,
+ `service_name` varchar(64),
+ `service_id` integer NOT NULL,
+ `consistency_level` varchar(8) NOT NULL DEFAULT 'weak',
+ `hbase_table_name` varchar(255) NOT NULL DEFAULT 's2graph',
+ `hbase_table_ttl` integer,
+ `schema_version` varchar(8) NOT NULL default 'v2',
+ `is_async` tinyint(4) NOT NULL default '0',
+ `compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4',
+ `options` text,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label` (`label`),
+ INDEX `idx_src_column_name` (`src_column_name`),
+ INDEX `idx_tgt_column_name` (`tgt_column_name`),
+ INDEX `idx_src_service_id` (`src_service_id`),
+ INDEX `idx_tgt_service_id` (`tgt_service_id`),
+ INDEX `idx_service_name` (`service_name`),
+ INDEX `idx_service_id` (`service_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id);
+
+
+
+-- ----------------------------
+-- Table structure for `label_metas`
+-- ----------------------------
+DROP TABLE IF EXISTS `label_metas`;
+CREATE TABLE `label_metas` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `label_id` integer NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `seq` tinyint NOT NULL,
+ `default_value` varchar(64) NOT NULL,
+ `data_type` varchar(8) NOT NULL DEFAULT 'long',
+ `used_in_index` tinyint NOT NULL DEFAULT 0,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label_id_name` (`label_id`, `name`),
+ INDEX `idx_label_id_seq` (`label_id`, `seq`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE label_metas ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `label_indices`
+-- ----------------------------
+DROP TABLE IF EXISTS `label_indices`;
+CREATE TABLE `label_indices` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `label_id` int(11) NOT NULL,
+ `name` varchar(64) NOT NULL DEFAULT '_PK',
+ `seq` tinyint(4) NOT NULL,
+ `meta_seqs` varchar(64) NOT NULL,
+ `formulars` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label_id_seq` (`label_id`,`meta_seqs`),
+ UNIQUE KEY `ux_label_id_name` (`label_id`,`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `experiments`
+-- ----------------------------
+DROP TABLE IF EXISTS `experiments`;
+CREATE TABLE `experiments` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_id` integer NOT NULL,
+ `service_name` varchar(128) NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `description` varchar(255) NOT NULL,
+ `experiment_type` varchar(8) NOT NULL DEFAULT 'u',
+ `total_modular` int NOT NULL DEFAULT 100,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_service_id_name` (`service_id`, `name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ALTER TABLE experiments ADD FOREIGN KEY(service_id) REFERENCES service(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `buckets`
+-- ----------------------------
+CREATE TABLE `buckets` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `experiment_id` integer NOT NULL,
+ `modular` varchar(64) NOT NULL,
+ `http_verb` varchar(8) NOT NULL,
+ `api_path` text NOT NULL,
+ `uuid_key` varchar(128),
+ `uuid_placeholder` varchar(64),
+ `request_body` text NOT NULL,
+ `timeout` int NOT NULL DEFAULT 1000,
+ `impression_id` varchar(64) NOT NULL,
+ `is_graph_query` tinyint NOT NULL DEFAULT 1,
+ `is_empty` tinyint NOT NULL DEFAULT 0,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_impression_id` (`impression_id`),
+ INDEX `idx_experiment_id` (`experiment_id`),
+ INDEX `idx_impression_id` (`impression_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+SET FOREIGN_KEY_CHECKS = 1;
+
+
+-- ----------------------------
+-- Table structure for `counter`
+-- ----------------------------
+DROP TABLE IF EXISTS `counter`;
+CREATE TABLE `counter` (
+ `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+ `use_flag` tinyint(1) NOT NULL DEFAULT '0',
+ `version` smallint(1) NOT NULL DEFAULT '1',
+ `service` varchar(64) NOT NULL DEFAULT '',
+ `action` varchar(64) NOT NULL DEFAULT '',
+ `item_type` int(11) NOT NULL DEFAULT '0',
+ `auto_comb` tinyint(1) NOT NULL DEFAULT '1',
+ `dimension` varchar(1024) NOT NULL,
+ `use_profile` tinyint(1) NOT NULL DEFAULT '0',
+ `bucket_imp_id` varchar(64) DEFAULT NULL,
+ `use_exact` tinyint(1) NOT NULL DEFAULT '1',
+ `use_rank` tinyint(1) NOT NULL DEFAULT '1',
+ `ttl` int(11) NOT NULL DEFAULT '172800',
+ `daily_ttl` int(11) DEFAULT NULL,
+ `hbase_table` varchar(1024) DEFAULT NULL,
+ `interval_unit` varchar(1024) DEFAULT NULL,
+ `rate_action_id` int(11) unsigned DEFAULT NULL,
+ `rate_base_id` int(11) unsigned DEFAULT NULL,
+ `rate_threshold` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `svc` (`service`,`action`),
+ KEY `rate_action_id` (`rate_action_id`),
+ KEY `rate_base_id` (`rate_base_id`),
+ CONSTRAINT `rate_action_id` FOREIGN KEY (`rate_action_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
+ CONSTRAINT `rate_base_id` FOREIGN KEY (`rate_base_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker-compose.yml b/docker-compose.yml
deleted file mode 100644
index 3149a89..0000000
--- a/docker-compose.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-graph:
- image: s2graph:0.12.0-SNAPSHOT
- container_name: graph
- net: container:graph_hbase
- links:
- - graph_mysql
- - graph_hbase
-
-graph_mysql:
- image: devsupport_graph_mysql
- container_name: graph_mysql
- environment:
- MYSQL_ROOT_PASSWORD: graph
- net: container:graph_hbase
-
-graph_hbase:
- image: nerdammer/hbase:0.98.10.1
- container_name: graph_hbase
- hostname: default
- ports:
- - "3306:3306"
- - "2181:2181"
- - "60010:60010"
- - "60000:60000"
- - "60020:60020"
- - "60030:60030"
- - "9000:9000"
- expose:
- - "3306:3306"
- - "9000"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/build.sbt
----------------------------------------------------------------------
diff --git a/loader/build.sbt b/loader/build.sbt
index b497a9c..43e1126 100644
--- a/loader/build.sbt
+++ b/loader/build.sbt
@@ -4,9 +4,14 @@ name := "s2loader"
scalacOptions ++= Seq("-deprecation")
+projectDependencies := Seq(
+ (projectID in "s2core").value exclude("org.mortbay.jetty", "*") exclude("javax.xml.stream", "*") exclude("javax.servlet", "*")
+)
+
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided",
+ "org.apache.spark" %% "spark-hive" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion,
"org.apache.httpcomponents" % "fluent-hc" % "4.2.5",
"org.specs2" %% "specs2-core" % "2.4.11" % "test",
@@ -30,3 +35,5 @@ excludedJars in assembly := {
}
test in assembly := {}
+
+parallelExecution in Test := false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/main/scala/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/GraphSubscriber.scala b/loader/src/main/scala/subscriber/GraphSubscriber.scala
index 88de3d7..f4f7865 100644
--- a/loader/src/main/scala/subscriber/GraphSubscriber.scala
+++ b/loader/src/main/scala/subscriber/GraphSubscriber.scala
@@ -22,7 +22,7 @@ object GraphConfig {
var kafkaBrokers = ""
var cacheTTL = s"${60 * 60 * 1}"
def apply(phase: String, dbUrl: Option[String], zkAddr: Option[String], kafkaBrokerList: Option[String]): Config = {
- database = dbUrl.getOrElse("jdbc:mysql://localhost:3306/graph")
+ database = dbUrl.getOrElse("jdbc:mysql://localhost:3306/graph_dev")
zkQuorum = zkAddr.getOrElse("localhost")
// val newConf = new util.HashMap[String, Object]()
@@ -50,6 +50,7 @@ object GraphSubscriberHelper extends WithKafka {
private val maxTryNum = 10
var g: Graph = null
+ var management: Management = null
val conns = new scala.collection.mutable.HashMap[String, Connection]()
def toOption(s: String) = {
@@ -63,7 +64,9 @@ object GraphSubscriberHelper extends WithKafka {
config = GraphConfig(phase, toOption(dbUrl), toOption(zkQuorum), toOption(kafkaBrokerList))
if (g == null) {
- g = new Graph(config)(ExecutionContext.Implicits.global)
+ val ec = ExecutionContext.Implicits.global
+ g = new Graph(config)(ec)
+ management = new Management(g)(ec)
}
}
@@ -95,6 +98,8 @@ object GraphSubscriberHelper extends WithKafka {
case Some(v) if v.isInstanceOf[Vertex] =>
statFunc("VertexParseOk", 1)
v.asInstanceOf[Vertex]
+ case Some(x) =>
+ throw new RuntimeException(s">>>>> GraphSubscriber.toGraphElements: parsing failed. ${x.serviceName}")
case None =>
throw new RuntimeException(s"GraphSubscriber.toGraphElements: parsing failed. $msg")
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/main/scala/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/TransferToHFile.scala b/loader/src/main/scala/subscriber/TransferToHFile.scala
index aba9673..516bb39 100644
--- a/loader/src/main/scala/subscriber/TransferToHFile.scala
+++ b/loader/src/main/scala/subscriber/TransferToHFile.scala
@@ -2,8 +2,8 @@ package subscriber
import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.Label
-import com.kakao.s2graph.core.types.{LabelWithDirection, SourceVertexId}
+import com.kakao.s2graph.core.mysqls.{LabelMeta, Label}
+import com.kakao.s2graph.core.types.{InnerValLikeWithTs, LabelWithDirection, SourceVertexId}
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
@@ -55,10 +55,7 @@ object TransferToHFile extends SparkApp with JSONParser {
direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
reverseDirection = if (direction == "out") "in" else "out"
convertedLabelName = labelMapping.get(tokens(5)).getOrElse(tokens(5))
- (vertexIdStr, vertexIdStrReversed) = direction match {
- case "out" => (tokens(3), tokens(4))
- case _ => (tokens(4), tokens(3))
- }
+ (vertexIdStr, vertexIdStrReversed) = (tokens(3), tokens(4))
degreeKey = DegreeKey(vertexIdStr, convertedLabelName, direction)
degreeKeyReversed = DegreeKey(vertexIdStrReversed, convertedLabelName, reverseDirection)
extra = if (edgeAutoCreate) List(degreeKeyReversed -> 1L) else Nil
@@ -80,8 +77,12 @@ object TransferToHFile extends SparkApp with JSONParser {
throw new RuntimeException(s"$vertexId can not be converted into innerval")
}
val vertex = Vertex(SourceVertexId(label.srcColumn.id.get, innerVal))
+
+ val ts = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
val labelWithDir = LabelWithDirection(label.id.get, dir)
- val edge = Edge(vertex, vertex, labelWithDir)
+ val edge = Edge(vertex, vertex, labelWithDir, propsWithTs=propsWithTs)
+
edge.edgesWithIndex.flatMap { indexEdge =>
GraphSubscriberHelper.g.storage.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
new PutRequest(kv.table, kv.row, kv.cf, Array.empty[Byte], Bytes.toBytes(degreeVal), kv.timestamp)
@@ -129,14 +130,14 @@ object TransferToHFile extends SparkApp with JSONParser {
val labelMapping = if (args.length >= 7) GraphSubscriberHelper.toLabelMapping(args(6)) else Map.empty[String, String]
val autoEdgeCreate = if (args.length >= 8) args(7).toBoolean else false
val buildDegree = if (args.length >= 9) args(8).toBoolean else true
-
+ val compressionAlgorithm = if (args.length >= 10) args(9) else "lz4"
val conf = sparkConf(s"$input: TransferToHFile")
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
conf.set("spark.kryoserializer.buffer.mb", "24")
val sc = new SparkContext(conf)
-
+ Management.createTable(zkQuorum, tableName, List("e", "v"), maxHFilePerResionServer, None, compressionAlgorithm)
/** set up hbase init */
val hbaseConf = HBaseConfiguration.create()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/main/scala/subscriber/WalLogStat.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/WalLogStat.scala b/loader/src/main/scala/subscriber/WalLogStat.scala
new file mode 100644
index 0000000..f5db2c1
--- /dev/null
+++ b/loader/src/main/scala/subscriber/WalLogStat.scala
@@ -0,0 +1,94 @@
+package subscriber
+
+import java.text.SimpleDateFormat
+import java.util.Date
+
+import com.kakao.s2graph.core.Graph
+import kafka.producer.KeyedMessage
+import kafka.serializer.StringDecoder
+import org.apache.spark.streaming.Durations._
+import org.apache.spark.streaming.kafka.HasOffsetRanges
+import s2.spark.{HashMapParam, SparkApp, WithKafka}
+
+import scala.collection.mutable.{HashMap => MutableHashMap}
+import scala.language.postfixOps
+
+object WalLogStat extends SparkApp with WithKafka {
+
+ override def run() = {
+
+ validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "statTopic")
+
+ val kafkaZkQuorum = args(0)
+ val brokerList = args(1)
+ val topics = args(2)
+ val intervalInSec = seconds(args(3).toLong)
+ val dbUrl = args(4)
+ val statTopic = args(5)
+
+
+ val conf = sparkConf(s"$topics: ${getClass.getSimpleName}")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val groupId = topics.replaceAll(",", "_") + "_stat"
+
+ val kafkaParams = Map(
+ "zookeeper.connect" -> kafkaZkQuorum,
+ "group.id" -> groupId,
+ "metadata.broker.list" -> brokerList,
+ "zookeeper.connection.timeout.ms" -> "10000",
+ "auto.offset.reset" -> "largest")
+
+ val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
+ val statProducer = getProducer[String, String](brokerList)
+
+ stream.foreachRDD { (rdd, time) =>
+
+ val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
+
+ val ts = time.milliseconds
+
+ val elements = rdd.mapPartitions { partition =>
+ // set executor setting.
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
+ partition.map { case (key, msg) =>
+ Graph.toGraphElement(msg) match {
+ case Some(elem) =>
+ val serviceName = elem.serviceName
+ msg.split("\t", 7) match {
+ case Array(_, operation, log_type, _, _, label, _*) =>
+ Seq(serviceName, label, operation, log_type).mkString("\t")
+ case _ =>
+ Seq("no_service_name", "no_label", "no_operation", "parsing_error").mkString("\t")
+ }
+ case None =>
+ Seq("no_service_name", "no_label", "no_operation", "no_element_error").mkString("\t")
+ }
+ }
+ }
+
+ val countByKey = elements.map(_ -> 1L).reduceByKey(_ + _).collect()
+ val totalCount = countByKey.map(_._2).sum
+ val keyedMessage = countByKey.map { case (key, value) =>
+ new KeyedMessage[String, String](statTopic, s"$ts\t$key\t$value\t$totalCount")
+ }
+
+ statProducer.send(keyedMessage: _*)
+
+ elements.mapPartitionsWithIndex { (i, part) =>
+ // commit offset range
+ val osr = offsets(i)
+ getStreamHelper(kafkaParams).commitConsumerOffset(osr)
+ Iterator.empty
+ }.foreach {
+ (_: Nothing) => ()
+ }
+
+ }
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/main/scala/subscriber/WalLogToHDFS.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/subscriber/WalLogToHDFS.scala b/loader/src/main/scala/subscriber/WalLogToHDFS.scala
index 2c37aef..44902c2 100644
--- a/loader/src/main/scala/subscriber/WalLogToHDFS.scala
+++ b/loader/src/main/scala/subscriber/WalLogToHDFS.scala
@@ -5,6 +5,7 @@ import java.util.Date
import com.kakao.s2graph.core.Graph
import kafka.serializer.StringDecoder
+import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.streaming.Durations._
import org.apache.spark.streaming.kafka.HasOffsetRanges
import s2.spark.{HashMapParam, SparkApp, WithKafka}
@@ -12,103 +13,137 @@ import s2.spark.{HashMapParam, SparkApp, WithKafka}
import scala.collection.mutable.{HashMap => MutableHashMap}
import scala.language.postfixOps
-//object WalLogToHDFS extends SparkApp with WithKafka {
-// private def toOutputPath(ts: Long): String = {
-// val dateId = new SimpleDateFormat("yyyy-MM-dd").format(new Date(ts))
-// s"date_id=$dateId/ts=$ts"
-// }
-//
-// val usages =
-// s"""
-// |/**
-// |this job consume edges/vertices from kafka topic then load them into s2graph.
-// |params:
-// | 1. kafkaZkQuorum: kafka zk address to consume events
-// | 2. brokerList: kafka cluster`s broker list.
-// | 3. topics: , delimited list of topics to consume
-// | 4. intervalInSec: batch interval for this job.
-// | 5. dbUrl:
-// | 6. outputPath:
-// |*/
-// """.stripMargin
-// override def run() = {
-// validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "outputPath")
-//// if (args.length != 7) {
-//// System.err.println(usages)
-//// System.exit(1)
-//// }
-// val kafkaZkQuorum = args(0)
-// val brokerList = args(1)
-// val topics = args(2)
-// val intervalInSec = seconds(args(3).toLong)
-// val dbUrl = args(4)
-// val outputPath = args(5)
-//
-// val conf = sparkConf(s"$topics: WalLogToHDFS")
-// val ssc = streamingContext(conf, intervalInSec)
-// val sc = ssc.sparkContext
-//
-// val groupId = topics.replaceAll(",", "_") + "_stream"
-// val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
-//
-// val kafkaParams = Map(
-// "zookeeper.connect" -> kafkaZkQuorum,
-// "group.id" -> groupId,
-// "metadata.broker.list" -> brokerList,
-// "zookeeper.connection.timeout.ms" -> "10000",
-// "auto.offset.reset" -> "largest")
-//
-// val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
-//
-// val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
-//
-// stream.foreachRDD { (rdd, time) =>
-// val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
-//
-// val elements = rdd.mapPartitions { partition =>
-// // set executor setting.
-// val phase = System.getProperty("phase")
-// GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
-//
-// partition.flatMap { case (key, msg) =>
-// val optMsg = Graph.toGraphElement(msg).map { element =>
-// val n = msg.split("\t", -1).length
-// if(n == 6) {
-// Seq(msg, "{}", element.serviceName).mkString("\t")
-// }
-// else if(n == 7) {
-// Seq(msg, element.serviceName).mkString("\t")
-// }
-// else {
-// null
-// }
-// }
-// optMsg
-// }
-// }
-//
-// val ts = time.milliseconds
-// val path = s"$outputPath/${toOutputPath(ts)}"
-//
-// /** make sure that `elements` are not running at the same time */
-// val elementsWritten = {
-// elements.saveAsTextFile(path)
-// elements
-// }
-//
-// elementsWritten.mapPartitionsWithIndex { (i, part) =>
-// // commit offset range
-// val osr = offsets(i)
-// getStreamHelper(kafkaParams).commitConsumerOffset(osr)
-// Iterator.empty
-// }.foreach {
-// (_: Nothing) => ()
-// }
-// }
-//
-// logInfo(s"counter: ${mapAcc.value}")
-// println(s"counter: ${mapAcc.value}")
-// ssc.start()
-// ssc.awaitTermination()
-// }
-//}
+object WalLogToHDFS extends SparkApp with WithKafka {
+
+ override def run() = {
+
+ validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "outputPath", "hiveDatabase", "hiveTable", "splitListPath")
+
+ val kafkaZkQuorum = args(0)
+ val brokerList = args(1)
+ val topics = args(2)
+ val intervalInSec = seconds(args(3).toLong)
+ val dbUrl = args(4)
+ val outputPath = args(5)
+ val hiveDatabase = args(6)
+ val hiveTable = args(7)
+ val splitListPath = args(8)
+
+ val conf = sparkConf(s"$topics: WalLogToHDFS")
+ val ssc = streamingContext(conf, intervalInSec)
+ val sc = ssc.sparkContext
+
+ val groupId = topics.replaceAll(",", "_") + "_stream"
+ val fallbackTopic = topics.replaceAll(",", "_") + "_stream_failed"
+
+ val kafkaParams = Map(
+ "zookeeper.connect" -> kafkaZkQuorum,
+ "group.id" -> groupId,
+ "metadata.broker.list" -> brokerList,
+ "zookeeper.connection.timeout.ms" -> "10000",
+ "auto.offset.reset" -> "largest")
+
+ val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
+
+ val mapAcc = sc.accumulable(new MutableHashMap[String, Long](), "Throughput")(HashMapParam[String, Long](_ + _))
+
+ val hdfsBlockSize = 134217728 // 128M
+ val hiveContext = new HiveContext(sc)
+ var splits = Array[String]()
+ var excludeLabels = Set[String]()
+ var excludeServices = Set[String]()
+ stream.foreachRDD { (rdd, time) =>
+ try {
+ val read = sc.textFile(splitListPath).collect().map(_.split("=")).flatMap {
+ case Array(value) => Some(("split", value))
+ case Array(key, value) => Some((key, value))
+ case _ => None
+ }
+ splits = read.filter(_._1 == "split").map(_._2)
+ excludeLabels = read.filter(_._1 == "exclude_label").map(_._2).toSet
+ excludeServices = read.filter(_._1 == "exclude_service").map(_._2).toSet
+ } catch {
+ case _: Throwable => // use previous information
+ }
+
+ val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
+
+ val elements = rdd.mapPartitions { partition =>
+ // set executor setting.
+ val phase = System.getProperty("phase")
+ GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
+
+ partition.flatMap { case (key, msg) =>
+ val optMsg = Graph.toGraphElement(msg).flatMap { element =>
+ val arr = msg.split("\t", 7)
+ val service = element.serviceName
+ val label = arr(5)
+ val n = arr.length
+
+ if (excludeServices.contains(service) || excludeLabels.contains(label)) {
+ None
+ } else if(n == 6) {
+ Some(Seq(msg, "{}", service).mkString("\t"))
+ }
+ else if(n == 7) {
+ Some(Seq(msg, service).mkString("\t"))
+ }
+ else {
+ None
+ }
+ }
+ optMsg
+ }
+ }
+
+ val ts = time.milliseconds
+ val dateId = new SimpleDateFormat("yyyy-MM-dd").format(new Date(ts))
+
+ /** make sure that `elements` are not running at the same time */
+ val elementsWritten = {
+ elements.cache()
+ (Array("all") ++ splits).foreach {
+ case split if split == "all" =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ elements.saveAsTextFile(path)
+ case split =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ val strlen = split.length
+ val splitData = elements.filter(_.takeRight(strlen) == split).cache()
+ val totalSize = splitData
+ .mapPartitions { iterator =>
+ val s = iterator.map(_.length.toLong).sum
+ Iterator.single(s)
+ }
+ .sum
+ .toLong
+ val numPartitions = math.max(1, (totalSize / hdfsBlockSize.toDouble).toInt)
+ splitData.coalesce(math.min(splitData.partitions.length, numPartitions)).saveAsTextFile(path)
+ splitData.unpersist()
+ }
+ elements.unpersist()
+ elements
+ }
+
+ elementsWritten.mapPartitionsWithIndex { (i, part) =>
+ // commit offset range
+ val osr = offsets(i)
+ getStreamHelper(kafkaParams).commitConsumerOffset(osr)
+ Iterator.empty
+ }.foreach {
+ (_: Nothing) => ()
+ }
+
+ (Array("all") ++ splits).foreach { split =>
+ val path = s"$outputPath/split=$split/date_id=$dateId/ts=$ts"
+ hiveContext.sql(s"use $hiveDatabase")
+ hiveContext.sql(s"alter table $hiveTable add partition (split='$split', date_id='$dateId', ts='$ts') location '$path'")
+ }
+ }
+
+ logInfo(s"counter: ${mapAcc.value}")
+ println(s"counter: ${mapAcc.value}")
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/subscriber/GraphSubscriberTest.scala b/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
index a70c0f6..f927509 100644
--- a/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
+++ b/loader/src/test/scala/subscriber/GraphSubscriberTest.scala
@@ -1,6 +1,6 @@
package subscriber
-import com.kakao.s2graph.core.{Label, Service, Management}
+import com.kakao.s2graph.core.Management
import org.scalatest.{ FunSuite, Matchers }
import play.api.libs.json.{JsBoolean, JsNumber}
import s2.spark.WithKafka
@@ -28,7 +28,7 @@ class GraphSubscriberTest extends FunSuite with Matchers with WithKafka {
test("GraphSubscriberHelper.store") {
// actually we need to delete labelToReplace first for each test.
val labelMapping = Map(testLabelName -> labelToReplace)
- Management.copyLabel(testLabelName, labelToReplace, Some(hTableName))
+ GraphSubscriberHelper.management.copyLabel(testLabelName, labelToReplace, Some(hTableName))
//
// val msgs = (for {
@@ -39,7 +39,7 @@ class GraphSubscriberTest extends FunSuite with Matchers with WithKafka {
// }).toSeq
val msgs = testStrings
- val stat = GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping = labelMapping, autoCreateEdge = false)(None)
- println(stat)
+// val stat = GraphSubscriberHelper.storeBulk(zkQuorum, hTableName)(msgs, labelMapping = labelMapping, autoCreateEdge = false)(None)
+// println(stat)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/loader/src/test/scala/subscriber/TransferToHFileTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/subscriber/TransferToHFileTest.scala
new file mode 100644
index 0000000..7b3f72f
--- /dev/null
+++ b/loader/src/test/scala/subscriber/TransferToHFileTest.scala
@@ -0,0 +1,169 @@
+package subscriber
+
+import com.kakao.s2graph.core.Management
+import com.kakao.s2graph.core.types.HBaseType
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest._
+import subscriber.TransferToHFile._
+
+/**
+ * Created by Eric on 2015. 12. 2..
+ */
+class TransferToHFileTest extends FlatSpec with BeforeAndAfterAll with Matchers {
+
+ private val master = "local[2]"
+ private val appName = "example-spark"
+
+ private var sc: SparkContext = _
+
+ val dataWithoutDir =
+ """
+ |1447686000000 insertBulk e a b friends_rel {}
+ |1447686000000 insertBulk e a c friends_rel {}
+ |1447686000000 insertBulk e a d friends_rel {}
+ |1447686000000 insertBulk e b d friends_rel {}
+ |1447686000000 insertBulk e b e friends_rel {}
+ """.stripMargin.trim
+
+ val dataWithDir =
+ """
+ |1447686000000 insertBulk e a b friends_rel {} out
+ |1447686000000 insertBulk e b a friends_rel {} in
+ |1447686000000 insertBulk e a c friends_rel {} out
+ |1447686000000 insertBulk e c a friends_rel {} in
+ |1447686000000 insertBulk e a d friends_rel {} out
+ |1447686000000 insertBulk e d a friends_rel {} in
+ |1447686000000 insertBulk e b d friends_rel {} out
+ |1447686000000 insertBulk e d b friends_rel {} in
+ |1447686000000 insertBulk e b e friends_rel {} out
+ |1447686000000 insertBulk e e b friends_rel {} in
+ """.stripMargin.trim
+
+ override def beforeAll(): Unit = {
+ println("### beforeAll")
+
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ // 1. create service
+ if(Management.findService("loader-test").isEmpty) {
+ println(">>> create service...")
+ Management.createService("loader-test", "localhost", "loader-test-dev", 1, None, "gz")
+ }
+
+ // 2. create label
+ if(Management.findLabel("friends_rel").isEmpty) {
+ println(">>> create label...")
+ Management.createLabel(
+ "friends_rel",
+ "loader-test", "user_id", "string",
+ "loader-test", "user_id", "string",
+ true,
+ "loader-test",
+ Seq(),
+ Seq(),
+ "weak",
+ None, None,
+ HBaseType.DEFAULT_VERSION,
+ false,
+ Management.defaultCompressionAlgorithm
+ )
+ }
+
+ // create spark context
+ val conf = new SparkConf()
+ .setMaster(master)
+ .setAppName(appName)
+
+ sc = new SparkContext(conf)
+ }
+
+ override def afterAll(): Unit = {
+ println("### afterALL")
+ if (sc != null) {
+ sc.stop()
+ }
+
+ Management.deleteLabel("friends_rel")
+ }
+
+ "buildDegreePutRequest" should "transform degree to PutRequest" in {
+ val putReqs = buildDegreePutRequests("a", "friends_rel", "out", 3L)
+ putReqs.size should equal(1)
+ }
+
+ "toKeyValues" should "transform edges to KeyValues on edge format data without direction" in {
+ val rdd = sc.parallelize(dataWithoutDir.split("\n"))
+
+ val kvs = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
+ }
+ kvs.foreach(println)
+ // edges * 2 (snapshot edges + indexed edges)
+ kvs.count() should equal(10)
+
+
+ val kvsAutoCreated = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], true)
+ }
+
+ // edges * 3 (snapshot edges + indexed edges + reverse edges)
+ kvsAutoCreated.count() should equal(15)
+ }
+
+ "toKeyValues" should "transform edges to KeyValues on edge format data with direction" in {
+ val rdd = sc.parallelize(dataWithDir.split("\n"))
+
+ val kvs = rdd.mapPartitions { iter =>
+ GraphSubscriberHelper.apply("dev", "none", "none", "none")
+ TransferToHFile.toKeyValues(iter.toSeq, Map.empty[String, String], false)
+ }
+
+ // edges * 2 (snapshot edges + indexed edges)
+ kvs.count() should equal(20)
+ }
+
+ "buildDegrees" should "build degrees on edge format data without direction" in {
+ val rdd = sc.parallelize(dataWithoutDir.split("\n"))
+
+ // autoCreate = false
+ val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+ degrees.size should equal(2)
+
+ degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+
+
+ // autoCreate = true
+ val degreesAutoCreated = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], true).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+ degreesAutoCreated.size should equal(6)
+
+ degreesAutoCreated should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+ degreesAutoCreated should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
+ degreesAutoCreated should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
+ degreesAutoCreated should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
+ degreesAutoCreated should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
+ }
+
+ "buildDegrees" should "build degrees on edge format data with direction" in {
+ val rdd = sc.parallelize(dataWithDir.split("\n"))
+
+ val degrees = TransferToHFile.buildDegrees(rdd, Map.empty[String, String], false).reduceByKey { (agg, current) =>
+ agg + current
+ }.collectAsMap()
+
+ degrees.size should equal(6)
+
+ degrees should contain(DegreeKey("a", "friends_rel", "out") -> 3L)
+ degrees should contain(DegreeKey("b", "friends_rel", "out") -> 2L)
+ degrees should contain(DegreeKey("b", "friends_rel", "in") -> 1L)
+ degrees should contain(DegreeKey("c", "friends_rel", "in") -> 1L)
+ degrees should contain(DegreeKey("d", "friends_rel", "in") -> 2L)
+ degrees should contain(DegreeKey("e", "friends_rel", "in") -> 1L)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 4c90a1f..7f33c0e 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,4 +1,5 @@
-// Use the Play sbt plugin for Play projects
+// use the Play sbt plugin for Play projects
+
addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10")
// http://www.scalastyle.org/sbt.html
@@ -8,3 +9,5 @@ addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.3")
+
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.0")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/build.sbt
----------------------------------------------------------------------
diff --git a/s2core/build.sbt b/s2core/build.sbt
index 7a3c2d5..ca40ca7 100644
--- a/s2core/build.sbt
+++ b/s2core/build.sbt
@@ -30,4 +30,3 @@ libraryDependencies := {
libraryDependencies.value
}
}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar b/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar
index 6dcc75d..801d87d 100644
Binary files a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar and b/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/migrate/mysql/schema.sql
----------------------------------------------------------------------
diff --git a/s2core/migrate/mysql/schema.sql b/s2core/migrate/mysql/schema.sql
index 06c029b..4911348 100644
--- a/s2core/migrate/mysql/schema.sql
+++ b/s2core/migrate/mysql/schema.sql
@@ -87,6 +87,7 @@ CREATE TABLE `labels` (
`schema_version` varchar(8) NOT NULL default 'v2',
`is_async` tinyint(4) NOT NULL default '0',
`compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4',
+ `options` text,
PRIMARY KEY (`id`),
UNIQUE KEY `ux_label` (`label`),
INDEX `idx_src_column_name` (`src_column_name`),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/resources/create_model.hql
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/create_model.hql b/s2core/src/main/resources/create_model.hql
index df8e4dc..18c870b 100644
--- a/s2core/src/main/resources/create_model.hql
+++ b/s2core/src/main/resources/create_model.hql
@@ -1,2 +1 @@
-create 'models-dev', {NAME => 'm'}
create 's2graph-dev', {NAME => 'e'}, {NAME => 'v'}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 543365c..18aa2bd 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -9,12 +9,17 @@
</encoder>
</appender>
- <root level="ERROR">
+ <!--<root level="INFO">-->
+ <!--<appender-ref ref="STDOUT"/>-->
+ <!--</root>-->
+
+ <logger name="application" level="DEBUG">
<appender-ref ref="STDOUT"/>
- </root>
+ </logger>
- <logger name="application" level="DEBUG"></logger>
+ <logger name="error" level="DEBUG">
+ <appender-ref ref="STDOUT"/>
+ </logger>
- <logger name="error" level="DEBUG"></logger>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf
index cef0284..c93d204 100644
--- a/s2core/src/main/resources/reference.conf
+++ b/s2core/src/main/resources/reference.conf
@@ -1,6 +1,7 @@
# APP PHASE
-phase=dev
-host=localhost
+phase = dev
+
+host = localhost
# Hbase
hbase.table.compression.algorithm="gz"
@@ -25,11 +26,12 @@ cache.ttl.seconds=60
cache.max.size=100000
# DB
-s2graph.models.table.name="models-dev"
-db.default.driver="com.mysql.jdbc.Driver"
-db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
-db.default.user="graph"
-db.default.password="graph"
+s2graph.models.table.name = "models-dev"
+db.default.driver = "com.mysql.jdbc.Driver"
+db.default.url = "jdbc:mysql://"${host}":3306/graph_dev"
+db.default.user = "graph"
+db.default.password = "graph"
+
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
index 10443c3..8e6ad7d 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Edge.scala
@@ -57,6 +57,7 @@ case class IndexEdge(srcVertex: Vertex,
// assert(props.containsKey(LabelMeta.timeStampSeq))
val ts = props(LabelMeta.timeStampSeq).toString.toLong
+ val degreeEdge = props.contains(LabelMeta.degreeSeq)
lazy val label = Label.findById(labelWithDir.labelId)
val schemaVer = label.schemaVersion
lazy val labelIndex = LabelIndex.findByLabelIdAndSeq(labelWithDir.labelId, labelIndexSeq).get
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
index 76d3151..fdc8553 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
@@ -42,7 +42,8 @@ object Graph {
"delete.all.fetch.size" -> java.lang.Integer.valueOf(1000),
"future.cache.max.size" -> java.lang.Integer.valueOf(100000),
"future.cache.expire.after.write" -> java.lang.Integer.valueOf(10000),
- "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000)
+ "future.cache.expire.after.access" -> java.lang.Integer.valueOf(5000),
+ "s2graph.storage.backend" -> "hbase"
)
var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs)
@@ -99,9 +100,6 @@ object Graph {
case _ => innerVal.toString().toLong
}
} getOrElse(edge.ts)
- // val innerVal = edge.propsWithTs(timeDecay.labelMetaSeq).innerVal
- //
- // edge.propsWithTs.get(timeDecay.labelMetaSeq).map(_.toString.toLong).getOrElse(edge.ts)
} catch {
case e: Exception =>
logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
@@ -332,19 +330,24 @@ object Graph {
logger.error(s"toVertex: $e", e)
throw e
} get
+
+ def initStorage(config: Config)(ec: ExecutionContext) = {
+ config.getString("s2graph.storage.backend") match {
+ case "hbase" => new AsynchbaseStorage(config)(ec)
+ case _ => throw new RuntimeException("not supported storage.")
+ }
+ }
}
-class Graph(_config: Config)(implicit ec: ExecutionContext) {
+class Graph(_config: Config)(implicit val ec: ExecutionContext) {
val config = _config.withFallback(Graph.DefaultConfig)
- val cacheSize = config.getInt("cache.max.size")
-// val cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Seq[QueryResult]]()
- val vertexCache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Option[Vertex]]()
Model.apply(config)
Model.loadCache()
// TODO: Make storage client by config param
- val storage: Storage = new AsynchbaseStorage(config, vertexCache)(ec)
+ val storage = Graph.initStorage(config)(ec)
+
for {
entry <- config.entrySet() if Graph.DefaultConfigs.contains(entry.getKey)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
index 32f7a5a..ccf9d1f 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Management.scala
@@ -4,7 +4,6 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.GraphExceptions.{InvalidHTableException, LabelAlreadyExistException, LabelNotExistException}
import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
import com.kakao.s2graph.core.mysqls._
-import com.kakao.s2graph.core.storage.Storage
import com.kakao.s2graph.core.types.HBaseType._
import com.kakao.s2graph.core.types._
import play.api.libs.json.Reads._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
index 42b0146..f301d68 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
@@ -165,6 +165,37 @@ object PostProcess extends JSONParser {
}
}
+ private def buildReplaceJson(jsValue: JsValue)(mapper: JsValue => JsValue): JsValue = {
+ def traverse(js: JsValue): JsValue = js match {
+ case JsNull => mapper(JsNull)
+ case JsUndefined() => mapper(JsUndefined(""))
+ case JsNumber(v) => mapper(js)
+ case JsString(v) => mapper(js)
+ case JsBoolean(v) => mapper(js)
+ case JsArray(elements) => JsArray(elements.map { t => traverse(mapper(t)) })
+ case JsObject(values) => JsObject(values.map { case (k, v) => k -> traverse(mapper(v)) })
+ }
+
+ traverse(jsValue)
+ }
+
+ /** test query with filterOut is not working since it can not diffrentate filterOut */
+ private def buildNextQuery(jsonQuery: JsValue, _cursors: Seq[Seq[String]]): JsValue = {
+ val cursors = _cursors.flatten.iterator
+
+ buildReplaceJson(jsonQuery) {
+ case js@JsObject(fields) =>
+ val isStep = fields.find { case (k, _) => k == "label" } // find label group
+ if (isStep.isEmpty) js
+ else {
+ // TODO: Order not ensured
+ val withCursor = js.fieldSet | Set("cursor" -> JsString(cursors.next))
+ JsObject(withCursor.toSeq)
+ }
+ case js => js
+ }
+ }
+
private def buildRawEdges(queryOption: QueryOption,
queryRequestWithResultLs: Seq[QueryRequestWithResult],
excludeIds: Map[Int, Boolean],
@@ -267,6 +298,7 @@ object PostProcess extends JSONParser {
Json.obj(
"size" -> edges.size,
"degrees" -> resultDegrees,
+// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
"results" -> edges
)
} else {
@@ -316,6 +348,7 @@ object PostProcess extends JSONParser {
Json.obj(
"size" -> groupedSortedJsons.size,
"degrees" -> resultDegrees,
+// "queryNext" -> buildNextQuery(q.jsonQuery, q.cursorStrings()),
"results" -> groupedSortedJsons
)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
index a9a5112..184cd08 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
@@ -6,7 +6,7 @@ import com.kakao.s2graph.core.parsers.{Where, WhereParser}
import com.kakao.s2graph.core.types._
import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.ColumnRangeFilter
-import play.api.libs.json.{JsNumber, JsValue, Json}
+import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
import scala.util.hashing.MurmurHash3
import scala.util.{Success, Try}
@@ -32,6 +32,11 @@ object Query {
}
}
+case class MultiQuery(queries: Seq[Query],
+ weights: Seq[Double],
+ queryOption: QueryOption,
+ jsonQuery: JsValue = JsNull)
+
case class QueryOption(removeCycle: Boolean = false,
selectColumns: Seq[String] = Seq.empty,
groupByColumns: Seq[String] = Seq.empty,
@@ -45,14 +50,15 @@ case class QueryOption(removeCycle: Boolean = false,
scoreThreshold: Double = Double.MinValue,
returnDegree: Boolean = true)
-case class MultiQuery(queries: Seq[Query], weights: Seq[Double], queryOption: QueryOption)
case class Query(vertices: Seq[Vertex] = Seq.empty[Vertex],
steps: IndexedSeq[Step] = Vector.empty[Step],
- queryOption: QueryOption = QueryOption()) {
+ queryOption: QueryOption = QueryOption(),
+ jsonQuery: JsValue = JsNull) {
val removeCycle = queryOption.removeCycle
val selectColumns = queryOption.selectColumns
+// val groupBy = queryOption.groupBy
val groupByColumns = queryOption.groupByColumns
val orderByColumns = queryOption.orderByColumns
val filterOutQuery = queryOption.filterOutQuery
@@ -477,13 +483,13 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System
this
}
- def whereRawOpt(sqlOpt: Option[String]): QueryParam = {
- this.whereRawOpt = sqlOpt
+ def shouldNormalize(shouldNormalize: Boolean): QueryParam = {
+ this.shouldNormalize = shouldNormalize
this
}
- def shouldNormalize(shouldNormalize: Boolean): QueryParam = {
- this.shouldNormalize = shouldNormalize
+ def whereRawOpt(sqlOpt: Option[String]): QueryParam = {
+ this.whereRawOpt = sqlOpt
this
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/QueryRequest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryRequest.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryRequest.scala
deleted file mode 100644
index 413bfdb..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryRequest.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-//package com.kakao.s2graph.core
-//
-//import scala.collection.Seq
-//
-//case class QueryRequest(query: Query,
-// stepIdx: Int,
-// vertex: Vertex,
-// queryParam: QueryParam,
-// prevStepScore: Double,
-// tgtVertexOpt: Option[Vertex] = None,
-// parentEdges: Seq[EdgeWithScore] = Nil,
-// isInnerCall: Boolean = false)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
index 36c5a01..0ca92b5 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/QueryResult.scala
@@ -7,18 +7,22 @@ import scala.collection.Seq
object QueryResult {
def fromVertices(query: Query): Seq[QueryRequestWithResult] = {
- val queryParam = query.steps.head.queryParams.head
- val label = queryParam.label
- val currentTs = System.currentTimeMillis()
- val propsWithTs = Map(LabelMeta.timeStampSeq ->
- InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs))
- for {
- vertex <- query.vertices
- } yield {
- val edge = Edge(vertex, vertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
- val edgeWithScore = EdgeWithScore(edge, Graph.DefaultScore)
- QueryRequestWithResult(QueryRequest(query, -1, vertex, queryParam),
- QueryResult(edgeWithScoreLs = Seq(edgeWithScore)))
+ if (query.steps.isEmpty || query.steps.head.queryParams.isEmpty) {
+ Seq.empty
+ } else {
+ val queryParam = query.steps.head.queryParams.head
+ val label = queryParam.label
+ val currentTs = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq ->
+ InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs))
+ for {
+ vertex <- query.vertices
+ } yield {
+ val edge = Edge(vertex, vertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
+ val edgeWithScore = EdgeWithScore(edge, Graph.DefaultScore)
+ QueryRequestWithResult(QueryRequest(query, -1, vertex, queryParam),
+ QueryResult(edgeWithScoreLs = Seq(edgeWithScore)))
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
index 9cc0607..c2b86c2 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Vertex.scala
@@ -1,5 +1,6 @@
package com.kakao.s2graph.core
+
import com.kakao.s2graph.core.mysqls._
//import com.kakao.s2graph.core.models._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
index 88c85b7..46b92ab 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala
@@ -1,6 +1,7 @@
package com.kakao.s2graph.core.mysqls
import com.kakao.s2graph.core.GraphUtil
+import com.kakao.s2graph.core.utils.logger
import scalikejdbc._
import scala.util.Random
@@ -50,6 +51,7 @@ case class Experiment(id: Option[Int],
totalModular: Int) {
def buckets = Bucket.finds(id.get)
+
def rangeBuckets = for {
bucket <- buckets
range <- bucket.rangeOpt
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index 2449082..22fbd8b 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -2,7 +2,7 @@ package com.kakao.s2graph.core.rest
import java.util.concurrent.{Callable, TimeUnit}
-import com.google.common.cache.CacheBuilder
+import com.google.common.cache.{CacheLoader, CacheBuilder}
import com.kakao.s2graph.core.GraphExceptions.{BadQueryException, ModelNotFoundException}
import com.kakao.s2graph.core._
import com.kakao.s2graph.core.mysqls._
@@ -15,11 +15,9 @@ import scala.util.{Failure, Success, Try}
object TemplateHelper {
val findVar = """\"?\$\{(.*?)\}\"?""".r
val num = """(next_day|next_hour|next_week|now)?\s*(-?\s*[0-9]+)?\s*(hour|day|week)?""".r
-
val hour = 60 * 60 * 1000L
val day = hour * 24L
val week = day * 7L
-
def calculate(now: Long, n: Int, unit: String): Long = {
val duration = unit match {
case "hour" | "HOUR" => n * hour
@@ -59,6 +57,7 @@ class RequestParser(config: Config) extends JSONParser {
val hardLimit = 100000
val defaultLimit = 100
+ val maxLimit = Int.MaxValue - 1
val DefaultRpcTimeout = config.getInt("hbase.rpc.timeout")
val DefaultMaxAttempt = config.getInt("hbase.client.retries.number")
val DefaultCluster = config.getString("hbase.zookeeper.quorum")
@@ -71,7 +70,6 @@ class RequestParser(config: Config) extends JSONParser {
.initialCapacity(1000)
.build[String, Try[Where]]
-
private def extractScoring(labelId: Int, value: JsValue) = {
val ret = for {
js <- parse[Option[JsObject]](value, "scoring")
@@ -149,7 +147,6 @@ class RequestParser(config: Config) extends JSONParser {
ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
}
-
def extractWhere(label: Label, whereClauseOpt: Option[String]): Try[Where] = {
whereClauseOpt match {
case None => Success(WhereParser.success)
@@ -186,7 +183,8 @@ class RequestParser(config: Config) extends JSONParser {
toQuery(queryJson, isEdgeQuery)
}
val weights = (jsValue \ "weights").asOpt[Seq[Double]].getOrElse(queries.map(_ => 1.0))
- MultiQuery(queries = queries, weights = weights, queryOption = toQueryOption(jsValue))
+ MultiQuery(queries = queries, weights = weights,
+ queryOption = toQueryOption(jsValue), jsonQuery = jsValue)
}
def toQueryOption(jsValue: JsValue): QueryOption = {
@@ -313,7 +311,7 @@ class RequestParser(config: Config) extends JSONParser {
}
- val ret = Query(vertices, querySteps, queryOption)
+ val ret = Query(vertices, querySteps, queryOption, jsValue)
// logger.debug(ret.toString)
ret
} catch {
@@ -335,7 +333,7 @@ class RequestParser(config: Config) extends JSONParser {
val limit = {
parse[Option[Int]](labelGroup, "limit") match {
case None => defaultLimit
- case Some(l) if l < 0 => Int.MaxValue
+ case Some(l) if l < 0 => maxLimit
case Some(l) if l >= 0 =>
val default = hardLimit
Math.min(l, default)
@@ -381,7 +379,6 @@ class RequestParser(config: Config) extends JSONParser {
val scorePropagateOp = (labelGroup \ "scorePropagateOp").asOpt[String].getOrElse("multiply")
val sample = (labelGroup \ "sample").asOpt[Int].getOrElse(-1)
val shouldNormalize = (labelGroup \ "normalize").asOpt[Boolean].getOrElse(false)
-
// FIXME: Order of command matter
QueryParam(labelWithDir)
.sample(sample)
@@ -432,31 +429,39 @@ class RequestParser(config: Config) extends JSONParser {
def toEdgesWithOrg(jsValue: JsValue, operation: String): (List[Edge], List[JsValue]) = {
val jsValues = toJsValues(jsValue)
- val edges = jsValues.map(toEdge(_, operation))
+ val edges = jsValues.flatMap(toEdge(_, operation))
(edges, jsValues)
}
def toEdges(jsValue: JsValue, operation: String): List[Edge] = {
- toJsValues(jsValue).map(toEdge(_, operation))
+ toJsValues(jsValue).flatMap { edgeJson =>
+ toEdge(edgeJson, operation)
+ }
}
- def toEdge(jsValue: JsValue, operation: String) = {
- val srcId = parse[JsValue](jsValue, "from") match {
- case s: JsString => s.as[String]
- case o@_ => s"${o}"
- }
- val tgtId = parse[JsValue](jsValue, "to") match {
+ private def toEdge(jsValue: JsValue, operation: String): List[Edge] = {
+
+ def parseId(js: JsValue) = js match {
case s: JsString => s.as[String]
case o@_ => s"${o}"
}
+ val srcId = (jsValue \ "from").asOpt[JsValue].toList.map(parseId(_))
+ val tgtId = (jsValue \ "to").asOpt[JsValue].toList.map(parseId(_))
+ val srcIds = (jsValue \ "froms").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ srcId
+ val tgtIds = (jsValue \ "tos").asOpt[List[JsValue]].toList.flatMap(froms => froms.map(js => parseId(js))) ++ tgtId
+
val label = parse[String](jsValue, "label")
val timestamp = parse[Long](jsValue, "timestamp")
val direction = parse[Option[String]](jsValue, "direction").getOrElse("")
val props = (jsValue \ "props").asOpt[JsValue].getOrElse("{}")
- Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
-
+ for {
+ srcId <- srcIds
+ tgtId <- tgtIds
+ } yield {
+ Management.toEdge(timestamp, operation, srcId, tgtId, label, direction, props.toString)
+ }
}
def toVertices(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None) = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
new file mode 100644
index 0000000..cff968f
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Deserializable.scala
@@ -0,0 +1,25 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.storage.{SKeyValue, StorageDeserializable}
+import com.kakao.s2graph.core.types.{LabelWithDirection, SourceVertexId, VertexId}
+import org.apache.hadoop.hbase.util.Bytes
+
+
+trait Deserializable[E] extends StorageDeserializable[E] {
+ import StorageDeserializable._
+
+ type RowKeyRaw = (VertexId, LabelWithDirection, Byte, Boolean, Int)
+
+ /** version 1 and version 2 share same code for parsing row key part */
+ def parseRow(kv: SKeyValue, version: String): RowKeyRaw = {
+ var pos = 0
+ val (srcVertexId, srcIdLen) = SourceVertexId.fromBytes(kv.row, pos, kv.row.length, version)
+ pos += srcIdLen
+ val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
+ pos += 4
+ val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
+
+ val rowLen = srcIdLen + 4 + 1
+ (srcVertexId, labelWithDir, labelIdxSeq, isInverted, rowLen)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeDeserializable.scala
new file mode 100644
index 0000000..2190222
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeDeserializable.scala
@@ -0,0 +1,128 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.LabelMeta
+import com.kakao.s2graph.core.storage.{CanSKeyValue, StorageDeserializable, SKeyValue}
+import com.kakao.s2graph.core.types._
+import org.apache.hadoop.hbase.util.Bytes
+import StorageDeserializable._
+
+class IndexEdgeDeserializable(bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[IndexEdge] {
+
+ import StorageDeserializable._
+
+ type QualifierRaw = (Array[(Byte, InnerValLike)], VertexId, Byte, Boolean, Int)
+ type ValueRaw = (Array[(Byte, InnerValLike)], Int)
+
+ private def parseDegreeQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+// val degree = Bytes.toLong(kv.value)
+ val degree = bytesToLongFunc(kv.value, 0)
+ val idxPropsRaw = Array(LabelMeta.degreeSeq -> InnerVal.withLong(degree, version))
+ val tgtVertexIdRaw = VertexId(HBaseType.DEFAULT_COL_ID, InnerVal.withStr("0", version))
+ (idxPropsRaw, tgtVertexIdRaw, GraphUtil.operations("insert"), false, 0)
+ }
+
+ private def parseQualifier(kv: SKeyValue, version: String): QualifierRaw = {
+ var qualifierLen = 0
+ var pos = 0
+ val (idxPropsRaw, idxPropsLen, tgtVertexIdRaw, tgtVertexIdLen) = {
+ val (props, endAt) = bytesToProps(kv.qualifier, pos, version)
+ pos = endAt
+ qualifierLen += endAt
+ val (tgtVertexId, tgtVertexIdLen) = if (endAt == kv.qualifier.length) {
+ (HBaseType.defaultTgtVertexId, 0)
+ } else {
+ TargetVertexId.fromBytes(kv.qualifier, endAt, kv.qualifier.length, version)
+ }
+ qualifierLen += tgtVertexIdLen
+ (props, endAt, tgtVertexId, tgtVertexIdLen)
+ }
+ val (op, opLen) =
+ if (kv.qualifier.length == qualifierLen) (GraphUtil.defaultOpByte, 0)
+ else (kv.qualifier(qualifierLen), 1)
+
+ qualifierLen += opLen
+
+ (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdLen != 0, qualifierLen)
+ }
+
+ private def parseValue(kv: SKeyValue, version: String): ValueRaw = {
+ val (props, endAt) = bytesToKeyValues(kv.value, 0, kv.value.length, version)
+ (props, endAt)
+ }
+
+ private def parseDegreeValue(kv: SKeyValue, version: String): ValueRaw = {
+ (Array.empty[(Byte, InnerValLike)], 0)
+ }
+
+
+
+ /** version 1 and version 2 is same logic */
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[IndexEdge] = None): IndexEdge = {
+ fromKeyValuesInnerOld(queryParam, _kvs, version, cacheElementOpt)
+ }
+
+ def fromKeyValuesInnerOld[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[IndexEdge] = None): IndexEdge = {
+ assert(_kvs.size == 1)
+
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+
+ val kv = kvs.head
+ val (srcVertexId, labelWithDir, labelIdxSeq, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.id, e.labelWithDir, e.labelIndexSeq, false, 0)
+ }.getOrElse(parseRow(kv, version))
+
+ val (idxPropsRaw, tgtVertexIdRaw, op, tgtVertexIdInQualifier, _) =
+ if (kv.qualifier.isEmpty) parseDegreeQualifier(kv, version)
+ else parseQualifier(kv, version)
+
+ val (props, _) = if (op == GraphUtil.operations("incrementCount")) {
+// val countVal = Bytes.toLong(kv.value)
+ val countVal = bytesToLongFunc(kv.value, 0)
+ val dummyProps = Array(LabelMeta.countSeq -> InnerVal.withLong(countVal, version))
+ (dummyProps, 8)
+ } else if (kv.qualifier.isEmpty) {
+ parseDegreeValue(kv, version)
+ } else {
+ parseValue(kv, version)
+ }
+
+ val index = queryParam.label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${queryParam.label.id.get}, ${labelIdxSeq}"))
+
+
+ // assert(kv.qualifier.nonEmpty && index.metaSeqs.size == idxPropsRaw.size)
+
+ val idxProps = for {
+ (seq, (k, v)) <- index.metaSeqs.zip(idxPropsRaw)
+ } yield {
+ if (k == LabelMeta.degreeSeq) k -> v
+ else seq -> v
+ }
+
+ val idxPropsMap = idxProps.toMap
+ val tgtVertexId = if (tgtVertexIdInQualifier) {
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => tgtVertexIdRaw
+ case Some(vId) => TargetVertexId(HBaseType.DEFAULT_COL_ID, vId)
+ }
+ } else tgtVertexIdRaw
+
+ val _mergedProps = (idxProps ++ props).toMap
+ val mergedProps =
+ if (_mergedProps.contains(LabelMeta.timeStampSeq)) _mergedProps
+ else _mergedProps + (LabelMeta.timeStampSeq -> InnerVal.withLong(kv.timestamp, version))
+
+ // logger.error(s"$mergedProps")
+ // val ts = mergedProps(LabelMeta.timeStampSeq).toString().toLong
+
+ val ts = kv.timestamp
+ IndexEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts), labelWithDir, op, ts, labelIdxSeq, mergedProps)
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeSerializable.scala
new file mode 100644
index 0000000..56f70b9
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/IndexEdgeSerializable.scala
@@ -0,0 +1,58 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.mysqls.LabelMeta
+import com.kakao.s2graph.core.storage.{StorageSerializable, SKeyValue}
+import com.kakao.s2graph.core.types.{HBaseType, VertexId}
+import com.kakao.s2graph.core.utils.logger
+import com.kakao.s2graph.core.{GraphUtil, IndexEdge}
+import org.apache.hadoop.hbase.util.Bytes
+
+case class IndexEdgeSerializable(indexEdge: IndexEdge) extends Serializable[IndexEdge] {
+
+ import StorageSerializable._
+
+ val label = indexEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ val idxPropsMap = indexEdge.orders.toMap
+ val idxPropsBytes = propsToBytes(indexEdge.orders)
+
+ /** version 1 and version 2 share same code for serialize row key part */
+ override def toKeyValues: Seq[SKeyValue] = {
+ toKeyValuesInner
+ }
+ def toKeyValuesInner: Seq[SKeyValue] = {
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+
+ val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+ // logger.error(s"${row.toList}\n${srcIdBytes.toList}\n${labelWithDirBytes.toList}\n${labelIndexSeqWithIsInvertedBytes.toList}")
+ val tgtIdBytes = VertexId.toTargetVertexId(indexEdge.tgtVertex.id).bytes
+ val qualifier =
+ if (indexEdge.degreeEdge) Array.empty[Byte]
+ else {
+ if (indexEdge.op == GraphUtil.operations("incrementCount")) {
+ Bytes.add(idxPropsBytes, tgtIdBytes, Array.fill(1)(indexEdge.op))
+ } else {
+ idxPropsMap.get(LabelMeta.toSeq) match {
+ case None => Bytes.add(idxPropsBytes, tgtIdBytes)
+ case Some(vId) => idxPropsBytes
+ }
+ }
+ }
+
+
+ val value =
+ if (indexEdge.degreeEdge)
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.degreeSeq).innerVal.toString().toLong)
+ else if (indexEdge.op == GraphUtil.operations("incrementCount"))
+ Bytes.toBytes(indexEdge.propsWithTs(LabelMeta.countSeq).innerVal.toString().toLong)
+ else propsToKeyValues(indexEdge.metas.toSeq)
+
+ val kv = SKeyValue(table, row, cf, qualifier, value, indexEdge.version)
+
+ Seq(kv)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/MutationBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/MutationBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/MutationBuilder.scala
deleted file mode 100644
index c5d4882..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/MutationBuilder.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.kakao.s2graph.core._
-
-import scala.collection.Seq
-import scala.concurrent.ExecutionContext
-
-
-abstract class MutationBuilder[T](storage: Storage)(implicit ex: ExecutionContext) {
- /** operation that needs to be supported by backend persistent storage system */
- def put(kvs: Seq[SKeyValue]): Seq[T]
-
- def increment(kvs: Seq[SKeyValue]): Seq[T]
-
- def delete(kvs: Seq[SKeyValue]): Seq[T]
-
-
- /** build mutation for backend persistent storage system */
-
- /** EdgeMutate */
- def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[T]
-
- def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[T]
-
- def increments(edgeMutate: EdgeMutate): Seq[T]
-
- /** IndexEdge */
- def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[T]
-
- def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[T]
-
- def buildDeletesAsync(indexedEdge: IndexEdge): Seq[T]
-
- def buildPutsAsync(indexedEdge: IndexEdge): Seq[T]
-
- /** SnapshotEdge */
- def buildPutAsync(snapshotEdge: SnapshotEdge): Seq[T]
-
- def buildDeleteAsync(snapshotEdge: SnapshotEdge): Seq[T]
-
- /** Vertex */
- def buildPutsAsync(vertex: Vertex): Seq[T]
-
- def buildDeleteAsync(vertex: Vertex): Seq[T]
-
- def buildDeleteBelongsToId(vertex: Vertex): Seq[T]
-
- def buildVertexPutsAsync(edge: Edge): Seq[T]
-
- def buildPutsAll(vertex: Vertex): Seq[T] = {
- vertex.op match {
- case d: Byte if d == GraphUtil.operations("delete") => buildDeleteAsync(vertex)
- case _ => buildPutsAsync(vertex)
- }
- }
-
-}
[3/4] incubator-s2graph git commit: [S2GRAPH-17]: Remove unnecessary
abstraction layer, Storage.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/QueryBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/QueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/QueryBuilder.scala
deleted file mode 100644
index f04e342..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/QueryBuilder.scala
+++ /dev/null
@@ -1,104 +0,0 @@
-package com.kakao.s2graph.core.storage
-
-import com.google.common.cache.Cache
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.types.{LabelWithDirection, VertexId}
-import com.kakao.s2graph.core.utils.logger
-import scala.collection.{Map, Seq}
-import scala.concurrent.{Future, ExecutionContext}
-import scala.util.Try
-
-abstract class QueryBuilder[R, T](storage: Storage)(implicit ec: ExecutionContext) {
-
- def buildRequest(queryRequest: QueryRequest): R
-
- def getEdge(srcVertex: Vertex, tgtVertex: Vertex, queryParam: QueryParam, isInnerCall: Boolean): T
-
- def fetch(queryRequest: QueryRequest,
- prevStepScore: Double,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): T
-
- def toCacheKeyBytes(request: R): Array[Byte]
-
- def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]]
-
-
- def fetchStep(orgQuery: Query, queryRequestWithResultsLs: Seq[QueryRequestWithResult]): Future[Seq[QueryRequestWithResult]] = {
- if (queryRequestWithResultsLs.isEmpty) Future.successful(Nil)
- else {
- val queryRequest = queryRequestWithResultsLs.head.queryRequest
- val q = orgQuery
- val queryResultsLs = queryRequestWithResultsLs.map(_.queryResult)
-
- val stepIdx = queryRequest.stepIdx + 1
-
- val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
- val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
- val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
- val step = q.steps(stepIdx)
- val alreadyVisited =
- if (stepIdx == 0) Map.empty[(LabelWithDirection, Vertex), Boolean]
- else Graph.alreadyVisitedVertices(queryResultsLs)
-
- val groupedBy = queryResultsLs.flatMap { queryResult =>
- queryResult.edgeWithScoreLs.map { case edgeWithScore =>
- edgeWithScore.edge.tgtVertex -> edgeWithScore
- }
- }.groupBy { case (vertex, edgeWithScore) => vertex }
-
- val groupedByFiltered = for {
- (vertex, edgesWithScore) <- groupedBy
- aggregatedScore = edgesWithScore.map(_._2.score).sum if aggregatedScore >= prevStepThreshold
- } yield vertex -> aggregatedScore
-
- val prevStepTgtVertexIdEdges = for {
- (vertex, edgesWithScore) <- groupedBy
- } yield vertex.id -> edgesWithScore.map { case (vertex, edgeWithScore) => edgeWithScore }
-
- val nextStepSrcVertices = if (prevStepLimit >= 0) {
- groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
- } else {
- groupedByFiltered.toSeq
- }
-
- val queryRequests = for {
- (vertex, prevStepScore) <- nextStepSrcVertices
- queryParam <- step.queryParams
- } yield (QueryRequest(q, stepIdx, vertex, queryParam), prevStepScore)
-
- Graph.filterEdges(fetches(queryRequests, prevStepTgtVertexIdEdges), alreadyVisited)(ec)
- }
- }
-
- def fetchStepFuture(orgQuery: Query, queryRequestWithResultLsFuture: Future[Seq[QueryRequestWithResult]]): Future[Seq[QueryRequestWithResult]] = {
- for {
- queryRequestWithResultLs <- queryRequestWithResultLsFuture
- ret <- fetchStep(orgQuery, queryRequestWithResultLs)
- } yield ret
- }
-
- def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = {
- val fallback = {
- val queryRequest = QueryRequest(query = q, stepIdx = 0, q.vertices.head, queryParam = QueryParam.Empty)
- Future.successful(q.vertices.map(v => QueryRequestWithResult(queryRequest, QueryResult())))
- }
- Try {
- if (q.steps.isEmpty) {
- // TODO: this should be get vertex query.
- fallback
- } else {
- // current stepIdx = -1
- val startQueryResultLs = QueryResult.fromVertices(q)
- q.steps.foldLeft(Future.successful(startQueryResultLs)) { case (acc, step) =>
- fetchStepFuture(q, acc)
- }
- }
- } recover {
- case e: Exception =>
- logger.error(s"getEdgesAsync: $e", e)
- fallback
- } get
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
index c2c822b..a0aa261 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SKeyValue.scala
@@ -1,11 +1,28 @@
package com.kakao.s2graph.core.storage
+import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.KeyValue
-case class SKeyValue(table: Array[Byte], row: Array[Byte], cf: Array[Byte], qualifier: Array[Byte], value: Array[Byte], timestamp: Long) {
+
+object SKeyValue {
+ val Put = 1
+ val Delete = 2
+ val Increment = 3
+ val Default = Put
+}
+case class SKeyValue(table: Array[Byte],
+ row: Array[Byte],
+ cf: Array[Byte],
+ qualifier: Array[Byte],
+ value: Array[Byte],
+ timestamp: Long,
+ operation: Int = SKeyValue.Default) {
def toLogString = {
- Map("table" -> table.toList, "row" -> row.toList, "cf" -> cf.toList, "qualifier" -> qualifier.toList, "value" -> value.toList, "timestamp" -> timestamp).toString
+ Map("table" -> table.toList, "row" -> row.toList, "cf" -> Bytes.toString(cf),
+ "qualifier" -> qualifier.toList, "value" -> value.toList, "timestamp" -> timestamp,
+ "operation" -> operation).toString
}
+ override def toString(): String = toLogString
}
trait CanSKeyValue[T] {
@@ -28,4 +45,3 @@ object CanSKeyValue {
// For hbase KeyValues
}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
new file mode 100644
index 0000000..bee064b
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Serializable.scala
@@ -0,0 +1,10 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.storage.StorageSerializable
+
+object Serializable {
+ val vertexCf = "v".getBytes()
+ val edgeCf = "e".getBytes()
+}
+
+trait Serializable[E] extends StorageSerializable[E]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeDeserializable.scala
new file mode 100644
index 0000000..d4f55f0
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeDeserializable.scala
@@ -0,0 +1,142 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.mysqls.{LabelIndex, LabelMeta}
+import com.kakao.s2graph.core.storage.{CanSKeyValue, SKeyValue, StorageDeserializable}
+import com.kakao.s2graph.core.types._
+import com.kakao.s2graph.core.{Edge, QueryParam, SnapshotEdge, Vertex}
+import org.apache.hadoop.hbase.util.Bytes
+
+class SnapshotEdgeDeserializable extends Deserializable[SnapshotEdge] {
+
+ import StorageDeserializable._
+
+ override def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
+ queryParam.label.schemaVersion match {
+ case HBaseType.VERSION2 | HBaseType.VERSION1 => fromKeyValuesInnerOld(queryParam, _kvs, version, cacheElementOpt)
+ case _ => fromKeyValuesInnerV3(queryParam, _kvs, version, cacheElementOpt)
+ }
+ }
+
+ def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
+ val statusCode = byte >> 4
+ val op = byte & ((1 << 4) - 1)
+ (statusCode.toByte, op.toByte)
+ }
+
+ def fromKeyValuesInnerOld[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+ assert(kvs.size == 1)
+
+ val kv = kvs.head
+ val schemaVer = queryParam.label.schemaVersion
+ val cellVersion = kv.timestamp
+
+ val (srcVertexId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.id, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
+ }.getOrElse(parseRow(kv, schemaVer))
+
+ val (tgtVertexId, props, op, ts, statusCode, _pendingEdgeOpt) = {
+ val (tgtVertexId, _) = TargetVertexId.fromBytes(kv.qualifier, 0, kv.qualifier.length, schemaVer)
+ var pos = 0
+ val (statusCode, op) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ val kvsMap = props.toMap
+ val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
+
+ pos = endAt
+ val _pendingEdgeOpt =
+ if (pos == kv.value.length) None
+ else {
+ val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ // val versionNum = Bytes.toLong(kv.value, pos, 8)
+ // pos += 8
+ val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ pos = endAt
+ val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
+
+ val pendingEdge =
+ Edge(Vertex(srcVertexId, cellVersion),
+ Vertex(tgtVertexId, cellVersion),
+ labelWithDir, pendingEdgeOp,
+ cellVersion, pendingEdgeProps.toMap,
+ statusCode = pendingEdgeStatusCode, lockTs = lockTs)
+ Option(pendingEdge)
+ }
+
+ (tgtVertexId, kvsMap, op, ts, statusCode, _pendingEdgeOpt)
+ }
+
+ SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
+ labelWithDir, op, cellVersion, props, statusCode = statusCode,
+ pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
+ }
+
+ private def fromKeyValuesInnerV3[T: CanSKeyValue](queryParam: QueryParam, _kvs: Seq[T], version: String, cacheElementOpt: Option[SnapshotEdge]): SnapshotEdge = {
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+ assert(kvs.size == 1)
+
+ val kv = kvs.head
+ val schemaVer = queryParam.label.schemaVersion
+ val cellVersion = kv.timestamp
+ /** rowKey */
+ def parseRowV3(kv: SKeyValue, version: String) = {
+ var pos = 0
+ val (srcIdAndTgtId, srcIdAndTgtIdLen) = SourceAndTargetVertexIdPair.fromBytes(kv.row, pos, kv.row.length, version)
+ pos += srcIdAndTgtIdLen
+ val labelWithDir = LabelWithDirection(Bytes.toInt(kv.row, pos, 4))
+ pos += 4
+ val (labelIdxSeq, isInverted) = bytesToLabelIndexSeqWithIsInverted(kv.row, pos)
+
+ val rowLen = srcIdAndTgtIdLen + 4 + 1
+ (srcIdAndTgtId.srcInnerId, srcIdAndTgtId.tgtInnerId, labelWithDir, labelIdxSeq, isInverted, rowLen)
+
+ }
+ val (srcInnerId, tgtInnerId, labelWithDir, _, _, _) = cacheElementOpt.map { e =>
+ (e.srcVertex.innerId, e.tgtVertex.innerId, e.labelWithDir, LabelIndex.DefaultSeq, true, 0)
+ }.getOrElse(parseRowV3(kv, schemaVer))
+
+ val srcVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, srcInnerId)
+ val tgtVertexId = SourceVertexId(HBaseType.DEFAULT_COL_ID, tgtInnerId)
+
+ val (props, op, ts, statusCode, _pendingEdgeOpt) = {
+ var pos = 0
+ val (statusCode, op) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ val (props, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ val kvsMap = props.toMap
+ val ts = kvsMap(LabelMeta.timeStampSeq).innerVal.toString.toLong
+
+ pos = endAt
+ val _pendingEdgeOpt =
+ if (pos == kv.value.length) None
+ else {
+ val (pendingEdgeStatusCode, pendingEdgeOp) = statusCodeWithOp(kv.value(pos))
+ pos += 1
+ // val versionNum = Bytes.toLong(kv.value, pos, 8)
+ // pos += 8
+ val (pendingEdgeProps, endAt) = bytesToKeyValuesWithTs(kv.value, pos, schemaVer)
+ pos = endAt
+ val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
+
+ val pendingEdge =
+ Edge(Vertex(srcVertexId, cellVersion),
+ Vertex(tgtVertexId, cellVersion),
+ labelWithDir, pendingEdgeOp,
+ cellVersion, pendingEdgeProps.toMap,
+ statusCode = pendingEdgeStatusCode, lockTs = lockTs)
+ Option(pendingEdge)
+ }
+
+ (kvsMap, op, ts, statusCode, _pendingEdgeOpt)
+ }
+
+ SnapshotEdge(Vertex(srcVertexId, ts), Vertex(tgtVertexId, ts),
+ labelWithDir, op, cellVersion, props, statusCode = statusCode,
+ pendingEdgeOpt = _pendingEdgeOpt, lockTs = None)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeSerializable.scala
new file mode 100644
index 0000000..9e6e1b7
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/SnapshotEdgeSerializable.scala
@@ -0,0 +1,76 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.SnapshotEdge
+import com.kakao.s2graph.core.mysqls.LabelIndex
+import com.kakao.s2graph.core.types.{HBaseType, SourceAndTargetVertexIdPair, VertexId}
+import org.apache.hadoop.hbase.util.Bytes
+
+
+class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[SnapshotEdge] {
+ import StorageSerializable._
+
+ val label = snapshotEdge.label
+ val table = label.hbaseTableName.getBytes()
+ val cf = Serializable.edgeCf
+
+ def statusCodeWithOp(statusCode: Byte, op: Byte): Array[Byte] = {
+ val byte = (((statusCode << 4) | op).toByte)
+ Array.fill(1)(byte.toByte)
+ }
+ def valueBytes() = Bytes.add(statusCodeWithOp(snapshotEdge.statusCode, snapshotEdge.op),
+ propsToKeyValuesWithTs(snapshotEdge.props.toList))
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ label.schemaVersion match {
+ case HBaseType.VERSION1 | HBaseType.VERSION2 => toKeyValuesInner
+ case _ => toKeyValuesInnerV3
+ }
+ }
+
+ private def toKeyValuesInner: Seq[SKeyValue] = {
+ val srcIdBytes = VertexId.toSourceVertexId(snapshotEdge.srcVertex.id).bytes
+ val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
+
+ val row = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+ val tgtIdBytes = VertexId.toTargetVertexId(snapshotEdge.tgtVertex.id).bytes
+
+ val qualifier = tgtIdBytes
+
+ val value = snapshotEdge.pendingEdgeOpt match {
+ case None => valueBytes()
+ case Some(pendingEdge) =>
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val versionBytes = Array.empty[Byte]
+ val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
+ val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+ Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
+ }
+ val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
+ Seq(kv)
+ }
+
+ private def toKeyValuesInnerV3: Seq[SKeyValue] = {
+ val srcIdAndTgtIdBytes = SourceAndTargetVertexIdPair(snapshotEdge.srcVertex.innerId, snapshotEdge.tgtVertex.innerId).bytes
+ val labelWithDirBytes = snapshotEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = labelOrderSeqWithIsInverted(LabelIndex.DefaultSeq, isInverted = true)
+
+ val row = Bytes.add(srcIdAndTgtIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+
+ val qualifier = Array.empty[Byte]
+
+ val value = snapshotEdge.pendingEdgeOpt match {
+ case None => valueBytes()
+ case Some(pendingEdge) =>
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val versionBytes = Array.empty[Byte]
+ val propsBytes = propsToKeyValuesWithTs(pendingEdge.propsWithTs.toSeq)
+ val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+
+ Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
+ }
+
+ val kv = SKeyValue(table, row, cf, qualifier, value, snapshotEdge.version)
+ Seq(kv)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
index c9e768e..bca8df3 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/Storage.scala
@@ -1,64 +1,324 @@
package com.kakao.s2graph.core.storage
-import com.google.common.cache.Cache
+import com.kakao.s2graph.core.ExceptionHandler.{Key, Val, KafkaMessage}
+import com.kakao.s2graph.core.GraphExceptions.FetchTimeoutException
import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls.{Service, Label}
-import com.kakao.s2graph.core.utils.logger
+import com.kakao.s2graph.core.mysqls._
+import com.kakao.s2graph.core.types._
+import com.kakao.s2graph.core.utils.{Extensions, logger}
import com.typesafe.config.Config
-
-
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.kafka.clients.producer.ProducerRecord
+import scala.annotation.tailrec
import scala.collection.Seq
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Try
+import scala.util.{Random, Try}
+
+abstract class Storage[R](val config: Config)(implicit ec: ExecutionContext) {
+ /** storage dependent configurations */
+ val MaxRetryNum = config.getInt("max.retry.number")
+ val MaxBackOff = config.getInt("max.back.off")
+ val DeleteAllFetchSize = config.getInt("delete.all.fetch.size")
+ val FailProb = config.getDouble("hbase.fail.prob")
+ val LockExpireDuration = Math.max(MaxRetryNum * MaxBackOff * 2, 10000)
+ val maxSize = config.getInt("future.cache.max.size")
+ val expireAfterWrite = config.getInt("future.cache.expire.after.write")
+ val expireAfterAccess = config.getInt("future.cache.expire.after.access")
+
+
+ /**
+ * create serializer that knows how to convert given snapshotEdge into kvs: Seq[SKeyValue]
+ * so we can store this kvs.
+ * @param snapshotEdge: snapshotEdge to serialize
+ * @return serializer implementation for StorageSerializable which has toKeyValues return Seq[SKeyValue]
+ */
+ def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge) = new SnapshotEdgeSerializable(snapshotEdge)
+
+ /**
+ * create serializer that knows how to convert given indexEdge into kvs: Seq[SKeyValue]
+ * @param indexedEdge: indexEdge to serialize
+ * @return serializer implementation
+ */
+ def indexEdgeSerializer(indexedEdge: IndexEdge) = new IndexEdgeSerializable(indexedEdge)
+
+ /**
+ * create serializer that knows how to convert given vertex into kvs: Seq[SKeyValue]
+ * @param vertex: vertex to serialize
+ * @return serializer implementation
+ */
+ def vertexSerializer(vertex: Vertex) = new VertexSerializable(vertex)
+
+ /**
+ * create deserializer that can parse stored CanSKeyValue into snapshotEdge.
+ * note that each storage implementation should implement implicit type class
+ * to convert storage dependent dataType into common SKeyValue type by implementing CanSKeyValue
+ *
+ * ex) Asynchbase use it's KeyValue class and CanSKeyValue object has implicit type conversion method.
+ * if any storaage use different class to represent stored byte array,
+ * then that storage implementation is responsible to provide implicit type conversion method on CanSKeyValue.
+ * */
+ val snapshotEdgeDeserializer = new SnapshotEdgeDeserializable
-abstract class Storage(val config: Config)(implicit ec: ExecutionContext) {
+ /** create deserializer that can parse stored CanSKeyValue into indexEdge. */
+ val indexEdgeDeserializer = new IndexEdgeDeserializable
- def cacheOpt: Option[Cache[Integer, Seq[QueryResult]]]
+ /** create deserializer that can parser stored CanSKeyValue into vertex. */
+ val vertexDeserializer = new VertexDeserializable
- def vertexCacheOpt: Option[Cache[Integer, Option[Vertex]]]
- // Serializer/Deserializer
- def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): StorageSerializable[SnapshotEdge]
+ /**
+ * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
+ * note that this should be return true on all success.
+ * we assumes that each storage implementation has client as member variable.
+ *
+ *
+ * @param cluster: where this key values should be stored.
+ * @param kvs: sequence of SKeyValue that need to be stored in storage.
+ * @param withWait: flag to control wait ack from storage.
+ * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
+ * it never block thread, but rather submit work and notified by event loop when storage send ack back.
+ * @return ack message from storage.
+ */
+ def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean]
- def indexEdgeSerializer(indexedEdge: IndexEdge): StorageSerializable[IndexEdge]
+// def writeToStorage(kv: SKeyValue, withWait: Boolean): Future[Boolean]
- def vertexSerializer(vertex: Vertex): StorageSerializable[Vertex]
+ /**
+ * fetch SnapshotEdge for given request from storage.
+ * also storage datatype should be converted into SKeyValue.
+ * note that return type is Sequence rather than single SKeyValue for simplicity,
+ * even though there is assertions sequence.length == 1.
+ * @param request
+ * @return
+ */
+ def fetchSnapshotEdgeKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
- def snapshotEdgeDeserializer: StorageDeserializable[SnapshotEdge]
+ /**
+ * write requestKeyValue into storage if the current value in storage that is stored matches.
+ * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
+ *
+ * Most important thing is this have to be 'atomic' operation.
+ * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
+ * either blocked or failed on write-write conflict case.
+ *
+ * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
+ * prevent wrong data for read.
+ *
+ * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
+ * compareAndSet to synchronize.
+ *
+ * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
+ * for storage that does not support concurrency control, then storage implementation
+ * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
+ * and write(writeLock).
+ * @param requestKeyValue
+ * @param expectedOpt
+ * @return
+ */
+ def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue]): Future[Boolean]
- def indexEdgeDeserializer: StorageDeserializable[IndexEdge]
+ /**
+ * build proper request which is specific into storage to call fetchIndexEdgeKeyValues or fetchSnapshotEdgeKeyValues.
+ * for example, Asynchbase use GetRequest, Scanner so this method is responsible to build
+ * client request(GetRequest, Scanner) based on user provided query.
+ * @param queryRequest
+ * @return
+ */
+ def buildRequest(queryRequest: QueryRequest): AnyRef
- def vertexDeserializer: StorageDeserializable[Vertex]
+ /**
+ * fetch IndexEdges for given queryParam in queryRequest.
+ * this expect previous step starting score to propagate score into next step.
+ * also parentEdges is necessary to return full bfs tree when query require it.
+ *
+ * note that return type is general type.
+ * for example, currently we wanted to use Asynchbase
+ * so single I/O return type should be Deferred[T].
+ *
+ * if we use native hbase client, then this return type can be Future[T] or just T.
+ * @param queryRequest
+ * @param prevStepScore
+ * @param isInnerCall
+ * @param parentEdges
+ * @return
+ */
+ def fetch(queryRequest: QueryRequest,
+ prevStepScore: Double,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore]): R
- // Interface
- def getEdges(q: Query): Future[Seq[QueryRequestWithResult]]
+ /**
+ * responsible to fire parallel fetch call into storage and create future that will return merged result.
+ * @param queryRequestWithScoreLs
+ * @param prevStepEdges
+ * @return
+ */
+ def fetches(queryRequestWithScoreLs: Seq[(QueryRequest, Double)],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[QueryRequestWithResult]]
- def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]]
+ /**
+ * fetch Vertex for given request from storage.
+ * @param request
+ * @return
+ */
+ def fetchVertexKeyValues(request: AnyRef): Future[Seq[SKeyValue]]
- def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]]
+ /**
+ * decide how to apply given edges(indexProps values + Map(_count -> countVal)) into storage.
+ * @param edges
+ * @param withWait
+ * @return
+ */
+ def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]]
+
+ /**
+ * this method need to be called when client shutdown. this is responsible to cleanUp the resources
+ * such as client into storage.
+ */
+ def flush(): Unit
+
+ /**
+ * create table on storage.
+ * if storage implementation does not support namespace or table, then there is nothing to be done
+ * @param zkAddr
+ * @param tableName
+ * @param cfs
+ * @param regionMultiplier
+ * @param ttl
+ * @param compressionAlgorithm
+ */
+ def createTable(zkAddr: String,
+ tableName: String,
+ cfs: List[String],
+ regionMultiplier: Int,
+ ttl: Option[Int],
+ compressionAlgorithm: String): Unit
+
+
+
+
+
+ /** Public Interface */
+
+ def getVertices(vertices: Seq[Vertex]): Future[Seq[Vertex]] = {
+ def fromResult(queryParam: QueryParam,
+ kvs: Seq[SKeyValue],
+ version: String): Option[Vertex] = {
+ if (kvs.isEmpty) None
+ else vertexDeserializer.fromKeyValues(queryParam, kvs, version, None)
+ }
+
+ val futures = vertices.map { vertex =>
+ val queryParam = QueryParam.Empty
+ val q = Query.toQuery(Seq(vertex), queryParam)
+ val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
+ fetchVertexKeyValues(buildRequest(queryRequest)).map { kvs =>
+ fromResult(queryParam, kvs, vertex.serviceColumn.schemaVersion)
+ } recoverWith { case ex: Throwable =>
+ Future.successful(None)
+ }
+ }
+
+ Future.sequence(futures).map { result => result.toList.flatten }
+ }
def mutateElements(elements: Seq[GraphElement],
withWait: Boolean = false): Future[Seq[Boolean]] = {
- val futures = elements.map {
- case edge: Edge => mutateEdge(edge, withWait)
- case vertex: Vertex => mutateVertex(vertex, withWait)
- case element => throw new RuntimeException(s"$element is not edge/vertex")
+
+ val edgeBuffer = ArrayBuffer[Edge]()
+ val vertexBuffer = ArrayBuffer[Vertex]()
+
+ elements.foreach {
+ case e: Edge => edgeBuffer += e
+ case v: Vertex => vertexBuffer += v
+ case any@_ => logger.error(s"Unknown type: ${any}")
+ }
+
+ val edgeFuture = mutateEdges(edgeBuffer, withWait)
+ val vertexFuture = mutateVertices(vertexBuffer, withWait)
+
+ val graphFuture = for {
+ edgesMutated <- edgeFuture
+ verticesMutated <- vertexFuture
+ } yield edgesMutated ++ verticesMutated
+
+ graphFuture
+ }
+
+ def mutateEdges(edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
+ val (strongEdges, weakEdges) =
+ (edges.partition(e => e.label.consistencyLevel == "strong" || e.op == GraphUtil.operations("insertBulk")))
+
+ val weakEdgesFutures = weakEdges.groupBy { e => e.label.hbaseZkAddr }.map { case (zkQuorum, edges) =>
+ val mutations = edges.flatMap { edge =>
+ val (_, edgeUpdate) =
+ if (edge.op == GraphUtil.operations("delete")) Edge.buildDeleteBulk(None, edge)
+ else Edge.buildOperation(None, Seq(edge))
+ buildVertexPutsAsync(edge) ++ indexedEdgeMutations(edgeUpdate) ++
+ snapshotEdgeMutations(edgeUpdate) ++ increments(edgeUpdate)
+ }
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ val strongEdgesFutures = mutateStrongEdges(strongEdges, withWait)
+ for {
+ weak <- Future.sequence(weakEdgesFutures)
+ strong <- strongEdgesFutures
+ } yield {
+ strong ++ weak
}
- Future.sequence(futures)
}
+ def mutateStrongEdges(_edges: Seq[Edge], withWait: Boolean): Future[Seq[Boolean]] = {
+
+ val grouped = _edges.groupBy { edge => (edge.label, edge.srcVertex.innerId, edge.tgtVertex.innerId) } toSeq
+ val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
+ val (deleteAllEdges, edges) = edgeGroup.partition(_.op == GraphUtil.operations("deleteAll"))
- def mutateEdge(edge: Edge, withWait: Boolean): Future[Boolean]
+ // DeleteAll first
+ val deleteAllFutures = deleteAllEdges.map { edge =>
+ deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.label), edge.labelWithDir.dir, edge.ts)
+ }
- def mutateEdges(edges: Seq[Edge], withWait: Boolean = false): Future[Seq[Boolean]]
+ // After deleteAll, process others
+ lazy val mutateEdgeFutures = edges.toList match {
+ case head :: tail =>
+ // val strongConsistency = edges.head.label.consistencyLevel == "strong"
+ // if (strongConsistency) {
+ val edgeFuture = mutateEdgesInner(edges, checkConsistency = true , withWait)(Edge.buildOperation)
-// def mutateEdges(edges: Seq[Edge],
-// withWait: Boolean = false): Future[Seq[Boolean]] = {
-// val futures = edges.map { edge => mutateEdge(edge, withWait) }
-// Future.sequence(futures)
-// }
+ //TODO: decide what we will do on failure on vertex put
+ val puts = buildVertexPutsAsync(head)
+ val vertexFuture = writeToStorage(head.label.hbaseZkAddr, puts, withWait)
+ Seq(edgeFuture, vertexFuture)
+ // } else {
+ // edges.map { edge => mutateEdge(edge, withWait = withWait) }
+ // }
+ case Nil => Nil
+ }
- def mutateVertex(vertex: Vertex, withWait: Boolean): Future[Boolean]
+ val composed = for {
+ deleteRet <- Future.sequence(deleteAllFutures)
+ mutateRet <- Future.sequence(mutateEdgeFutures)
+ } yield deleteRet ++ mutateRet
+
+ composed.map(_.forall(identity))
+ }
+
+ Future.sequence(mutateEdges)
+ }
+
+ def mutateVertex(vertex: Vertex, withWait: Boolean): Future[Boolean] = {
+ if (vertex.op == GraphUtil.operations("delete")) {
+ writeToStorage(vertex.hbaseZkAddr,
+ vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
+ } else if (vertex.op == GraphUtil.operations("deleteAll")) {
+ logger.info(s"deleteAll for vertex is truncated. $vertex")
+ Future.successful(true) // Ignore withWait parameter, because deleteAll operation may takes long time
+ } else {
+ writeToStorage(vertex.hbaseZkAddr, buildPutsAll(vertex), withWait)
+ }
+ }
def mutateVertices(vertices: Seq[Vertex],
withWait: Boolean = false): Future[Seq[Boolean]] = {
@@ -66,30 +326,272 @@ abstract class Storage(val config: Config)(implicit ec: ExecutionContext) {
Future.sequence(futures)
}
- def deleteAllAdjacentEdges(srcVertices: Seq[Vertex], labels: Seq[Label], dir: Int, ts: Long): Future[Boolean]
- def incrementCounts(edges: Seq[Edge], withWait: Boolean): Future[Seq[(Boolean, Long)]]
+ def mutateEdgesInner(edges: Seq[Edge],
+ checkConsistency: Boolean,
+ withWait: Boolean)(f: (Option[Edge], Seq[Edge]) => (Edge, EdgeMutate)): Future[Boolean] = {
+ if (!checkConsistency) {
+ val zkQuorum = edges.head.label.hbaseZkAddr
+ val futures = edges.map { edge =>
+ val (_, edgeUpdate) = f(None, Seq(edge))
+ val mutations =
+ indexedEdgeMutations(edgeUpdate) ++
+ snapshotEdgeMutations(edgeUpdate) ++
+ increments(edgeUpdate)
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ Future.sequence(futures).map { rets => rets.forall(identity) }
+ } else {
+ def commit(_edges: Seq[Edge], statusCode: Byte): Future[Boolean] = {
- def flush(): Unit
+ fetchSnapshotEdge(_edges.head) flatMap { case (queryParam, snapshotEdgeOpt, kvOpt) =>
- def createTable(zkAddr: String,
- tableName: String,
- cfs: List[String],
- regionMultiplier: Int,
- ttl: Option[Int],
- compressionAlgorithm: String): Unit
+ val (newEdge, edgeUpdate) = f(snapshotEdgeOpt, _edges)
+ logger.debug(s"${snapshotEdgeOpt}\n${edgeUpdate.toLogString}")
+ //shouldReplace false.
+ if (edgeUpdate.newSnapshotEdge.isEmpty && statusCode <= 0) {
+ logger.debug(s"${newEdge.toLogString} drop.")
+ Future.successful(true)
+ } else {
+ commitUpdate(newEdge, statusCode)(snapshotEdgeOpt, kvOpt, edgeUpdate).map { ret =>
+ if (ret) {
+ logger.info(s"[Success] commit: \n${_edges.map(_.toLogString).mkString("\n")}")
+ } else {
+ throw new PartialFailureException(newEdge, 3, "commit failed.")
+ }
+ true
+ }
+ }
+ }
+ }
+ def retry(tryNum: Int)(edges: Seq[Edge], statusCode: Byte)(fn: (Seq[Edge], Byte) => Future[Boolean]): Future[Boolean] = {
+ if (tryNum >= MaxRetryNum) {
+ edges.foreach { edge =>
+ logger.error(s"commit failed after $MaxRetryNum\n${edge.toLogString}")
+ ExceptionHandler.enqueue(ExceptionHandler.toKafkaMessage(element = edge))
+ }
+ Future.successful(false)
+ } else {
+ val future = fn(edges, statusCode)
+ future.onSuccess {
+ case success =>
+ logger.debug(s"Finished. [$tryNum]\n${edges.head.toLogString}\n")
+ }
+ future recoverWith {
+ case FetchTimeoutException(retryEdge) =>
+ logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}")
+ retry(tryNum + 1)(edges, statusCode)(fn)
+
+ case PartialFailureException(retryEdge, failedStatusCode, faileReason) =>
+ val status = failedStatusCode match {
+ case 0 => "AcquireLock failed."
+ case 1 => "Mutation failed."
+ case 2 => "Increment failed."
+ case 3 => "ReleaseLock failed."
+ case 4 => "Unknown"
+ }
+
+ Thread.sleep(Random.nextInt(MaxBackOff))
+ logger.info(s"[Try: $tryNum], [Status: $status] partial fail.\n${retryEdge.toLogString}\nFailReason: ${faileReason}")
+ retry(tryNum + 1)(Seq(retryEdge), failedStatusCode)(fn)
+ case ex: Exception =>
+ logger.error("Unknown exception", ex)
+ Future.successful(false)
+ }
+ }
+ }
+ retry(1)(edges, 0)(commit)
+ }
+ }
+
+ def mutateLog(snapshotEdgeOpt: Option[Edge], edges: Seq[Edge],
+ newEdge: Edge, edgeMutate: EdgeMutate) =
+ Seq("----------------------------------------------",
+ s"SnapshotEdge: ${snapshotEdgeOpt.map(_.toLogString)}",
+ s"requestEdges: ${edges.map(_.toLogString).mkString("\n")}",
+ s"newEdge: ${newEdge.toLogString}",
+ s"mutation: \n${edgeMutate.toLogString}",
+ "----------------------------------------------").mkString("\n")
+
+
+ /** Delete All */
+ protected def deleteAllFetchedEdgesAsyncOld(queryRequest: QueryRequest,
+ queryResult: QueryResult,
+ requestTs: Long,
+ retryNum: Int): Future[Boolean] = {
+ val queryParam = queryRequest.queryParam
+ val zkQuorum = queryParam.label.hbaseZkAddr
+ val futures = for {
+ edgeWithScore <- queryResult.edgeWithScoreLs
+ (edge, score) = EdgeWithScore.unapply(edgeWithScore).get
+ } yield {
+ /** reverted direction */
+ val reversedIndexedEdgesMutations = edge.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ buildIncrementsAsync(indexEdge, -1L)
+ }
+ val reversedSnapshotEdgeMutations = snapshotEdgeSerializer(edge.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ val forwardIndexedEdgeMutations = edge.edgesWithIndex.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ buildIncrementsAsync(indexEdge, -1L)
+ }
+ val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
+ writeToStorage(zkQuorum, mutations, withWait = true)
+ }
+ Future.sequence(futures).map { rets => rets.forall(identity) }
+ }
+
+ protected def buildEdgesToDelete(queryRequestWithResultLs: QueryRequestWithResult, requestTs: Long): QueryResult = {
+ val (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResultLs).get
+ val edgeWithScoreLs = queryResult.edgeWithScoreLs.filter { edgeWithScore =>
+ (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
+ }.map { edgeWithScore =>
+ val label = queryRequest.queryParam.label
+ val (newOp, newVersion, newPropsWithTs) = label.consistencyLevel match {
+ case "strong" =>
+ val _newPropsWithTs = edgeWithScore.edge.propsWithTs ++
+ Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(requestTs, requestTs, label.schemaVersion))
+ (GraphUtil.operations("delete"), requestTs, _newPropsWithTs)
+ case _ =>
+ val oldEdge = edgeWithScore.edge
+ (oldEdge.op, oldEdge.version, oldEdge.propsWithTs)
+ }
+
+ val copiedEdge =
+ edgeWithScore.edge.copy(op = newOp, version = newVersion, propsWithTs = newPropsWithTs)
+
+ val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
+// logger.debug(s"delete edge from deleteAll: ${edgeToDelete.edge.toLogString}")
+ edgeToDelete
+ }
+
+ queryResult.copy(edgeWithScoreLs = edgeWithScoreLs)
+ }
+
+ protected def deleteAllFetchedEdgesLs(queryRequestWithResultLs: Seq[QueryRequestWithResult], requestTs: Long): Future[(Boolean, Boolean)] = {
+ val queryResultLs = queryRequestWithResultLs.map(_.queryResult)
+ queryResultLs.foreach { queryResult =>
+ if (queryResult.isFailure) throw new RuntimeException("fetched result is fallback.")
+ }
+ val futures = for {
+ queryRequestWithResult <- queryRequestWithResultLs
+ (queryRequest, _) = QueryRequestWithResult.unapply(queryRequestWithResult).get
+ deleteQueryResult = buildEdgesToDelete(queryRequestWithResult, requestTs)
+ if deleteQueryResult.edgeWithScoreLs.nonEmpty
+ } yield {
+ val label = queryRequest.queryParam.label
+ label.schemaVersion match {
+ case HBaseType.VERSION3 =>
+ if (label.consistencyLevel == "strong") {
+ /**
+ * read: snapshotEdge on queryResult = O(N)
+ * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
+ */
+ mutateEdges(deleteQueryResult.edgeWithScoreLs.map(_.edge), withWait = true).map(_.forall(identity))
+ } else {
+ deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
+ }
+ case _ =>
+
+ /**
+ * read: x
+ * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
+ */
+ deleteAllFetchedEdgesAsyncOld(queryRequest, deleteQueryResult, requestTs, MaxRetryNum)
+ }
+ }
+
+ if (futures.isEmpty) {
+ // all deleted.
+ Future.successful(true -> true)
+ } else {
+ Future.sequence(futures).map { rets => false -> rets.forall(identity) }
+ }
+ }
+
+ protected def fetchAndDeleteAll(query: Query, requestTs: Long): Future[(Boolean, Boolean)] = {
+ val future = for {
+ queryRequestWithResultLs <- getEdges(query)
+ (allDeleted, ret) <- deleteAllFetchedEdgesLs(queryRequestWithResultLs, requestTs)
+ } yield {
+// logger.debug(s"fetchAndDeleteAll: ${allDeleted}, ${ret}")
+ (allDeleted, ret)
+ }
+
+ Extensions.retryOnFailure(MaxRetryNum) {
+ future
+ } {
+ logger.error(s"fetch and deleteAll failed.")
+ (true, false)
+ }
+
+ }
+
+ def deleteAllAdjacentEdges(srcVertices: Seq[Vertex],
+ labels: Seq[Label],
+ dir: Int,
+ ts: Long): Future[Boolean] = {
+
+ def enqueueLogMessage() = {
+ val kafkaMessages = for {
+ vertice <- srcVertices
+ id = vertice.innerId.toIdString()
+ label <- labels
+ } yield {
+ val tsv = Seq(ts, "deleteAll", "e", id, id, label.label, "{}", GraphUtil.fromOp(dir.toByte)).mkString("\t")
+ val topic = ExceptionHandler.failTopic
+ val kafkaMsg = KafkaMessage(new ProducerRecord[Key, Val](topic, null, tsv))
+ kafkaMsg
+ }
+
+ ExceptionHandler.enqueues(kafkaMessages)
+ }
+
+ val requestTs = ts
+ val queryParams = for {
+ label <- labels
+ } yield {
+ val labelWithDir = LabelWithDirection(label.id.get, dir)
+ QueryParam(labelWithDir).limit(0, DeleteAllFetchSize).duplicatePolicy(Option(Query.DuplicatePolicy.Raw))
+ }
+
+ val step = Step(queryParams.toList)
+ val q = Query(srcVertices, Vector(step))
+
+ // Extensions.retryOnSuccessWithBackoff(MaxRetryNum, Random.nextInt(MaxBackOff) + 1) {
+ val retryFuture = Extensions.retryOnSuccess(MaxRetryNum) {
+ fetchAndDeleteAll(q, requestTs)
+ } { case (allDeleted, deleteSuccess) =>
+ allDeleted && deleteSuccess
+ }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
+
+ retryFuture onFailure {
+ case ex =>
+ logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
+ enqueueLogMessage()
+ }
+
+ retryFuture
+ }
+
+ /** End Of Delete All */
+
+
+
+
+ /** Parsing Logic: parse from kv from Storage into Edge */
def toEdge[K: CanSKeyValue](kv: K,
queryParam: QueryParam,
cacheElementOpt: Option[IndexEdge],
parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
+// logger.debug(s"toEdge: $kv")
try {
- val indexEdge = indexEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
-
- Option(indexEdge.toEdge.copy(parentEdges = parentEdges))
+ val indexEdgeOpt = indexEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
+ indexEdgeOpt.map(indexEdge => indexEdge.toEdge.copy(parentEdges = parentEdges))
} catch {
case ex: Exception =>
- logger.error(s"Fail on toEdge: ${kv.toString}, ${queryParam}")
+ logger.error(s"Fail on toEdge: ${kv.toString}, ${queryParam}", ex)
None
}
}
@@ -99,19 +601,24 @@ abstract class Storage(val config: Config)(implicit ec: ExecutionContext) {
cacheElementOpt: Option[SnapshotEdge] = None,
isInnerCall: Boolean,
parentEdges: Seq[EdgeWithScore]): Option[Edge] = {
- val snapshotEdge = snapshotEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
+// logger.debug(s"SnapshottoEdge: $kv")
+ val snapshotEdgeOpt = snapshotEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, cacheElementOpt)
if (isInnerCall) {
- val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
- if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
- else None
- } else {
- if (Edge.allPropsDeleted(snapshotEdge.props)) None
- else {
+ snapshotEdgeOpt.flatMap { snapshotEdge =>
val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
else None
}
+ } else {
+ snapshotEdgeOpt.flatMap { snapshotEdge =>
+ if (Edge.allPropsDeleted(snapshotEdge.props)) None
+ else {
+ val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
+ if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
+ else None
+ }
+ }
}
}
@@ -126,7 +633,7 @@ abstract class Storage(val config: Config)(implicit ec: ExecutionContext) {
val kv = first
val cacheElementOpt =
if (queryParam.isSnapshotEdge) None
- else Option(indexEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, None))
+ else indexEdgeDeserializer.fromKeyValues(queryParam, Seq(kv), queryParam.label.schemaVersion, None)
for {
kv <- kvs
@@ -145,6 +652,526 @@ abstract class Storage(val config: Config)(implicit ec: ExecutionContext) {
}
}
+ /** End Of Parse Logic */
+
+// /** methods for consistency */
+// protected def writeAsyncSimple(zkQuorum: String, elementRpcs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
+// if (elementRpcs.isEmpty) {
+// Future.successful(true)
+// } else {
+// val futures = elementRpcs.map { rpc => writeToStorage(rpc, withWait) }
+// Future.sequence(futures).map(_.forall(identity))
+// }
+// }
+
+ case class PartialFailureException(edge: Edge, statusCode: Byte, failReason: String) extends Exception
+
+ protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge) = {
+ val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}").mkString("\n")
+ logger.debug(msg)
+ }
+
+ protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge, edgeMutate: EdgeMutate) = {
+ val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}",
+ s"${edgeMutate.toLogString}").mkString("\n")
+ logger.debug(msg)
+ }
+
+ protected def buildLockEdge(snapshotEdgeOpt: Option[Edge], edge: Edge, kvOpt: Option[SKeyValue]) = {
+ val currentTs = System.currentTimeMillis()
+ val lockTs = snapshotEdgeOpt match {
+ case None => Option(currentTs)
+ case Some(snapshotEdge) =>
+ snapshotEdge.pendingEdgeOpt match {
+ case None => Option(currentTs)
+ case Some(pendingEdge) => pendingEdge.lockTs
+ }
+ }
+ val newVersion = kvOpt.map(_.timestamp).getOrElse(edge.ts) + 1
+ // snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
+ val pendingEdge = edge.copy(version = newVersion, statusCode = 1, lockTs = lockTs)
+ val base = snapshotEdgeOpt match {
+ case None =>
+ // no one ever mutated on this snapshotEdge.
+ edge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
+ case Some(snapshotEdge) =>
+ // there is at least one mutation have been succeed.
+ snapshotEdgeOpt.get.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
+ }
+ base.copy(version = newVersion, statusCode = 1, lockTs = None)
+ }
+
+ protected def buildReleaseLockEdge(snapshotEdgeOpt: Option[Edge], lockEdge: SnapshotEdge,
+ edgeMutate: EdgeMutate) = {
+ val newVersion = lockEdge.version + 1
+ val base = edgeMutate.newSnapshotEdge match {
+ case None =>
+ // shouldReplace false
+ assert(snapshotEdgeOpt.isDefined)
+ snapshotEdgeOpt.get.toSnapshotEdge
+ case Some(newSnapshotEdge) => newSnapshotEdge
+ }
+ base.copy(version = newVersion, statusCode = 0, pendingEdgeOpt = None)
+ }
+
+ protected def acquireLock(statusCode: Byte,
+ edge: Edge,
+ oldSnapshotEdgeOpt: Option[Edge],
+ lockEdge: SnapshotEdge,
+ oldBytes: Array[Byte]): Future[Boolean] = {
+ if (statusCode >= 1) {
+ logger.debug(s"skip acquireLock: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 0, s"$p")
+ else {
+ val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
+ val oldPut = oldSnapshotEdgeOpt.map(e => snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head)
+// val lockEdgePut = buildPutAsync(lockEdge).head
+// val oldPut = oldSnapshotEdgeOpt.map(e => buildPutAsync(e.toSnapshotEdge).head)
+ writeLock(lockEdgePut, oldPut).recoverWith { case ex: Exception =>
+ logger.error(s"AcquireLock RPC Failed.")
+ throw new PartialFailureException(edge, 0, "AcquireLock RPC Failed")
+ }.map { ret =>
+ if (ret) {
+ val log = Seq(
+ "\n",
+ "=" * 50,
+ s"[Success]: acquireLock",
+ s"[RequestEdge]: ${edge.toLogString}",
+ s"[LockEdge]: ${lockEdge.toLogString()}",
+ s"[PendingEdge]: ${lockEdge.pendingEdgeOpt.map(_.toLogString).getOrElse("")}",
+ "=" * 50, "\n").mkString("\n")
+
+ logger.debug(log)
+ // debug(ret, "acquireLock", edge.toSnapshotEdge)
+ } else {
+ throw new PartialFailureException(edge, 0, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+ }
+
+
+
+ protected def releaseLock(predicate: Boolean,
+ edge: Edge,
+ lockEdge: SnapshotEdge,
+ releaseLockEdge: SnapshotEdge,
+ _edgeMutate: EdgeMutate,
+ oldBytes: Array[Byte]): Future[Boolean] = {
+ if (!predicate) {
+ throw new PartialFailureException(edge, 3, "predicate failed.")
+ }
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 3, s"$p")
+ else {
+ val releaseLockEdgePut = snapshotEdgeSerializer(releaseLockEdge).toKeyValues.head
+ val lockEdgePut = snapshotEdgeSerializer(lockEdge).toKeyValues.head
+ writeLock(releaseLockEdgePut, Option(lockEdgePut)).recoverWith {
+ case ex: Exception =>
+ logger.error(s"ReleaseLock RPC Failed.")
+ throw new PartialFailureException(edge, 3, "ReleaseLock RPC Failed")
+ }.map { ret =>
+ if (ret) {
+ debug(ret, "releaseLock", edge.toSnapshotEdge)
+ } else {
+ val msg = Seq("\nFATAL ERROR\n",
+ "=" * 50,
+ oldBytes.toList,
+ lockEdgePut,
+ releaseLockEdgePut,
+ // lockEdgePut.value.toList,
+ // releaseLockEdgePut.value().toList,
+ "=" * 50,
+ "\n"
+ )
+ logger.error(msg.mkString("\n"))
+ // error(ret, "releaseLock", edge.toSnapshotEdge)
+ throw new PartialFailureException(edge, 3, "hbase fail.")
+ }
+ true
+ }
+ }
+ Future.successful(true)
+ }
-}
+ protected def mutate(predicate: Boolean,
+ edge: Edge,
+ statusCode: Byte,
+ _edgeMutate: EdgeMutate): Future[Boolean] = {
+ if (!predicate) throw new PartialFailureException(edge, 1, "predicate failed.")
+
+ if (statusCode >= 2) {
+ logger.debug(s"skip mutate: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 1, s"$p")
+ else
+ writeToStorage(edge.label.hbaseZkAddr, indexedEdgeMutations(_edgeMutate), withWait = true).map { ret =>
+ if (ret) {
+ debug(ret, "mutate", edge.toSnapshotEdge, _edgeMutate)
+ } else {
+ throw new PartialFailureException(edge, 1, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+
+ protected def increment(predicate: Boolean,
+ edge: Edge,
+ statusCode: Byte, _edgeMutate: EdgeMutate): Future[Boolean] = {
+ if (!predicate) throw new PartialFailureException(edge, 2, "predicate failed.")
+ if (statusCode >= 3) {
+ logger.debug(s"skip increment: [$statusCode]\n${edge.toLogString}")
+ Future.successful(true)
+ } else {
+ val p = Random.nextDouble()
+ if (p < FailProb) throw new PartialFailureException(edge, 2, s"$p")
+ else
+ writeToStorage(edge.label.hbaseZkAddr, increments(_edgeMutate), withWait = true).map { ret =>
+ if (ret) {
+ debug(ret, "increment", edge.toSnapshotEdge, _edgeMutate)
+ } else {
+ throw new PartialFailureException(edge, 2, "hbase fail.")
+ }
+ true
+ }
+ }
+ }
+
+
+ /** this may be overrided by specific storage implementation */
+ protected def commitProcess(edge: Edge, statusCode: Byte)
+ (snapshotEdgeOpt: Option[Edge], kvOpt: Option[SKeyValue])
+ (lockEdge: SnapshotEdge, releaseLockEdge: SnapshotEdge, _edgeMutate: EdgeMutate): Future[Boolean] = {
+ val oldBytes = kvOpt.map(kv => kv.value).getOrElse(Array.empty[Byte])
+ for {
+ locked <- acquireLock(statusCode, edge, snapshotEdgeOpt, lockEdge, oldBytes)
+ mutated <- mutate(locked, edge, statusCode, _edgeMutate)
+ incremented <- increment(mutated, edge, statusCode, _edgeMutate)
+ released <- releaseLock(incremented, edge, lockEdge, releaseLockEdge, _edgeMutate, oldBytes)
+ } yield {
+ released
+ }
+ }
+
+ protected def commitUpdate(edge: Edge,
+ statusCode: Byte)(snapshotEdgeOpt: Option[Edge],
+ kvOpt: Option[SKeyValue],
+ edgeUpdate: EdgeMutate): Future[Boolean] = {
+ val label = edge.label
+ def oldBytes = kvOpt.map(_.value).getOrElse(Array.empty)
+
+ val lockEdge = buildLockEdge(snapshotEdgeOpt, edge, kvOpt)
+ val releaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, edgeUpdate)
+ val _process = commitProcess(edge, statusCode)(snapshotEdgeOpt, kvOpt)_
+ snapshotEdgeOpt match {
+ case None =>
+ // no one ever did success on acquire lock.
+ _process(lockEdge, releaseLockEdge, edgeUpdate)
+ // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
+ case Some(snapshotEdge) =>
+ // someone did success on acquire lock at least one.
+ snapshotEdge.pendingEdgeOpt match {
+ case None =>
+ // not locked
+ _process(lockEdge, releaseLockEdge, edgeUpdate)
+ // process(lockEdge, releaseLockEdge, edgeUpdate, statusCode)
+ case Some(pendingEdge) =>
+ def isLockExpired = pendingEdge.lockTs.get + LockExpireDuration < System.currentTimeMillis()
+ if (isLockExpired) {
+ val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
+ val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(pendingEdge))
+ val newLockEdge = buildLockEdge(snapshotEdgeOpt, pendingEdge, kvOpt)
+ val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, newLockEdge, newEdgeUpdate)
+ commitProcess(edge, statusCode = 0)(snapshotEdgeOpt, kvOpt)(newLockEdge, newReleaseLockEdge, newEdgeUpdate).flatMap { ret =>
+ // process(newLockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode = 0).flatMap { ret =>
+ val log = s"[Success]: Resolving expired pending edge.\n${pendingEdge.toLogString}"
+ throw new PartialFailureException(edge, 0, log)
+ }
+ } else {
+ // locked
+ if (pendingEdge.ts == edge.ts && statusCode > 0) {
+ // self locked
+ val oldSnapshotEdge = if (snapshotEdge.ts == pendingEdge.ts) None else Option(snapshotEdge)
+ val (_, newEdgeUpdate) = Edge.buildOperation(oldSnapshotEdge, Seq(edge))
+ val newReleaseLockEdge = buildReleaseLockEdge(snapshotEdgeOpt, lockEdge, newEdgeUpdate)
+
+ /** lockEdge will be ignored */
+ _process(lockEdge, newReleaseLockEdge, newEdgeUpdate)
+ // process(lockEdge, newReleaseLockEdge, newEdgeUpdate, statusCode)
+ } else {
+ throw new PartialFailureException(edge, statusCode, s"others[${pendingEdge.ts}] is mutating. me[${edge.ts}]")
+ }
+ }
+ }
+ }
+ }
+
+ /** end of methods for consistency */
+
+
+ // def futureCache[T] = Cache[Long, (Long, T)]
+
+ protected def toRequestEdge(queryRequest: QueryRequest): Edge = {
+ val srcVertex = queryRequest.vertex
+ // val tgtVertexOpt = queryRequest.tgtVertexOpt
+ val edgeCf = Serializable.edgeCf
+ val queryParam = queryRequest.queryParam
+ val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
+ val label = queryParam.label
+ val labelWithDir = queryParam.labelWithDir
+ val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
+ val (srcInnerId, tgtInnerId) = tgtVertexIdOpt match {
+ case Some(tgtVertexId) => // _to is given.
+ /** we use toSnapshotEdge so dont need to swap src, tgt */
+ val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ val tgt = InnerVal.convertVersion(tgtVertexId, tgtColumn.columnType, label.schemaVersion)
+ (src, tgt)
+ case None =>
+ val src = InnerVal.convertVersion(srcVertex.innerId, srcColumn.columnType, label.schemaVersion)
+ (src, src)
+ }
+
+ val (srcVId, tgtVId) = (SourceVertexId(srcColumn.id.get, srcInnerId), TargetVertexId(tgtColumn.id.get, tgtInnerId))
+ val (srcV, tgtV) = (Vertex(srcVId), Vertex(tgtVId))
+ val currentTs = System.currentTimeMillis()
+ val propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs)).toMap
+ Edge(srcV, tgtV, labelWithDir, propsWithTs = propsWithTs)
+ }
+
+
+
+ protected def fetchSnapshotEdge(edge: Edge): Future[(QueryParam, Option[Edge], Option[SKeyValue])] = {
+ val labelWithDir = edge.labelWithDir
+ val queryParam = QueryParam(labelWithDir)
+ val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
+ val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
+ val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
+
+ fetchSnapshotEdgeKeyValues(buildRequest(queryRequest)).map { kvs =>
+ val (edgeOpt, kvOpt) =
+ if (kvs.isEmpty) (None, None)
+ else {
+ val _edgeOpt = toEdges(kvs, queryParam, 1.0, isInnerCall = true, parentEdges = Nil).headOption.map(_.edge)
+ val _kvOpt = kvs.headOption
+ (_edgeOpt, _kvOpt)
+ }
+ (queryParam, edgeOpt, kvOpt)
+ } recoverWith { case ex: Throwable =>
+ logger.error(s"fetchQueryParam failed. fallback return.", ex)
+ throw new FetchTimeoutException(s"${edge.toLogString}")
+ }
+ }
+
+ protected def fetchStep(orgQuery: Query, queryRequestWithResultsLs: Seq[QueryRequestWithResult]): Future[Seq[QueryRequestWithResult]] = {
+ if (queryRequestWithResultsLs.isEmpty) Future.successful(Nil)
+ else {
+ val queryRequest = queryRequestWithResultsLs.head.queryRequest
+ val q = orgQuery
+ val queryResultsLs = queryRequestWithResultsLs.map(_.queryResult)
+
+ val stepIdx = queryRequest.stepIdx + 1
+
+ val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
+ val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
+ val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
+ val step = q.steps(stepIdx)
+ val alreadyVisited =
+ if (stepIdx == 0) Map.empty[(LabelWithDirection, Vertex), Boolean]
+ else Graph.alreadyVisitedVertices(queryResultsLs)
+
+ val groupedBy = queryResultsLs.flatMap { queryResult =>
+ queryResult.edgeWithScoreLs.map { case edgeWithScore =>
+ edgeWithScore.edge.tgtVertex -> edgeWithScore
+ }
+ }.groupBy { case (vertex, edgeWithScore) => vertex }
+
+ val groupedByFiltered = for {
+ (vertex, edgesWithScore) <- groupedBy
+ aggregatedScore = edgesWithScore.map(_._2.score).sum if aggregatedScore >= prevStepThreshold
+ } yield vertex -> aggregatedScore
+
+ val prevStepTgtVertexIdEdges = for {
+ (vertex, edgesWithScore) <- groupedBy
+ } yield vertex.id -> edgesWithScore.map { case (vertex, edgeWithScore) => edgeWithScore }
+
+ val nextStepSrcVertices = if (prevStepLimit >= 0) {
+ groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
+ } else {
+ groupedByFiltered.toSeq
+ }
+
+ val queryRequests = for {
+ (vertex, prevStepScore) <- nextStepSrcVertices
+ queryParam <- step.queryParams
+ } yield (QueryRequest(q, stepIdx, vertex, queryParam), prevStepScore)
+
+ Graph.filterEdges(fetches(queryRequests, prevStepTgtVertexIdEdges), alreadyVisited)(ec)
+ }
+ }
+
+ protected def fetchStepFuture(orgQuery: Query, queryRequestWithResultLsFuture: Future[Seq[QueryRequestWithResult]]): Future[Seq[QueryRequestWithResult]] = {
+ for {
+ queryRequestWithResultLs <- queryRequestWithResultLsFuture
+ ret <- fetchStep(orgQuery, queryRequestWithResultLs)
+ } yield ret
+ }
+
+ def getEdges(q: Query): Future[Seq[QueryRequestWithResult]] = {
+ val fallback = {
+ val queryRequest = QueryRequest(query = q, stepIdx = 0, q.vertices.head, queryParam = QueryParam.Empty)
+ Future.successful(q.vertices.map(v => QueryRequestWithResult(queryRequest, QueryResult())))
+ }
+ Try {
+
+ if (q.steps.isEmpty) {
+ // TODO: this should be get vertex query.
+ fallback
+ } else {
+ // current stepIdx = -1
+ val startQueryResultLs = QueryResult.fromVertices(q)
+ q.steps.foldLeft(Future.successful(startQueryResultLs)) { case (acc, step) =>
+ fetchStepFuture(q, acc)
+// fetchStepFuture(q, acc).map { stepResults =>
+// step.queryParams.zip(stepResults).foreach { case (qParam, queryRequestWithResult) =>
+// val cursor = Base64.getEncoder.encodeToString(queryRequestWithResult.queryResult.tailCursor)
+// qParam.cursorOpt = Option(cursor)
+// }
+// stepResults
+// }
+ }
+ }
+ } recover {
+ case e: Exception =>
+ logger.error(s"getEdgesAsync: $e", e)
+ fallback
+ } get
+ }
+
+ def checkEdges(params: Seq[(Vertex, Vertex, QueryParam)]): Future[Seq[QueryRequestWithResult]] = {
+ val ts = System.currentTimeMillis()
+ val futures = for {
+ (srcVertex, tgtVertex, queryParam) <- params
+ propsWithTs = Map(LabelMeta.timeStampSeq -> InnerValLikeWithTs.withLong(ts, ts, queryParam.label.schemaVersion))
+ edge = Edge(srcVertex, tgtVertex, queryParam.labelWithDir, propsWithTs = propsWithTs)
+ } yield {
+ fetchSnapshotEdge(edge).map { case (queryParam, edgeOpt, kvOpt) =>
+ val _queryParam = queryParam.tgtVertexInnerIdOpt(Option(edge.tgtVertex.innerId))
+ val q = Query.toQuery(Seq(edge.srcVertex), _queryParam)
+ val queryRequest = QueryRequest(q, 0, edge.srcVertex, _queryParam)
+ val queryResult = QueryResult(edgeOpt.toSeq.map(e => EdgeWithScore(e, 1.0)))
+ QueryRequestWithResult(queryRequest, queryResult)
+ }
+ }
+
+ Future.sequence(futures)
+ }
+
+
+
+ @tailrec
+ final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (range < sampleNumber || set.size == sampleNumber) set
+ else randomInt(sampleNumber, range, set + Random.nextInt(range))
+ }
+
+ protected def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
+ if (edges.size <= n){
+ edges
+ }else{
+ val plainEdges = if (queryRequest.queryParam.offset == 0) {
+ edges.tail
+ } else edges
+
+ val randoms = randomInt(n, plainEdges.size)
+ var samples = List.empty[EdgeWithScore]
+ var idx = 0
+ plainEdges.foreach { e =>
+ if (randoms.contains(idx)) samples = e :: samples
+ idx += 1
+ }
+ samples.toSeq
+ }
+
+ }
+ /** end of query */
+
+ /** Mutation Builder */
+
+
+ /** EdgeMutate */
+ def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] = {
+ val deleteMutations = edgeMutate.edgesToDelete.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
+ }
+ val insertMutations = edgeMutate.edgesToInsert.flatMap { indexEdge =>
+ indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ }
+
+ deleteMutations ++ insertMutations
+ }
+
+ def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+ edgeMutate.newSnapshotEdge.map(e => snapshotEdgeSerializer(e).toKeyValues).getOrElse(Nil)
+
+ def increments(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+ (edgeMutate.edgesToDelete.isEmpty, edgeMutate.edgesToInsert.isEmpty) match {
+ case (true, true) =>
+
+ /** when there is no need to update. shouldUpdate == false */
+ List.empty
+ case (true, false) =>
+
+ /** no edges to delete but there is new edges to insert so increase degree by 1 */
+ edgeMutate.edgesToInsert.flatMap { e => buildIncrementsAsync(e) }
+ case (false, true) =>
+
+ /** no edges to insert but there is old edges to delete so decrease degree by 1 */
+ edgeMutate.edgesToDelete.flatMap { e => buildIncrementsAsync(e, -1L) }
+ case (false, false) =>
+
+ /** update on existing edges so no change on degree */
+ List.empty
+ }
+
+ /** IndexEdge */
+ def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
+ val newProps = indexedEdge.props ++ Map(LabelMeta.degreeSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
+ val _indexedEdge = indexedEdge.copy(props = newProps)
+ indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
+ }
+
+ def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] = {
+ val newProps = indexedEdge.props ++ Map(LabelMeta.countSeq -> InnerVal.withLong(amount, indexedEdge.schemaVer))
+ val _indexedEdge = indexedEdge.copy(props = newProps)
+ indexEdgeSerializer(_indexedEdge).toKeyValues.map(_.copy(operation = SKeyValue.Increment))
+ }
+ def buildDeleteBelongsToId(vertex: Vertex): Seq[SKeyValue] = {
+ val kvs = vertexSerializer(vertex).toKeyValues
+ val kv = kvs.head
+ vertex.belongLabelIds.map { id =>
+ kv.copy(qualifier = Bytes.toBytes(Vertex.toPropKey(id)), operation = SKeyValue.Delete)
+ }
+ }
+
+ def buildVertexPutsAsync(edge: Edge): Seq[SKeyValue] =
+ if (edge.op == GraphUtil.operations("delete"))
+ buildDeleteBelongsToId(edge.srcForVertex) ++ buildDeleteBelongsToId(edge.tgtForVertex)
+ else
+ vertexSerializer(edge.srcForVertex).toKeyValues ++ vertexSerializer(edge.tgtForVertex).toKeyValues
+
+ def buildPutsAll(vertex: Vertex): Seq[SKeyValue] = {
+ vertex.op match {
+ case d: Byte if d == GraphUtil.operations("delete") => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
+ case _ => vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
index 791a080..4b3300a 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/StorageDeserializable.scala
@@ -1,7 +1,9 @@
package com.kakao.s2graph.core.storage
-import com.kakao.s2graph.core.{Edge, IndexEdge, QueryParam}
+import com.kakao.s2graph.core.utils.logger
+import com.kakao.s2graph.core.QueryParam
import com.kakao.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, InnerValLikeWithTs}
+import org.apache.hadoop.hbase.util.Bytes
object StorageDeserializable {
/** Deserializer */
@@ -75,8 +77,19 @@ object StorageDeserializable {
ret
}
+
+ def bytesToLong(bytes: Array[Byte], offset: Int): Long = Bytes.toLong(bytes, offset)
}
trait StorageDeserializable[E] {
- def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): E
+ def fromKeyValues[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): Option[E] = {
+ try {
+ Option(fromKeyValuesInner(queryParam, kvs, version, cacheElementOpt))
+ } catch {
+ case e: Exception =>
+ logger.error(s"${this.getClass.getName} fromKeyValues failed.", e)
+ None
+ }
+ }
+ def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam, kvs: Seq[T], version: String, cacheElementOpt: Option[E]): E
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexDeserializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexDeserializable.scala
new file mode 100644
index 0000000..699981d
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexDeserializable.scala
@@ -0,0 +1,46 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.types.{InnerVal, InnerValLike, VertexId}
+import com.kakao.s2graph.core.{QueryParam, Vertex}
+import org.apache.hadoop.hbase.util.Bytes
+
+import scala.collection.mutable.ListBuffer
+
+class VertexDeserializable extends Deserializable[Vertex] {
+ def fromKeyValuesInner[T: CanSKeyValue](queryParam: QueryParam,
+ _kvs: Seq[T],
+ version: String,
+ cacheElementOpt: Option[Vertex]): Vertex = {
+
+ val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
+
+ val kv = kvs.head
+ val (vertexId, _) = VertexId.fromBytes(kv.row, 0, kv.row.length, version)
+
+ var maxTs = Long.MinValue
+ val propsMap = new collection.mutable.HashMap[Int, InnerValLike]
+ val belongLabelIds = new ListBuffer[Int]
+
+ for {
+ kv <- kvs
+ } {
+ val propKey =
+ if (kv.qualifier.length == 1) kv.qualifier.head.toInt
+ else Bytes.toInt(kv.qualifier)
+
+ val ts = kv.timestamp
+ if (ts > maxTs) maxTs = ts
+
+ if (Vertex.isLabelId(propKey)) {
+ belongLabelIds += Vertex.toLabelId(propKey)
+ } else {
+ val v = kv.value
+ val (value, _) = InnerVal.fromBytes(v, 0, v.length, version)
+ propsMap += (propKey -> value)
+ }
+ }
+ assert(maxTs != Long.MinValue)
+ Vertex(vertexId, maxTs, propsMap.toMap, belongLabelIds = belongLabelIds)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexSerializable.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexSerializable.scala
new file mode 100644
index 0000000..bda909d
--- /dev/null
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/VertexSerializable.scala
@@ -0,0 +1,18 @@
+package com.kakao.s2graph.core.storage
+
+import com.kakao.s2graph.core.Vertex
+import org.apache.hadoop.hbase.util.Bytes
+
+case class VertexSerializable(vertex: Vertex) extends Serializable[Vertex] {
+
+ val cf = Serializable.vertexCf
+
+ override def toKeyValues: Seq[SKeyValue] = {
+ val row = vertex.id.bytes
+ val base = for ((k, v) <- vertex.props ++ vertex.defaultProps) yield Bytes.toBytes(k) -> v.bytes
+ val belongsTo = vertex.belongLabelIds.map { labelId => Bytes.toBytes(Vertex.toPropKey(labelId)) -> Array.empty[Byte] }
+ (base ++ belongsTo).map { case (qualifier, value) =>
+ SKeyValue(vertex.hbaseTableName.getBytes, row, cf, qualifier, value, vertex.ts)
+ } toSeq
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e207f676/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseMutationBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseMutationBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseMutationBuilder.scala
deleted file mode 100644
index 504f369..0000000
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseMutationBuilder.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-package com.kakao.s2graph.core.storage.hbase
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.storage.{SKeyValue, MutationBuilder}
-import org.apache.hadoop.hbase.util.Bytes
-import org.hbase.async.{DeleteRequest, AtomicIncrementRequest, PutRequest, HBaseRpc}
-
-import scala.collection.Seq
-import scala.concurrent.ExecutionContext
-
-class AsynchbaseMutationBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionContext)
- extends MutationBuilder[HBaseRpc](storage) {
-
- def put(kvs: Seq[SKeyValue]): Seq[HBaseRpc] =
- kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
-
- def increment(kvs: Seq[SKeyValue]): Seq[HBaseRpc] =
- kvs.map { kv => new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, Bytes.toLong(kv.value)) }
-
- def delete(kvs: Seq[SKeyValue]): Seq[HBaseRpc] =
- kvs.map { kv =>
- if (kv.qualifier == null) new DeleteRequest(kv.table, kv.row, kv.cf, kv.timestamp)
- else new DeleteRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.timestamp)
- }
-
- /** EdgeMutate */
- def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[HBaseRpc] = {
- val deleteMutations = edgeMutate.edgesToDelete.flatMap(edge => buildDeletesAsync(edge))
- val insertMutations = edgeMutate.edgesToInsert.flatMap(edge => buildPutsAsync(edge))
-
- deleteMutations ++ insertMutations
- }
-
- def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[HBaseRpc] =
- edgeMutate.newSnapshotEdge.map(e => buildPutAsync(e)).getOrElse(Nil)
-
-
- def increments(edgeMutate: EdgeMutate): Seq[HBaseRpc] = {
- (edgeMutate.edgesToDelete.isEmpty, edgeMutate.edgesToInsert.isEmpty) match {
- case (true, true) =>
-
- /** when there is no need to update. shouldUpdate == false */
- List.empty[AtomicIncrementRequest]
- case (true, false) =>
-
- /** no edges to delete but there is new edges to insert so increase degree by 1 */
- edgeMutate.edgesToInsert.flatMap { e => buildIncrementsAsync(e) }
- case (false, true) =>
-
- /** no edges to insert but there is old edges to delete so decrease degree by 1 */
- edgeMutate.edgesToDelete.flatMap { e => buildIncrementsAsync(e, -1L) }
- case (false, false) =>
-
- /** update on existing edges so no change on degree */
- List.empty[AtomicIncrementRequest]
- }
- }
-
- /** IndexEdge */
- def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[HBaseRpc] =
- storage.indexEdgeSerializer(indexedEdge).toKeyValues.headOption match {
- case None => Nil
- case Some(kv) =>
- val copiedKV = kv.copy(qualifier = Array.empty[Byte], value = Bytes.toBytes(amount))
- increment(Seq(copiedKV))
- }
-
-
- def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[HBaseRpc] =
- storage.indexEdgeSerializer(indexedEdge).toKeyValues.headOption match {
- case None => Nil
- case Some(kv) =>
- val copiedKV = kv.copy(value = Bytes.toBytes(amount))
- increment(Seq(copiedKV))
- }
-
- def buildDeletesAsync(indexedEdge: IndexEdge): Seq[HBaseRpc] =
- delete(storage.indexEdgeSerializer(indexedEdge).toKeyValues)
-
- def buildPutsAsync(indexedEdge: IndexEdge): Seq[HBaseRpc] =
- put(storage.indexEdgeSerializer(indexedEdge).toKeyValues)
-
- /** SnapshotEdge */
- def buildPutAsync(snapshotEdge: SnapshotEdge): Seq[HBaseRpc] =
- put(storage.snapshotEdgeSerializer(snapshotEdge).toKeyValues)
-
- def buildDeleteAsync(snapshotEdge: SnapshotEdge): Seq[HBaseRpc] =
- delete(storage.snapshotEdgeSerializer(snapshotEdge).toKeyValues)
-
- /** Vertex */
- def buildPutsAsync(vertex: Vertex): Seq[HBaseRpc] = {
- val kvs = storage.vertexSerializer(vertex).toKeyValues
- put(kvs)
- }
-
-
- def buildDeleteAsync(vertex: Vertex): Seq[HBaseRpc] = {
- val kvs = storage.vertexSerializer(vertex).toKeyValues
- val kv = kvs.head
- delete(Seq(kv.copy(qualifier = null)))
- }
-
- def buildDeleteBelongsToId(vertex: Vertex): Seq[HBaseRpc] = {
- val kvs = storage.vertexSerializer(vertex).toKeyValues
- val kv = kvs.head
-
- import org.apache.hadoop.hbase.util.Bytes
- val newKVs = vertex.belongLabelIds.map { id =>
- kv.copy(qualifier = Bytes.toBytes(Vertex.toPropKey(id)))
- }
- delete(newKVs)
- }
-
- def buildVertexPutsAsync(edge: Edge): Seq[HBaseRpc] =
- if (edge.op == GraphUtil.operations("delete"))
- buildDeleteBelongsToId(edge.srcForVertex) ++ buildDeleteBelongsToId(edge.tgtForVertex)
- else
- buildPutsAsync(edge.srcForVertex) ++ buildPutsAsync(edge.tgtForVertex)
-}