You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2018/05/11 03:05:41 UTC
[01/11] incubator-s2graph git commit: - Add Fetcher/Mutator interface
for query/mutation. - Refactor Storage to use Fetcher/Mutator interface.
Repository: incubator-s2graph
Updated Branches:
refs/heads/master 7af37dbd3 -> 33f4d0550
- Add Fetcher/Mutator interface for query/mutation.
- Refactor Storage to use Fetcher/Mutator interface.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/2357d810
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/2357d810
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/2357d810
Branch: refs/heads/master
Commit: 2357d810a6419011d4fd38af248089d9551a200c
Parents: 7af37db
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue May 8 16:27:07 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue May 8 16:27:07 2018 +0900
----------------------------------------------------------------------
project/Common.scala | 2 +
s2core/build.sbt | 2 +-
.../scala/org/apache/s2graph/core/Fetcher.scala | 36 ++++++
.../org/apache/s2graph/core/Management.scala | 15 ++-
.../scala/org/apache/s2graph/core/Mutator.scala | 41 +++++++
.../org/apache/s2graph/core/QueryResult.scala | 4 +-
.../scala/org/apache/s2graph/core/S2Graph.scala | 39 ++++--
.../org/apache/s2graph/core/S2GraphLike.scala | 11 ++
.../apache/s2graph/core/TraversalHelper.scala | 8 +-
.../s2graph/core/model/ImportStatus.scala | 59 +++++++++
.../apache/s2graph/core/model/Importer.scala | 122 +++++++++++++++++++
.../s2graph/core/model/MemoryModelFetcher.scala | 59 +++++++++
.../s2graph/core/model/ModelManager.scala | 103 ++++++++++++++++
.../org/apache/s2graph/core/schema/Label.scala | 35 +++---
.../org/apache/s2graph/core/schema/Schema.scala | 4 +-
.../apache/s2graph/core/schema/Service.scala | 2 +-
.../apache/s2graph/core/storage/Storage.scala | 45 +++----
.../s2graph/core/storage/StorageReadable.scala | 22 ++--
.../s2graph/core/storage/StorageWritable.scala | 19 +--
.../storage/WriteWriteConflictResolver.scala | 2 +-
.../core/storage/hbase/AsynchbaseStorage.scala | 12 +-
.../hbase/AsynchbaseStorageWritable.scala | 11 +-
.../core/storage/rocks/RocksStorage.scala | 11 +-
.../storage/rocks/RocksStorageReadable.scala | 2 +-
.../storage/rocks/RocksStorageWritable.scala | 10 +-
.../core/storage/serde/MutationHelper.scala | 32 +++--
.../apache/s2graph/core/model/FetcherTest.scala | 87 +++++++++++++
.../apache/s2graph/graphql/GraphQLServer.scala | 18 ++-
.../org/apache/s2graph/graphql/HttpServer.scala | 2 +
29 files changed, 695 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index 96109d3..b46d190 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -33,6 +33,8 @@ object Common {
val KafkaVersion = "0.10.2.1"
+ val rocksVersion = "5.11.3"
+
/** use Log4j 1.2.17 as the SLF4j backend in runtime, with bridging libraries to forward JCL and JUL logs to SLF4j */
val loggingRuntime = Seq(
"log4j" % "log4j" % "1.2.17",
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/build.sbt
----------------------------------------------------------------------
diff --git a/s2core/build.sbt b/s2core/build.sbt
index cc70e97..12319d8 100644
--- a/s2core/build.sbt
+++ b/s2core/build.sbt
@@ -50,7 +50,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion ,
"org.apache.lucene" % "lucene-core" % "6.6.0",
"org.apache.lucene" % "lucene-queryparser" % "6.6.0",
- "org.rocksdb" % "rocksdbjni" % "5.8.0",
+ "org.rocksdb" % "rocksdbjni" % rocksVersion,
"org.scala-lang.modules" %% "scala-java8-compat" % "0.8.0",
"com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion excludeLogging(),
"com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion excludeLogging(),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
new file mode 100644
index 0000000..57d2f29
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.types.VertexId
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait Fetcher {
+
+ def init(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] =
+ Future.successful(this)
+
+ def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
+
+ def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index d026e5b..7ff5a9e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -20,6 +20,7 @@
package org.apache.s2graph.core
import java.util
+import java.util.concurrent.Executors
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.s2graph.core.GraphExceptions.{InvalidHTableException, LabelAlreadyExistException, LabelNameTooLongException, LabelNotExistException}
@@ -28,8 +29,10 @@ import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.types.HBaseType._
import org.apache.s2graph.core.types._
import org.apache.s2graph.core.JSONParser._
+import org.apache.s2graph.core.model.Importer
import play.api.libs.json._
+import scala.concurrent.{ExecutionContext, Future}
import scala.util.Try
/**
@@ -70,7 +73,6 @@ object Management {
case class Index(name: String, propNames: Seq[String], direction: Option[Int] = None, options: Option[String] = None)
}
-
def findService(serviceName: String) = {
Service.findByName(serviceName, useCache = false)
}
@@ -298,9 +300,18 @@ object Management {
class Management(graph: S2GraphLike) {
+ val importEx = ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor())
import Management._
- import scala.collection.JavaConversions._
+
+ def importModel(labelName: String, options: String): Future[Importer] = {
+ Label.updateOption(labelName, options)
+
+ val label = Label.findByName(labelName, false).getOrElse(throw new LabelNotExistException(labelName))
+ val config = ConfigFactory.parseString(options)
+
+ graph.modelManager.importModel(label, config)(importEx)
+ }
def createStorageTable(zkAddr: String,
tableName: String,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
new file mode 100644
index 0000000..53161e1
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.storage.{MutateResponse, SKeyValue}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait Mutator {
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
+
+ def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]]
+
+ def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]]
+
+ def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]]
+
+ def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse]
+
+ def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean]
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index be57017..4a1018f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -237,7 +237,7 @@ object StepResult {
// val newOrderByValues = updateScoreOnOrderByValues(globalQueryOption.scoreFieldIdx, t.orderByValues, newScore)
val newOrderByValues =
- if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTsInnerValValue(), None, None)
+ if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTs(), None, None)
else toTuple4(newT.toValues(globalQueryOption.orderByKeys))
val newGroupByValues = newT.toValues(globalQueryOption.groupBy.keys)
@@ -262,7 +262,7 @@ object StepResult {
// val newOrderByValues = updateScoreOnOrderByValues(globalQueryOption.scoreFieldIdx, t.orderByValues, newScore)
val newOrderByValues =
- if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTsInnerValValue(), None, None)
+ if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTs(), None, None)
else toTuple4(newT.toValues(globalQueryOption.orderByKeys))
val newGroupByValues = newT.toValues(globalQueryOption.groupBy.keys)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 7816a63..4b2274a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -27,6 +27,7 @@ import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
import org.apache.s2graph.core.index.IndexProvider
import org.apache.s2graph.core.io.tinkerpop.optimize.S2GraphStepStrategy
+import org.apache.s2graph.core.model.ModelManager
import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
import org.apache.s2graph.core.storage.rocks.RocksStorage
@@ -186,6 +187,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override val management = new Management(this)
+ override val modelManager = new ModelManager(this)
+
override val indexProvider = IndexProvider.apply(config)
override val elementBuilder = new GraphElementBuilder(this)
@@ -247,6 +250,25 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
storagePool.getOrElse(s"label:${label.label}", defaultStorage)
}
+ //TODO:
+ override def getFetcher(column: ServiceColumn): Fetcher = {
+ getStorage(column.service).reader
+ }
+
+ override def getFetcher(label: Label): Fetcher = {
+ if (label.fetchConfigExist) modelManager.getFetcher(label)
+ else getStorage(label).reader
+ }
+
+ override def getMutator(column: ServiceColumn): Mutator = {
+ getStorage(column.service).mutator
+ }
+
+ override def getMutator(label: Label): Mutator = {
+ getStorage(label).mutator
+ }
+
+ //TODO:
override def flushStorage(): Unit = {
storagePool.foreach { case (_, storage) =>
@@ -302,7 +324,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
def mutateVertices(storage: Storage)(zkQuorum: String, vertices: Seq[S2VertexLike],
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val futures = vertices.map { vertex =>
- storage.mutateVertex(zkQuorum, vertex, withWait)
+ getMutator(vertex.serviceColumn).mutateVertex(zkQuorum, vertex, withWait)
}
Future.sequence(futures)
}
@@ -329,12 +351,12 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val weakEdgesFutures = weakEdges.groupBy { case (edge, idx) => edge.innerLabel.hbaseZkAddr }.map { case (zkQuorum, edgeWithIdxs) =>
val futures = edgeWithIdxs.groupBy(_._1.innerLabel).map { case (label, edgeGroup) =>
- val storage = getStorage(label)
+ val mutator = getMutator(label)
val edges = edgeGroup.map(_._1)
val idxs = edgeGroup.map(_._2)
/* multiple edges with weak consistency level will be processed as batch */
- storage.mutateWeakEdges(zkQuorum, edges, withWait)
+ mutator.mutateWeakEdges(zkQuorum, edges, withWait)
}
Future.sequence(futures)
}
@@ -347,9 +369,10 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val strongEdgesFutures = strongEdgesAll.groupBy { case (edge, idx) => edge.innerLabel }.map { case (label, edgeGroup) =>
val edges = edgeGroup.map(_._1)
val idxs = edgeGroup.map(_._2)
- val storage = getStorage(label)
+ val mutator = getMutator(label)
val zkQuorum = label.hbaseZkAddr
- storage.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
+
+ mutator.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
idxs.zip(rets)
}
}
@@ -474,7 +497,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
val edgesWithIdx = edges.zipWithIndex
val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
- getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
+ getMutator(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
}
Future.sequence(futures).map { ls =>
@@ -484,9 +507,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
val label = edge.innerLabel
- val storage = getStorage(label)
+ val mutator = getMutator(label)
- storage.updateDegree(label.hbaseZkAddr, edge, degreeVal)
+ mutator.updateDegree(label.hbaseZkAddr, edge, degreeVal)
}
override def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index cbd31cc..6ed78b0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -31,6 +31,7 @@ import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core.features.{S2Features, S2GraphVariables}
import org.apache.s2graph.core.index.IndexProvider
+import org.apache.s2graph.core.model.ModelManager
import org.apache.s2graph.core.schema.{Label, LabelMeta, Service, ServiceColumn}
import org.apache.s2graph.core.storage.{MutateResponse, Storage}
import org.apache.s2graph.core.types.{InnerValLike, VertexId}
@@ -68,6 +69,8 @@ trait S2GraphLike extends Graph {
val traversalHelper: TraversalHelper
+ val modelManager: ModelManager
+
lazy val MaxRetryNum: Int = config.getInt("max.retry.number")
lazy val MaxBackOff: Int = config.getInt("max.back.off")
lazy val BackoffTimeout: Int = config.getInt("back.off.timeout")
@@ -90,6 +93,14 @@ trait S2GraphLike extends Graph {
def getStorage(label: Label): Storage
+ def getFetcher(column: ServiceColumn): Fetcher
+
+ def getFetcher(label: Label): Fetcher
+
+ def getMutator(label: Label): Mutator
+
+ def getMutator(column: ServiceColumn): Mutator
+
def flushStorage(): Unit
def shutdown(modelDataDelete: Boolean = false): Unit
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
index 0dc2aa2..d19dd1f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -204,7 +204,7 @@ class TraversalHelper(graph: S2GraphLike) {
val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
for {
prev <- prevFuture
- cur <- graph.getStorage(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
+ cur <- graph.getFetcher(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
} yield {
prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
}
@@ -256,7 +256,7 @@ class TraversalHelper(graph: S2GraphLike) {
*/
graph.mutateEdges(edgesToDelete.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
} else {
- graph.getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ graph.getMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
case _ =>
@@ -264,7 +264,7 @@ class TraversalHelper(graph: S2GraphLike) {
* read: x
* write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
*/
- graph.getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ graph.getMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
ret
}
@@ -389,7 +389,7 @@ class TraversalHelper(graph: S2GraphLike) {
val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
/* OrderBy */
val orderByValues =
- if (queryOption.orderByKeys.isEmpty) (score, edge.getTsInnerValValue(), None, None)
+ if (queryOption.orderByKeys.isEmpty) (score, edge.getTs(), None, None)
else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
/* StepGroupBy */
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
new file mode 100644
index 0000000..189a6d0
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.model
+
+import java.util.concurrent.atomic.AtomicInteger
+
+trait ImportStatus {
+ val done: AtomicInteger
+
+ def isCompleted: Boolean
+
+ def percentage: Int
+
+ val total: Int
+}
+
+class ImportRunningStatus(val total: Int) extends ImportStatus {
+ require(total > 0, s"Total should be positive: $total")
+
+ val done = new AtomicInteger(0)
+
+ def isCompleted: Boolean = total == done.get
+
+ def percentage = 100 * done.get / total
+}
+
+case object ImportDoneStatus extends ImportStatus {
+ val total = 1
+
+ val done = new AtomicInteger(1)
+
+ def isCompleted: Boolean = true
+
+ def percentage = 100
+}
+
+object ImportStatus {
+ def apply(total: Int): ImportStatus = new ImportRunningStatus(total)
+
+ def unapply(importResult: ImportStatus): Option[(Boolean, Int, Int)] =
+ Some((importResult.isCompleted, importResult.total, importResult.done.get))
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
new file mode 100644
index 0000000..e3084dd
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.model
+
+import java.io.File
+
+import com.typesafe.config.Config
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.s2graph.core.{Fetcher, S2GraphLike}
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object Importer {
+ def toHDFSConfiguration(hdfsConfDir: String): Configuration = {
+ val conf = new Configuration
+
+ val hdfsConfDirectory = new File(hdfsConfDir)
+ if (hdfsConfDirectory.exists()) {
+ if (!hdfsConfDirectory.isDirectory || !hdfsConfDirectory.canRead) {
+ throw new IllegalStateException(s"HDFS configuration directory ($hdfsConfDirectory) cannot be read.")
+ }
+
+ val path = hdfsConfDirectory.getAbsolutePath
+ conf.addResource(new Path(s"file:///$path/core-site.xml"))
+ conf.addResource(new Path(s"file:///$path/hdfs-site.xml"))
+ } else {
+ logger.warn("RocksDBImporter doesn't have valid hadoop configuration directory..")
+ }
+ conf
+ }
+}
+
+trait Importer {
+ @volatile var isFinished: Boolean = false
+
+ def run(config: Config)(implicit ec: ExecutionContext): Future[Importer]
+
+ def status: Boolean = isFinished
+
+ def setStatus(otherStatus: Boolean): Boolean = {
+ this.isFinished = otherStatus
+ this.isFinished
+ }
+
+ def close(): Unit
+}
+
+case class IdentityImporter(graph: S2GraphLike) extends Importer {
+ override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ Future.successful(this)
+ }
+
+ override def close(): Unit = {}
+}
+
+object HDFSImporter {
+
+ import scala.collection.JavaConverters._
+
+ val PathsKey = "paths"
+ val HDFSConfDirKey = "hdfsConfDir"
+
+ def extractPaths(config: Config): Map[String, String] = {
+ config.getConfigList(PathsKey).asScala.map { e =>
+ val key = e.getString("src")
+ val value = e.getString("tgt")
+
+ key -> value
+ }.toMap
+ }
+}
+
+case class HDFSImporter(graph: S2GraphLike) extends Importer {
+
+ import HDFSImporter._
+
+ override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ Future {
+ val paths = extractPaths(config)
+ val hdfsConfiDir = config.getString(HDFSConfDirKey)
+
+ val hadoopConfig = Importer.toHDFSConfiguration(hdfsConfiDir)
+ val fs = FileSystem.get(hadoopConfig)
+
+ def copyToLocal(remoteSrc: String, localSrc: String): Unit = {
+ val remoteSrcPath = new Path(remoteSrc)
+ val localSrcPath = new Path(localSrc)
+
+ fs.copyToLocalFile(remoteSrcPath, localSrcPath)
+ }
+
+ paths.foreach { case (srcPath, tgtPath) =>
+ copyToLocal(srcPath, tgtPath)
+ }
+
+ this
+ }
+ }
+
+ // override def status: ImportStatus = ???
+
+ override def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
new file mode 100644
index 0000000..2130066
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.model
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.types.VertexId
+
+import scala.concurrent.{ExecutionContext, Future}
+
+/**
+ * Reference implementation for Fetcher interface.
+ * it only produce constant edges.
+ */
+class MemoryModelFetcher(val graph: S2GraphLike) extends Fetcher {
+ val builder = graph.elementBuilder
+ val ranges = (0 until 10)
+
+ override def init(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] = {
+ Future.successful(this)
+ }
+
+ override def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
+ val stepResultLs = queryRequests.map { queryRequest =>
+ val queryParam = queryRequest.queryParam
+ val edges = ranges.map { ith =>
+ val tgtVertexId = builder.newVertexId(queryParam.label.service, queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), ith.toString)
+
+ graph.toEdge(queryRequest.vertex.innerIdVal,
+ tgtVertexId.innerId.value, queryParam.label.label, queryParam.direction)
+ }
+
+ val edgeWithScores = edges.map(e => EdgeWithScore(e, 1.0, queryParam.label))
+ StepResult(edgeWithScores, Nil, Nil)
+ }
+
+ Future.successful(stepResultLs)
+ }
+
+ override def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
new file mode 100644
index 0000000..3cad13c
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.model
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{Fetcher, S2GraphLike}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object ModelManager {
+ val ClassNameKey = "className"
+}
+
+class ModelManager(s2GraphLike: S2GraphLike) {
+
+ import ModelManager._
+
+ private val fetcherPool = scala.collection.mutable.Map.empty[String, Fetcher]
+ private val ImportLock = new java.util.concurrent.ConcurrentHashMap[String, Importer]
+
+ def toImportLockKey(label: Label): String = label.label
+
+ def getFetcher(label: Label): Fetcher = {
+ fetcherPool.getOrElse(toImportLockKey(label), throw new IllegalStateException(s"$label is not imported."))
+ }
+
+ def initImporter(config: Config): Importer = {
+ val className = config.getString(ClassNameKey)
+
+ Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(s2GraphLike)
+ .asInstanceOf[Importer]
+ }
+
+ def initFetcher(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] = {
+ val className = config.getString(ClassNameKey)
+
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(s2GraphLike)
+ .asInstanceOf[Fetcher]
+
+ fetcher.init(config)
+ }
+
+ def importModel(label: Label, config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ val importer = ImportLock.computeIfAbsent(toImportLockKey(label), new java.util.function.Function[String, Importer] {
+ override def apply(k: String): Importer = {
+ val importer = initImporter(config.getConfig("importer"))
+
+ //TODO: Update Label's extra options.
+ importer
+ .run(config.getConfig("importer"))
+ .map { importer =>
+ logger.info(s"Close importer")
+ importer.close()
+
+ initFetcher(config.getConfig("fetcher")).map { fetcher =>
+ importer.setStatus(true)
+
+ fetcherPool
+ .remove(k)
+ .foreach { oldFetcher =>
+ logger.info(s"Delete old storage ($k) => $oldFetcher")
+ oldFetcher.close()
+ }
+
+ fetcherPool += (k -> fetcher)
+ }
+ }
+ .onComplete { _ =>
+ logger.info(s"ImportLock release: $k")
+ ImportLock.remove(k)
+ }
+
+ importer
+ }
+ })
+
+ Future.successful(importer)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
index 7fb1183..cca1769 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
@@ -259,6 +259,20 @@ object Label extends SQLSyntaxSupport[Label] {
cnt
}
+ def updateOption(labelName: String, options: String)(implicit session: DBSession = AutoSession) = {
+ scala.util.Try(Json.parse(options)).getOrElse(throw new RuntimeException("invalid Json option"))
+ logger.info(s"update options of label $labelName, ${options}")
+ val cnt = sql"""update labels set options = $options where label = $labelName""".update().apply()
+ val label = Label.findByName(labelName, useCache = false).get
+
+ val cacheKeys = List(s"id=${label.id.get}", s"label=${label.label}")
+ cacheKeys.foreach { key =>
+ expireCache(className + key)
+ expireCaches(className + key)
+ }
+ cnt
+ }
+
def delete(id: Int)(implicit session: DBSession = AutoSession) = {
val label = findById(id)
logger.info(s"delete label: $label")
@@ -369,19 +383,6 @@ case class Label(id: Option[Int], label: String,
prop <- metaProps if LabelMeta.isValidSeq(prop.seq)
jsValue <- innerValToJsValue(toInnerVal(prop.defaultValue, prop.dataType, schemaVersion), prop.dataType)
} yield prop -> jsValue).toMap
-// lazy val extraOptions = Model.extraOptions(Option("""{
-// "storage": {
-// "s2graph.storage.backend": "rocks",
-// "rocks.db.path": "/tmp/db"
-// }
-// }"""))
-
- lazy val tokens: Set[String] = extraOptions.get("tokens").fold(Set.empty[String]) {
- case JsArray(tokens) => tokens.map(_.as[String]).toSet
- case _ =>
- logger.error("Invalid token JSON")
- Set.empty[String]
- }
lazy val extraOptions = Schema.extraOptions(options)
@@ -389,8 +390,14 @@ case class Label(id: Option[Int], label: String,
lazy val storageConfigOpt: Option[Config] = toStorageConfig
+ lazy val fetchConfigExist: Boolean = toFetcherConfig.isDefined
+
+ def toFetcherConfig: Option[Config] = {
+ Schema.toConfig(extraOptions, "fetcher")
+ }
+
def toStorageConfig: Option[Config] = {
- Schema.toStorageConfig(extraOptions)
+ Schema.toConfig(extraOptions, "storage")
}
def srcColumnWithDir(dir: Int) = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala
index c28df80..50c1b7f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Schema.scala
@@ -175,9 +175,9 @@ object Schema {
}
}
- def toStorageConfig(options: Map[String, JsValue]): Option[Config] = {
+ def toConfig(options: Map[String, JsValue], key: String): Option[Config] = {
try {
- options.get("storage").map { jsValue =>
+ options.get(key).map { jsValue =>
import scala.collection.JavaConverters._
val configMap = jsValue.as[JsObject].fieldSet.toMap.map { case (key, value) =>
key -> JSONParser.jsValueToAny(value).getOrElse(throw new RuntimeException("!!"))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/schema/Service.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Service.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Service.scala
index 611a746..dbbfed7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Service.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Service.scala
@@ -129,5 +129,5 @@ case class Service(id: Option[Int],
lazy val extraOptions = Schema.extraOptions(options)
lazy val storageConfigOpt: Option[Config] = toStorageConfig
def serviceColumns(useCache: Boolean): Seq[ServiceColumn] = ServiceColumn.findByServiceId(id.get, useCache = useCache)
- def toStorageConfig: Option[Config] = Schema.toStorageConfig(extraOptions)
+ def toStorageConfig: Option[Config] = Schema.toConfig(extraOptions, "storage")
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index 18f6b1e..d2500a6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -22,7 +22,7 @@ package org.apache.s2graph.core.storage
import com.typesafe.config.Config
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.serde.{Deserializable, MutationHelper}
+import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.indexedge.tall.IndexEdgeDeserializable
import org.apache.s2graph.core.types._
@@ -33,14 +33,11 @@ abstract class Storage(val graph: S2GraphLike,
/* Storage backend specific resource management */
val management: StorageManagement
- /* Physically store given KeyValue into backend storage. */
- val mutator: StorageWritable
-
/*
* Given QueryRequest/Vertex/Edge, fetch KeyValue from storage
* then convert them into Edge/Vertex
*/
- val fetcher: StorageReadable
+ val reader: StorageReadable
/*
* Serialize Edge/Vertex, to common KeyValue, SKeyValue that
@@ -50,6 +47,11 @@ abstract class Storage(val graph: S2GraphLike,
val serDe: StorageSerDe
/*
+ * Responsible to connect physical storage backend to store GraphElement(Edge/Vertex).
+ */
+ val mutator: Mutator
+
+ /*
* Common helper to translate SKeyValue to Edge/Vertex and vice versa.
* Note that it require storage backend specific implementation for serialize/deserialize.
*/
@@ -60,31 +62,24 @@ abstract class Storage(val graph: S2GraphLike,
* Note that it require storage backend specific implementations for
* all of StorageWritable, StorageReadable, StorageSerDe, StorageIO
*/
- lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, mutator, fetcher)
-
- lazy val mutationHelper: MutationHelper = new MutationHelper(this)
-
- /** Mutation **/
- def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutator.writeToStorage(cluster, kvs, withWait)
+// lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, mutator, reader)
+// lazy val mutationHelper: MutationHelper = new MutationHelper(this)
- def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutator.writeLock(requestKeyValue, expectedOpt)
/** Fetch **/
def fetches(queryRequests: Seq[QueryRequest],
prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] =
- fetcher.fetches(queryRequests, prevStepEdges)
+ reader.fetches(queryRequests, prevStepEdges)
def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] =
- fetcher.fetchVertices(vertices)
+ reader.fetchVertices(vertices)
- def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = fetcher.fetchEdgesAll()
+ def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = reader.fetchEdgesAll()
- def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = fetcher.fetchVerticesAll()
+ def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = reader.fetchVerticesAll()
def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] =
- fetcher.fetchSnapshotEdgeInner(edge)
+ reader.fetchSnapshotEdgeInner(edge)
/** Management **/
def flush(): Unit = management.flush()
@@ -102,21 +97,21 @@ abstract class Storage(val graph: S2GraphLike,
def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
requestTs: Long,
retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] =
- mutationHelper.deleteAllFetchedEdgesAsyncOld(stepInnerResult, requestTs, retryNum)
+ mutator.deleteAllFetchedEdgesAsyncOld(stepInnerResult, requestTs, retryNum)
def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutationHelper.mutateVertex(zkQuorum: String, vertex, withWait)
+ mutator.mutateVertex(zkQuorum: String, vertex, withWait)
def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] =
- mutationHelper.mutateStrongEdges(zkQuorum, _edges, withWait)
+ mutator.mutateStrongEdges(zkQuorum, _edges, withWait)
def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] =
- mutationHelper.mutateWeakEdges(zkQuorum, _edges, withWait)
+ mutator.mutateWeakEdges(zkQuorum, _edges, withWait)
def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] =
- mutationHelper.incrementCounts(zkQuorum, edges, withWait)
+ mutator.incrementCounts(zkQuorum, edges, withWait)
def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutationHelper.updateDegree(zkQuorum, edge, degreeVal)
+ mutator.updateDegree(zkQuorum, edge, degreeVal)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
index 0965f68..b10feb9 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
@@ -19,6 +19,7 @@
package org.apache.s2graph.core.storage
+import com.typesafe.config.Config
import org.apache.s2graph.core.GraphExceptions.FetchTimeoutException
import org.apache.s2graph.core._
import org.apache.s2graph.core.types.VertexId
@@ -26,18 +27,18 @@ import org.apache.s2graph.core.utils.logger
import scala.concurrent.{ExecutionContext, Future}
-trait StorageReadable {
+trait StorageReadable extends Fetcher {
val io: StorageIO
val serDe: StorageSerDe
- /**
- * responsible to fire parallel fetch call into storage and create future that will return merged result.
- *
- * @param queryRequests
- * @param prevStepEdges
- * @return
- */
- def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
+// /**
+// * responsible to fire parallel fetch call into storage and create future that will return merged result.
+// *
+// * @param queryRequests
+// * @param prevStepEdges
+// * @return
+// */
+// def fetches(queryRequests: Seq[QueryRequest],
+// prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]]
@@ -92,4 +93,5 @@ trait StorageReadable {
Future.sequence(futures).map(_.flatten)
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
index 80da3a9..8c2fb27 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
@@ -19,20 +19,21 @@
package org.apache.s2graph.core.storage
+import org.apache.s2graph.core.Mutator
+
import scala.concurrent.{ExecutionContext, Future}
-trait StorageWritable {
+trait OptimisticMutator extends Mutator {
/**
* decide how to store given key values Seq[SKeyValue] into storage using storage's client.
* note that this should be return true on all success.
* we assumes that each storage implementation has client as member variable.
*
- *
- * @param cluster: where this key values should be stored.
- * @param kvs: sequence of SKeyValue that need to be stored in storage.
- * @param withWait: flag to control wait ack from storage.
- * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
- * it never block thread, but rather submit work and notified by event loop when storage send ack back.
+ * @param cluster : where this key values should be stored.
+ * @param kvs : sequence of SKeyValue that need to be stored in storage.
+ * @param withWait : flag to control wait ack from storage.
+ * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
+ * it never block thread, but rather submit work and notified by event loop when storage send ack back.
* @return ack message from storage.
*/
def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
@@ -55,10 +56,10 @@ trait StorageWritable {
* for storage that does not support concurrency control, then storage implementation
* itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
* and write(writeLock).
+ *
* @param requestKeyValue
* @param expectedOpt
* @return
*/
def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse]
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
index dcef1cc..bfc5bc6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
@@ -31,7 +31,7 @@ import scala.util.Random
class WriteWriteConflictResolver(graph: S2GraphLike,
serDe: StorageSerDe,
io: StorageIO,
- mutator: StorageWritable,
+ mutator: OptimisticMutator,
fetcher: StorageReadable) {
val BackoffTimeout = graph.BackoffTimeout
val MaxRetryNum = graph.MaxRetryNum
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index 8b3d862..4be3767 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -151,17 +151,9 @@ class AsynchbaseStorage(override val graph: S2GraphLike,
override val management: StorageManagement = new AsynchbaseStorageManagement(config, clients)
- override val mutator: StorageWritable = new AsynchbaseStorageWritable(client, clientWithFlush)
-
override val serDe: StorageSerDe = new AsynchbaseStorageSerDe(graph)
- override val fetcher: StorageReadable = new AsynchbaseStorageReadable(graph, config, client, serDe, io)
-
- // val hbaseExecutor: ExecutorService =
- // if (config.getString("hbase.zookeeper.quorum") == "localhost")
- // AsynchbaseStorage.initLocalHBase(config)
- // else
- // null
-
+ override val reader: StorageReadable = new AsynchbaseStorageReadable(graph, config, client, serDe, io)
+ override val mutator: Mutator = new AsynchbaseStorageWritable(graph, serDe, reader, client, clientWithFlush)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
index 7ca3782..b4236b9 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
@@ -20,14 +20,19 @@
package org.apache.s2graph.core.storage.hbase
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.storage.{IncrementResponse, MutateResponse, SKeyValue, StorageWritable}
+import org.apache.s2graph.core.S2GraphLike
+import org.apache.s2graph.core.storage._
import org.apache.s2graph.core.utils.{Extensions, logger}
import org.hbase.async.{AtomicIncrementRequest, DeleteRequest, HBaseClient, PutRequest}
+
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{ExecutionContext, Future}
-class AsynchbaseStorageWritable(val client: HBaseClient,
- val clientWithFlush: HBaseClient) extends StorageWritable {
+class AsynchbaseStorageWritable(val graph: S2GraphLike,
+ val serDe: StorageSerDe,
+ val reader: StorageReadable,
+ val client: HBaseClient,
+ val clientWithFlush: HBaseClient) extends DefaultOptimisticMutator(graph, serDe, reader) {
import Extensions.DeferOps
private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
index 11fae17..b24e375 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
@@ -26,7 +26,7 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
import com.google.common.hash.Hashing
import com.typesafe.config.Config
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.Storage
+import org.apache.s2graph.core.storage.{Storage, StorageManagement, StorageReadable, StorageSerDe}
import org.apache.s2graph.core.storage.rocks.RocksHelper.RocksRPC
import org.apache.s2graph.core.utils.logger
import org.rocksdb._
@@ -150,11 +150,12 @@ class RocksStorage(override val graph: S2GraphLike,
.maximumSize(1000 * 10 * 10 * 10 * 10)
.build[String, ReentrantLock](cacheLoader)
- override val management = new RocksStorageManagement(config, vdb, db)
+ override val management: StorageManagement = new RocksStorageManagement(config, vdb, db)
- override val mutator = new RocksStorageWritable(db, vdb, lockMap)
+ override val serDe: StorageSerDe = new RocksStorageSerDe(graph)
- override val serDe = new RocksStorageSerDe(graph)
+ override val reader: StorageReadable = new RocksStorageReadable(graph, config, db, vdb, serDe, io)
+
+ override val mutator: Mutator = new RocksStorageWritable(graph, serDe, reader, db, vdb, lockMap)
- override val fetcher = new RocksStorageReadable(graph, config, db, vdb, serDe, io)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
index 5db02cc..27e3efd 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
@@ -27,7 +27,7 @@ import org.apache.s2graph.core._
import org.apache.s2graph.core.schema.{Label, ServiceColumn}
import org.apache.s2graph.core.storage.rocks.RocksHelper.{GetRequest, RocksRPC, ScanWithRange}
import org.apache.s2graph.core.storage.serde.StorageSerializable
-import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageReadable, StorageSerDe}
+import org.apache.s2graph.core.storage._
import org.apache.s2graph.core.types.{HBaseType, VertexId}
import org.rocksdb.RocksDB
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
index 7ec147d..d29ccce 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
@@ -23,15 +23,19 @@ import java.util.concurrent.locks.ReentrantLock
import com.google.common.cache.{Cache, LoadingCache}
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.storage.{MutateResponse, SKeyValue, StorageWritable}
+import org.apache.s2graph.core.S2GraphLike
+import org.apache.s2graph.core.storage._
import org.apache.s2graph.core.utils.logger
import org.rocksdb.{RocksDB, RocksDBException, WriteBatch, WriteOptions}
import scala.concurrent.{ExecutionContext, Future}
-class RocksStorageWritable(val db: RocksDB,
+class RocksStorageWritable(val graph: S2GraphLike,
+ val serDe: StorageSerDe,
+ val reader: StorageReadable,
+ val db: RocksDB,
val vdb: RocksDB,
- val lockMap: LoadingCache[String, ReentrantLock]) extends StorageWritable {
+ val lockMap: LoadingCache[String, ReentrantLock]) extends DefaultOptimisticMutator(graph, serDe, reader) {
override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext) = {
if (kvs.isEmpty) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
index 0748efb..8cd32d4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
@@ -16,25 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-
-package org.apache.s2graph.core.storage.serde
-
-import org.apache.s2graph.core.schema.LabelMeta
+package org.apache.s2graph.core.storage
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.schema.LabelMeta
import org.apache.s2graph.core.utils.logger
import scala.concurrent.{ExecutionContext, Future}
-class MutationHelper(storage: Storage) {
- val serDe = storage.serDe
- val io = storage.io
- val fetcher = storage.fetcher
- val mutator = storage.mutator
- val conflictResolver = storage.conflictResolver
+abstract class DefaultOptimisticMutator(graph: S2GraphLike,
+ serDe: StorageSerDe,
+ reader: StorageReadable) extends OptimisticMutator {
+ val fetcher = reader
+
+ lazy val io: StorageIO = new StorageIO(graph, serDe)
+ lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, this, reader)
- private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutator.writeToStorage(cluster, kvs, withWait)
+// private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
+// mutator.writeToStorage(cluster, kvs, withWait)
def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
requestTs: Long,
@@ -93,7 +91,7 @@ class MutationHelper(storage: Storage) {
val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
- if (bufferIncr.nonEmpty) storage.writeToStorage(zkQuorum, bufferIncr, withWait = false)
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
}
@@ -152,7 +150,7 @@ class MutationHelper(storage: Storage) {
}
val composed = for {
- // deleteRet <- Future.sequence(deleteAllFutures)
+ // deleteRet <- Future.sequence(deleteAllFutures)
mutateRet <- Future.sequence(mutateEdgeFutures)
} yield mutateRet
@@ -185,6 +183,6 @@ class MutationHelper(storage: Storage) {
def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
val kvs = io.buildDegreePuts(edge, degreeVal)
- mutator.writeToStorage(zkQuorum, kvs, withWait = true)
+ writeToStorage(zkQuorum, kvs, withWait = true)
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
new file mode 100644
index 0000000..6c76cdf
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.model
+
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.Integrate.IntegrateCommon
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.{Query, QueryParam}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext}
+
+class FetcherTest extends IntegrateCommon {
+
+ import scala.collection.JavaConverters._
+
+ test("MemoryModelFetcher") {
+ // 1. create label.
+ // 2. importLabel.
+ // 3. fetch.
+ val service = management.createService("s2graph", "localhost", "s2graph_htable", -1, None).get
+ val serviceColumn =
+ management.createServiceColumn("s2graph", "user", "string", Seq(Prop("age", "0", "int", true)))
+ val labelName = "fetcher_test"
+ val options =
+ s"""{
+ |
+ | "importer": {
+ | "${ModelManager.ClassNameKey}": "org.apache.s2graph.core.model.IdentityImporter"
+ | },
+ | "fetcher": {
+ | "${ModelManager.ClassNameKey}": "org.apache.s2graph.core.model.MemoryModelFetcher"
+ | }
+ |}""".stripMargin
+
+ Label.findByName(labelName, useCache = false).foreach { label => Label.delete(label.id.get) }
+
+ val label = management.createLabel(
+ labelName,
+ serviceColumn,
+ serviceColumn,
+ true,
+ service.serviceName,
+ Seq.empty[Index].asJava,
+ Seq.empty[Prop].asJava,
+ "strong",
+ null,
+ -1,
+ "v3",
+ "gz",
+ options
+ )
+ val config = ConfigFactory.parseString(options)
+ val importerFuture = graph.modelManager.importModel(label, config)(ExecutionContext.Implicits.global)
+ Await.ready(importerFuture, Duration("60 seconds"))
+
+ Thread.sleep(1000)
+
+ val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "daewon")
+ val queryParam = QueryParam(labelName = labelName)
+
+ val query = Query.toQuery(srcVertices = Seq(vertex), queryParams = Seq(queryParam))
+ val stepResult = Await.result(graph.getEdges(query), Duration("60 seconds"))
+
+ stepResult.edgeWithScores.foreach { es =>
+ println(es.edge)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
index 391a99f..0bf62d9 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
@@ -37,7 +37,7 @@ import sangria.execution.deferred.DeferredResolver
import sangria.marshalling.sprayJson._
import sangria.parser.QueryParser
import sangria.schema.Schema
-import spray.json.{JsObject, JsString}
+import spray.json.{JsBoolean, JsObject, JsString}
import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext
@@ -63,8 +63,22 @@ object GraphQLServer {
val schemaCache = new SafeUpdateCache(schemaConfig)
- def endpoint(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = {
+ def importModel(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = {
+ val ret = Try {
+ val spray.json.JsObject(fields) = requestJSON
+ val spray.json.JsString(labelName) = fields("label")
+ val jsOptions = fields("options")
+
+ s2graph.management.importModel(labelName, jsOptions.compactPrint)
+ }
+ ret match {
+ case Success(f) => complete(f.map(i => OK -> JsString("start")))
+ case Failure(e) => complete(InternalServerError -> spray.json.JsObject("message" -> JsString(e.toString)))
+ }
+ }
+
+ def endpoint(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = {
val spray.json.JsObject(fields) = requestJSON
val spray.json.JsString(query) = fields("query")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2357d810/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
index 685e87b..38cdce3 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
@@ -44,6 +44,8 @@ object Server extends App {
val route: Flow[HttpRequest, HttpResponse, Any] = (post & path("graphql")) {
entity(as[spray.json.JsValue])(GraphQLServer.endpoint)
+ } ~ (post & path("importModel")) {
+ entity(as[spray.json.JsValue])(GraphQLServer.importModel)
} ~ {
getFromResource("assets/graphiql.html")
}
[03/11] incubator-s2graph git commit: separate Storage into multiple
small interfaces such as EdgeFetcher/VertexMutator, ...
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
new file mode 100644
index 0000000..4239d15
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import java.util
+
+import com.stumbleupon.async.Deferred
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.{HBaseType, VertexId}
+import org.apache.s2graph.core.utils.{CanDefer, DeferCache, Extensions, logger}
+import org.hbase.async._
+
+import scala.concurrent.ExecutionContext
+
+class AsynchbaseEdgeFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val client: HBaseClient,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends EdgeFetcher {
+
+ import AsynchbaseStorage._
+ import CanDefer._
+ import Extensions.DeferOps
+
+ import scala.collection.JavaConverters._
+
+ /** Future Cache to squash request */
+ lazy private val futureCache = new DeferCache[StepResult, Deferred, Deferred](config, StepResult.Empty, "AsyncHbaseFutureCache", useMetric = true)
+
+ override def fetches(queryRequests: Seq[QueryRequest], prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext) = {
+ val defers: Seq[Deferred[StepResult]] = for {
+ queryRequest <- queryRequests
+ } yield {
+ val queryOption = queryRequest.query.queryOption
+ val queryParam = queryRequest.queryParam
+ val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
+ val parentEdges = if (shouldBuildParents) prevStepEdges.getOrElse(queryRequest.vertex.id, Nil) else Nil
+ fetch(queryRequest, isInnerCall = false, parentEdges)
+ }
+
+ val grouped: Deferred[util.ArrayList[StepResult]] = Deferred.groupInOrder(defers.asJava)
+ grouped.map(emptyStepResult) { queryResults: util.ArrayList[StepResult] =>
+ queryResults
+ }.toFuture(emptyStepResult).map(_.asScala)
+ }
+
+ /**
+ * we are using future cache to squash requests into same key on storage.
+ *
+ * @param queryRequest
+ * @param isInnerCall
+ * @param parentEdges
+ * @return we use Deferred here since it has much better performrance compared to scala.concurrent.Future.
+ * seems like map, flatMap on scala.concurrent.Future is slower than Deferred's addCallback
+ */
+ private def fetch(queryRequest: QueryRequest,
+ isInnerCall: Boolean,
+ parentEdges: Seq[EdgeWithScore])(implicit ec: ExecutionContext): Deferred[StepResult] = {
+
+ def fetchInner(hbaseRpc: AsyncRPC): Deferred[StepResult] = {
+ val prevStepScore = queryRequest.prevStepScore
+ val fallbackFn: (Exception => StepResult) = { ex =>
+ logger.error(s"fetchInner failed. fallback return. $hbaseRpc}", ex)
+ StepResult.Failure
+ }
+
+ val queryParam = queryRequest.queryParam
+ AsynchbaseStorage.fetchKeyValuesInner(client, hbaseRpc).mapWithFallback(emptyKeyValues)(fallbackFn) { _kvs =>
+ val kvs = _kvs.asScala
+ val (startOffset, len) = queryParam.label.schemaVersion match {
+ case HBaseType.VERSION4 =>
+ val offset = if (queryParam.cursorOpt.isDefined) 0 else queryParam.offset
+ (offset, queryParam.limit)
+ case _ => (0, kvs.length)
+ }
+
+ io.toEdges(kvs, queryRequest, prevStepScore, isInnerCall, parentEdges, startOffset, len)
+ }
+ }
+
+ val queryParam = queryRequest.queryParam
+ val cacheTTL = queryParam.cacheTTLInMillis
+ /* with version 4, request's type is (Scanner, (Int, Int)). otherwise GetRequest. */
+
+ val edge = graph.elementBuilder.toRequestEdge(queryRequest, parentEdges)
+ val request = buildRequest(client, serDe, queryRequest, edge)
+
+ val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
+ val requestCacheKey = Bytes.add(toCacheKeyBytes(request), intervalMaxBytes, intervalMinBytes)
+
+ if (cacheTTL <= 0) fetchInner(request)
+ else {
+ val cacheKeyBytes = Bytes.add(queryRequest.query.queryOption.cacheKeyBytes, requestCacheKey)
+
+ // val cacheKeyBytes = toCacheKeyBytes(request)
+ val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
+ futureCache.getOrElseUpdate(cacheKey, cacheTTL)(fetchInner(request))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticEdgeFetcher.scala
new file mode 100644
index 0000000..f03310c
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticEdgeFetcher.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import org.apache.s2graph.core.{QueryRequest, S2EdgeLike}
+import org.apache.s2graph.core.storage.{OptimisticEdgeFetcher, SKeyValue, StorageIO, StorageSerDe}
+import org.hbase.async.HBaseClient
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class AsynchbaseOptimisticEdgeFetcher(val client: HBaseClient,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends OptimisticEdgeFetcher {
+ override protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ val request = AsynchbaseStorage.buildRequest(client, serDe, queryRequest, edge)
+ AsynchbaseStorage.fetchKeyValues(client, request)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticMutator.scala
new file mode 100644
index 0000000..8305e04
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseOptimisticMutator.scala
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.S2GraphLike
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.utils.{Extensions, logger}
+import org.hbase.async.{AtomicIncrementRequest, DeleteRequest, HBaseClient, PutRequest}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{ExecutionContext, Future}
+
+class AsynchbaseOptimisticMutator(val graph: S2GraphLike,
+ val serDe: StorageSerDe,
+ val optimisticEdgeFetcher: OptimisticEdgeFetcher,
+ val client: HBaseClient,
+ val clientWithFlush: HBaseClient) extends OptimisticMutator {
+ import Extensions.DeferOps
+
+ private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
+ /**
+ * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
+ * note that this should be return true on all success.
+ * we assumes that each storage implementation has client as member variable.
+ *
+ * @param cluster : where this key values should be stored.
+ * @param kvs : sequence of SKeyValue that need to be stored in storage.
+ * @param withWait : flag to control wait ack from storage.
+ * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
+ * it never block thread, but rather submit work and notified by event loop when storage send ack back.
+ * @return ack message from storage.
+ */
+ override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext) = {
+ if (kvs.isEmpty) Future.successful(MutateResponse.Success)
+ else {
+ val _client = client(withWait)
+ val (increments, putAndDeletes) = kvs.partition(_.operation == SKeyValue.Increment)
+
+ /* Asynchbase IncrementRequest does not implement HasQualifiers */
+ val incrementsFutures = increments.map { kv =>
+ val countVal = Bytes.toLong(kv.value)
+ val request = new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, countVal)
+ val fallbackFn: (Exception => MutateResponse) = { ex =>
+ logger.error(s"mutation failed. $request", ex)
+ new IncrementResponse(false, -1L, -1L)
+ }
+ val future = _client.bufferAtomicIncrement(request).mapWithFallback(0L)(fallbackFn) { resultCount: java.lang.Long =>
+ new IncrementResponse(true, resultCount.longValue(), countVal)
+ }.toFuture(MutateResponse.IncrementFailure)
+
+ if (withWait) future else Future.successful(MutateResponse.IncrementSuccess)
+ }
+
+ /* PutRequest and DeleteRequest accept byte[][] qualifiers/values. */
+ val othersFutures = putAndDeletes.groupBy { kv =>
+ (kv.table.toSeq, kv.row.toSeq, kv.cf.toSeq, kv.operation, kv.timestamp)
+ }.map { case ((table, row, cf, operation, timestamp), groupedKeyValues) =>
+
+ val durability = groupedKeyValues.head.durability
+ val qualifiers = new ArrayBuffer[Array[Byte]]()
+ val values = new ArrayBuffer[Array[Byte]]()
+
+ groupedKeyValues.foreach { kv =>
+ if (kv.qualifier != null) qualifiers += kv.qualifier
+ if (kv.value != null) values += kv.value
+ }
+ val defer = operation match {
+ case SKeyValue.Put =>
+ val put = new PutRequest(table.toArray, row.toArray, cf.toArray, qualifiers.toArray, values.toArray, timestamp)
+ put.setDurable(durability)
+ _client.put(put)
+ case SKeyValue.Delete =>
+ val delete =
+ if (qualifiers.isEmpty)
+ new DeleteRequest(table.toArray, row.toArray, cf.toArray, timestamp)
+ else
+ new DeleteRequest(table.toArray, row.toArray, cf.toArray, qualifiers.toArray, timestamp)
+ delete.setDurable(durability)
+ _client.delete(delete)
+ }
+ if (withWait) {
+ defer.toFuture(new AnyRef()).map(_ => MutateResponse.Success).recover { case ex: Exception =>
+ groupedKeyValues.foreach { kv => logger.error(s"mutation failed. $kv", ex) }
+ MutateResponse.Failure
+ }
+ } else Future.successful(MutateResponse.Success)
+ }
+ for {
+ incrementRets <- Future.sequence(incrementsFutures)
+ otherRets <- Future.sequence(othersFutures)
+ } yield new MutateResponse(isSuccess = (incrementRets ++ otherRets).forall(_.isSuccess))
+ }
+ }
+
+ /**
+ * write requestKeyValue into storage if the current value in storage that is stored matches.
+ * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
+ *
+ * Most important thing is this have to be 'atomic' operation.
+ * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
+ * either blocked or failed on write-write conflict case.
+ *
+ * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
+ * prevent wrong data for read.
+ *
+ * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
+ * compareAndSet to synchronize.
+ *
+ * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
+ * for storage that does not support concurrency control, then storage implementation
+ * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
+ * and write(writeLock).
+ *
+ * @param rpc
+ * @param expectedOpt
+ * @return
+ */
+ override def writeLock(rpc: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ val put = new PutRequest(rpc.table, rpc.row, rpc.cf, rpc.qualifier, rpc.value, rpc.timestamp)
+ val expected = expectedOpt.map(_.value).getOrElse(Array.empty)
+ client(withWait = true).compareAndSet(put, expected).map(true.booleanValue())(ret => ret.booleanValue()).toFuture(true)
+ .map(r => new MutateResponse(isSuccess = r))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index 4be3767..f65ee20 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -21,25 +21,40 @@ package org.apache.s2graph.core.storage.hbase
import java.util
+import java.util.Base64
import java.util.concurrent.{ExecutorService, Executors}
+import com.stumbleupon.async.Deferred
import com.typesafe.config.Config
import org.apache.commons.io.FileUtils
+import org.apache.hadoop.hbase.util.Bytes
import org.apache.s2graph.core._
import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.storage.serde._
+import org.apache.s2graph.core.types.{HBaseType, VertexId}
import org.apache.s2graph.core.utils._
+import org.hbase.async.FilterList.Operator.MUST_PASS_ALL
import org.hbase.async._
-import org.apache.s2graph.core.storage.serde._
+
import scala.collection.JavaConversions._
+import scala.concurrent.{ExecutionContext, Future}
object AsynchbaseStorage {
+ import Extensions.DeferOps
+ import CanDefer._
+
val vertexCf = Serializable.vertexCf
val edgeCf = Serializable.edgeCf
+
val emptyKVs = new util.ArrayList[KeyValue]()
+ val emptyKeyValues = new util.ArrayList[KeyValue]()
+ val emptyKeyValuesLs = new util.ArrayList[util.ArrayList[KeyValue]]()
+ val emptyStepResult = new util.ArrayList[StepResult]()
AsynchbasePatcher.init()
+
def makeClient(config: Config, overrideKv: (String, String)*) = {
val asyncConfig: org.hbase.async.Config =
if (config.hasPath("hbase.security.auth.enable") && config.getBoolean("hbase.security.auth.enable")) {
@@ -135,6 +150,161 @@ object AsynchbaseStorage {
hbaseExecutor
}
+
+ def fetchKeyValues(client: HBaseClient, rpc: AsyncRPC)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ val defer = fetchKeyValuesInner(client, rpc)
+ defer.toFuture(emptyKeyValues).map { kvsArr =>
+ kvsArr.map { kv =>
+ implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
+ }
+ }
+ }
+
+ def fetchKeyValuesInner(client: HBaseClient, rpc: AsyncRPC)(implicit ec: ExecutionContext): Deferred[util.ArrayList[KeyValue]] = {
+ rpc match {
+ case Left(get) => client.get(get)
+ case Right(ScanWithRange(scanner, offset, limit)) =>
+ val fallbackFn: (Exception => util.ArrayList[KeyValue]) = { ex =>
+ logger.error(s"fetchKeyValuesInner failed.", ex)
+ scanner.close()
+ emptyKeyValues
+ }
+
+ scanner.nextRows().mapWithFallback(new util.ArrayList[util.ArrayList[KeyValue]]())(fallbackFn) { kvsLs =>
+ val ls = new util.ArrayList[KeyValue]
+ if (kvsLs == null) {
+ } else {
+ kvsLs.foreach { kvs =>
+ if (kvs != null) kvs.foreach { kv => ls.add(kv) }
+ else {
+
+ }
+ }
+ }
+
+ scanner.close()
+ val toIndex = Math.min(ls.size(), offset + limit)
+ new util.ArrayList[KeyValue](ls.subList(offset, toIndex))
+ }
+ case _ => Deferred.fromError(new RuntimeException(s"fetchKeyValues failed. $rpc"))
+ }
+ }
+
+ def toCacheKeyBytes(hbaseRpc: AsyncRPC): Array[Byte] = {
+ /* with version 4, request's type is (Scanner, (Int, Int)). otherwise GetRequest. */
+ hbaseRpc match {
+ case Left(getRequest) => getRequest.key
+ case Right(ScanWithRange(scanner, offset, limit)) =>
+ Bytes.add(scanner.getCurrentKey, Bytes.add(Bytes.toBytes(offset), Bytes.toBytes(limit)))
+ case _ =>
+ logger.error(s"toCacheKeyBytes failed. not supported class type. $hbaseRpc")
+ throw new RuntimeException(s"toCacheKeyBytes: $hbaseRpc")
+ }
+ }
+
+ def buildRequest(serDe: StorageSerDe, queryRequest: QueryRequest, vertex: S2VertexLike) = {
+ val kvs = serDe.vertexSerializer(vertex).toKeyValues
+ val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
+ // get.setTimeout(this.singleGetTimeout.toShort)
+ get.setFailfast(true)
+ get.maxVersions(1)
+
+ Left(get)
+ }
+
+ def buildRequest(client: HBaseClient, serDe: StorageSerDe, queryRequest: QueryRequest, edge: S2EdgeLike) = {
+ val queryParam = queryRequest.queryParam
+ val label = queryParam.label
+
+ val serializer = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
+ val snapshotEdge = edge.toSnapshotEdge
+ serDe.snapshotEdgeSerializer(snapshotEdge)
+ } else {
+ val indexEdge = edge.toIndexEdge(queryParam.labelOrderSeq)
+ serDe.indexEdgeSerializer(indexEdge)
+ }
+
+ val rowKey = serializer.toRowKey
+ val (minTs, maxTs) = queryParam.durationOpt.getOrElse((0L, Long.MaxValue))
+
+ val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
+
+ label.schemaVersion match {
+ case HBaseType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty =>
+ val scanner = AsynchbasePatcher.newScanner(client, label.hbaseTableName)
+ scanner.setFamily(SKeyValue.EdgeCf)
+
+ /*
+ * TODO: remove this part.
+ */
+ val indexEdgeOpt = edge.edgesWithIndex.find(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq)
+ val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
+
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+ val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+
+ val (startKey, stopKey) =
+ if (queryParam.intervalOpt.isDefined) {
+ // interval is set.
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Base64.getDecoder.decode(cursor)
+ case None => Bytes.add(baseKey, intervalMaxBytes)
+ }
+ (_startKey , Bytes.add(baseKey, intervalMinBytes))
+ } else {
+ /*
+ * note: since propsToBytes encode size of property map at first byte, we are sure about max value here
+ */
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Base64.getDecoder.decode(cursor)
+ case None => baseKey
+ }
+ (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
+ }
+
+ scanner.setStartKey(startKey)
+ scanner.setStopKey(stopKey)
+
+ if (queryParam.limit == Int.MinValue) logger.debug(s"MinValue: $queryParam")
+
+ scanner.setMaxVersions(1)
+ // TODO: exclusive condition innerOffset with cursorOpt
+ if (queryParam.cursorOpt.isDefined) {
+ scanner.setMaxNumRows(queryParam.limit)
+ } else {
+ scanner.setMaxNumRows(queryParam.innerOffset + queryParam.innerLimit)
+ }
+ scanner.setMaxTimestamp(maxTs)
+ scanner.setMinTimestamp(minTs)
+ scanner.setRpcTimeout(queryParam.rpcTimeout)
+
+ // SET option for this rpc properly.
+ if (queryParam.cursorOpt.isDefined) Right(ScanWithRange(scanner, 0, queryParam.limit))
+ else Right(ScanWithRange(scanner, 0, queryParam.innerOffset + queryParam.innerLimit))
+
+ case _ =>
+ val get = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
+ new GetRequest(label.hbaseTableName.getBytes, rowKey, SKeyValue.EdgeCf, serializer.toQualifier)
+ } else {
+ new GetRequest(label.hbaseTableName.getBytes, rowKey, SKeyValue.EdgeCf)
+ }
+
+ get.maxVersions(1)
+ get.setFailfast(true)
+ get.setMinTimestamp(minTs)
+ get.setMaxTimestamp(maxTs)
+ get.setTimeout(queryParam.rpcTimeout)
+
+ val pagination = new ColumnPaginationFilter(queryParam.limit, queryParam.offset)
+ val columnRangeFilterOpt = queryParam.intervalOpt.map { interval =>
+ new ColumnRangeFilter(intervalMaxBytes, true, intervalMinBytes, true)
+ }
+ get.setFilter(new FilterList(pagination +: columnRangeFilterOpt.toSeq, MUST_PASS_ALL))
+ Left(get)
+ }
+ }
}
@@ -149,11 +319,21 @@ class AsynchbaseStorage(override val graph: S2GraphLike,
val clientWithFlush = AsynchbaseStorage.makeClient(config, "hbase.rpcs.buffered_flush_interval" -> "0")
val clients = Seq(client, clientWithFlush)
- override val management: StorageManagement = new AsynchbaseStorageManagement(config, clients)
+// private lazy val _fetcher = new AsynchbaseStorageFetcher(graph, config, client, serDe, io)
+
+ private lazy val optimisticEdgeFetcher = new AsynchbaseOptimisticEdgeFetcher(client, serDe, io)
+ private lazy val optimisticMutator = new AsynchbaseOptimisticMutator(graph, serDe, optimisticEdgeFetcher, client, clientWithFlush)
+ private lazy val _mutator = new DefaultOptimisticMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator)
+ override val management: StorageManagement = new AsynchbaseStorageManagement(config, clients)
override val serDe: StorageSerDe = new AsynchbaseStorageSerDe(graph)
- override val reader: StorageReadable = new AsynchbaseStorageReadable(graph, config, client, serDe, io)
+ override val edgeFetcher: EdgeFetcher = new AsynchbaseEdgeFetcher(graph, config, client, serDe, io)
+ override val edgeBulkFetcher: EdgeBulkFetcher = new AsynchbaseEdgeBulkFetcher(graph, config, client, serDe, io)
+ override val vertexFetcher: VertexFetcher = new AsynchbaseVertexFetcher(graph, config, client, serDe, io)
+ override val vertexBulkFetcher: VertexBulkFetcher = new AsynchbaseVertexBulkFetcher(graph, config, client, serDe, io)
+
+ override val edgeMutator: EdgeMutator = _mutator
+ override val vertexMutator: VertexMutator = _mutator
- override val mutator: Mutator = new AsynchbaseStorageWritable(graph, serDe, reader, client, clientWithFlush)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
deleted file mode 100644
index 0526042..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- *
- */
-
-package org.apache.s2graph.core.storage.hbase
-
-import java.util
-import java.util.Base64
-
-import com.stumbleupon.async.Deferred
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core._
-import org.apache.s2graph.core.schema.{Label, ServiceColumn}
-import org.apache.s2graph.core.storage._
-import org.apache.s2graph.core.storage.serde._
-import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage.{AsyncRPC, ScanWithRange}
-import org.apache.s2graph.core.types.{HBaseType, VertexId}
-import org.apache.s2graph.core.utils.{CanDefer, DeferCache, Extensions, logger}
-import org.hbase.async.FilterList.Operator.MUST_PASS_ALL
-import org.hbase.async._
-
-import scala.collection.JavaConversions._
-import scala.concurrent.{ExecutionContext, Future}
-
-class AsynchbaseStorageReadable(val graph: S2GraphLike,
- val config: Config,
- val client: HBaseClient,
- override val serDe: StorageSerDe,
- override val io: StorageIO) extends StorageReadable {
- import Extensions.DeferOps
- import CanDefer._
-
- private val emptyKeyValues = new util.ArrayList[KeyValue]()
- private val emptyKeyValuesLs = new util.ArrayList[util.ArrayList[KeyValue]]()
- private val emptyStepResult = new util.ArrayList[StepResult]()
-
- /** Future Cache to squash request */
- lazy private val futureCache = new DeferCache[StepResult, Deferred, Deferred](config, StepResult.Empty, "AsyncHbaseFutureCache", useMetric = true)
- /** v4 max next row size */
- private val v4_max_num_rows = 10000
- private def getV4MaxNumRows(limit : Int): Int = {
- if (limit < v4_max_num_rows) limit
- else v4_max_num_rows
- }
-
- /**
- * build proper request which is specific into storage to call fetchIndexEdgeKeyValues or fetchSnapshotEdgeKeyValues.
- * for example, Asynchbase use GetRequest, Scanner so this method is responsible to build
- * client request(GetRequest, Scanner) based on user provided query.
- *
- * @param queryRequest
- * @return
- */
- private def buildRequest(queryRequest: QueryRequest, edge: S2EdgeLike) = {
- import Serializable._
- val queryParam = queryRequest.queryParam
- val label = queryParam.label
-
- val serializer = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
- val snapshotEdge = edge.toSnapshotEdge
- serDe.snapshotEdgeSerializer(snapshotEdge)
- } else {
- val indexEdge = edge.toIndexEdge(queryParam.labelOrderSeq)
- serDe.indexEdgeSerializer(indexEdge)
- }
-
- val rowKey = serializer.toRowKey
- val (minTs, maxTs) = queryParam.durationOpt.getOrElse((0L, Long.MaxValue))
-
- val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
-
- label.schemaVersion match {
- case HBaseType.VERSION4 if queryParam.tgtVertexInnerIdOpt.isEmpty =>
- val scanner = AsynchbasePatcher.newScanner(client, label.hbaseTableName)
- scanner.setFamily(edgeCf)
-
- /*
- * TODO: remove this part.
- */
- val indexEdgeOpt = edge.edgesWithIndex.find(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq)
- val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
-
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
- val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
-
- val (startKey, stopKey) =
- if (queryParam.intervalOpt.isDefined) {
- // interval is set.
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Base64.getDecoder.decode(cursor)
- case None => Bytes.add(baseKey, intervalMaxBytes)
- }
- (_startKey , Bytes.add(baseKey, intervalMinBytes))
- } else {
- /*
- * note: since propsToBytes encode size of property map at first byte, we are sure about max value here
- */
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Base64.getDecoder.decode(cursor)
- case None => baseKey
- }
- (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
- }
-
- scanner.setStartKey(startKey)
- scanner.setStopKey(stopKey)
-
- if (queryParam.limit == Int.MinValue) logger.debug(s"MinValue: $queryParam")
-
- scanner.setMaxVersions(1)
- // TODO: exclusive condition innerOffset with cursorOpt
- if (queryParam.cursorOpt.isDefined) {
- scanner.setMaxNumRows(getV4MaxNumRows(queryParam.limit))
- } else {
- scanner.setMaxNumRows(getV4MaxNumRows(queryParam.innerOffset + queryParam.innerLimit))
- }
- scanner.setMaxTimestamp(maxTs)
- scanner.setMinTimestamp(minTs)
- scanner.setRpcTimeout(queryParam.rpcTimeout)
-
- // SET option for this rpc properly.
- if (queryParam.cursorOpt.isDefined) Right(ScanWithRange(scanner, 0, queryParam.limit))
- else Right(ScanWithRange(scanner, 0, queryParam.innerOffset + queryParam.innerLimit))
-
- case _ =>
- val get = if (queryParam.tgtVertexInnerIdOpt.isDefined) {
- new GetRequest(label.hbaseTableName.getBytes, rowKey, edgeCf, serializer.toQualifier)
- } else {
- new GetRequest(label.hbaseTableName.getBytes, rowKey, edgeCf)
- }
-
- get.maxVersions(1)
- get.setFailfast(true)
- get.setMinTimestamp(minTs)
- get.setMaxTimestamp(maxTs)
- get.setTimeout(queryParam.rpcTimeout)
-
- val pagination = new ColumnPaginationFilter(queryParam.limit, queryParam.offset)
- val columnRangeFilterOpt = queryParam.intervalOpt.map { interval =>
- new ColumnRangeFilter(intervalMaxBytes, true, intervalMinBytes, true)
- }
- get.setFilter(new FilterList(pagination +: columnRangeFilterOpt.toSeq, MUST_PASS_ALL))
- Left(get)
- }
- }
-
- /**
- *
- * @param queryRequest
- * @param vertex
- * @return
- */
- private def buildRequest(queryRequest: QueryRequest, vertex: S2VertexLike) = {
- val kvs = serDe.vertexSerializer(vertex).toKeyValues
- val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
- // get.setTimeout(this.singleGetTimeout.toShort)
- get.setFailfast(true)
- get.maxVersions(1)
-
- Left(get)
- }
-
- override def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext) = {
- val rpc = buildRequest(queryRequest, edge)
- fetchKeyValues(rpc)
- }
-
- override def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext) = {
- val rpc = buildRequest(queryRequest, vertex)
- fetchKeyValues(rpc)
- }
-
- /**
- * responsible to fire parallel fetch call into storage and create future that will return merged result.
- *
- * @param queryRequests
- * @param prevStepEdges
- * @return
- */
- override def fetches(queryRequests: Seq[QueryRequest], prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext) = {
- val defers: Seq[Deferred[StepResult]] = for {
- queryRequest <- queryRequests
- } yield {
- val queryOption = queryRequest.query.queryOption
- val queryParam = queryRequest.queryParam
- val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
- val parentEdges = if (shouldBuildParents) prevStepEdges.getOrElse(queryRequest.vertex.id, Nil) else Nil
- fetch(queryRequest, isInnerCall = false, parentEdges)
- }
-
- val grouped: Deferred[util.ArrayList[StepResult]] = Deferred.groupInOrder(defers)
- grouped.map(emptyStepResult) { queryResults: util.ArrayList[StepResult] =>
- queryResults.toSeq
- }.toFuture(emptyStepResult)
- }
-
- def fetchKeyValues(rpc: AsyncRPC)(implicit ec: ExecutionContext) = {
- val defer = fetchKeyValuesInner(rpc)
- defer.toFuture(emptyKeyValues).map { kvsArr =>
- kvsArr.map { kv =>
- implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
- }
- }
- }
-
- override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
- val futures = Label.findAll().groupBy(_.hbaseTableName).toSeq.map { case (hTableName, labels) =>
- val distinctLabels = labels.toSet
- val scan = AsynchbasePatcher.newScanner(client, hTableName)
- scan.setFamily(Serializable.edgeCf)
- scan.setMaxVersions(1)
-
- scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
- case null => Seq.empty
- case kvsLs =>
- kvsLs.flatMap { kvs =>
- kvs.flatMap { kv =>
- val sKV = implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
-
- serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION)
- .fromKeyValues(Seq(kv), None)
- .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
- }
- }
- }
- }
-
- Future.sequence(futures).map(_.flatten)
- }
-
- override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
- val futures = ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.map { case (hTableName, columns) =>
- val distinctColumns = columns.toSet
- val scan = AsynchbasePatcher.newScanner(client, hTableName)
- scan.setFamily(Serializable.vertexCf)
- scan.setMaxVersions(1)
-
- scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
- case null => Seq.empty
- case kvsLs =>
- kvsLs.flatMap { kvs =>
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(kvs, None)
- .filter(v => distinctColumns(v.serviceColumn))
- }
- }
- }
- Future.sequence(futures).map(_.flatten)
- }
-
-
- /**
- * we are using future cache to squash requests into same key on storage.
- *
- * @param queryRequest
- * @param isInnerCall
- * @param parentEdges
- * @return we use Deferred here since it has much better performrance compared to scala.concurrent.Future.
- * seems like map, flatMap on scala.concurrent.Future is slower than Deferred's addCallback
- */
- private def fetch(queryRequest: QueryRequest,
- isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore])(implicit ec: ExecutionContext): Deferred[StepResult] = {
-
- def fetchInner(hbaseRpc: AsyncRPC): Deferred[StepResult] = {
- val prevStepScore = queryRequest.prevStepScore
- val fallbackFn: (Exception => StepResult) = { ex =>
- logger.error(s"fetchInner failed. fallback return. $hbaseRpc}", ex)
- StepResult.Failure
- }
-
- val queryParam = queryRequest.queryParam
- fetchKeyValuesInner(hbaseRpc).mapWithFallback(emptyKeyValues)(fallbackFn) { kvs =>
- val (startOffset, len) = queryParam.label.schemaVersion match {
- case HBaseType.VERSION4 =>
- val offset = if (queryParam.cursorOpt.isDefined) 0 else queryParam.offset
- (offset, queryParam.limit)
- case _ => (0, kvs.length)
- }
-
- io.toEdges(kvs, queryRequest, prevStepScore, isInnerCall, parentEdges, startOffset, len)
- }
- }
-
- val queryParam = queryRequest.queryParam
- val cacheTTL = queryParam.cacheTTLInMillis
- /* with version 4, request's type is (Scanner, (Int, Int)). otherwise GetRequest. */
-
- val edge = graph.elementBuilder.toRequestEdge(queryRequest, parentEdges)
- val request = buildRequest(queryRequest, edge)
-
- val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
- val requestCacheKey = Bytes.add(toCacheKeyBytes(request), intervalMaxBytes, intervalMinBytes)
-
- if (cacheTTL <= 0) fetchInner(request)
- else {
- val cacheKeyBytes = Bytes.add(queryRequest.query.queryOption.cacheKeyBytes, requestCacheKey)
-
- // val cacheKeyBytes = toCacheKeyBytes(request)
- val cacheKey = queryParam.toCacheKey(cacheKeyBytes)
- futureCache.getOrElseUpdate(cacheKey, cacheTTL)(fetchInner(request))
- }
- }
-
- /**
- * Private Methods which is specific to Asynchbase implementation.
- */
- private def fetchKeyValuesInner(rpc: AsyncRPC)(implicit ec: ExecutionContext): Deferred[util.ArrayList[KeyValue]] = {
- rpc match {
- case Left(get) => client.get(get)
- case Right(ScanWithRange(scanner, offset, limit)) =>
- val fallbackFn: (Exception => util.ArrayList[KeyValue]) = { ex =>
- logger.error(s"fetchKeyValuesInner failed.", ex)
- scanner.close()
- emptyKeyValues
- }
-
- scanner.nextRows().mapWithFallback(new util.ArrayList[util.ArrayList[KeyValue]]())(fallbackFn) { kvsLs =>
- val ls = new util.ArrayList[KeyValue]
- if (kvsLs == null) {
- } else {
- kvsLs.foreach { kvs =>
- if (kvs != null) kvs.foreach { kv => ls.add(kv) }
- else {
-
- }
- }
- }
-
- scanner.close()
- val toIndex = Math.min(ls.size(), offset + limit)
- new util.ArrayList[KeyValue](ls.subList(offset, toIndex))
- }
- case _ => Deferred.fromError(new RuntimeException(s"fetchKeyValues failed. $rpc"))
- }
- }
-
- private def toCacheKeyBytes(hbaseRpc: AsyncRPC): Array[Byte] = {
- /* with version 4, request's type is (Scanner, (Int, Int)). otherwise GetRequest. */
- hbaseRpc match {
- case Left(getRequest) => getRequest.key
- case Right(ScanWithRange(scanner, offset, limit)) =>
- Bytes.add(scanner.getCurrentKey, Bytes.add(Bytes.toBytes(offset), Bytes.toBytes(limit)))
- case _ =>
- logger.error(s"toCacheKeyBytes failed. not supported class type. $hbaseRpc")
- throw new RuntimeException(s"toCacheKeyBytes: $hbaseRpc")
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
deleted file mode 100644
index b4236b9..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageWritable.scala
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.hbase
-
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.S2GraphLike
-import org.apache.s2graph.core.storage._
-import org.apache.s2graph.core.utils.{Extensions, logger}
-import org.hbase.async.{AtomicIncrementRequest, DeleteRequest, HBaseClient, PutRequest}
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-
-class AsynchbaseStorageWritable(val graph: S2GraphLike,
- val serDe: StorageSerDe,
- val reader: StorageReadable,
- val client: HBaseClient,
- val clientWithFlush: HBaseClient) extends DefaultOptimisticMutator(graph, serDe, reader) {
- import Extensions.DeferOps
-
- private def client(withWait: Boolean): HBaseClient = if (withWait) clientWithFlush else client
- /**
- * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
- * note that this should be return true on all success.
- * we assumes that each storage implementation has client as member variable.
- *
- * @param cluster : where this key values should be stored.
- * @param kvs : sequence of SKeyValue that need to be stored in storage.
- * @param withWait : flag to control wait ack from storage.
- * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
- * it never block thread, but rather submit work and notified by event loop when storage send ack back.
- * @return ack message from storage.
- */
- override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext) = {
- if (kvs.isEmpty) Future.successful(MutateResponse.Success)
- else {
- val _client = client(withWait)
- val (increments, putAndDeletes) = kvs.partition(_.operation == SKeyValue.Increment)
-
- /* Asynchbase IncrementRequest does not implement HasQualifiers */
- val incrementsFutures = increments.map { kv =>
- val countVal = Bytes.toLong(kv.value)
- val request = new AtomicIncrementRequest(kv.table, kv.row, kv.cf, kv.qualifier, countVal)
- val fallbackFn: (Exception => MutateResponse) = { ex =>
- logger.error(s"mutation failed. $request", ex)
- new IncrementResponse(false, -1L, -1L)
- }
- val future = _client.bufferAtomicIncrement(request).mapWithFallback(0L)(fallbackFn) { resultCount: java.lang.Long =>
- new IncrementResponse(true, resultCount.longValue(), countVal)
- }.toFuture(MutateResponse.IncrementFailure)
-
- if (withWait) future else Future.successful(MutateResponse.IncrementSuccess)
- }
-
- /* PutRequest and DeleteRequest accept byte[][] qualifiers/values. */
- val othersFutures = putAndDeletes.groupBy { kv =>
- (kv.table.toSeq, kv.row.toSeq, kv.cf.toSeq, kv.operation, kv.timestamp)
- }.map { case ((table, row, cf, operation, timestamp), groupedKeyValues) =>
-
- val durability = groupedKeyValues.head.durability
- val qualifiers = new ArrayBuffer[Array[Byte]]()
- val values = new ArrayBuffer[Array[Byte]]()
-
- groupedKeyValues.foreach { kv =>
- if (kv.qualifier != null) qualifiers += kv.qualifier
- if (kv.value != null) values += kv.value
- }
- val defer = operation match {
- case SKeyValue.Put =>
- val put = new PutRequest(table.toArray, row.toArray, cf.toArray, qualifiers.toArray, values.toArray, timestamp)
- put.setDurable(durability)
- _client.put(put)
- case SKeyValue.Delete =>
- val delete =
- if (qualifiers.isEmpty)
- new DeleteRequest(table.toArray, row.toArray, cf.toArray, timestamp)
- else
- new DeleteRequest(table.toArray, row.toArray, cf.toArray, qualifiers.toArray, timestamp)
- delete.setDurable(durability)
- _client.delete(delete)
- }
- if (withWait) {
- defer.toFuture(new AnyRef()).map(_ => MutateResponse.Success).recover { case ex: Exception =>
- groupedKeyValues.foreach { kv => logger.error(s"mutation failed. $kv", ex) }
- MutateResponse.Failure
- }
- } else Future.successful(MutateResponse.Success)
- }
- for {
- incrementRets <- Future.sequence(incrementsFutures)
- otherRets <- Future.sequence(othersFutures)
- } yield new MutateResponse(isSuccess = (incrementRets ++ otherRets).forall(_.isSuccess))
- }
- }
-
- /**
- * write requestKeyValue into storage if the current value in storage that is stored matches.
- * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
- *
- * Most important thing is this have to be 'atomic' operation.
- * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
- * either blocked or failed on write-write conflict case.
- *
- * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
- * prevent wrong data for read.
- *
- * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
- * compareAndSet to synchronize.
- *
- * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
- * for storage that does not support concurrency control, then storage implementation
- * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
- * and write(writeLock).
- *
- * @param rpc
- * @param expectedOpt
- * @return
- */
- override def writeLock(rpc: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse] = {
- val put = new PutRequest(rpc.table, rpc.row, rpc.cf, rpc.qualifier, rpc.value, rpc.timestamp)
- val expected = expectedOpt.map(_.value).getOrElse(Array.empty)
- client(withWait = true).compareAndSet(put, expected).map(true.booleanValue())(ret => ret.booleanValue()).toFuture(true)
- .map(r => new MutateResponse(isSuccess = r))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
new file mode 100644
index 0000000..e6bf4e6
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.schema.ServiceColumn
+import org.apache.s2graph.core.storage.serde.Serializable
+import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.s2graph.core.utils.Extensions
+import org.apache.s2graph.core.{S2Graph, S2GraphLike, VertexBulkFetcher}
+import org.hbase.async.HBaseClient
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class AsynchbaseVertexBulkFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val client: HBaseClient,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends VertexBulkFetcher {
+
+ import AsynchbaseStorage._
+ import Extensions.DeferOps
+
+ import scala.collection.JavaConverters._
+
+ override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
+ val futures = ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.map { case (hTableName, columns) =>
+ val distinctColumns = columns.toSet
+ val scan = AsynchbasePatcher.newScanner(client, hTableName)
+ scan.setFamily(Serializable.vertexCf)
+ scan.setMaxVersions(1)
+
+ scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
+ case null => Seq.empty
+ case kvsLs =>
+ kvsLs.asScala.flatMap { kvs =>
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(kvs.asScala, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ }
+ }
+ }
+ Future.sequence(futures).map(_.flatten)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
new file mode 100644
index 0000000..560dd2b
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.hbase.async.HBaseClient
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class AsynchbaseVertexFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val client: HBaseClient,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends VertexFetcher {
+ import AsynchbaseStorage._
+
+ private def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ val rpc = buildRequest(serDe, queryRequest, vertex)
+ AsynchbaseStorage.fetchKeyValues(client, rpc)
+ }
+
+ override def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = {
+ def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2VertexLike] = {
+ if (kvs.isEmpty) Nil
+ else serDe.vertexDeserializer(version).fromKeyValues(kvs, None).toSeq
+ }
+
+ val futures = vertices.map { vertex =>
+ val queryParam = QueryParam.Empty
+ val q = Query.toQuery(Seq(vertex), Seq(queryParam))
+ val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
+
+ fetchKeyValues(queryRequest, vertex).map { kvs =>
+ fromResult(kvs, vertex.serviceColumn.schemaVersion)
+ } recoverWith {
+ case ex: Throwable => Future.successful(Nil)
+ }
+ }
+
+ Future.sequence(futures).map(_.flatten)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
new file mode 100644
index 0000000..2ca4b35
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.{EdgeBulkFetcher, S2EdgeLike, S2GraphLike}
+import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
+import org.rocksdb.RocksDB
+
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksEdgeBulkFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends EdgeBulkFetcher {
+ import RocksStorage._
+
+ override def fetchEdgesAll()(implicit ec: ExecutionContext) = {
+ val edges = new ArrayBuffer[S2EdgeLike]()
+ Label.findAll().groupBy(_.hbaseTableName).toSeq.foreach { case (hTableName, labels) =>
+ val distinctLabels = labels.toSet
+
+ val iter = db.newIterator()
+ try {
+ iter.seekToFirst()
+ while (iter.isValid) {
+ val kv = SKeyValue(table, iter.key(), SKeyValue.EdgeCf, qualifier, iter.value, System.currentTimeMillis())
+
+ serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(Seq(kv), None)
+ .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
+ .foreach { edge =>
+ edges += edge
+ }
+
+
+ iter.next()
+ }
+
+ } finally {
+ iter.close()
+ }
+ }
+
+ Future.successful(edges)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
new file mode 100644
index 0000000..628c5e1
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.VertexId
+import org.rocksdb.RocksDB
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksEdgeFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends EdgeFetcher {
+ import RocksStorage._
+
+ override def fetches(queryRequests: Seq[QueryRequest], prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
+ val futures = for {
+ queryRequest <- queryRequests
+ } yield {
+ val parentEdges = prevStepEdges.getOrElse(queryRequest.vertex.id, Nil)
+ val edge = graph.elementBuilder.toRequestEdge(queryRequest, parentEdges)
+ val rpc = buildRequest(graph, serDe, queryRequest, edge)
+ fetchKeyValues(vdb, db, rpc).map { kvs =>
+ val queryParam = queryRequest.queryParam
+ val stepResult = io.toEdges(kvs, queryRequest, queryRequest.prevStepScore, false, parentEdges)
+ val edgeWithScores = stepResult.edgeWithScores.filter { case edgeWithScore =>
+ val edge = edgeWithScore.edge
+ val duration = queryParam.durationOpt.getOrElse((Long.MinValue, Long.MaxValue))
+ edge.ts >= duration._1 && edge.ts < duration._2
+ }
+
+ stepResult.copy(edgeWithScores = edgeWithScores)
+ }
+ }
+
+ Future.sequence(futures)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticEdgeFetcher.scala
new file mode 100644
index 0000000..6513442
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticEdgeFetcher.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.{QueryRequest, S2EdgeLike, S2GraphLike}
+import org.apache.s2graph.core.storage.{OptimisticEdgeFetcher, SKeyValue, StorageIO, StorageSerDe}
+import org.rocksdb.RocksDB
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksOptimisticEdgeFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends OptimisticEdgeFetcher {
+
+ override protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ val request = RocksStorage.buildRequest(graph, serDe, queryRequest, edge)
+
+ RocksStorage.fetchKeyValues(vdb, db, request)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticMutator.scala
new file mode 100644
index 0000000..ec77d3f
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksOptimisticMutator.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import java.util.concurrent.locks.ReentrantLock
+
+import com.google.common.cache.{Cache, LoadingCache}
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.S2GraphLike
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.utils.logger
+import org.rocksdb.{RocksDB, RocksDBException, WriteBatch, WriteOptions}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksOptimisticMutator(val graph: S2GraphLike,
+ val serDe: StorageSerDe,
+ val optimisticEdgeFetcher: OptimisticEdgeFetcher,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val lockMap: LoadingCache[String, ReentrantLock]) extends OptimisticMutator {
+
+ override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext) = {
+ if (kvs.isEmpty) {
+ Future.successful(MutateResponse.Success)
+ } else {
+ val ret = {
+ val (kvsV, kvsE) = kvs.partition(kv => Bytes.equals(kv.cf, SKeyValue.VertexCf))
+ val writeBatchV = buildWriteBatch(kvsV)
+ val writeBatchE = buildWriteBatch(kvsE)
+ val writeOptions = new WriteOptions
+ try {
+ vdb.write(writeOptions, writeBatchV)
+ db.write(writeOptions, writeBatchE)
+ true
+ } catch {
+ case e: Exception =>
+ logger.error(s"writeAsyncSimple failed.", e)
+ false
+ } finally {
+ writeBatchV.close()
+ writeBatchE.close()
+ writeOptions.close()
+ }
+ }
+
+ Future.successful(new MutateResponse(ret))
+ }
+ }
+
+
+ override def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext) = {
+ def op = {
+ val writeOptions = new WriteOptions
+ try {
+ val fetchedValue = db.get(requestKeyValue.row)
+ val innerRet = expectedOpt match {
+ case None =>
+ if (fetchedValue == null) {
+
+ db.put(writeOptions, requestKeyValue.row, requestKeyValue.value)
+ true
+ } else {
+ false
+ }
+ case Some(kv) =>
+ if (fetchedValue == null) {
+ false
+ } else {
+ if (Bytes.compareTo(fetchedValue, kv.value) == 0) {
+ db.put(writeOptions, requestKeyValue.row, requestKeyValue.value)
+ true
+ } else {
+ false
+ }
+ }
+ }
+
+ Future.successful(new MutateResponse(innerRet))
+ } catch {
+ case e: RocksDBException =>
+ logger.error(s"Write lock failed", e)
+ Future.successful(MutateResponse.Failure)
+ } finally {
+ writeOptions.close()
+ }
+ }
+
+ withLock(requestKeyValue.row)(op)
+ }
+
+ private def buildWriteBatch(kvs: Seq[SKeyValue]): WriteBatch = {
+ val writeBatch = new WriteBatch()
+ kvs.foreach { kv =>
+ kv.operation match {
+ case SKeyValue.Put => writeBatch.put(kv.row, kv.value)
+ case SKeyValue.Delete => writeBatch.remove(kv.row)
+ case SKeyValue.Increment => writeBatch.merge(kv.row, kv.value)
+ case _ => throw new RuntimeException(s"not supported rpc operation. ${kv.operation}")
+ }
+ }
+ writeBatch
+ }
+
+ private def withLock[A](key: Array[Byte])(op: => A): A = {
+ val lockKey = Bytes.toString(key)
+ val lock = lockMap.get(lockKey)
+
+ try {
+ lock.lock
+ op
+ } finally {
+ lock.unlock()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
index b24e375..8948e13 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
@@ -19,22 +19,30 @@
package org.apache.s2graph.core.storage.rocks
+import java.util.Base64
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
import com.google.common.hash.Hashing
import com.typesafe.config.Config
+import org.apache.hadoop.hbase.util.Bytes
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.{Storage, StorageManagement, StorageReadable, StorageSerDe}
-import org.apache.s2graph.core.storage.rocks.RocksHelper.RocksRPC
+import org.apache.s2graph.core.storage.rocks.RocksHelper.{GetRequest, RocksRPC, ScanWithRange}
+import org.apache.s2graph.core.storage.serde.StorageSerializable
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.types.VertexId
import org.apache.s2graph.core.utils.logger
import org.rocksdb._
import org.rocksdb.util.SizeUnit
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Random, Try}
object RocksStorage {
+ val table = Array.emptyByteArray
+ val qualifier = Array.emptyByteArray
RocksDB.loadLibrary()
@@ -129,6 +137,84 @@ object RocksStorage {
throw e
}
}
+
+ def buildRequest(graph: S2GraphLike, serDe: StorageSerDe, queryRequest: QueryRequest, edge: S2EdgeLike): RocksRPC = {
+ queryRequest.queryParam.tgtVertexInnerIdOpt match {
+ case None => // indexEdges
+ val queryParam = queryRequest.queryParam
+ val indexEdgeOpt = edge.edgesWithIndex.filter(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq).headOption
+ val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
+ val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
+ val labelWithDirBytes = indexEdge.labelWithDir.bytes
+ val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
+
+ val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
+
+ val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
+ val (startKey, stopKey) =
+ if (queryParam.intervalOpt.isDefined) {
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Base64.getDecoder.decode(cursor)
+ case None => Bytes.add(baseKey, intervalMaxBytes)
+ }
+ (_startKey, Bytes.add(baseKey, intervalMinBytes))
+ } else {
+ val _startKey = queryParam.cursorOpt match {
+ case Some(cursor) => Base64.getDecoder.decode(cursor)
+ case None => baseKey
+ }
+ (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
+ }
+
+ Right(ScanWithRange(SKeyValue.EdgeCf, startKey, stopKey, queryParam.innerOffset, queryParam.innerLimit))
+
+ case Some(tgtId) => // snapshotEdge
+ val kv = serDe.snapshotEdgeSerializer(graph.elementBuilder.toRequestEdge(queryRequest, Nil).toSnapshotEdge).toKeyValues.head
+ Left(GetRequest(SKeyValue.EdgeCf, kv.row))
+ }
+ }
+
+ def buildRequest(queryRequest: QueryRequest, vertex: S2VertexLike): RocksRPC = {
+ val startKey = vertex.id.bytes
+ val stopKey = Bytes.add(startKey, Array.fill(1)(Byte.MaxValue))
+
+ Right(ScanWithRange(SKeyValue.VertexCf, startKey, stopKey, 0, Byte.MaxValue))
+ }
+
+ def fetchKeyValues(vdb: RocksDB, db: RocksDB, rpc: RocksRPC)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ rpc match {
+ case Left(GetRequest(cf, key)) =>
+ val _db = if (Bytes.equals(cf, SKeyValue.VertexCf)) vdb else db
+ val v = _db.get(key)
+
+ val kvs =
+ if (v == null) Seq.empty
+ else Seq(SKeyValue(table, key, cf, qualifier, v, System.currentTimeMillis()))
+
+ Future.successful(kvs)
+ case Right(ScanWithRange(cf, startKey, stopKey, offset, limit)) =>
+ val _db = if (Bytes.equals(cf, SKeyValue.VertexCf)) vdb else db
+ val kvs = new ArrayBuffer[SKeyValue]()
+ val iter = _db.newIterator()
+
+ try {
+ var idx = 0
+ iter.seek(startKey)
+ val (startOffset, len) = (offset, limit)
+ while (iter.isValid && Bytes.compareTo(iter.key, stopKey) <= 0 && idx < startOffset + len) {
+ if (idx >= startOffset) {
+ kvs += SKeyValue(table, iter.key, cf, qualifier, iter.value, System.currentTimeMillis())
+ }
+
+ iter.next()
+ idx += 1
+ }
+ } finally {
+ iter.close()
+ }
+ Future.successful(kvs)
+ }
+ }
}
class RocksStorage(override val graph: S2GraphLike,
@@ -150,12 +236,18 @@ class RocksStorage(override val graph: S2GraphLike,
.maximumSize(1000 * 10 * 10 * 10 * 10)
.build[String, ReentrantLock](cacheLoader)
- override val management: StorageManagement = new RocksStorageManagement(config, vdb, db)
+ private lazy val optimisticEdgeFetcher = new RocksOptimisticEdgeFetcher(graph, config, db, vdb, serDe, io)
+ private lazy val optimisticMutator = new RocksOptimisticMutator(graph, serDe, optimisticEdgeFetcher, db, vdb, lockMap)
+ private lazy val _mutator = new DefaultOptimisticMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator)
+ override val management: StorageManagement = new RocksStorageManagement(config, vdb, db)
override val serDe: StorageSerDe = new RocksStorageSerDe(graph)
- override val reader: StorageReadable = new RocksStorageReadable(graph, config, db, vdb, serDe, io)
-
- override val mutator: Mutator = new RocksStorageWritable(graph, serDe, reader, db, vdb, lockMap)
+ override val edgeFetcher: EdgeFetcher = new RocksEdgeFetcher(graph, config, db, vdb, serDe, io)
+ override val edgeBulkFetcher: EdgeBulkFetcher = new RocksEdgeBulkFetcher(graph, config, db, vdb, serDe, io)
+ override val vertexFetcher: VertexFetcher = new RocksVertexFetcher(graph, config, db, vdb, serDe, io)
+ override val vertexBulkFetcher: VertexBulkFetcher = new RocksVertexBulkFetcher(graph, config, db, vdb, serDe, io)
+ override val edgeMutator: EdgeMutator = _mutator
+ override val vertexMutator: VertexMutator = _mutator
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
deleted file mode 100644
index 27e3efd..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageReadable.scala
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.rocks
-
-import java.util.Base64
-
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core._
-import org.apache.s2graph.core.schema.{Label, ServiceColumn}
-import org.apache.s2graph.core.storage.rocks.RocksHelper.{GetRequest, RocksRPC, ScanWithRange}
-import org.apache.s2graph.core.storage.serde.StorageSerializable
-import org.apache.s2graph.core.storage._
-import org.apache.s2graph.core.types.{HBaseType, VertexId}
-import org.rocksdb.RocksDB
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-
-class RocksStorageReadable(val graph: S2GraphLike,
- val config: Config,
- val db: RocksDB,
- val vdb: RocksDB,
- val serDe: StorageSerDe,
- override val io: StorageIO) extends StorageReadable {
-
- private val table = Array.emptyByteArray
- private val qualifier = Array.emptyByteArray
-
- private def buildRequest(queryRequest: QueryRequest, edge: S2EdgeLike): RocksRPC = {
- queryRequest.queryParam.tgtVertexInnerIdOpt match {
- case None => // indexEdges
- val queryParam = queryRequest.queryParam
- val indexEdgeOpt = edge.edgesWithIndex.filter(edgeWithIndex => edgeWithIndex.labelIndex.seq == queryParam.labelOrderSeq).headOption
- val indexEdge = indexEdgeOpt.getOrElse(throw new RuntimeException(s"Can`t find index for query $queryParam"))
- val srcIdBytes = VertexId.toSourceVertexId(indexEdge.srcVertex.id).bytes
- val labelWithDirBytes = indexEdge.labelWithDir.bytes
- val labelIndexSeqWithIsInvertedBytes = StorageSerializable.labelOrderSeqWithIsInverted(indexEdge.labelIndexSeq, isInverted = false)
-
- val baseKey = Bytes.add(srcIdBytes, labelWithDirBytes, labelIndexSeqWithIsInvertedBytes)
-
- val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
- val (startKey, stopKey) =
- if (queryParam.intervalOpt.isDefined) {
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Base64.getDecoder.decode(cursor)
- case None => Bytes.add(baseKey, intervalMaxBytes)
- }
- (_startKey, Bytes.add(baseKey, intervalMinBytes))
- } else {
- val _startKey = queryParam.cursorOpt match {
- case Some(cursor) => Base64.getDecoder.decode(cursor)
- case None => baseKey
- }
- (_startKey, Bytes.add(baseKey, Array.fill(1)(-1)))
- }
-
- Right(ScanWithRange(SKeyValue.EdgeCf, startKey, stopKey, queryParam.innerOffset, queryParam.innerLimit))
-
- case Some(tgtId) => // snapshotEdge
- val kv = serDe.snapshotEdgeSerializer(graph.elementBuilder.toRequestEdge(queryRequest, Nil).toSnapshotEdge).toKeyValues.head
- Left(GetRequest(SKeyValue.EdgeCf, kv.row))
- }
- }
-
- private def buildRequest(queryRequest: QueryRequest, vertex: S2VertexLike): RocksRPC = {
- val startKey = vertex.id.bytes
- val stopKey = Bytes.add(startKey, Array.fill(1)(Byte.MaxValue))
-
- Right(ScanWithRange(SKeyValue.VertexCf, startKey, stopKey, 0, Byte.MaxValue))
-// val kv = serDe.vertexSerializer(vertex).toKeyValues.head
-// Left(GetRequest(SKeyValue.VertexCf, kv.row))
- }
-
- override def fetches(queryRequests: Seq[QueryRequest], prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
- val futures = for {
- queryRequest <- queryRequests
- } yield {
- val parentEdges = prevStepEdges.getOrElse(queryRequest.vertex.id, Nil)
- val edge = graph.elementBuilder.toRequestEdge(queryRequest, parentEdges)
- val rpc = buildRequest(queryRequest, edge)
- fetchKeyValues(rpc).map { kvs =>
- val queryParam = queryRequest.queryParam
- val stepResult = io.toEdges(kvs, queryRequest, queryRequest.prevStepScore, false, parentEdges)
- val edgeWithScores = stepResult.edgeWithScores.filter { case edgeWithScore =>
- val edge = edgeWithScore.edge
- val duration = queryParam.durationOpt.getOrElse((Long.MinValue, Long.MaxValue))
- edge.ts >= duration._1 && edge.ts < duration._2
- }
-
- stepResult.copy(edgeWithScores = edgeWithScores)
- }
- }
-
- Future.sequence(futures)
- }
-
- private def fetchKeyValues(rpc: RocksRPC)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
- rpc match {
- case Left(GetRequest(cf, key)) =>
- val _db = if (Bytes.equals(cf, SKeyValue.VertexCf)) vdb else db
- val v = _db.get(key)
-
- val kvs =
- if (v == null) Seq.empty
- else Seq(SKeyValue(table, key, cf, qualifier, v, System.currentTimeMillis()))
-
- Future.successful(kvs)
- case Right(ScanWithRange(cf, startKey, stopKey, offset, limit)) =>
- val _db = if (Bytes.equals(cf, SKeyValue.VertexCf)) vdb else db
- val kvs = new ArrayBuffer[SKeyValue]()
- val iter = _db.newIterator()
-
- try {
- var idx = 0
- iter.seek(startKey)
- val (startOffset, len) = (offset, limit)
- while (iter.isValid && Bytes.compareTo(iter.key, stopKey) <= 0 && idx < startOffset + len) {
- if (idx >= startOffset) {
- kvs += SKeyValue(table, iter.key, cf, qualifier, iter.value, System.currentTimeMillis())
- }
-
- iter.next()
- idx += 1
- }
- } finally {
- iter.close()
- }
- Future.successful(kvs)
- }
- }
-
- override def fetchEdgesAll()(implicit ec: ExecutionContext) = {
- val edges = new ArrayBuffer[S2EdgeLike]()
- Label.findAll().groupBy(_.hbaseTableName).toSeq.foreach { case (hTableName, labels) =>
- val distinctLabels = labels.toSet
-
- val iter = db.newIterator()
- try {
- iter.seekToFirst()
- while (iter.isValid) {
- val kv = SKeyValue(table, iter.key(), SKeyValue.EdgeCf, qualifier, iter.value, System.currentTimeMillis())
-
- serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(Seq(kv), None)
- .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
- .foreach { edge =>
- edges += edge
- }
-
-
- iter.next()
- }
-
- } finally {
- iter.close()
- }
- }
-
- Future.successful(edges)
- }
-
- override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
- import scala.collection.mutable
-
- val vertices = new ArrayBuffer[S2VertexLike]()
- ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.foreach { case (hTableName, columns) =>
- val distinctColumns = columns.toSet
-
- val iter = vdb.newIterator()
- val buffer = mutable.ListBuffer.empty[SKeyValue]
- var oldVertexIdBytes = Array.empty[Byte]
- var minusPos = 0
-
- try {
- iter.seekToFirst()
- while (iter.isValid) {
- val row = iter.key()
- if (!Bytes.equals(oldVertexIdBytes, 0, oldVertexIdBytes.length - minusPos, row, 0, row.length - 1)) {
- if (buffer.nonEmpty)
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
- .filter(v => distinctColumns(v.serviceColumn))
- .foreach { vertex =>
- vertices += vertex
- }
-
- oldVertexIdBytes = row
- minusPos = 1
- buffer.clear()
- }
- val kv = SKeyValue(table, iter.key(), SKeyValue.VertexCf, qualifier, iter.value(), System.currentTimeMillis())
- buffer += kv
-
- iter.next()
- }
- if (buffer.nonEmpty)
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
- .filter(v => distinctColumns(v.serviceColumn))
- .foreach { vertex =>
- vertices += vertex
- }
-
- } finally {
- iter.close()
- }
- }
-
- Future.successful(vertices)
- }
-
- override def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext) = {
- fetchKeyValues(buildRequest(queryRequest, edge))
- }
-
- override def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext) = {
- fetchKeyValues(buildRequest(queryRequest, vertex))
- }
-}
[02/11] incubator-s2graph git commit: separate Storage into multiple
small interfaces such as EdgeFetcher/VertexMutator, ...
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
deleted file mode 100644
index d29ccce..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorageWritable.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.rocks
-
-import java.util.concurrent.locks.ReentrantLock
-
-import com.google.common.cache.{Cache, LoadingCache}
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.S2GraphLike
-import org.apache.s2graph.core.storage._
-import org.apache.s2graph.core.utils.logger
-import org.rocksdb.{RocksDB, RocksDBException, WriteBatch, WriteOptions}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-class RocksStorageWritable(val graph: S2GraphLike,
- val serDe: StorageSerDe,
- val reader: StorageReadable,
- val db: RocksDB,
- val vdb: RocksDB,
- val lockMap: LoadingCache[String, ReentrantLock]) extends DefaultOptimisticMutator(graph, serDe, reader) {
-
- override def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext) = {
- if (kvs.isEmpty) {
- Future.successful(MutateResponse.Success)
- } else {
- val ret = {
- val (kvsV, kvsE) = kvs.partition(kv => Bytes.equals(kv.cf, SKeyValue.VertexCf))
- val writeBatchV = buildWriteBatch(kvsV)
- val writeBatchE = buildWriteBatch(kvsE)
- val writeOptions = new WriteOptions
- try {
- vdb.write(writeOptions, writeBatchV)
- db.write(writeOptions, writeBatchE)
- true
- } catch {
- case e: Exception =>
- logger.error(s"writeAsyncSimple failed.", e)
- false
- } finally {
- writeBatchV.close()
- writeBatchE.close()
- writeOptions.close()
- }
- }
-
- Future.successful(new MutateResponse(ret))
- }
- }
-
-
- override def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext) = {
- def op = {
- val writeOptions = new WriteOptions
- try {
- val fetchedValue = db.get(requestKeyValue.row)
- val innerRet = expectedOpt match {
- case None =>
- if (fetchedValue == null) {
-
- db.put(writeOptions, requestKeyValue.row, requestKeyValue.value)
- true
- } else {
- false
- }
- case Some(kv) =>
- if (fetchedValue == null) {
- false
- } else {
- if (Bytes.compareTo(fetchedValue, kv.value) == 0) {
- db.put(writeOptions, requestKeyValue.row, requestKeyValue.value)
- true
- } else {
- false
- }
- }
- }
-
- Future.successful(new MutateResponse(innerRet))
- } catch {
- case e: RocksDBException =>
- logger.error(s"Write lock failed", e)
- Future.successful(MutateResponse.Failure)
- } finally {
- writeOptions.close()
- }
- }
-
- withLock(requestKeyValue.row)(op)
- }
-
- private def buildWriteBatch(kvs: Seq[SKeyValue]): WriteBatch = {
- val writeBatch = new WriteBatch()
- kvs.foreach { kv =>
- kv.operation match {
- case SKeyValue.Put => writeBatch.put(kv.row, kv.value)
- case SKeyValue.Delete => writeBatch.remove(kv.row)
- case SKeyValue.Increment => writeBatch.merge(kv.row, kv.value)
- case _ => throw new RuntimeException(s"not supported rpc operation. ${kv.operation}")
- }
- }
- writeBatch
- }
-
- private def withLock[A](key: Array[Byte])(op: => A): A = {
- val lockKey = Bytes.toString(key)
- val lock = lockMap.get(lockKey)
-
- try {
- lock.lock
- op
- } finally {
- lock.unlock()
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
new file mode 100644
index 0000000..20acfaa
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import com.typesafe.config.Config
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.s2graph.core.schema.ServiceColumn
+import org.apache.s2graph.core.{S2GraphLike, S2VertexLike, VertexBulkFetcher}
+import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
+import org.rocksdb.RocksDB
+
+import scala.collection.mutable.ArrayBuffer
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksVertexBulkFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends VertexBulkFetcher {
+ import RocksStorage._
+
+ override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
+ import scala.collection.mutable
+
+ val vertices = new ArrayBuffer[S2VertexLike]()
+ ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.foreach { case (hTableName, columns) =>
+ val distinctColumns = columns.toSet
+
+ val iter = vdb.newIterator()
+ val buffer = mutable.ListBuffer.empty[SKeyValue]
+ var oldVertexIdBytes = Array.empty[Byte]
+ var minusPos = 0
+
+ try {
+ iter.seekToFirst()
+ while (iter.isValid) {
+ val row = iter.key()
+ if (!Bytes.equals(oldVertexIdBytes, 0, oldVertexIdBytes.length - minusPos, row, 0, row.length - 1)) {
+ if (buffer.nonEmpty)
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ .foreach { vertex =>
+ vertices += vertex
+ }
+
+ oldVertexIdBytes = row
+ minusPos = 1
+ buffer.clear()
+ }
+ val kv = SKeyValue(table, iter.key(), SKeyValue.VertexCf, qualifier, iter.value(), System.currentTimeMillis())
+ buffer += kv
+
+ iter.next()
+ }
+ if (buffer.nonEmpty)
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ .foreach { vertex =>
+ vertices += vertex
+ }
+
+ } finally {
+ iter.close()
+ }
+ }
+
+ Future.successful(vertices)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
new file mode 100644
index 0000000..6becd98
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.rocks
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.rocksdb.RocksDB
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class RocksVertexFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val db: RocksDB,
+ val vdb: RocksDB,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends VertexFetcher {
+ private def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
+ val rpc = RocksStorage.buildRequest(queryRequest, vertex)
+
+ RocksStorage.fetchKeyValues(vdb, db, rpc)
+ }
+
+ override def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = {
+ def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2VertexLike] = {
+ if (kvs.isEmpty) Nil
+ else serDe.vertexDeserializer(version).fromKeyValues(kvs, None).toSeq
+ }
+
+ val futures = vertices.map { vertex =>
+ val queryParam = QueryParam.Empty
+ val q = Query.toQuery(Seq(vertex), Seq(queryParam))
+ val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
+
+ fetchKeyValues(queryRequest, vertex).map { kvs =>
+ fromResult(kvs, vertex.serviceColumn.schemaVersion)
+ } recoverWith {
+ case ex: Throwable => Future.successful(Nil)
+ }
+ }
+
+ Future.sequence(futures).map(_.flatten)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
deleted file mode 100644
index 8cd32d4..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.s2graph.core.storage
-import org.apache.s2graph.core._
-import org.apache.s2graph.core.schema.LabelMeta
-import org.apache.s2graph.core.utils.logger
-
-import scala.concurrent.{ExecutionContext, Future}
-
-abstract class DefaultOptimisticMutator(graph: S2GraphLike,
- serDe: StorageSerDe,
- reader: StorageReadable) extends OptimisticMutator {
- val fetcher = reader
-
- lazy val io: StorageIO = new StorageIO(graph, serDe)
- lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, this, reader)
-
-// private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
-// mutator.writeToStorage(cluster, kvs, withWait)
-
- def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
- requestTs: Long,
- retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
- if (stepInnerResult.isEmpty) Future.successful(true)
- else {
- val head = stepInnerResult.edgeWithScores.head
- val zkQuorum = head.edge.innerLabel.hbaseZkAddr
- val futures = for {
- edgeWithScore <- stepInnerResult.edgeWithScores
- } yield {
- val edge = edgeWithScore.edge
-
- val edgeSnapshot = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
-
- val edgeForward = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
- serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- io.buildIncrementsAsync(indexEdge, -1L)
- }
-
- /* reverted direction */
- val edgeRevert = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
- serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- io.buildIncrementsAsync(indexEdge, -1L)
- }
-
- val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
-
- writeToStorage(zkQuorum, mutations, withWait = true)
- }
-
- Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
- }
- }
-
- def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- if (vertex.op == GraphUtil.operations("delete")) {
- writeToStorage(zkQuorum,
- serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
- } else if (vertex.op == GraphUtil.operations("deleteAll")) {
- logger.info(s"deleteAll for vertex is truncated. $vertex")
- Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
- } else {
- writeToStorage(zkQuorum, io.buildPutsAll(vertex), withWait)
- }
- }
-
- def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] = {
- val mutations = _edges.flatMap { edge =>
- val (_, edgeUpdate) =
- if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
- else S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
-
- if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
- io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
- }
-
- writeToStorage(zkQuorum, mutations, withWait).map { ret =>
- _edges.zipWithIndex.map { case (edge, idx) =>
- idx -> ret.isSuccess
- }
- }
- }
-
- def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
- def mutateEdgesInner(edges: Seq[S2EdgeLike],
- checkConsistency: Boolean,
- withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- assert(edges.nonEmpty)
- // TODO:: remove after code review: unreachable code
- if (!checkConsistency) {
-
- val futures = edges.map { edge =>
- val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
- val mutations =
- io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
-
- if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
-
- writeToStorage(zkQuorum, mutations, withWait)
- }
- Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
- } else {
- fetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
- conflictResolver.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
- }
- }
- }
-
- val edgeWithIdxs = _edges.zipWithIndex
- val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
- (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
- } toSeq
-
- val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
- val edges = edgeGroup.map(_._1)
- val idxs = edgeGroup.map(_._2)
- // After deleteAll, process others
- val mutateEdgeFutures = edges.toList match {
- case head :: tail =>
- val edgeFuture = mutateEdgesInner(edges, checkConsistency = true, withWait)
-
- //TODO: decide what we will do on failure on vertex put
- val puts = io.buildVertexPutsAsync(head)
- val vertexFuture = writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
- Seq(edgeFuture, vertexFuture)
- case Nil => Nil
- }
-
- val composed = for {
- // deleteRet <- Future.sequence(deleteAllFutures)
- mutateRet <- Future.sequence(mutateEdgeFutures)
- } yield mutateRet
-
- composed.map(_.forall(_.isSuccess)).map { ret => idxs.map(idx => idx -> ret) }
- }
-
- Future.sequence(mutateEdges).map { squashedRets =>
- squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
- }
- }
-
- def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] = {
- val futures = for {
- edge <- edges
- } yield {
- val kvs = for {
- relEdge <- edge.relatedEdges
- edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
- } yield {
- val countWithTs = edge.propertyValueInner(LabelMeta.count)
- val countVal = countWithTs.innerVal.toString().toLong
- io.buildIncrementsCountAsync(edgeWithIndex, countVal).head
- }
- writeToStorage(zkQuorum, kvs, withWait = withWait)
- }
-
- Future.sequence(futures)
- }
-
- def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- val kvs = io.buildDegreePuts(edge, degreeVal)
-
- writeToStorage(zkQuorum, kvs, withWait = true)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/utils/ImportStatus.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/ImportStatus.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/ImportStatus.scala
new file mode 100644
index 0000000..aa0c6b5
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/ImportStatus.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.utils
+
+import java.util.concurrent.atomic.AtomicInteger
+
+trait ImportStatus {
+ val done: AtomicInteger
+
+ def isCompleted: Boolean
+
+ def percentage: Int
+
+ val total: Int
+}
+
+class ImportRunningStatus(val total: Int) extends ImportStatus {
+ require(total > 0, s"Total should be positive: $total")
+
+ val done = new AtomicInteger(0)
+
+ def isCompleted: Boolean = total == done.get
+
+ def percentage = 100 * done.get / total
+}
+
+case object ImportDoneStatus extends ImportStatus {
+ val total = 1
+
+ val done = new AtomicInteger(1)
+
+ def isCompleted: Boolean = true
+
+ def percentage = 100
+}
+
+object ImportStatus {
+ def apply(total: Int): ImportStatus = new ImportRunningStatus(total)
+
+ def unapply(importResult: ImportStatus): Option[(Boolean, Int, Int)] =
+ Some((importResult.isCompleted, importResult.total, importResult.done.get))
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/utils/Importer.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/Importer.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/Importer.scala
new file mode 100644
index 0000000..300106a
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/Importer.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.utils
+
+import java.io.File
+
+import com.typesafe.config.Config
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.s2graph.core.{EdgeFetcher, S2GraphLike}
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object Importer {
+ def toHDFSConfiguration(hdfsConfDir: String): Configuration = {
+ val conf = new Configuration
+
+ val hdfsConfDirectory = new File(hdfsConfDir)
+ if (hdfsConfDirectory.exists()) {
+ if (!hdfsConfDirectory.isDirectory || !hdfsConfDirectory.canRead) {
+ throw new IllegalStateException(s"HDFS configuration directory ($hdfsConfDirectory) cannot be read.")
+ }
+
+ val path = hdfsConfDirectory.getAbsolutePath
+ conf.addResource(new Path(s"file:///$path/core-site.xml"))
+ conf.addResource(new Path(s"file:///$path/hdfs-site.xml"))
+ } else {
+ logger.warn("RocksDBImporter doesn't have valid hadoop configuration directory..")
+ }
+ conf
+ }
+}
+
+trait Importer {
+ @volatile var isFinished: Boolean = false
+
+ def run(config: Config)(implicit ec: ExecutionContext): Future[Importer]
+
+ def status: Boolean = isFinished
+
+ def setStatus(otherStatus: Boolean): Boolean = {
+ this.isFinished = otherStatus
+ this.isFinished
+ }
+
+ def close(): Unit
+}
+
+case class IdentityImporter(graph: S2GraphLike) extends Importer {
+ override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ Future.successful(this)
+ }
+
+ override def close(): Unit = {}
+}
+
+object HDFSImporter {
+
+ import scala.collection.JavaConverters._
+
+ val PathsKey = "paths"
+ val HDFSConfDirKey = "hdfsConfDir"
+
+ def extractPaths(config: Config): Map[String, String] = {
+ config.getConfigList(PathsKey).asScala.map { e =>
+ val key = e.getString("src")
+ val value = e.getString("tgt")
+
+ key -> value
+ }.toMap
+ }
+}
+
+case class HDFSImporter(graph: S2GraphLike) extends Importer {
+
+ import HDFSImporter._
+
+ override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ Future {
+ val paths = extractPaths(config)
+ val hdfsConfiDir = config.getString(HDFSConfDirKey)
+
+ val hadoopConfig = Importer.toHDFSConfiguration(hdfsConfiDir)
+ val fs = FileSystem.get(hadoopConfig)
+
+ def copyToLocal(remoteSrc: String, localSrc: String): Unit = {
+ val remoteSrcPath = new Path(remoteSrc)
+ val localSrcPath = new Path(localSrc)
+
+ fs.copyToLocalFile(remoteSrcPath, localSrcPath)
+ }
+
+ paths.foreach { case (srcPath, tgtPath) =>
+ copyToLocal(srcPath, tgtPath)
+ }
+
+ this
+ }
+ }
+
+ // override def status: ImportStatus = ???
+
+ override def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
new file mode 100644
index 0000000..6d95c93
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.fetcher
+
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.Integrate.IntegrateCommon
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.{Query, QueryParam}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext}
+
+class EdgeFetcherTest extends IntegrateCommon {
+
+ import scala.collection.JavaConverters._
+
+ test("MemoryModelFetcher") {
+ // 1. create label.
+ // 2. importLabel.
+ // 3. fetch.
+ val service = management.createService("s2graph", "localhost", "s2graph_htable", -1, None).get
+ val serviceColumn =
+ management.createServiceColumn("s2graph", "user", "string", Seq(Prop("age", "0", "int", true)))
+ val labelName = "fetcher_test"
+ val options =
+ s"""{
+ |
+ | "importer": {
+ | "${FetcherManager.ClassNameKey}": "org.apache.s2graph.core.utils.IdentityImporter"
+ | },
+ | "fetcher": {
+ | "${FetcherManager.ClassNameKey}": "org.apache.s2graph.core.fetcher.MemoryModelFetcher"
+ | }
+ |}""".stripMargin
+
+ Label.findByName(labelName, useCache = false).foreach { label => Label.delete(label.id.get) }
+
+ val label = management.createLabel(
+ labelName,
+ serviceColumn,
+ serviceColumn,
+ true,
+ service.serviceName,
+ Seq.empty[Index].asJava,
+ Seq.empty[Prop].asJava,
+ "strong",
+ null,
+ -1,
+ "v3",
+ "gz",
+ options
+ )
+ val config = ConfigFactory.parseString(options)
+ val importerFuture = graph.modelManager.importModel(label, config)(ExecutionContext.Implicits.global)
+ Await.ready(importerFuture, Duration("60 seconds"))
+
+ Thread.sleep(1000)
+
+ val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "daewon")
+ val queryParam = QueryParam(labelName = labelName)
+
+ val query = Query.toQuery(srcVertices = Seq(vertex), queryParams = Seq(queryParam))
+ val stepResult = Await.result(graph.getEdges(query), Duration("60 seconds"))
+
+ stepResult.edgeWithScores.foreach { es =>
+ println(es.edge)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
deleted file mode 100644
index 6c76cdf..0000000
--- a/s2core/src/test/scala/org/apache/s2graph/core/model/FetcherTest.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.model
-
-import com.typesafe.config.ConfigFactory
-import org.apache.s2graph.core.Integrate.IntegrateCommon
-import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
-import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.{Query, QueryParam}
-
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext}
-
-class FetcherTest extends IntegrateCommon {
-
- import scala.collection.JavaConverters._
-
- test("MemoryModelFetcher") {
- // 1. create label.
- // 2. importLabel.
- // 3. fetch.
- val service = management.createService("s2graph", "localhost", "s2graph_htable", -1, None).get
- val serviceColumn =
- management.createServiceColumn("s2graph", "user", "string", Seq(Prop("age", "0", "int", true)))
- val labelName = "fetcher_test"
- val options =
- s"""{
- |
- | "importer": {
- | "${ModelManager.ClassNameKey}": "org.apache.s2graph.core.model.IdentityImporter"
- | },
- | "fetcher": {
- | "${ModelManager.ClassNameKey}": "org.apache.s2graph.core.model.MemoryModelFetcher"
- | }
- |}""".stripMargin
-
- Label.findByName(labelName, useCache = false).foreach { label => Label.delete(label.id.get) }
-
- val label = management.createLabel(
- labelName,
- serviceColumn,
- serviceColumn,
- true,
- service.serviceName,
- Seq.empty[Index].asJava,
- Seq.empty[Prop].asJava,
- "strong",
- null,
- -1,
- "v3",
- "gz",
- options
- )
- val config = ConfigFactory.parseString(options)
- val importerFuture = graph.modelManager.importModel(label, config)(ExecutionContext.Implicits.global)
- Await.ready(importerFuture, Duration("60 seconds"))
-
- Thread.sleep(1000)
-
- val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "daewon")
- val queryParam = QueryParam(labelName = labelName)
-
- val query = Query.toQuery(srcVertices = Seq(vertex), queryParams = Seq(queryParam))
- val stepResult = Await.result(graph.getEdges(query), Duration("60 seconds"))
-
- stepResult.edgeWithScores.foreach { es =>
- println(es.edge)
- }
- }
-}
[09/11] incubator-s2graph git commit: add cacheTTLInSecs on
SafeUpdateCache.withCache.
Posted by st...@apache.org.
add cacheTTLInSecs on SafeUpdateCache.withCache.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e8230417
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e8230417
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e8230417
Branch: refs/heads/master
Commit: e823041715dcca2d330bb25410a0a7cf420bc522
Parents: 9132f74
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu May 10 10:35:35 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu May 10 10:35:35 2018 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/Management.scala | 8 ++++----
.../apache/s2graph/core/ResourceManager.scala | 20 ++++++++++++--------
.../s2graph/core/utils/SafeUpdateCache.scala | 9 ++++++---
3 files changed, 22 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e8230417/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index c3aef7a..f534423 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -313,7 +313,7 @@ class Management(graph: S2GraphLike) {
def updateEdgeFetcher(label: Label, options: String): Unit = {
val newLabel = Label.updateOption(label, options)
- graph.resourceManager.getOrElseUpdateEdgeFetcher(newLabel, forceUpdate = true)
+ graph.resourceManager.getOrElseUpdateEdgeFetcher(newLabel, cacheTTLInSecs = Option(-1))
}
def updateVertexFetcher(serviceName: String, columnName: String, options: String): Unit = {
@@ -325,7 +325,7 @@ class Management(graph: S2GraphLike) {
def updateVertexFetcher(column: ServiceColumn, options: String): Unit = {
val newColumn = ServiceColumn.updateOption(column, options)
- graph.resourceManager.getOrElseUpdateVertexFetcher(newColumn, forceUpdate = true)
+ graph.resourceManager.getOrElseUpdateVertexFetcher(newColumn, cacheTTLInSecs = Option(-1))
}
def updateEdgeMutator(labelName: String, options: String): Unit = {
@@ -336,7 +336,7 @@ class Management(graph: S2GraphLike) {
def updateEdgeMutator(label: Label, options: String): Unit = {
val newLabel = Label.updateOption(label, options)
- graph.resourceManager.getOrElseUpdateEdgeMutator(newLabel, forceUpdate = true)
+ graph.resourceManager.getOrElseUpdateEdgeMutator(newLabel, cacheTTLInSecs = Option(-1))
}
def updateVertexMutator(serviceName: String, columnName: String, options: String): Unit = {
@@ -348,7 +348,7 @@ class Management(graph: S2GraphLike) {
def updateVertexMutator(column: ServiceColumn, options: String): Unit = {
val newColumn = ServiceColumn.updateOption(column, options)
- graph.resourceManager.getOrElseUpdateVertexMutator(newColumn, forceUpdate = true)
+ graph.resourceManager.getOrElseUpdateVertexMutator(newColumn, cacheTTLInSecs = Option(-1))
}
def createStorageTable(zkAddr: String,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e8230417/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
index b877603..a8a3a34 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
@@ -59,9 +59,10 @@ class ResourceManager(graph: S2GraphLike,
cache.asMap().asScala.toSeq.collect { case (_, (obj: EdgeFetcher, _, _)) => obj }
}
- def getOrElseUpdateVertexFetcher(column: ServiceColumn, forceUpdate: Boolean = false): Option[VertexFetcher] = {
+ def getOrElseUpdateVertexFetcher(column: ServiceColumn,
+ cacheTTLInSecs: Option[Int] = None): Option[VertexFetcher] = {
val cacheKey = VertexFetcherKey + "_" + column.service.serviceName + "_" + column.columnName
- cache.withCache(cacheKey, false, forceUpdate) {
+ cache.withCache(cacheKey, false, cacheTTLInSecs) {
column.toFetcherConfig.map { fetcherConfig =>
val className = fetcherConfig.getString(ClassNameKey)
val fetcher = Class.forName(className)
@@ -76,10 +77,11 @@ class ResourceManager(graph: S2GraphLike,
}
}
- def getOrElseUpdateEdgeFetcher(label: Label, forceUpdate: Boolean = false): Option[EdgeFetcher] = {
+ def getOrElseUpdateEdgeFetcher(label: Label,
+ cacheTTLInSecs: Option[Int] = None): Option[EdgeFetcher] = {
val cacheKey = EdgeFetcherKey + "_" + label.label
- cache.withCache(cacheKey, false, forceUpdate) {
+ cache.withCache(cacheKey, false, cacheTTLInSecs) {
label.toFetcherConfig.map { fetcherConfig =>
val className = fetcherConfig.getString(ClassNameKey)
val fetcher = Class.forName(className)
@@ -94,9 +96,10 @@ class ResourceManager(graph: S2GraphLike,
}
}
- def getOrElseUpdateVertexMutator(column: ServiceColumn, forceUpdate: Boolean = false): Option[VertexMutator] = {
+ def getOrElseUpdateVertexMutator(column: ServiceColumn,
+ cacheTTLInSecs: Option[Int] = None): Option[VertexMutator] = {
val cacheKey = VertexMutatorKey + "_" + column.service.serviceName + "_" + column.columnName
- cache.withCache(cacheKey, false, forceUpdate) {
+ cache.withCache(cacheKey, false, cacheTTLInSecs) {
column.toMutatorConfig.map { mutatorConfig =>
val className = mutatorConfig.getString(ClassNameKey)
val fetcher = Class.forName(className)
@@ -111,9 +114,10 @@ class ResourceManager(graph: S2GraphLike,
}
}
- def getOrElseUpdateEdgeMutator(label: Label, forceUpdate: Boolean = false): Option[EdgeMutator] = {
+ def getOrElseUpdateEdgeMutator(label: Label,
+ cacheTTLInSecs: Option[Int] = None): Option[EdgeMutator] = {
val cacheKey = EdgeMutatorKey + "_" + label.label
- cache.withCache(cacheKey, false, forceUpdate) {
+ cache.withCache(cacheKey, false, cacheTTLInSecs) {
label.toMutatorConfig.map { mutatorConfig =>
val className = mutatorConfig.getString(ClassNameKey)
val fetcher = Class.forName(className)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e8230417/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
index 51e45c2..3ecb02f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
@@ -86,7 +86,7 @@ class SafeUpdateCache(val config: Config)
import java.lang.{Long => JLong}
import SafeUpdateCache._
val maxSize = config.getInt(SafeUpdateCache.MaxSizeKey)
- val ttl = config.getInt(SafeUpdateCache.TtlKey)
+ val systemTtl = config.getInt(SafeUpdateCache.TtlKey)
private val cache = CacheBuilder.newBuilder().maximumSize(maxSize)
.build[JLong, (AnyRef, Int, AtomicBoolean)]()
@@ -116,7 +116,7 @@ class SafeUpdateCache(val config: Config)
def withCache[T <: AnyRef](key: String,
broadcast: Boolean,
- forceUpdate: Boolean = false)(op: => T): T = {
+ cacheTTLInSecs: Option[Int] = None)(op: => T): T = {
val cacheKey = toCacheKey(key)
val cachedValWithTs = cache.getIfPresent(cacheKey)
@@ -129,7 +129,10 @@ class SafeUpdateCache(val config: Config)
val (_cachedVal, updatedAt, isUpdating) = cachedValWithTs
val cachedVal = _cachedVal.asInstanceOf[T]
- if (!forceUpdate && toTs() < updatedAt + ttl) cachedVal // in cache TTL
+ val ttl = cacheTTLInSecs.getOrElse(systemTtl)
+ val isValidCacheVal = toTs() < updatedAt + ttl
+
+ if (isValidCacheVal) cachedVal // in cache TTL
else {
val running = isUpdating.getAndSet(true)
[04/11] incubator-s2graph git commit: separate Storage into multiple
small interfaces such as EdgeFetcher/VertexMutator, ...
Posted by st...@apache.org.
separate Storage into multiple small interfaces such as EdgeFetcher/VertexMutator, ...
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/43f627e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/43f627e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/43f627e5
Branch: refs/heads/master
Commit: 43f627e551fd0b744a858c9e6e7feba7fd68e58c
Parents: 2357d81
Author: DO YUNG YOON <st...@apache.org>
Authored: Wed May 9 15:58:19 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Wed May 9 16:56:37 2018 +0900
----------------------------------------------------------------------
.../apache/s2graph/core/EdgeBulkFetcher.scala | 28 ++
.../org/apache/s2graph/core/EdgeFetcher.scala | 35 ++
.../org/apache/s2graph/core/EdgeMutator.scala | 38 ++
.../scala/org/apache/s2graph/core/Fetcher.scala | 36 --
.../org/apache/s2graph/core/Management.scala | 4 +-
.../scala/org/apache/s2graph/core/Mutator.scala | 41 ---
.../scala/org/apache/s2graph/core/S2Graph.scala | 51 ++-
.../org/apache/s2graph/core/S2GraphLike.scala | 25 +-
.../apache/s2graph/core/TraversalHelper.scala | 6 +-
.../apache/s2graph/core/VertexBulkFetcher.scala | 26 ++
.../org/apache/s2graph/core/VertexFetcher.scala | 31 ++
.../org/apache/s2graph/core/VertexMutator.scala | 28 ++
.../s2graph/core/fetcher/FetcherManager.scala | 106 ++++++
.../core/fetcher/MemoryModelEdgeFetcher.scala | 54 +++
.../s2graph/core/model/ImportStatus.scala | 59 ---
.../apache/s2graph/core/model/Importer.scala | 122 ------
.../s2graph/core/model/MemoryModelFetcher.scala | 59 ---
.../s2graph/core/model/ModelManager.scala | 103 ------
.../core/storage/DefaultOptimisticMutator.scala | 190 ++++++++++
.../core/storage/OptimisticEdgeFetcher.scala | 56 +++
.../core/storage/OptimisticMutator.scala | 63 ++++
.../apache/s2graph/core/storage/Storage.scala | 72 +---
.../s2graph/core/storage/StorageReadable.scala | 49 +--
.../s2graph/core/storage/StorageWritable.scala | 65 ----
.../storage/WriteWriteConflictResolver.scala | 6 +-
.../hbase/AsynchbaseEdgeBulkFetcher.scala | 69 ++++
.../storage/hbase/AsynchbaseEdgeFetcher.scala | 120 ++++++
.../hbase/AsynchbaseOptimisticEdgeFetcher.scala | 35 ++
.../hbase/AsynchbaseOptimisticMutator.scala | 142 +++++++
.../core/storage/hbase/AsynchbaseStorage.scala | 188 +++++++++-
.../hbase/AsynchbaseStorageReadable.scala | 367 -------------------
.../hbase/AsynchbaseStorageWritable.scala | 142 -------
.../hbase/AsynchbaseVertexBulkFetcher.scala | 63 ++++
.../storage/hbase/AsynchbaseVertexFetcher.scala | 61 +++
.../storage/rocks/RocksEdgeBulkFetcher.scala | 68 ++++
.../core/storage/rocks/RocksEdgeFetcher.scala | 60 +++
.../rocks/RocksOptimisticEdgeFetcher.scala | 41 +++
.../storage/rocks/RocksOptimisticMutator.scala | 133 +++++++
.../core/storage/rocks/RocksStorage.scala | 104 +++++-
.../storage/rocks/RocksStorageReadable.scala | 234 ------------
.../storage/rocks/RocksStorageWritable.scala | 133 -------
.../storage/rocks/RocksVertexBulkFetcher.scala | 88 +++++
.../core/storage/rocks/RocksVertexFetcher.scala | 61 +++
.../core/storage/serde/MutationHelper.scala | 188 ----------
.../s2graph/core/utils/ImportStatus.scala | 59 +++
.../apache/s2graph/core/utils/Importer.scala | 122 ++++++
.../s2graph/core/fetcher/EdgeFetcherTest.scala | 87 +++++
.../apache/s2graph/core/model/FetcherTest.scala | 87 -----
48 files changed, 2244 insertions(+), 1761 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
new file mode 100644
index 0000000..646f5f4
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import com.typesafe.config.Config
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait EdgeBulkFetcher {
+ def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
new file mode 100644
index 0000000..f28a161
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.types.VertexId
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait EdgeFetcher {
+
+ def init(config: Config)(implicit ec: ExecutionContext): Unit = {}
+
+ def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
+
+ def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
new file mode 100644
index 0000000..dc0099e
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.storage.MutateResponse
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait EdgeMutator {
+ def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]]
+
+ def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]]
+
+ def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]]
+
+ def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse]
+
+ def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
deleted file mode 100644
index 57d2f29..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/Fetcher.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.types.VertexId
-
-import scala.concurrent.{ExecutionContext, Future}
-
-trait Fetcher {
-
- def init(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] =
- Future.successful(this)
-
- def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
-
- def close(): Unit = {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index 7ff5a9e..9046449 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -19,7 +19,7 @@
package org.apache.s2graph.core
-import java.util
+
import java.util.concurrent.Executors
import com.typesafe.config.{Config, ConfigFactory}
@@ -29,7 +29,7 @@ import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.types.HBaseType._
import org.apache.s2graph.core.types._
import org.apache.s2graph.core.JSONParser._
-import org.apache.s2graph.core.model.Importer
+import org.apache.s2graph.core.utils.Importer
import play.api.libs.json._
import scala.concurrent.{ExecutionContext, Future}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
deleted file mode 100644
index 53161e1..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/Mutator.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core
-
-import org.apache.s2graph.core.storage.{MutateResponse, SKeyValue}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-trait Mutator {
- def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
-
- def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]]
-
- def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]]
-
- def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]]
-
- def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse]
-
- def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
- requestTs: Long,
- retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean]
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 4b2274a..c4cb48f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -27,11 +27,11 @@ import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
import org.apache.s2graph.core.index.IndexProvider
import org.apache.s2graph.core.io.tinkerpop.optimize.S2GraphStepStrategy
-import org.apache.s2graph.core.model.ModelManager
+import org.apache.s2graph.core.fetcher.FetcherManager
import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
import org.apache.s2graph.core.storage.rocks.RocksStorage
-import org.apache.s2graph.core.storage.{MutateResponse, Storage}
+import org.apache.s2graph.core.storage.{MutateResponse, OptimisticEdgeFetcher, Storage}
import org.apache.s2graph.core.types._
import org.apache.s2graph.core.utils.{DeferCache, Extensions, logger}
import org.apache.tinkerpop.gremlin.process.traversal.{P, TraversalStrategies}
@@ -187,7 +187,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override val management = new Management(this)
- override val modelManager = new ModelManager(this)
+ override val modelManager = new FetcherManager(this)
override val indexProvider = IndexProvider.apply(config)
@@ -251,21 +251,34 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
//TODO:
- override def getFetcher(column: ServiceColumn): Fetcher = {
- getStorage(column.service).reader
+ override def getVertexFetcher(column: ServiceColumn): VertexFetcher = {
+ getStorage(column.service).vertexFetcher
+ }
+ override def getVertexBulkFetcher: VertexBulkFetcher = {
+ defaultStorage.vertexBulkFetcher
}
- override def getFetcher(label: Label): Fetcher = {
+ override def getEdgeFetcher(label: Label): EdgeFetcher = {
if (label.fetchConfigExist) modelManager.getFetcher(label)
- else getStorage(label).reader
+ else getStorage(label).edgeFetcher
+ }
+
+ override def getEdgeBulkFetcher: EdgeBulkFetcher = {
+ defaultStorage.edgeBulkFetcher
}
- override def getMutator(column: ServiceColumn): Mutator = {
- getStorage(column.service).mutator
+ override def getVertexMutator(column: ServiceColumn): VertexMutator = {
+ getStorage(column.service).vertexMutator
}
- override def getMutator(label: Label): Mutator = {
- getStorage(label).mutator
+ override def getEdgeMutator(label: Label): EdgeMutator = {
+ getStorage(label).edgeMutator
+ }
+
+ /** optional */
+ override def getOptimisticEdgeFetcher(label: Label): OptimisticEdgeFetcher = {
+// getStorage(label).optimisticEdgeFetcher
+ null
}
//TODO:
@@ -296,8 +309,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]] = {
val verticesWithIdx = vertices.zipWithIndex
- val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
- getStorage(service).fetchVertices(vertices).map(_.zip(vertexGroup.map(_._2)))
+ val futures = verticesWithIdx.groupBy { case (v, idx) => v.serviceColumn }.map { case (serviceColumn, vertexGroup) =>
+ getVertexFetcher(serviceColumn).fetchVertices(vertices).map(_.zip(vertexGroup.map(_._2)))
}
Future.sequence(futures).map { ls =>
@@ -309,7 +322,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val futures = for {
edge <- edges
} yield {
- getStorage(edge.innerLabel).fetchSnapshotEdgeInner(edge).map { case (edgeOpt, _) =>
+ getOptimisticEdgeFetcher(edge.innerLabel).fetchSnapshotEdgeInner(edge).map { case (edgeOpt, _) =>
edgeOpt.toSeq.map(e => EdgeWithScore(e, 1.0, edge.innerLabel))
}
}
@@ -324,7 +337,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
def mutateVertices(storage: Storage)(zkQuorum: String, vertices: Seq[S2VertexLike],
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val futures = vertices.map { vertex =>
- getMutator(vertex.serviceColumn).mutateVertex(zkQuorum, vertex, withWait)
+ getVertexMutator(vertex.serviceColumn).mutateVertex(zkQuorum, vertex, withWait)
}
Future.sequence(futures)
}
@@ -351,7 +364,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val weakEdgesFutures = weakEdges.groupBy { case (edge, idx) => edge.innerLabel.hbaseZkAddr }.map { case (zkQuorum, edgeWithIdxs) =>
val futures = edgeWithIdxs.groupBy(_._1.innerLabel).map { case (label, edgeGroup) =>
- val mutator = getMutator(label)
+ val mutator = getEdgeMutator(label)
val edges = edgeGroup.map(_._1)
val idxs = edgeGroup.map(_._2)
@@ -369,7 +382,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val strongEdgesFutures = strongEdgesAll.groupBy { case (edge, idx) => edge.innerLabel }.map { case (label, edgeGroup) =>
val edges = edgeGroup.map(_._1)
val idxs = edgeGroup.map(_._2)
- val mutator = getMutator(label)
+ val mutator = getEdgeMutator(label)
val zkQuorum = label.hbaseZkAddr
mutator.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
@@ -497,7 +510,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
val edgesWithIdx = edges.zipWithIndex
val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
- getMutator(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
+ getEdgeMutator(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
}
Future.sequence(futures).map { ls =>
@@ -507,7 +520,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
val label = edge.innerLabel
- val mutator = getMutator(label)
+ val mutator = getEdgeMutator(label)
mutator.updateDegree(label.hbaseZkAddr, edge, degreeVal)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 6ed78b0..5e2c168 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -31,9 +31,9 @@ import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core.features.{S2Features, S2GraphVariables}
import org.apache.s2graph.core.index.IndexProvider
-import org.apache.s2graph.core.model.ModelManager
+import org.apache.s2graph.core.fetcher.FetcherManager
import org.apache.s2graph.core.schema.{Label, LabelMeta, Service, ServiceColumn}
-import org.apache.s2graph.core.storage.{MutateResponse, Storage}
+import org.apache.s2graph.core.storage.{MutateResponse, OptimisticEdgeFetcher, Storage}
import org.apache.s2graph.core.types.{InnerValLike, VertexId}
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
import org.apache.tinkerpop.gremlin.structure
@@ -69,7 +69,7 @@ trait S2GraphLike extends Graph {
val traversalHelper: TraversalHelper
- val modelManager: ModelManager
+ val modelManager: FetcherManager
lazy val MaxRetryNum: Int = config.getInt("max.retry.number")
lazy val MaxBackOff: Int = config.getInt("max.back.off")
@@ -93,13 +93,20 @@ trait S2GraphLike extends Graph {
def getStorage(label: Label): Storage
- def getFetcher(column: ServiceColumn): Fetcher
+ def getVertexFetcher(column: ServiceColumn): VertexFetcher
- def getFetcher(label: Label): Fetcher
+ def getVertexBulkFetcher(): VertexBulkFetcher
- def getMutator(label: Label): Mutator
+ def getEdgeFetcher(label: Label): EdgeFetcher
- def getMutator(column: ServiceColumn): Mutator
+ def getEdgeBulkFetcher(): EdgeBulkFetcher
+
+ /** optional */
+ def getOptimisticEdgeFetcher(label: Label): OptimisticEdgeFetcher
+
+ def getEdgeMutator(label: Label): EdgeMutator
+
+ def getVertexMutator(column: ServiceColumn): VertexMutator
def flushStorage(): Unit
@@ -204,7 +211,7 @@ trait S2GraphLike extends Graph {
if (ids.isEmpty) {
//TODO: default storage need to be fixed.
- Await.result(defaultStorage.fetchVerticesAll(), WaitTimeout).iterator
+ Await.result(getVertexBulkFetcher().fetchVerticesAll(), WaitTimeout).iterator
} else {
val vertices = ids.collect {
case s2Vertex: S2VertexLike => s2Vertex
@@ -229,7 +236,7 @@ trait S2GraphLike extends Graph {
def edges(edgeIds: AnyRef*): util.Iterator[structure.Edge] = {
if (edgeIds.isEmpty) {
// FIXME
- Await.result(defaultStorage.fetchEdgesAll(), WaitTimeout).iterator
+ Await.result(getEdgeBulkFetcher().fetchEdgesAll(), WaitTimeout).iterator
} else {
Await.result(edgesAsync(edgeIds: _*), WaitTimeout)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
index d19dd1f..0a4a49b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -204,7 +204,7 @@ class TraversalHelper(graph: S2GraphLike) {
val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
for {
prev <- prevFuture
- cur <- graph.getFetcher(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
+ cur <- graph.getEdgeFetcher(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
} yield {
prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
}
@@ -256,7 +256,7 @@ class TraversalHelper(graph: S2GraphLike) {
*/
graph.mutateEdges(edgesToDelete.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
} else {
- graph.getMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ graph.getEdgeMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
case _ =>
@@ -264,7 +264,7 @@ class TraversalHelper(graph: S2GraphLike) {
* read: x
* write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
*/
- graph.getMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ graph.getEdgeMutator(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
ret
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
new file mode 100644
index 0000000..cbebab5
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait VertexBulkFetcher {
+ def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
new file mode 100644
index 0000000..5c10d18
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.types.VertexId
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait VertexFetcher {
+ def init(config: Config)(implicit ec: ExecutionContext): Unit = {}
+ def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
+ def close(): Unit = {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
new file mode 100644
index 0000000..18be890
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.storage.MutateResponse
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait VertexMutator {
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
new file mode 100644
index 0000000..26db7ff
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.fetcher
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.utils.{Importer, logger}
+import org.apache.s2graph.core.{EdgeFetcher, S2GraphLike}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object FetcherManager {
+ val ClassNameKey = "className"
+}
+
+class FetcherManager(s2GraphLike: S2GraphLike) {
+
+ import FetcherManager._
+
+ private val fetcherPool = scala.collection.mutable.Map.empty[String, EdgeFetcher]
+
+ private val ImportLock = new java.util.concurrent.ConcurrentHashMap[String, Importer]
+
+ def toImportLockKey(label: Label): String = label.label
+
+ def getFetcher(label: Label): EdgeFetcher = {
+ fetcherPool.getOrElse(toImportLockKey(label), throw new IllegalStateException(s"$label is not imported."))
+ }
+
+ def initImporter(config: Config): Importer = {
+ val className = config.getString(ClassNameKey)
+
+ Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(s2GraphLike)
+ .asInstanceOf[Importer]
+ }
+
+ def initFetcher(config: Config)(implicit ec: ExecutionContext): Future[EdgeFetcher] = {
+ val className = config.getString(ClassNameKey)
+
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(s2GraphLike)
+ .asInstanceOf[EdgeFetcher]
+
+ fetcher.init(config)
+
+ Future.successful(fetcher)
+ }
+
+ def importModel(label: Label, config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
+ val importer = ImportLock.computeIfAbsent(toImportLockKey(label), new java.util.function.Function[String, Importer] {
+ override def apply(k: String): Importer = {
+ val importer = initImporter(config.getConfig("importer"))
+
+ //TODO: Update Label's extra options.
+ importer
+ .run(config.getConfig("importer"))
+ .map { importer =>
+ logger.info(s"Close importer")
+ importer.close()
+
+ initFetcher(config.getConfig("fetcher")).map { fetcher =>
+ importer.setStatus(true)
+
+ fetcherPool
+ .remove(k)
+ .foreach { oldFetcher =>
+ logger.info(s"Delete old storage ($k) => $oldFetcher")
+ oldFetcher.close()
+ }
+
+ fetcherPool += (k -> fetcher)
+ }
+ }
+ .onComplete { _ =>
+ logger.info(s"ImportLock release: $k")
+ ImportLock.remove(k)
+ }
+
+ importer
+ }
+ })
+
+ Future.successful(importer)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
new file mode 100644
index 0000000..bf90d69
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.fetcher
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.types.VertexId
+
+import scala.concurrent.{ExecutionContext, Future}
+
+/**
+ * Reference implementation for Fetcher interface.
+ * it only produce constant edges.
+ */
+class MemoryModelEdgeFetcher(val graph: S2GraphLike) extends EdgeFetcher {
+ val builder = graph.elementBuilder
+ val ranges = (0 until 10)
+
+
+ override def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
+ val stepResultLs = queryRequests.map { queryRequest =>
+ val queryParam = queryRequest.queryParam
+ val edges = ranges.map { ith =>
+ val tgtVertexId = builder.newVertexId(queryParam.label.service, queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), ith.toString)
+
+ graph.toEdge(queryRequest.vertex.innerIdVal,
+ tgtVertexId.innerId.value, queryParam.label.label, queryParam.direction)
+ }
+
+ val edgeWithScores = edges.map(e => EdgeWithScore(e, 1.0, queryParam.label))
+ StepResult(edgeWithScores, Nil, Nil)
+ }
+
+ Future.successful(stepResultLs)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
deleted file mode 100644
index 189a6d0..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/model/ImportStatus.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.model
-
-import java.util.concurrent.atomic.AtomicInteger
-
-trait ImportStatus {
- val done: AtomicInteger
-
- def isCompleted: Boolean
-
- def percentage: Int
-
- val total: Int
-}
-
-class ImportRunningStatus(val total: Int) extends ImportStatus {
- require(total > 0, s"Total should be positive: $total")
-
- val done = new AtomicInteger(0)
-
- def isCompleted: Boolean = total == done.get
-
- def percentage = 100 * done.get / total
-}
-
-case object ImportDoneStatus extends ImportStatus {
- val total = 1
-
- val done = new AtomicInteger(1)
-
- def isCompleted: Boolean = true
-
- def percentage = 100
-}
-
-object ImportStatus {
- def apply(total: Int): ImportStatus = new ImportRunningStatus(total)
-
- def unapply(importResult: ImportStatus): Option[(Boolean, Int, Int)] =
- Some((importResult.isCompleted, importResult.total, importResult.done.get))
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
deleted file mode 100644
index e3084dd..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/model/Importer.scala
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.model
-
-import java.io.File
-
-import com.typesafe.config.Config
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.s2graph.core.{Fetcher, S2GraphLike}
-import org.apache.s2graph.core.utils.logger
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object Importer {
- def toHDFSConfiguration(hdfsConfDir: String): Configuration = {
- val conf = new Configuration
-
- val hdfsConfDirectory = new File(hdfsConfDir)
- if (hdfsConfDirectory.exists()) {
- if (!hdfsConfDirectory.isDirectory || !hdfsConfDirectory.canRead) {
- throw new IllegalStateException(s"HDFS configuration directory ($hdfsConfDirectory) cannot be read.")
- }
-
- val path = hdfsConfDirectory.getAbsolutePath
- conf.addResource(new Path(s"file:///$path/core-site.xml"))
- conf.addResource(new Path(s"file:///$path/hdfs-site.xml"))
- } else {
- logger.warn("RocksDBImporter doesn't have valid hadoop configuration directory..")
- }
- conf
- }
-}
-
-trait Importer {
- @volatile var isFinished: Boolean = false
-
- def run(config: Config)(implicit ec: ExecutionContext): Future[Importer]
-
- def status: Boolean = isFinished
-
- def setStatus(otherStatus: Boolean): Boolean = {
- this.isFinished = otherStatus
- this.isFinished
- }
-
- def close(): Unit
-}
-
-case class IdentityImporter(graph: S2GraphLike) extends Importer {
- override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
- Future.successful(this)
- }
-
- override def close(): Unit = {}
-}
-
-object HDFSImporter {
-
- import scala.collection.JavaConverters._
-
- val PathsKey = "paths"
- val HDFSConfDirKey = "hdfsConfDir"
-
- def extractPaths(config: Config): Map[String, String] = {
- config.getConfigList(PathsKey).asScala.map { e =>
- val key = e.getString("src")
- val value = e.getString("tgt")
-
- key -> value
- }.toMap
- }
-}
-
-case class HDFSImporter(graph: S2GraphLike) extends Importer {
-
- import HDFSImporter._
-
- override def run(config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
- Future {
- val paths = extractPaths(config)
- val hdfsConfiDir = config.getString(HDFSConfDirKey)
-
- val hadoopConfig = Importer.toHDFSConfiguration(hdfsConfiDir)
- val fs = FileSystem.get(hadoopConfig)
-
- def copyToLocal(remoteSrc: String, localSrc: String): Unit = {
- val remoteSrcPath = new Path(remoteSrc)
- val localSrcPath = new Path(localSrc)
-
- fs.copyToLocalFile(remoteSrcPath, localSrcPath)
- }
-
- paths.foreach { case (srcPath, tgtPath) =>
- copyToLocal(srcPath, tgtPath)
- }
-
- this
- }
- }
-
- // override def status: ImportStatus = ???
-
- override def close(): Unit = {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
deleted file mode 100644
index 2130066..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/model/MemoryModelFetcher.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.model
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core._
-import org.apache.s2graph.core.types.VertexId
-
-import scala.concurrent.{ExecutionContext, Future}
-
-/**
- * Reference implementation for Fetcher interface.
- * it only produce constant edges.
- */
-class MemoryModelFetcher(val graph: S2GraphLike) extends Fetcher {
- val builder = graph.elementBuilder
- val ranges = (0 until 10)
-
- override def init(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] = {
- Future.successful(this)
- }
-
- override def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
- val stepResultLs = queryRequests.map { queryRequest =>
- val queryParam = queryRequest.queryParam
- val edges = ranges.map { ith =>
- val tgtVertexId = builder.newVertexId(queryParam.label.service, queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), ith.toString)
-
- graph.toEdge(queryRequest.vertex.innerIdVal,
- tgtVertexId.innerId.value, queryParam.label.label, queryParam.direction)
- }
-
- val edgeWithScores = edges.map(e => EdgeWithScore(e, 1.0, queryParam.label))
- StepResult(edgeWithScores, Nil, Nil)
- }
-
- Future.successful(stepResultLs)
- }
-
- override def close(): Unit = {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
deleted file mode 100644
index 3cad13c..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/model/ModelManager.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.model
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.utils.logger
-import org.apache.s2graph.core.{Fetcher, S2GraphLike}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object ModelManager {
- val ClassNameKey = "className"
-}
-
-class ModelManager(s2GraphLike: S2GraphLike) {
-
- import ModelManager._
-
- private val fetcherPool = scala.collection.mutable.Map.empty[String, Fetcher]
- private val ImportLock = new java.util.concurrent.ConcurrentHashMap[String, Importer]
-
- def toImportLockKey(label: Label): String = label.label
-
- def getFetcher(label: Label): Fetcher = {
- fetcherPool.getOrElse(toImportLockKey(label), throw new IllegalStateException(s"$label is not imported."))
- }
-
- def initImporter(config: Config): Importer = {
- val className = config.getString(ClassNameKey)
-
- Class.forName(className)
- .getConstructor(classOf[S2GraphLike])
- .newInstance(s2GraphLike)
- .asInstanceOf[Importer]
- }
-
- def initFetcher(config: Config)(implicit ec: ExecutionContext): Future[Fetcher] = {
- val className = config.getString(ClassNameKey)
-
- val fetcher = Class.forName(className)
- .getConstructor(classOf[S2GraphLike])
- .newInstance(s2GraphLike)
- .asInstanceOf[Fetcher]
-
- fetcher.init(config)
- }
-
- def importModel(label: Label, config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
- val importer = ImportLock.computeIfAbsent(toImportLockKey(label), new java.util.function.Function[String, Importer] {
- override def apply(k: String): Importer = {
- val importer = initImporter(config.getConfig("importer"))
-
- //TODO: Update Label's extra options.
- importer
- .run(config.getConfig("importer"))
- .map { importer =>
- logger.info(s"Close importer")
- importer.close()
-
- initFetcher(config.getConfig("fetcher")).map { fetcher =>
- importer.setStatus(true)
-
- fetcherPool
- .remove(k)
- .foreach { oldFetcher =>
- logger.info(s"Delete old storage ($k) => $oldFetcher")
- oldFetcher.close()
- }
-
- fetcherPool += (k -> fetcher)
- }
- }
- .onComplete { _ =>
- logger.info(s"ImportLock release: $k")
- ImportLock.remove(k)
- }
-
- importer
- }
- })
-
- Future.successful(importer)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
new file mode 100644
index 0000000..82cc27a
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.s2graph.core.storage
+
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.schema.LabelMeta
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class DefaultOptimisticMutator(graph: S2GraphLike,
+ serDe: StorageSerDe,
+ optimisticEdgeFetcher: OptimisticEdgeFetcher,
+ optimisticMutator: OptimisticMutator) extends VertexMutator with EdgeMutator {
+
+ lazy val io: StorageIO = new StorageIO(graph, serDe)
+
+ lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, optimisticMutator, optimisticEdgeFetcher)
+
+ private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
+ optimisticMutator.writeToStorage(cluster, kvs, withWait)
+
+ def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
+ if (stepInnerResult.isEmpty) Future.successful(true)
+ else {
+ val head = stepInnerResult.edgeWithScores.head
+ val zkQuorum = head.edge.innerLabel.hbaseZkAddr
+ val futures = for {
+ edgeWithScore <- stepInnerResult.edgeWithScores
+ } yield {
+ val edge = edgeWithScore.edge
+
+ val edgeSnapshot = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+
+ val edgeForward = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ /* reverted direction */
+ val edgeRevert = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
+
+ writeToStorage(zkQuorum, mutations, withWait = true)
+ }
+
+ Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
+ }
+ }
+
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ if (vertex.op == GraphUtil.operations("delete")) {
+ writeToStorage(zkQuorum,
+ serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
+ } else if (vertex.op == GraphUtil.operations("deleteAll")) {
+ logger.info(s"deleteAll for vertex is truncated. $vertex")
+ Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
+ } else {
+ writeToStorage(zkQuorum, io.buildPutsAll(vertex), withWait)
+ }
+ }
+
+ def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] = {
+ val mutations = _edges.flatMap { edge =>
+ val (_, edgeUpdate) =
+ if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
+ else S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
+ io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+ }
+
+ writeToStorage(zkQuorum, mutations, withWait).map { ret =>
+ _edges.zipWithIndex.map { case (edge, idx) =>
+ idx -> ret.isSuccess
+ }
+ }
+ }
+
+ def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ def mutateEdgesInner(edges: Seq[S2EdgeLike],
+ checkConsistency: Boolean,
+ withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ assert(edges.nonEmpty)
+ // TODO:: remove after code review: unreachable code
+ if (!checkConsistency) {
+
+ val futures = edges.map { edge =>
+ val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+ val mutations =
+ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
+
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
+ } else {
+ optimisticEdgeFetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
+ conflictResolver.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
+ }
+ }
+ }
+
+ val edgeWithIdxs = _edges.zipWithIndex
+ val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
+ (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
+ } toSeq
+
+ val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
+ val edges = edgeGroup.map(_._1)
+ val idxs = edgeGroup.map(_._2)
+ // After deleteAll, process others
+ val mutateEdgeFutures = edges.toList match {
+ case head :: tail =>
+ val edgeFuture = mutateEdgesInner(edges, checkConsistency = true, withWait)
+
+ //TODO: decide what we will do on failure on vertex put
+ val puts = io.buildVertexPutsAsync(head)
+ val vertexFuture = writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
+ Seq(edgeFuture, vertexFuture)
+ case Nil => Nil
+ }
+
+ val composed = for {
+ // deleteRet <- Future.sequence(deleteAllFutures)
+ mutateRet <- Future.sequence(mutateEdgeFutures)
+ } yield mutateRet
+
+ composed.map(_.forall(_.isSuccess)).map { ret => idxs.map(idx => idx -> ret) }
+ }
+
+ Future.sequence(mutateEdges).map { squashedRets =>
+ squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
+ }
+ }
+
+ def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] = {
+ val futures = for {
+ edge <- edges
+ } yield {
+ val kvs = for {
+ relEdge <- edge.relatedEdges
+ edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
+ } yield {
+ val countWithTs = edge.propertyValueInner(LabelMeta.count)
+ val countVal = countWithTs.innerVal.toString().toLong
+ io.buildIncrementsCountAsync(edgeWithIndex, countVal).head
+ }
+ writeToStorage(zkQuorum, kvs, withWait = withWait)
+ }
+
+ Future.sequence(futures)
+ }
+
+ def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ val kvs = io.buildDegreePuts(edge, degreeVal)
+
+ writeToStorage(zkQuorum, kvs, withWait = true)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticEdgeFetcher.scala
new file mode 100644
index 0000000..4111cc4
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticEdgeFetcher.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage
+
+import org.apache.s2graph.core.GraphExceptions.FetchTimeoutException
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait OptimisticEdgeFetcher {
+ val io: StorageIO
+ protected def fetchKeyValues(queryRequest: QueryRequest,
+ edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
+
+ def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] = {
+ val queryParam = QueryParam(labelName = edge.innerLabel.label,
+ direction = GraphUtil.fromDirection(edge.getDir()),
+ tgtVertexIdOpt = Option(edge.tgtVertex.innerIdVal),
+ cacheTTLInMillis = -1)
+ val q = Query.toQuery(Seq(edge.srcVertex), Seq(queryParam))
+ val queryRequest = QueryRequest(q, 0, edge.srcVertex, queryParam)
+
+ fetchKeyValues(queryRequest, edge).map { kvs =>
+ val (edgeOpt, kvOpt) =
+ if (kvs.isEmpty) (None, None)
+ else {
+ import CanSKeyValue._
+ val snapshotEdgeOpt = io.toSnapshotEdge(kvs.head, queryRequest, isInnerCall = true, parentEdges = Nil)
+ val _kvOpt = kvs.headOption
+ (snapshotEdgeOpt, _kvOpt)
+ }
+ (edgeOpt, kvOpt)
+ } recoverWith { case ex: Throwable =>
+ logger.error(s"fetchQueryParam failed. fallback return.", ex)
+ throw new FetchTimeoutException(s"${edge.toLogString}")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticMutator.scala
new file mode 100644
index 0000000..22269df
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/OptimisticMutator.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait OptimisticMutator {
+ /**
+ * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
+ * note that this should be return true on all success.
+ * we assumes that each storage implementation has client as member variable.
+ *
+ * @param cluster : where this key values should be stored.
+ * @param kvs : sequence of SKeyValue that need to be stored in storage.
+ * @param withWait : flag to control wait ack from storage.
+ * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
+ * it never block thread, but rather submit work and notified by event loop when storage send ack back.
+ * @return ack message from storage.
+ */
+ def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
+
+ /**
+ * write requestKeyValue into storage if the current value in storage that is stored matches.
+ * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
+ *
+ * Most important thing is this have to be 'atomic' operation.
+ * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
+ * either blocked or failed on write-write conflict case.
+ *
+ * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
+ * prevent wrong data for read.
+ *
+ * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
+ * compareAndSet to synchronize.
+ *
+ * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
+ * for storage that does not support concurrency control, then storage implementation
+ * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
+ * and write(writeLock).
+ *
+ * @param requestKeyValue
+ * @param expectedOpt
+ * @return
+ */
+ def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index d2500a6..36ecfcb 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -34,52 +34,30 @@ abstract class Storage(val graph: S2GraphLike,
val management: StorageManagement
/*
- * Given QueryRequest/Vertex/Edge, fetch KeyValue from storage
- * then convert them into Edge/Vertex
- */
- val reader: StorageReadable
-
- /*
- * Serialize Edge/Vertex, to common KeyValue, SKeyValue that
- * can be stored aligned to backend storage's physical schema.
- * Also Deserialize storage backend's KeyValue to SKeyValue.
- */
+ * Serialize Edge/Vertex, to common KeyValue, SKeyValue that
+ * can be stored aligned to backend storage's physical schema.
+ * Also Deserialize storage backend's KeyValue to SKeyValue.
+ */
val serDe: StorageSerDe
- /*
- * Responsible to connect physical storage backend to store GraphElement(Edge/Vertex).
- */
- val mutator: Mutator
+ val edgeFetcher: EdgeFetcher
- /*
- * Common helper to translate SKeyValue to Edge/Vertex and vice versa.
- * Note that it require storage backend specific implementation for serialize/deserialize.
- */
- lazy val io: StorageIO = new StorageIO(graph, serDe)
+ val edgeBulkFetcher: EdgeBulkFetcher
- /*
- * Common helper to resolve write-write conflict on snapshot edge with same EdgeId.
- * Note that it require storage backend specific implementations for
- * all of StorageWritable, StorageReadable, StorageSerDe, StorageIO
- */
-// lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, mutator, reader)
-// lazy val mutationHelper: MutationHelper = new MutationHelper(this)
+ val vertexFetcher: VertexFetcher
+ val vertexBulkFetcher: VertexBulkFetcher
- /** Fetch **/
- def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] =
- reader.fetches(queryRequests, prevStepEdges)
+ val edgeMutator: EdgeMutator
- def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] =
- reader.fetchVertices(vertices)
+ val vertexMutator: VertexMutator
- def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = reader.fetchEdgesAll()
-
- def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = reader.fetchVerticesAll()
+ /*
+ * Common helper to translate SKeyValue to Edge/Vertex and vice versa.
+ * Note that it require storage backend specific implementation for serialize/deserialize.
+ */
+ lazy val io: StorageIO = new StorageIO(graph, serDe)
- def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] =
- reader.fetchSnapshotEdgeInner(edge)
/** Management **/
def flush(): Unit = management.flush()
@@ -94,24 +72,4 @@ abstract class Storage(val graph: S2GraphLike,
def info: Map[String, String] = Map("className" -> this.getClass.getSimpleName)
- def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
- requestTs: Long,
- retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] =
- mutator.deleteAllFetchedEdgesAsyncOld(stepInnerResult, requestTs, retryNum)
-
- def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutator.mutateVertex(zkQuorum: String, vertex, withWait)
-
- def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] =
- mutator.mutateStrongEdges(zkQuorum, _edges, withWait)
-
-
- def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] =
- mutator.mutateWeakEdges(zkQuorum, _edges, withWait)
-
- def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] =
- mutator.incrementCounts(zkQuorum, edges, withWait)
-
- def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] =
- mutator.updateDegree(zkQuorum, edge, degreeVal)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
index b10feb9..c3abd03 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
@@ -27,7 +27,7 @@ import org.apache.s2graph.core.utils.logger
import scala.concurrent.{ExecutionContext, Future}
-trait StorageReadable extends Fetcher {
+trait StorageReadable extends EdgeFetcher {
val io: StorageIO
val serDe: StorageSerDe
// /**
@@ -44,9 +44,14 @@ trait StorageReadable extends Fetcher {
def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
+
+
+
+
+
protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
- protected def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
+// protected def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] = {
@@ -73,25 +78,25 @@ trait StorageReadable extends Fetcher {
}
}
- def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = {
- def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2VertexLike] = {
- if (kvs.isEmpty) Nil
- else serDe.vertexDeserializer(version).fromKeyValues(kvs, None).toSeq
- }
-
- val futures = vertices.map { vertex =>
- val queryParam = QueryParam.Empty
- val q = Query.toQuery(Seq(vertex), Seq(queryParam))
- val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
-
- fetchKeyValues(queryRequest, vertex).map { kvs =>
- fromResult(kvs, vertex.serviceColumn.schemaVersion)
- } recoverWith {
- case ex: Throwable => Future.successful(Nil)
- }
- }
-
- Future.sequence(futures).map(_.flatten)
- }
+// def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = {
+// def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2VertexLike] = {
+// if (kvs.isEmpty) Nil
+// else serDe.vertexDeserializer(version).fromKeyValues(kvs, None).toSeq
+// }
+//
+// val futures = vertices.map { vertex =>
+// val queryParam = QueryParam.Empty
+// val q = Query.toQuery(Seq(vertex), Seq(queryParam))
+// val queryRequest = QueryRequest(q, stepIdx = -1, vertex, queryParam)
+//
+// fetchKeyValues(queryRequest, vertex).map { kvs =>
+// fromResult(kvs, vertex.serviceColumn.schemaVersion)
+// } recoverWith {
+// case ex: Throwable => Future.successful(Nil)
+// }
+// }
+//
+// Future.sequence(futures).map(_.flatten)
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
deleted file mode 100644
index 8c2fb27..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageWritable.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage
-
-import org.apache.s2graph.core.Mutator
-
-import scala.concurrent.{ExecutionContext, Future}
-
-trait OptimisticMutator extends Mutator {
- /**
- * decide how to store given key values Seq[SKeyValue] into storage using storage's client.
- * note that this should be return true on all success.
- * we assumes that each storage implementation has client as member variable.
- *
- * @param cluster : where this key values should be stored.
- * @param kvs : sequence of SKeyValue that need to be stored in storage.
- * @param withWait : flag to control wait ack from storage.
- * note that in AsynchbaseStorage(which support asynchronous operations), even with true,
- * it never block thread, but rather submit work and notified by event loop when storage send ack back.
- * @return ack message from storage.
- */
- def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
-
- /**
- * write requestKeyValue into storage if the current value in storage that is stored matches.
- * note that we only use SnapshotEdge as place for lock, so this method only change SnapshotEdge.
- *
- * Most important thing is this have to be 'atomic' operation.
- * When this operation is mutating requestKeyValue's snapshotEdge, then other thread need to be
- * either blocked or failed on write-write conflict case.
- *
- * Also while this method is still running, then fetchSnapshotEdgeKeyValues should be synchronized to
- * prevent wrong data for read.
- *
- * Best is use storage's concurrency control(either pessimistic or optimistic) such as transaction,
- * compareAndSet to synchronize.
- *
- * for example, AsynchbaseStorage use HBase's CheckAndSet atomic operation to guarantee 'atomicity'.
- * for storage that does not support concurrency control, then storage implementation
- * itself can maintain manual locks that synchronize read(fetchSnapshotEdgeKeyValues)
- * and write(writeLock).
- *
- * @param requestKeyValue
- * @param expectedOpt
- * @return
- */
- def writeLock(requestKeyValue: SKeyValue, expectedOpt: Option[SKeyValue])(implicit ec: ExecutionContext): Future[MutateResponse]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
index bfc5bc6..18159f6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
@@ -32,7 +32,7 @@ class WriteWriteConflictResolver(graph: S2GraphLike,
serDe: StorageSerDe,
io: StorageIO,
mutator: OptimisticMutator,
- fetcher: StorageReadable) {
+ optimisticEdgeFetcher: OptimisticEdgeFetcher) {
val BackoffTimeout = graph.BackoffTimeout
val MaxRetryNum = graph.MaxRetryNum
val MaxBackOff = graph.MaxBackOff
@@ -68,7 +68,7 @@ class WriteWriteConflictResolver(graph: S2GraphLike,
case FetchTimeoutException(retryEdge) =>
logger.info(s"[Try: $tryNum], Fetch fail.\n${retryEdge}")
/* fetch failed. re-fetch should be done */
- fetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
+ optimisticEdgeFetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
retry(tryNum + 1)(edges, statusCode, snapshotEdgeOpt)
}
@@ -90,7 +90,7 @@ class WriteWriteConflictResolver(graph: S2GraphLike,
val future = if (failedStatusCode == 0) {
// acquire Lock failed. other is mutating so this thead need to re-fetch snapshotEdge.
/* fetch failed. re-fetch should be done */
- fetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
+ optimisticEdgeFetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
retry(tryNum + 1)(edges, statusCode, snapshotEdgeOpt)
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/43f627e5/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
new file mode 100644
index 0000000..3d25dd9
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage.hbase
+
+import java.util
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.storage.serde.Serializable
+import org.apache.s2graph.core.{EdgeBulkFetcher, S2EdgeLike, S2Graph, S2GraphLike}
+import org.apache.s2graph.core.storage.{CanSKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.s2graph.core.utils.{CanDefer, Extensions}
+import org.hbase.async.{HBaseClient, KeyValue}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class AsynchbaseEdgeBulkFetcher(val graph: S2GraphLike,
+ val config: Config,
+ val client: HBaseClient,
+ val serDe: StorageSerDe,
+ val io: StorageIO) extends EdgeBulkFetcher {
+ import Extensions.DeferOps
+ import CanDefer._
+ import scala.collection.JavaConverters._
+ import AsynchbaseStorage._
+
+ override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
+ val futures = Label.findAll().groupBy(_.hbaseTableName).toSeq.map { case (hTableName, labels) =>
+ val distinctLabels = labels.toSet
+ val scan = AsynchbasePatcher.newScanner(client, hTableName)
+ scan.setFamily(Serializable.edgeCf)
+ scan.setMaxVersions(1)
+
+ scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
+ case null => Seq.empty
+ case kvsLs =>
+ kvsLs.asScala.flatMap { kvs =>
+ kvs.asScala.flatMap { kv =>
+ val sKV = implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
+
+ serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION)
+ .fromKeyValues(Seq(kv), None)
+ .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
+ }
+ }
+ }
+ }
+
+ Future.sequence(futures).map(_.flatten)
+ }
+}
[10/11] incubator-s2graph git commit: Add preload all fetchers and
mutators that registered in serviceColumn/label while initialize new S2Graph
instance.
Posted by st...@apache.org.
Add preload all fetchers and mutators that registered in serviceColumn/label while initialize new S2Graph instance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/16feda80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/16feda80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/16feda80
Branch: refs/heads/master
Commit: 16feda80fceb7604932b32a2884c7551f570a38b
Parents: e823041
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu May 10 15:40:06 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu May 10 15:41:26 2018 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/GraphUtil.scala | 7 ++
.../org/apache/s2graph/core/Management.scala | 34 ++++-----
.../scala/org/apache/s2graph/core/S2Graph.scala | 28 ++++++++
.../apache/s2graph/core/S2GraphFactory.scala | 9 ++-
.../s2graph/core/rest/RequestParser.scala | 8 ++-
.../org/apache/s2graph/core/schema/Label.scala | 6 +-
.../s2graph/core/TestCommonWithModels.scala | 14 ++--
.../s2graph/core/fetcher/EdgeFetcherTest.scala | 30 ++++----
.../core/tinkerpop/S2GraphProvider.scala | 73 +++++++++-----------
.../core/tinkerpop/structure/S2GraphTest.scala | 6 +-
.../graphql/repository/GraphRepository.scala | 2 +-
.../apache/s2graph/s2jobs/BaseSparkTest.scala | 11 +--
12 files changed, 136 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
index a4f6bde..e08bb4e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
@@ -164,4 +164,11 @@ object GraphUtil {
}
}
+ def stringToOption(s: Option[String]): Option[String] = {
+ s.filter(_.trim.nonEmpty)
+ }
+
+ def stringToOption(s: String): Option[String] = {
+ Option(s).filter(_.trim.nonEmpty)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index f534423..f64e058 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -304,15 +304,16 @@ class Management(graph: S2GraphLike) {
val importEx = ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor())
import Management._
+ import GraphUtil._
def updateEdgeFetcher(labelName: String, options: String): Unit = {
val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
- updateEdgeFetcher(label, options)
+ updateEdgeFetcher(label, stringToOption(options))
}
- def updateEdgeFetcher(label: Label, options: String): Unit = {
- val newLabel = Label.updateOption(label, options)
+ def updateEdgeFetcher(label: Label, options: Option[String]): Unit = {
+ val newLabel = options.map(Label.updateOption(label, _)).getOrElse(label)
graph.resourceManager.getOrElseUpdateEdgeFetcher(newLabel, cacheTTLInSecs = Option(-1))
}
@@ -320,22 +321,22 @@ class Management(graph: S2GraphLike) {
val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName is not exist."))
val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName is not exist."))
- updateVertexFetcher(column, options)
+ updateVertexFetcher(column, stringToOption(options))
}
- def updateVertexFetcher(column: ServiceColumn, options: String): Unit = {
- val newColumn = ServiceColumn.updateOption(column, options)
+ def updateVertexFetcher(column: ServiceColumn, options: Option[String]): Unit = {
+ val newColumn = options.map(ServiceColumn.updateOption(column, _)).getOrElse(column)
graph.resourceManager.getOrElseUpdateVertexFetcher(newColumn, cacheTTLInSecs = Option(-1))
}
def updateEdgeMutator(labelName: String, options: String): Unit = {
val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
- updateEdgeMutator(label, options)
+ updateEdgeMutator(label, stringToOption(options))
}
- def updateEdgeMutator(label: Label, options: String): Unit = {
- val newLabel = Label.updateOption(label, options)
+ def updateEdgeMutator(label: Label, options: Option[String]): Unit = {
+ val newLabel = options.map(Label.updateOption(label, _)).getOrElse(label)
graph.resourceManager.getOrElseUpdateEdgeMutator(newLabel, cacheTTLInSecs = Option(-1))
}
@@ -343,11 +344,11 @@ class Management(graph: S2GraphLike) {
val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName is not exist."))
val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName is not exist."))
- updateVertexMutator(column, options)
+ updateVertexMutator(column, stringToOption(options))
}
- def updateVertexMutator(column: ServiceColumn, options: String): Unit = {
- val newColumn = ServiceColumn.updateOption(column, options)
+ def updateVertexMutator(column: ServiceColumn, options: Option[String]): Unit = {
+ val newColumn = options.map(ServiceColumn.updateOption(column, _)).getOrElse(column)
graph.resourceManager.getOrElseUpdateVertexMutator(newColumn, cacheTTLInSecs = Option(-1))
}
@@ -452,9 +453,9 @@ class Management(graph: S2GraphLike) {
createLabel(labelName,
srcColumn.service.serviceName, srcColumn.columnName, srcColumn.columnType,
tgtColumn.service.serviceName, tgtColumn.columnName, tgtColumn.columnType,
- isDirected, serviceName, indices, props, consistencyLevel,
+ serviceName, indices, props, isDirected, consistencyLevel,
Option(hTableName), Option(hTableTTL).filter(_ > -1),
- schemaVersion, false, compressionAlgorithm, Option(options)
+ schemaVersion, false, compressionAlgorithm, stringToOption(options)
).get
}
@@ -466,10 +467,10 @@ class Management(graph: S2GraphLike) {
tgtServiceName: String,
tgtColumnName: String,
tgtColumnType: String,
- isDirected: Boolean = true,
serviceName: String,
indices: Seq[Index],
props: Seq[Prop],
+ isDirected: Boolean = true,
consistencyLevel: String = "weak",
hTableName: Option[String] = None,
hTableTTL: Option[Int] = None,
@@ -523,8 +524,9 @@ class Management(graph: S2GraphLike) {
createLabel(newLabelName, old.srcService.serviceName, old.srcColumnName, old.srcColumnType,
old.tgtService.serviceName, old.tgtColumnName, old.tgtColumnType,
- old.isDirected, old.serviceName,
+ old.serviceName,
allIndices, allProps,
+ old.isDirected,
old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm, old.options)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 09fd55e..004b6e8 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -159,6 +159,30 @@ object S2Graph {
}
ConfigFactory.parseMap(kvs)
}
+
+ def initMutators(graph: S2GraphLike): Unit = {
+ val management = graph.management
+
+ ServiceColumn.findAll().foreach { column =>
+ management.updateVertexMutator(column, column.options)
+ }
+
+ Label.findAll().foreach { label =>
+ management.updateEdgeMutator(label, label.options)
+ }
+ }
+
+ def initFetchers(graph: S2GraphLike): Unit = {
+ val management = graph.management
+
+ ServiceColumn.findAll().foreach { column =>
+ management.updateVertexFetcher(column, column.options)
+ }
+
+ Label.findAll().foreach { label =>
+ management.updateEdgeFetcher(label, label.options)
+ }
+ }
}
class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2GraphLike {
@@ -197,6 +221,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
conf.withFallback(config)
}
+ S2Graph.initMutators(this)
+ S2Graph.initFetchers(this)
+
val defaultStorage: Storage = S2Graph.initStorage(this, config)(ec)
for {
@@ -248,6 +275,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
storagePool.getOrElse(s"label:${label.label}", defaultStorage)
}
+
/* Currently, each getter on Fetcher and Mutator missing proper implementation
* Please discuss what is proper way to maintain resources here and provide
* right implementation(S2GRAPH-213).
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
index 0a667ef..e79b831 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
@@ -97,7 +97,8 @@ object S2GraphFactory {
// Management.deleteLabel("_s2graph")
val DefaultLabel = management.createLabel("_s2graph", DefaultService.serviceName, DefaultColumn.columnName, DefaultColumn.columnType,
- DefaultService.serviceName, DefaultColumn.columnName, DefaultColumn.columnType, true, DefaultService.serviceName, Nil, Nil, "weak", None, None,
+ DefaultService.serviceName, DefaultColumn.columnName, DefaultColumn.columnType, DefaultService.serviceName, Nil, Nil,
+ isDirected = true, consistencyLevel = "weak", hTableName = None, hTableTTL = None,
options = Option("""{"skipReverse": false}""")
)
}
@@ -111,12 +112,14 @@ object S2GraphFactory {
val knows = mnt.createLabel("knows",
S2Graph.DefaultServiceName, "person", "integer",
S2Graph.DefaultServiceName, "person", "integer",
- true, S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double"), Prop("year", "0", "integer")), consistencyLevel = "strong", None, None)
+ S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double"), Prop("year", "0", "integer")),
+ isDirected = true, consistencyLevel = "strong", hTableName = None, hTableTTL = None)
val created = mnt.createLabel("created",
S2Graph.DefaultServiceName, "person", "integer",
S2Graph.DefaultServiceName, "software", "integer",
- true, S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double")), "strong", None, None)
+ S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double")),
+ isDirected = true, consistencyLevel = "strong", hTableName = None, hTableTTL = None)
}
def cleanupDefaultSchema(): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index f59abc0..7f8b280 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -652,9 +652,11 @@ class RequestParser(graph: S2GraphLike) {
val compressionAlgorithm = (jsValue \ "compressionAlgorithm").asOpt[String].getOrElse(DefaultCompressionAlgorithm)
val options = (jsValue \ "options").asOpt[JsValue].map(_.toString())
- graph.management.createLabel(labelName, srcServiceName, srcColumnName, srcColumnType,
- tgtServiceName, tgtColumnName, tgtColumnType, isDirected, serviceName,
- indices, allProps, consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm, options)
+ graph.management.createLabel(labelName,
+ srcServiceName, srcColumnName, srcColumnType,
+ tgtServiceName, tgtColumnName, tgtColumnType, serviceName,
+ indices, allProps, isDirected,
+ consistencyLevel, hTableName, hTableTTL, schemaVersion, isAsync, compressionAlgorithm, options)
}
def toIndexElements(jsValue: JsValue) = Try {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
index f3ce5e0..a359958 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
@@ -33,6 +33,8 @@ import scalikejdbc._
object Label extends SQLSyntaxSupport[Label] {
import Schema._
+ import GraphUtil._
+
val className = Label.getClass.getSimpleName
val maxHBaseTableNames = 2
@@ -43,7 +45,7 @@ object Label extends SQLSyntaxSupport[Label] {
rs.int("tgt_service_id"), rs.string("tgt_column_name"), rs.string("tgt_column_type"),
rs.boolean("is_directed"), rs.string("service_name"), rs.int("service_id"), rs.string("consistency_level"),
rs.string("hbase_table_name"), rs.intOpt("hbase_table_ttl"), rs.string("schema_version"), rs.boolean("is_async"),
- rs.string("compressionAlgorithm"), rs.stringOpt("options"))
+ rs.string("compressionAlgorithm"), stringToOption(rs.stringOpt("options")))
}
def deleteAll(label: Label)(implicit session: DBSession) = {
@@ -264,7 +266,7 @@ object Label extends SQLSyntaxSupport[Label] {
}
def updateOption(label: Label, options: String)(implicit session: DBSession = AutoSession) = {
- scala.util.Try(Json.parse(options)).getOrElse(throw new RuntimeException("invalid Json option"))
+ scala.util.Try(Json.parse(options)).getOrElse(throw new RuntimeException(s"invalid Json option: $options"))
logger.info(s"update options of label ${label.label}, ${options}")
val cnt = sql"""update labels set options = $options where id = ${label.id.get}""".update().apply()
val updatedLabel = findById(label.id.get, useCache = false)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
index 488b294..fb8de32 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
@@ -139,25 +139,25 @@ trait TestCommonWithModels {
implicit val session = AutoSession
management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
- isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
+ serviceName, testIdxProps, testProps, isDirected = true, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
management.createLabel(labelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
- isDirected = true, serviceNameV2, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
+ serviceNameV2, testIdxProps, testProps, isDirected = true, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
management.createLabel(labelNameV3, serviceNameV3, columnNameV3, columnTypeV3, serviceNameV3, tgtColumnNameV3, tgtColumnTypeV3,
- isDirected = true, serviceNameV3, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4", None)
+ serviceNameV3, testIdxProps, testProps, isDirected = true, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4", None)
management.createLabel(labelNameV4, serviceNameV4, columnNameV4, columnTypeV4, serviceNameV4, tgtColumnNameV4, tgtColumnTypeV4,
- isDirected = true, serviceNameV4, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4", None)
+ serviceNameV4, testIdxProps, testProps, isDirected = true, consistencyLevel, Some(hTableName), hTableTTL, VERSION4, false, "lg4", None)
management.createLabel(undirectedLabelName, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType,
- isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4", None)
+ serviceName, testIdxProps, testProps, isDirected = false, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4", None)
management.createLabel(undirectedLabelNameV2, serviceNameV2, columnNameV2, columnTypeV2, serviceNameV2, tgtColumnNameV2, tgtColumnTypeV2,
- isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
+ serviceName, testIdxProps, testProps, isDirected = false, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4", None)
management.createLabel(labelNameSecure, serviceName, columnName, columnType, serviceName, tgtColumnName, tgtColumnType,
- isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4",
+ serviceName, testIdxProps, testProps, isDirected = false, consistencyLevel, Some(hTableName), hTableTTL, VERSION3, false, "lg4",
Option("""{ "tokens": ["xxx-yyy", "aaa-bbb"] }"""))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
index 930c517..d78538d 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
@@ -55,21 +55,25 @@ class EdgeFetcherTest extends IntegrateCommon {
val label = management.createLabel(
labelName,
- serviceColumn,
- serviceColumn,
- true,
service.serviceName,
- Seq.empty[Index].asJava,
- Seq.empty[Prop].asJava,
- "strong",
- null,
- -1,
- "v3",
- "gz",
- options
- )
+ serviceColumn.columnName,
+ serviceColumn.columnType,
+ service.serviceName,
+ serviceColumn.columnName,
+ serviceColumn.columnType,
+ service.serviceName,
+ Seq.empty[Index],
+ Seq.empty[Prop],
+ isDirected = true,
+ consistencyLevel = "strong",
+ hTableName = None,
+ hTableTTL = None,
+ schemaVersion = "v3",
+ compressionAlgorithm = "gz",
+ options = Option(options)
+ ).get
- graph.management.updateEdgeFetcher(label, options)
+ graph.management.updateEdgeFetcher(label, Option(options))
val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "daewon")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
index 0467f7d..766e3c4 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
@@ -147,10 +147,10 @@ class S2GraphProvider extends AbstractGraphProvider {
}
if (loadGraphWith != null && loadGraphWith.value() == GraphData.MODERN) {
mnt.createLabel("knows", defaultService.serviceName, "person", "integer", defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil, knowsProp, "strong", None, None, options = Option("""{"skipReverse": false}"""))
+ defaultService.serviceName, Nil, knowsProp, true, "strong", None, None, options = Option("""{"skipReverse": false}"""))
} else {
mnt.createLabel("knows", defaultService.serviceName, "vertex", "integer", defaultService.serviceName, "vertex", "integer",
- true, defaultService.serviceName, Nil, knowsProp, "strong", None, None, options = Option("""{"skipReverse": false}"""))
+ defaultService.serviceName, Nil, knowsProp, true, "strong", None, None, options = Option("""{"skipReverse": false}"""))
}
// columns
@@ -193,24 +193,24 @@ class S2GraphProvider extends AbstractGraphProvider {
mnt.createLabel("created",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "software", "integer",
- true, defaultService.serviceName, Nil, createdProps, "strong", None, None)
+ defaultService.serviceName, Nil, createdProps, true, "strong", None, None)
} else {
mnt.createLabel("created",
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, createdProps, "strong", None, None)
+ defaultService.serviceName, Nil, createdProps, true, "strong", None, None)
}
val boughtProps = Seq(Prop("x", "-", "string"), Prop("y", "-", "string"))
allProps ++= boughtProps
val bought = mnt.createLabel("bought", defaultService.serviceName, "person", "integer", defaultService.serviceName, "product", "integer",
- true, defaultService.serviceName, Nil, boughtProps, "strong", None, None,
+ defaultService.serviceName, Nil, boughtProps, true, "strong", None, None,
options = Option("""{"skipReverse": true}"""))
val testProps = Seq(Prop("xxx", "-", "string"))
allProps ++= testProps
val test = mnt.createLabel("test", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, testProps, "weak", None, None,
+ defaultService.serviceName, Nil, testProps, true, "weak", None, None,
options = Option("""{"skipReverse": true}"""))
val selfProps = Seq(Prop("__id", "-", "string"), Prop("acl", "-", "string"),
@@ -222,12 +222,12 @@ class S2GraphProvider extends AbstractGraphProvider {
mnt.createLabel("self",
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, selfProps, "strong", None, None,
+ defaultService.serviceName, Nil, selfProps, true, "strong", None, None,
options = Option("""{"skipReverse": true}"""))
} else {
mnt.createLabel("self", defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil, selfProps, "strong", None, None,
+ defaultService.serviceName, Nil, selfProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}"""))
}
@@ -239,13 +239,13 @@ class S2GraphProvider extends AbstractGraphProvider {
mnt.createLabel("friends",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil, friendsProps,
+ defaultService.serviceName, Nil, friendsProps, true,
"strong", None, None,
options = Option("""{"skipReverse": false}"""))
} else {
mnt.createLabel("friends", defaultService.serviceName, defaultServiceColumn.columnName,
defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, Nil,
+ defaultService.serviceName, Nil, Nil, true,
"strong", None, None,
options = Option("""{"skipReverse": false}"""))
}
@@ -264,15 +264,13 @@ class S2GraphProvider extends AbstractGraphProvider {
mnt.createLabel("friend",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- friendProps, "strong", None, None,
+ defaultService.serviceName, Nil, friendProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}"""))
} else {
mnt.createLabel("friend",
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil,
- friendProps, "strong", None, None,
+ defaultService.serviceName, Nil, friendProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
}
@@ -280,7 +278,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val hateProps = Nil
val hate = mnt.createLabel("hate", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, hateProps, "strong", None, None,
+ defaultService.serviceName, Nil, hateProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -288,36 +286,35 @@ class S2GraphProvider extends AbstractGraphProvider {
allProps ++= collaboratorProps
val collaborator = mnt.createLabel("collaborator", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil,
- collaboratorProps, "strong", None, None,
+ defaultService.serviceName, Nil, collaboratorProps, true, "strong", None, None,
options = Option("""{"skipReverse": true}""")
)
val test1Props = Nil
val test1 = mnt.createLabel("test1", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, test1Props, "weak", None, None,
+ defaultService.serviceName, Nil, test1Props, true, "weak", None, None,
options = Option("""{"skipReverse": false}""")
)
val test2Props = Nil
val test2 = mnt.createLabel("test2", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, test2Props, "weak", None, None,
+ defaultService.serviceName, Nil, test2Props, true, "weak", None, None,
options = Option("""{"skipReverse": false}""")
)
val test3Props = Nil
val test3 = mnt.createLabel("test3", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, test3Props, "weak", None, None,
+ defaultService.serviceName, Nil, test3Props, true, "weak", None, None,
options = Option("""{"skipReverse": false}""")
)
val petsProps = Nil
val pets = mnt.createLabel("pets", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, petsProps, "strong", None, None,
+ defaultService.serviceName, Nil, petsProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -326,26 +323,25 @@ class S2GraphProvider extends AbstractGraphProvider {
val walks = mnt.createLabel("walks", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil,
- walksProps, "strong", None, None,
+ defaultService.serviceName, Nil, walksProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
val livesWithProps = Nil
val livesWith = mnt.createLabel("livesWith", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, livesWithProps, "strong", None, None,
+ defaultService.serviceName, Nil, livesWithProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
val hatesProps = Nil
val hates = mnt.createLabel("hates", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, hatesProps, "weak", None, None,
+ defaultService.serviceName, Nil, hatesProps, true, "weak", None, None,
options = Option("""{"skipReverse": false}""")
)
val linkProps = Nil
val link = mnt.createLabel("link", defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType, defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, linkProps, "strong", None, None,
+ defaultService.serviceName, Nil, linkProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -355,8 +351,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val codeveloper = mnt.createLabel("codeveloper",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- codeveloperProps, "strong", None, None,
+ defaultService.serviceName, Nil, codeveloperProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -370,8 +365,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val createdBy = mnt.createLabel("createdBy",
defaultService.serviceName, "software", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- createdByProps, "strong", None, None)
+ defaultService.serviceName, Nil, createdByProps, true, "strong", None, None)
val existsWithProps = Seq(Prop("time", "-", "string"))
allProps ++= existsWithProps
@@ -379,7 +373,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val existsWith = mnt.createLabel("existsWith",
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
defaultService.serviceName, defaultServiceColumn.columnName, defaultServiceColumn.columnType,
- true, defaultService.serviceName, Nil, existsWithProps, "strong", None, None,
+ defaultService.serviceName, Nil, existsWithProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -389,7 +383,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val followedBy = mnt.createLabel("followedBy",
defaultService.serviceName, "song", "integer",
defaultService.serviceName, "song", "integer",
- true, defaultService.serviceName, Nil, followedByProps, "strong", None, None,
+ defaultService.serviceName, Nil, followedByProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -399,7 +393,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val writtenBy = mnt.createLabel("writtenBy",
defaultService.serviceName, "song", "integer",
defaultService.serviceName, "artist", "integer",
- true, defaultService.serviceName, Nil, writtenByProps, "strong", None, None,
+ defaultService.serviceName, Nil, writtenByProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -409,7 +403,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val sungBy = mnt.createLabel("sungBy",
defaultService.serviceName, "song", "integer",
defaultService.serviceName, "artist", "integer",
- true, defaultService.serviceName, Nil, sungByProps, "strong", None, None,
+ defaultService.serviceName, Nil, sungByProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -417,7 +411,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val uses = mnt.createLabel("uses",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "software", "integer",
- true, defaultService.serviceName, Nil, usesProps, "strong", None, None,
+ defaultService.serviceName, Nil, usesProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -427,8 +421,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val likes = mnt.createLabel("likes",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- likesProps, "strong", None, None,
+ defaultService.serviceName, Nil, likesProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -438,8 +431,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val foo = mnt.createLabel("foo",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- fooProps, "strong", None, None,
+ defaultService.serviceName, Nil, fooProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
@@ -449,8 +441,7 @@ class S2GraphProvider extends AbstractGraphProvider {
val bar = mnt.createLabel("bar",
defaultService.serviceName, "person", "integer",
defaultService.serviceName, "person", "integer",
- true, defaultService.serviceName, Nil,
- barProps, "strong", None, None,
+ defaultService.serviceName, Nil, barProps, true, "strong", None, None,
options = Option("""{"skipReverse": false}""")
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
index a77c6f3..51ddba2 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
@@ -428,12 +428,14 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels {
val knows = mnt.createLabel("knows",
S2Graph.DefaultServiceName, "person", "integer",
S2Graph.DefaultServiceName, "person", "integer",
- true, S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double"), Prop("year", "0", "integer")), consistencyLevel = "strong", None, None)
+ S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double"), Prop("year", "0", "integer")),
+ true, consistencyLevel = "strong", None, None)
val created = mnt.createLabel("created",
S2Graph.DefaultServiceName, "person", "integer",
S2Graph.DefaultServiceName, "person", "integer",
- true, S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double")), "strong", None, None)
+ S2Graph.DefaultServiceName, Nil, Seq(Prop("weight", "0.0", "double")),
+ true, "strong", None, None)
val g = graph.traversal()
val v1 = graph.addVertex(T.label, "person", T.id, Int.box(1), "name", "marko", "age", Int.box(29))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
index 90037cf..0b5a2e9 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
@@ -256,10 +256,10 @@ class GraphRepository(val graph: S2GraphLike) {
tgtServiceProp.serviceName,
tgtServiceColumn.columnName,
tgtServiceColumn.columnType,
- isDirected,
serviceName,
indices,
allProps,
+ isDirected,
consistencyLevel,
hTableName,
hTableTTL,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16feda80/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala
index 8dfbe1e..e6779fa 100644
--- a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala
+++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/BaseSparkTest.scala
@@ -121,10 +121,13 @@ class BaseSparkTest extends FunSuite with Matchers with BeforeAndAfterAll with D
val serviceColumn = management.createServiceColumn(service.serviceName, "user", "string", Nil)
Try {
- management.createLabel("friends", serviceColumn, serviceColumn, isDirected = true,
- serviceName = service.serviceName, indices = new java.util.ArrayList[Index],
- props = Seq(Prop("since", "0", "long"), Prop("score", "0", "integer")).asJava, consistencyLevel = "strong", hTableName = tableName,
- hTableTTL = -1, schemaVersion = schemaVersion, compressionAlgorithm = compressionAlgorithm, options = "")
+ management.createLabel("friends",
+ serviceColumn.service.serviceName, serviceColumn.columnName, serviceColumn.columnType,
+ serviceColumn.service.serviceName, serviceColumn.columnName, serviceColumn.columnType,
+ serviceName = service.serviceName, indices = Nil,
+ props = Seq(Prop("since", "0", "long"), Prop("score", "0", "integer")),
+ isDirected = true, consistencyLevel = "strong", hTableName = Option(tableName),
+ hTableTTL = None, schemaVersion = schemaVersion, compressionAlgorithm = compressionAlgorithm, options = None)
}
Label.findByName("friends").getOrElse(throw new IllegalArgumentException("friends label is not initialized."))
[07/11] incubator-s2graph git commit: add options on ServiceColumn at
schema.sql
Posted by st...@apache.org.
add options on ServiceColumn at schema.sql
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f6b0740f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f6b0740f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f6b0740f
Branch: refs/heads/master
Commit: f6b0740f50541f875779d730f0df2c1e0df421e0
Parents: be83d07
Author: DO YUNG YOON <st...@apache.org>
Authored: Wed May 9 23:19:52 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Wed May 9 23:19:52 2018 +0900
----------------------------------------------------------------------
dev_support/graph_mysql/schema.sql | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f6b0740f/dev_support/graph_mysql/schema.sql
----------------------------------------------------------------------
diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql
index 5781f3b..be9500e 100644
--- a/dev_support/graph_mysql/schema.sql
+++ b/dev_support/graph_mysql/schema.sql
@@ -59,6 +59,7 @@ CREATE TABLE `service_columns` (
`column_name` varchar(64) NOT NULL,
`column_type` varchar(8) NOT NULL,
`schema_version` varchar(8) NOT NULL default 'v2',
+ `options` text,
PRIMARY KEY (`id`),
UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
[06/11] incubator-s2graph git commit: Start implement ResourceManager.
Posted by st...@apache.org.
Start implement ResourceManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/be83d07c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/be83d07c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/be83d07c
Branch: refs/heads/master
Commit: be83d07ca5ecb271bd3678c38734d1176182c286
Parents: 43f627e
Author: DO YUNG YOON <st...@apache.org>
Authored: Wed May 9 19:25:15 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Wed May 9 19:25:15 2018 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/schema/schema.sql | 1 +
.../apache/s2graph/core/EdgeBulkFetcher.scala | 28 ---
.../org/apache/s2graph/core/EdgeFetcher.scala | 3 +
.../org/apache/s2graph/core/EdgeMutator.scala | 7 +
.../org/apache/s2graph/core/Management.scala | 57 +++++-
.../apache/s2graph/core/ResourceManager.scala | 130 +++++++++++++
.../scala/org/apache/s2graph/core/S2Graph.scala | 49 ++---
.../apache/s2graph/core/S2GraphFactory.scala | 2 +-
.../org/apache/s2graph/core/S2GraphLike.scala | 38 ++--
.../apache/s2graph/core/VertexBulkFetcher.scala | 26 ---
.../org/apache/s2graph/core/VertexFetcher.scala | 4 +
.../org/apache/s2graph/core/VertexMutator.scala | 5 +
.../s2graph/core/fetcher/FetcherManager.scala | 106 -----------
.../core/fetcher/MemoryModelEdgeFetcher.scala | 26 ++-
.../apache/s2graph/core/io/Conversions.scala | 6 +-
.../org/apache/s2graph/core/schema/Label.scala | 34 ++--
.../s2graph/core/schema/ServiceColumn.scala | 47 ++++-
.../storage/DefaultOptimisticEdgeMutator.scala | 176 +++++++++++++++++
.../core/storage/DefaultOptimisticMutator.scala | 190 -------------------
.../DefaultOptimisticVertexMutator.scala | 44 +++++
.../apache/s2graph/core/storage/Storage.scala | 4 -
.../hbase/AsynchbaseEdgeBulkFetcher.scala | 69 -------
.../storage/hbase/AsynchbaseEdgeFetcher.scala | 31 ++-
.../core/storage/hbase/AsynchbaseStorage.scala | 7 +-
.../hbase/AsynchbaseVertexBulkFetcher.scala | 63 ------
.../storage/hbase/AsynchbaseVertexFetcher.scala | 26 +++
.../storage/rocks/RocksEdgeBulkFetcher.scala | 68 -------
.../core/storage/rocks/RocksEdgeFetcher.scala | 35 +++-
.../core/storage/rocks/RocksStorage.scala | 7 +-
.../storage/rocks/RocksVertexBulkFetcher.scala | 88 ---------
.../core/storage/rocks/RocksVertexFetcher.scala | 53 ++++++
.../s2graph/core/utils/SafeUpdateCache.scala | 6 +-
.../s2graph/core/fetcher/EdgeFetcherTest.scala | 12 +-
.../apache/s2graph/graphql/GraphQLServer.scala | 8 +-
.../org/apache/s2graph/graphql/HttpServer.scala | 4 +-
35 files changed, 710 insertions(+), 750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/resources/org/apache/s2graph/core/schema/schema.sql
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/org/apache/s2graph/core/schema/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/schema/schema.sql
index 6b9b71e..4f7f832 100644
--- a/s2core/src/main/resources/org/apache/s2graph/core/schema/schema.sql
+++ b/s2core/src/main/resources/org/apache/s2graph/core/schema/schema.sql
@@ -48,6 +48,7 @@ CREATE TABLE `service_columns` (
`column_name` varchar(64) NOT NULL,
`column_type` varchar(8) NOT NULL,
`schema_version` varchar(8) NOT NULL default 'v2',
+ `options` text,
PRIMARY KEY (`id`),
UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
deleted file mode 100644
index 646f5f4..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/EdgeBulkFetcher.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core
-
-import com.typesafe.config.Config
-
-import scala.concurrent.{ExecutionContext, Future}
-
-trait EdgeBulkFetcher {
- def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
index f28a161..c3760e0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/EdgeFetcher.scala
@@ -31,5 +31,8 @@ trait EdgeFetcher {
def fetches(queryRequests: Seq[QueryRequest],
prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
+ def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]]
+
def close(): Unit = {}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
index dc0099e..252d129 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/EdgeMutator.scala
@@ -19,11 +19,13 @@
package org.apache.s2graph.core
+import com.typesafe.config.Config
import org.apache.s2graph.core.storage.MutateResponse
import scala.concurrent.{ExecutionContext, Future}
trait EdgeMutator {
+
def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]]
def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]]
@@ -35,4 +37,9 @@ trait EdgeMutator {
def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
requestTs: Long,
retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean]
+
+ def close(): Unit = {}
+
+ def init(config: Config)(implicit ec: ExecutionContext): Unit = {}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index 9046449..c3aef7a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -95,14 +95,15 @@ object Management {
columnName: String,
columnType: String,
props: Seq[Prop],
- schemaVersion: String = DEFAULT_VERSION) = {
+ schemaVersion: String = DEFAULT_VERSION,
+ options: Option[String] = None) = {
Schema withTx { implicit session =>
val serviceOpt = Service.findByName(serviceName, useCache = false)
serviceOpt match {
case None => throw new RuntimeException(s"create service $serviceName has not been created.")
case Some(service) =>
- val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion, useCache = false)
+ val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion, options, useCache = false)
for {
Prop(propName, defaultValue, dataType, storeInGlobalIndex) <- props
} yield {
@@ -304,13 +305,50 @@ class Management(graph: S2GraphLike) {
import Management._
- def importModel(labelName: String, options: String): Future[Importer] = {
- Label.updateOption(labelName, options)
+ def updateEdgeFetcher(labelName: String, options: String): Unit = {
+ val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
- val label = Label.findByName(labelName, false).getOrElse(throw new LabelNotExistException(labelName))
- val config = ConfigFactory.parseString(options)
+ updateEdgeFetcher(label, options)
+ }
+
+ def updateEdgeFetcher(label: Label, options: String): Unit = {
+ val newLabel = Label.updateOption(label, options)
+ graph.resourceManager.getOrElseUpdateEdgeFetcher(newLabel, forceUpdate = true)
+ }
+
+ def updateVertexFetcher(serviceName: String, columnName: String, options: String): Unit = {
+ val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName is not exist."))
+ val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName is not exist."))
+
+ updateVertexFetcher(column, options)
+ }
+
+ def updateVertexFetcher(column: ServiceColumn, options: String): Unit = {
+ val newColumn = ServiceColumn.updateOption(column, options)
+ graph.resourceManager.getOrElseUpdateVertexFetcher(newColumn, forceUpdate = true)
+ }
+
+ def updateEdgeMutator(labelName: String, options: String): Unit = {
+ val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
+
+ updateEdgeMutator(label, options)
+ }
+
+ def updateEdgeMutator(label: Label, options: String): Unit = {
+ val newLabel = Label.updateOption(label, options)
+ graph.resourceManager.getOrElseUpdateEdgeMutator(newLabel, forceUpdate = true)
+ }
+
+ def updateVertexMutator(serviceName: String, columnName: String, options: String): Unit = {
+ val service = Service.findByName(serviceName).getOrElse(throw new IllegalArgumentException(s"$serviceName is not exist."))
+ val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new IllegalArgumentException(s"$columnName is not exist."))
+
+ updateVertexMutator(column, options)
+ }
- graph.modelManager.importModel(label, config)(importEx)
+ def updateVertexMutator(column: ServiceColumn, options: String): Unit = {
+ val newColumn = ServiceColumn.updateOption(column, options)
+ graph.resourceManager.getOrElseUpdateVertexMutator(newColumn, forceUpdate = true)
}
def createStorageTable(zkAddr: String,
@@ -375,14 +413,15 @@ class Management(graph: S2GraphLike) {
columnName: String,
columnType: String,
props: Seq[Prop],
- schemaVersion: String = DEFAULT_VERSION): ServiceColumn = {
+ schemaVersion: String = DEFAULT_VERSION,
+ options: Option[String] = None): ServiceColumn = {
val serviceColumnTry = Schema withTx { implicit session =>
val serviceOpt = Service.findByName(serviceName, useCache = false)
serviceOpt match {
case None => throw new RuntimeException(s"create service $serviceName has not been created.")
case Some(service) =>
- val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion, useCache = false)
+ val serviceColumn = ServiceColumn.findOrInsert(service.id.get, columnName, Some(columnType), schemaVersion, options, useCache = false)
for {
Prop(propName, defaultValue, dataType, storeInGlobalIndex) <- props
} yield {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
new file mode 100644
index 0000000..b877603
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/ResourceManager.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.s2graph.core.schema.{Label, ServiceColumn}
+import org.apache.s2graph.core.utils.SafeUpdateCache
+import scala.concurrent.ExecutionContext
+
+
+object ResourceManager {
+
+ import SafeUpdateCache._
+
+ import scala.collection.JavaConverters._
+
+ val ClassNameKey = "className"
+ val EdgeFetcherKey = classOf[EdgeFetcher].getClass().getName
+
+ val VertexFetcherKey = classOf[VertexFetcher].getClass().getName
+
+ val EdgeMutatorKey = classOf[EdgeMutator].getClass.getName
+ val VertexMutatorKey = classOf[VertexMutator].getClass.getName
+
+ val DefaultConfig = ConfigFactory.parseMap(Map(MaxSizeKey -> 1000, TtlKey -> -1).asJava)
+}
+
+class ResourceManager(graph: S2GraphLike,
+ _config: Config)(implicit ec: ExecutionContext) {
+
+ import ResourceManager._
+
+ import scala.collection.JavaConverters._
+
+ val cache = new SafeUpdateCache(_config)
+
+ def getAllVertexFetchers(): Seq[VertexFetcher] = {
+ cache.asMap().asScala.toSeq.collect { case (_, (obj: VertexFetcher, _, _)) => obj }
+ }
+
+ def getAllEdgeFetchers(): Seq[EdgeFetcher] = {
+ cache.asMap().asScala.toSeq.collect { case (_, (obj: EdgeFetcher, _, _)) => obj }
+ }
+
+ def getOrElseUpdateVertexFetcher(column: ServiceColumn, forceUpdate: Boolean = false): Option[VertexFetcher] = {
+ val cacheKey = VertexFetcherKey + "_" + column.service.serviceName + "_" + column.columnName
+ cache.withCache(cacheKey, false, forceUpdate) {
+ column.toFetcherConfig.map { fetcherConfig =>
+ val className = fetcherConfig.getString(ClassNameKey)
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(graph)
+ .asInstanceOf[VertexFetcher]
+
+ fetcher.init(fetcherConfig)
+
+ fetcher
+ }
+ }
+ }
+
+ def getOrElseUpdateEdgeFetcher(label: Label, forceUpdate: Boolean = false): Option[EdgeFetcher] = {
+ val cacheKey = EdgeFetcherKey + "_" + label.label
+
+ cache.withCache(cacheKey, false, forceUpdate) {
+ label.toFetcherConfig.map { fetcherConfig =>
+ val className = fetcherConfig.getString(ClassNameKey)
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(graph)
+ .asInstanceOf[EdgeFetcher]
+
+ fetcher.init(fetcherConfig)
+
+ fetcher
+ }
+ }
+ }
+
+ def getOrElseUpdateVertexMutator(column: ServiceColumn, forceUpdate: Boolean = false): Option[VertexMutator] = {
+ val cacheKey = VertexMutatorKey + "_" + column.service.serviceName + "_" + column.columnName
+ cache.withCache(cacheKey, false, forceUpdate) {
+ column.toMutatorConfig.map { mutatorConfig =>
+ val className = mutatorConfig.getString(ClassNameKey)
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(graph)
+ .asInstanceOf[VertexMutator]
+
+ fetcher.init(mutatorConfig)
+
+ fetcher
+ }
+ }
+ }
+
+ def getOrElseUpdateEdgeMutator(label: Label, forceUpdate: Boolean = false): Option[EdgeMutator] = {
+ val cacheKey = EdgeMutatorKey + "_" + label.label
+ cache.withCache(cacheKey, false, forceUpdate) {
+ label.toMutatorConfig.map { mutatorConfig =>
+ val className = mutatorConfig.getString(ClassNameKey)
+ val fetcher = Class.forName(className)
+ .getConstructor(classOf[S2GraphLike])
+ .newInstance(graph)
+ .asInstanceOf[EdgeMutator]
+
+ fetcher.init(mutatorConfig)
+
+ fetcher
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index c4cb48f..09fd55e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -20,30 +20,27 @@
package org.apache.s2graph.core
import java.util
-import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
-import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.{ExecutorService, Executors}
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
import org.apache.s2graph.core.index.IndexProvider
import org.apache.s2graph.core.io.tinkerpop.optimize.S2GraphStepStrategy
-import org.apache.s2graph.core.fetcher.FetcherManager
import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
import org.apache.s2graph.core.storage.rocks.RocksStorage
import org.apache.s2graph.core.storage.{MutateResponse, OptimisticEdgeFetcher, Storage}
import org.apache.s2graph.core.types._
-import org.apache.s2graph.core.utils.{DeferCache, Extensions, logger}
-import org.apache.tinkerpop.gremlin.process.traversal.{P, TraversalStrategies}
-import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer
+import org.apache.s2graph.core.utils.{Extensions, Importer, logger}
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies
import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Graph}
import scala.collection.JavaConversions._
import scala.collection.mutable
-import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent._
-import scala.concurrent.duration.Duration
-import scala.util.{Random, Try}
+import scala.util.Try
object S2Graph {
@@ -94,6 +91,7 @@ object S2Graph {
val numOfThread = Runtime.getRuntime.availableProcessors()
val threadPool = Executors.newFixedThreadPool(numOfThread)
val ec = ExecutionContext.fromExecutor(threadPool)
+ val resourceManagerEc = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(numOfThread))
val DefaultServiceName = ""
val DefaultColumnName = "vertex"
@@ -187,7 +185,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override val management = new Management(this)
- override val modelManager = new FetcherManager(this)
+ override val resourceManager: ResourceManager = new ResourceManager(this, config)(S2Graph.resourceManagerEc)
override val indexProvider = IndexProvider.apply(config)
@@ -250,32 +248,39 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
storagePool.getOrElse(s"label:${label.label}", defaultStorage)
}
- //TODO:
+ /* Currently, each getter on Fetcher and Mutator missing proper implementation
+ * Please discuss what is proper way to maintain resources here and provide
+ * right implementation(S2GRAPH-213).
+ * */
override def getVertexFetcher(column: ServiceColumn): VertexFetcher = {
- getStorage(column.service).vertexFetcher
- }
- override def getVertexBulkFetcher: VertexBulkFetcher = {
- defaultStorage.vertexBulkFetcher
+ resourceManager.getOrElseUpdateVertexFetcher(column)
+ .getOrElse(defaultStorage.vertexFetcher)
}
override def getEdgeFetcher(label: Label): EdgeFetcher = {
- if (label.fetchConfigExist) modelManager.getFetcher(label)
- else getStorage(label).edgeFetcher
+ resourceManager.getOrElseUpdateEdgeFetcher(label)
+ .getOrElse(defaultStorage.edgeFetcher)
}
- override def getEdgeBulkFetcher: EdgeBulkFetcher = {
- defaultStorage.edgeBulkFetcher
+ override def getAllVertexFetchers(): Seq[VertexFetcher] = {
+ resourceManager.getAllVertexFetchers()
+ }
+
+ override def getAllEdgeFetchers(): Seq[EdgeFetcher] = {
+ resourceManager.getAllEdgeFetchers()
}
override def getVertexMutator(column: ServiceColumn): VertexMutator = {
- getStorage(column.service).vertexMutator
+ resourceManager.getOrElseUpdateVertexMutator(column)
+ .getOrElse(defaultStorage.vertexMutator)
}
override def getEdgeMutator(label: Label): EdgeMutator = {
- getStorage(label).edgeMutator
+ resourceManager.getOrElseUpdateEdgeMutator(label)
+ .getOrElse(defaultStorage.edgeMutator)
}
- /** optional */
+ //TODO:
override def getOptimisticEdgeFetcher(label: Label): OptimisticEdgeFetcher = {
// getStorage(label).optimisticEdgeFetcher
null
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
index cce05af..0a667ef 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphFactory.scala
@@ -66,7 +66,7 @@ object S2GraphFactory {
val DefaultService = management.createService(DefaultServiceName, "localhost", "s2graph", 0, None).get
// Management.deleteColumn(DefaultServiceName, DefaultColumnName)
- val DefaultColumn = ServiceColumn.findOrInsert(DefaultService.id.get, DefaultColumnName, Some("integer"), HBaseType.DEFAULT_VERSION, useCache = false)
+ val DefaultColumn = ServiceColumn.findOrInsert(DefaultService.id.get, DefaultColumnName, Some("integer"), HBaseType.DEFAULT_VERSION, options = None, useCache = false)
val DefaultColumnMetas = {
ColumnMeta.findOrInsert(DefaultColumn.id.get, "test", "string", "-", storeInGlobalIndex = true, useCache = false)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 5e2c168..99423d6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -19,22 +19,20 @@
package org.apache.s2graph.core
-import java.util
-import java.util.concurrent.{CompletableFuture, TimeUnit}
-import java.util.concurrent.atomic.AtomicLong
import java.lang.{Boolean => JBoolean, Long => JLong}
+import java.util
import java.util.Optional
+import java.util.concurrent.atomic.AtomicLong
+import java.util.concurrent.{CompletableFuture, TimeUnit}
import com.typesafe.config.Config
import org.apache.commons.configuration.Configuration
import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
-import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core.features.{S2Features, S2GraphVariables}
import org.apache.s2graph.core.index.IndexProvider
-import org.apache.s2graph.core.fetcher.FetcherManager
import org.apache.s2graph.core.schema.{Label, LabelMeta, Service, ServiceColumn}
import org.apache.s2graph.core.storage.{MutateResponse, OptimisticEdgeFetcher, Storage}
-import org.apache.s2graph.core.types.{InnerValLike, VertexId}
+import org.apache.s2graph.core.types.VertexId
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
import org.apache.tinkerpop.gremlin.structure
import org.apache.tinkerpop.gremlin.structure.Edge.Exceptions
@@ -44,10 +42,10 @@ import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Element, Graph,
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext, Future}
import scala.compat.java8.FutureConverters._
import scala.compat.java8.OptionConverters._
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext, Future}
trait S2GraphLike extends Graph {
@@ -69,7 +67,7 @@ trait S2GraphLike extends Graph {
val traversalHelper: TraversalHelper
- val modelManager: FetcherManager
+ val resourceManager: ResourceManager
lazy val MaxRetryNum: Int = config.getInt("max.retry.number")
lazy val MaxBackOff: Int = config.getInt("max.back.off")
@@ -95,11 +93,11 @@ trait S2GraphLike extends Graph {
def getVertexFetcher(column: ServiceColumn): VertexFetcher
- def getVertexBulkFetcher(): VertexBulkFetcher
-
def getEdgeFetcher(label: Label): EdgeFetcher
- def getEdgeBulkFetcher(): EdgeBulkFetcher
+ def getAllVertexFetchers(): Seq[VertexFetcher]
+
+ def getAllEdgeFetchers(): Seq[EdgeFetcher]
/** optional */
def getOptimisticEdgeFetcher(label: Label): OptimisticEdgeFetcher
@@ -211,7 +209,13 @@ trait S2GraphLike extends Graph {
if (ids.isEmpty) {
//TODO: default storage need to be fixed.
- Await.result(getVertexBulkFetcher().fetchVerticesAll(), WaitTimeout).iterator
+ val futures = getAllVertexFetchers.map { vertexFetcher =>
+ vertexFetcher.fetchVerticesAll()
+ }
+
+ val future = Future.sequence(futures)
+
+ Await.result(future, WaitTimeout).flatten.iterator
} else {
val vertices = ids.collect {
case s2Vertex: S2VertexLike => s2Vertex
@@ -236,7 +240,13 @@ trait S2GraphLike extends Graph {
def edges(edgeIds: AnyRef*): util.Iterator[structure.Edge] = {
if (edgeIds.isEmpty) {
// FIXME
- Await.result(getEdgeBulkFetcher().fetchEdgesAll(), WaitTimeout).iterator
+ val futures = getAllEdgeFetchers().map { edgeFetcher =>
+ edgeFetcher.fetchEdgesAll()
+ }
+
+ val future = Future.sequence(futures)
+
+ Await.result(future, WaitTimeout).flatten.iterator
} else {
Await.result(edgesAsync(edgeIds: _*), WaitTimeout)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
deleted file mode 100644
index cbebab5..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/VertexBulkFetcher.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core
-
-import scala.concurrent.{ExecutionContext, Future}
-
-trait VertexBulkFetcher {
- def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
index 5c10d18..b641e7f 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/VertexFetcher.scala
@@ -26,6 +26,10 @@ import scala.concurrent.{ExecutionContext, Future}
trait VertexFetcher {
def init(config: Config)(implicit ec: ExecutionContext): Unit = {}
+
def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
+
+ def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
+
def close(): Unit = {}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
index 18be890..d1c8ecf 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/VertexMutator.scala
@@ -19,10 +19,15 @@
package org.apache.s2graph.core
+import com.typesafe.config.Config
import org.apache.s2graph.core.storage.MutateResponse
import scala.concurrent.{ExecutionContext, Future}
trait VertexMutator {
+ def close(): Unit = {}
+
+ def init(config: Config)(implicit ec: ExecutionContext): Unit = {}
+
def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse]
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
deleted file mode 100644
index 26db7ff..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/FetcherManager.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.fetcher
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.utils.{Importer, logger}
-import org.apache.s2graph.core.{EdgeFetcher, S2GraphLike}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object FetcherManager {
- val ClassNameKey = "className"
-}
-
-class FetcherManager(s2GraphLike: S2GraphLike) {
-
- import FetcherManager._
-
- private val fetcherPool = scala.collection.mutable.Map.empty[String, EdgeFetcher]
-
- private val ImportLock = new java.util.concurrent.ConcurrentHashMap[String, Importer]
-
- def toImportLockKey(label: Label): String = label.label
-
- def getFetcher(label: Label): EdgeFetcher = {
- fetcherPool.getOrElse(toImportLockKey(label), throw new IllegalStateException(s"$label is not imported."))
- }
-
- def initImporter(config: Config): Importer = {
- val className = config.getString(ClassNameKey)
-
- Class.forName(className)
- .getConstructor(classOf[S2GraphLike])
- .newInstance(s2GraphLike)
- .asInstanceOf[Importer]
- }
-
- def initFetcher(config: Config)(implicit ec: ExecutionContext): Future[EdgeFetcher] = {
- val className = config.getString(ClassNameKey)
-
- val fetcher = Class.forName(className)
- .getConstructor(classOf[S2GraphLike])
- .newInstance(s2GraphLike)
- .asInstanceOf[EdgeFetcher]
-
- fetcher.init(config)
-
- Future.successful(fetcher)
- }
-
- def importModel(label: Label, config: Config)(implicit ec: ExecutionContext): Future[Importer] = {
- val importer = ImportLock.computeIfAbsent(toImportLockKey(label), new java.util.function.Function[String, Importer] {
- override def apply(k: String): Importer = {
- val importer = initImporter(config.getConfig("importer"))
-
- //TODO: Update Label's extra options.
- importer
- .run(config.getConfig("importer"))
- .map { importer =>
- logger.info(s"Close importer")
- importer.close()
-
- initFetcher(config.getConfig("fetcher")).map { fetcher =>
- importer.setStatus(true)
-
- fetcherPool
- .remove(k)
- .foreach { oldFetcher =>
- logger.info(s"Delete old storage ($k) => $oldFetcher")
- oldFetcher.close()
- }
-
- fetcherPool += (k -> fetcher)
- }
- }
- .onComplete { _ =>
- logger.info(s"ImportLock release: $k")
- ImportLock.remove(k)
- }
-
- importer
- }
- })
-
- Future.successful(importer)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
index bf90d69..110d615 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/fetcher/MemoryModelEdgeFetcher.scala
@@ -37,18 +37,26 @@ class MemoryModelEdgeFetcher(val graph: S2GraphLike) extends EdgeFetcher {
override def fetches(queryRequests: Seq[QueryRequest],
prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
val stepResultLs = queryRequests.map { queryRequest =>
- val queryParam = queryRequest.queryParam
- val edges = ranges.map { ith =>
- val tgtVertexId = builder.newVertexId(queryParam.label.service, queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), ith.toString)
+ toEdges(queryRequest)
+ }
- graph.toEdge(queryRequest.vertex.innerIdVal,
- tgtVertexId.innerId.value, queryParam.label.label, queryParam.direction)
- }
+ Future.successful(stepResultLs)
+ }
- val edgeWithScores = edges.map(e => EdgeWithScore(e, 1.0, queryParam.label))
- StepResult(edgeWithScores, Nil, Nil)
+ override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
+ Future.successful(Nil)
+ }
+
+ private def toEdges(queryRequest: QueryRequest) = {
+ val queryParam = queryRequest.queryParam
+ val edges = ranges.map { ith =>
+ val tgtVertexId = builder.newVertexId(queryParam.label.service, queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), ith.toString)
+
+ graph.toEdge(queryRequest.vertex.innerIdVal,
+ tgtVertexId.innerId.value, queryParam.label.label, queryParam.direction)
}
- Future.successful(stepResultLs)
+ val edgeWithScores = edges.map(e => EdgeWithScore(e, 1.0, queryParam.label))
+ StepResult(edgeWithScores, Nil, Nil)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/io/Conversions.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/io/Conversions.scala b/s2core/src/main/scala/org/apache/s2graph/core/io/Conversions.scala
index 15f1231..948cdd8 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/io/Conversions.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/io/Conversions.scala
@@ -75,7 +75,8 @@ object Conversions {
(JsPath \ "serviceId").read[Int] and
(JsPath \ "columnName").read[String] and
(JsPath \ "columnType").read[String] and
- (JsPath \ "schemaVersion").read[String]
+ (JsPath \ "schemaVersion").read[String] and
+ (JsPath \ "options").readNullable[String]
)(ServiceColumn.apply _)
implicit val serviceColumnWrites: Writes[ServiceColumn] = (
@@ -83,7 +84,8 @@ object Conversions {
(JsPath \ "serviceId").write[Int] and
(JsPath \ "columnName").write[String] and
(JsPath \ "columnType").write[String] and
- (JsPath \ "schemaVersion").write[String]
+ (JsPath \ "schemaVersion").write[String] and
+ (JsPath \ "options").writeNullable[String]
)(unlift(ServiceColumn.unapply))
implicit val columnMetaReads: Reads[ColumnMeta] = (
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
index cca1769..f3ce5e0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
@@ -60,7 +60,9 @@ object Label extends SQLSyntaxSupport[Label] {
select *
from labels
where label = ${labelName}
- and deleted_at is null """.map { rs => Label(rs) }.single.apply()
+ and deleted_at is null """.map { rs =>
+ Label(rs)
+ }.single.apply()
if (useCache) withCache(cacheKey)(labelOpt)
else labelOpt
@@ -109,15 +111,17 @@ object Label extends SQLSyntaxSupport[Label] {
.map { rs => Label(rs) }.single.apply())
}
- def findById(id: Int)(implicit session: DBSession = AutoSession): Label = {
+ def findById(id: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Label = {
val cacheKey = className + "id=" + id
- withCache(cacheKey)(
- sql"""
+ lazy val sql = sql"""
select *
from labels
where id = ${id}
and deleted_at is null"""
- .map { rs => Label(rs) }.single.apply()).get
+ .map { rs => Label(rs) }.single.apply()
+
+ if (useCache) withCache(cacheKey)(sql).get
+ else sql.get
}
def findByTgtColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
@@ -191,8 +195,8 @@ object Label extends SQLSyntaxSupport[Label] {
val serviceId = service.id.get
/** insert serviceColumn */
- val srcCol = ServiceColumn.findOrInsert(srcServiceId, srcColumnName, Some(srcColumnType), schemaVersion)
- val tgtCol = ServiceColumn.findOrInsert(tgtServiceId, tgtColumnName, Some(tgtColumnType), schemaVersion)
+ val srcCol = ServiceColumn.findOrInsert(srcServiceId, srcColumnName, Some(srcColumnType), schemaVersion, None)
+ val tgtCol = ServiceColumn.findOrInsert(tgtServiceId, tgtColumnName, Some(tgtColumnType), schemaVersion, None)
if (srcCol.columnType != srcColumnType) throw new RuntimeException(s"source service column type not matched ${srcCol.columnType} != ${srcColumnType}")
if (tgtCol.columnType != tgtColumnType) throw new RuntimeException(s"target service column type not matched ${tgtCol.columnType} != ${tgtColumnType}")
@@ -259,18 +263,18 @@ object Label extends SQLSyntaxSupport[Label] {
cnt
}
- def updateOption(labelName: String, options: String)(implicit session: DBSession = AutoSession) = {
+ def updateOption(label: Label, options: String)(implicit session: DBSession = AutoSession) = {
scala.util.Try(Json.parse(options)).getOrElse(throw new RuntimeException("invalid Json option"))
- logger.info(s"update options of label $labelName, ${options}")
- val cnt = sql"""update labels set options = $options where label = $labelName""".update().apply()
- val label = Label.findByName(labelName, useCache = false).get
+ logger.info(s"update options of label ${label.label}, ${options}")
+ val cnt = sql"""update labels set options = $options where id = ${label.id.get}""".update().apply()
+ val updatedLabel = findById(label.id.get, useCache = false)
val cacheKeys = List(s"id=${label.id.get}", s"label=${label.label}")
cacheKeys.foreach { key =>
expireCache(className + key)
expireCaches(className + key)
}
- cnt
+ updatedLabel
}
def delete(id: Int)(implicit session: DBSession = AutoSession) = {
@@ -390,12 +394,14 @@ case class Label(id: Option[Int], label: String,
lazy val storageConfigOpt: Option[Config] = toStorageConfig
- lazy val fetchConfigExist: Boolean = toFetcherConfig.isDefined
-
def toFetcherConfig: Option[Config] = {
Schema.toConfig(extraOptions, "fetcher")
}
+ def toMutatorConfig: Option[Config] = {
+ Schema.toConfig(extraOptions, "mutator")
+ }
+
def toStorageConfig: Option[Config] = {
Schema.toConfig(extraOptions, "storage")
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/schema/ServiceColumn.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/ServiceColumn.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/ServiceColumn.scala
index cc1698a..61f1a09 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/ServiceColumn.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/ServiceColumn.scala
@@ -19,9 +19,11 @@
package org.apache.s2graph.core.schema
+import com.typesafe.config.Config
import org.apache.s2graph.core.JSONParser
import org.apache.s2graph.core.JSONParser._
import org.apache.s2graph.core.types.{HBaseType, InnerValLike, InnerValLikeWithTs}
+import org.apache.s2graph.core.utils.logger
import play.api.libs.json.Json
import scalikejdbc._
@@ -29,10 +31,11 @@ object ServiceColumn extends SQLSyntaxSupport[ServiceColumn] {
import Schema._
val className = ServiceColumn.getClass.getSimpleName
- val Default = ServiceColumn(Option(0), -1, "default", "string", "v4")
+ val Default = ServiceColumn(Option(0), -1, "default", "string", "v4", None)
def valueOf(rs: WrappedResultSet): ServiceColumn = {
- ServiceColumn(rs.intOpt("id"), rs.int("service_id"), rs.string("column_name"), rs.string("column_type").toLowerCase(), rs.string("schema_version"))
+ ServiceColumn(rs.intOpt("id"), rs.int("service_id"), rs.string("column_name"),
+ rs.string("column_type").toLowerCase(), rs.string("schema_version"), rs.stringOpt("options"))
}
def findByServiceId(serviceId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[ServiceColumn] = {
@@ -65,9 +68,9 @@ object ServiceColumn extends SQLSyntaxSupport[ServiceColumn] {
""".map { rs => ServiceColumn.valueOf(rs) }.single.apply()
}
}
- def insert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String)(implicit session: DBSession = AutoSession) = {
- sql"""insert into service_columns(service_id, column_name, column_type, schema_version)
- values(${serviceId}, ${columnName}, ${columnType}, ${schemaVersion})""".execute.apply()
+ def insert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String, options: Option[String])(implicit session: DBSession = AutoSession) = {
+ sql"""insert into service_columns(service_id, column_name, column_type, schema_version, options)
+ values(${serviceId}, ${columnName}, ${columnType}, ${schemaVersion}, ${options})""".execute.apply()
}
def delete(id: Int)(implicit session: DBSession = AutoSession) = {
val serviceColumn = findById(id, useCache = false)
@@ -79,11 +82,14 @@ object ServiceColumn extends SQLSyntaxSupport[ServiceColumn] {
expireCaches(className + key)
}
}
- def findOrInsert(serviceId: Int, columnName: String, columnType: Option[String], schemaVersion: String = HBaseType.DEFAULT_VERSION, useCache: Boolean = true)(implicit session: DBSession = AutoSession): ServiceColumn = {
+ def findOrInsert(serviceId: Int, columnName: String, columnType: Option[String],
+ schemaVersion: String = HBaseType.DEFAULT_VERSION,
+ options: Option[String],
+ useCache: Boolean = true)(implicit session: DBSession = AutoSession): ServiceColumn = {
find(serviceId, columnName, useCache) match {
case Some(sc) => sc
case None =>
- insert(serviceId, columnName, columnType, schemaVersion)
+ insert(serviceId, columnName, columnType, schemaVersion, options)
// val cacheKey = s"serviceId=$serviceId:columnName=$columnName"
val cacheKey = "serviceId=" + serviceId + ":columnName=" + columnName
expireCache(className + cacheKey)
@@ -101,12 +107,29 @@ object ServiceColumn extends SQLSyntaxSupport[ServiceColumn] {
ls
}
+ def updateOption(serviceColumn: ServiceColumn, options: String)(implicit session: DBSession = AutoSession) = {
+ scala.util.Try(Json.parse(options)).getOrElse(throw new RuntimeException("invalid Json option"))
+ logger.info(s"update options of service column ${serviceColumn.service.serviceName} ${serviceColumn.columnName}, ${options}")
+ val cnt = sql"""update service_columns set options = $options where id = ${serviceColumn.id.get}""".update().apply()
+ val column = findById(serviceColumn.id.get, useCache = false)
+
+ val cacheKeys = List(s"id=${column.id.get}",
+ s"serviceId=${serviceColumn.serviceId}:columnName=${serviceColumn.columnName}")
+
+ cacheKeys.foreach { key =>
+ expireCache(className + key)
+ expireCaches(className + key)
+ }
+
+ column
+ }
}
case class ServiceColumn(id: Option[Int],
serviceId: Int,
columnName: String,
columnType: String,
- schemaVersion: String) {
+ schemaVersion: String,
+ options: Option[String]) {
lazy val service = Service.findById(serviceId)
lazy val metasWithoutCache = ColumnMeta.timestamp +: ColumnMeta.findAllByColumn(id.get, false) :+ ColumnMeta.lastModifiedAtColumn
@@ -147,5 +170,13 @@ case class ServiceColumn(id: Option[Int],
}
}
+ lazy val extraOptions = Schema.extraOptions(options)
+ def toFetcherConfig: Option[Config] = {
+ Schema.toConfig(extraOptions, "fetcher")
+ }
+
+ def toMutatorConfig: Option[Config] = {
+ Schema.toConfig(extraOptions, "mutator")
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticEdgeMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticEdgeMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticEdgeMutator.scala
new file mode 100644
index 0000000..4deecf5
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticEdgeMutator.scala
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.s2graph.core.storage
+
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.schema.LabelMeta
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class DefaultOptimisticEdgeMutator(graph: S2GraphLike,
+ serDe: StorageSerDe,
+ optimisticEdgeFetcher: OptimisticEdgeFetcher,
+ optimisticMutator: OptimisticMutator,
+ io: StorageIO) extends EdgeMutator {
+ lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, optimisticMutator, optimisticEdgeFetcher)
+
+ private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
+ optimisticMutator.writeToStorage(cluster, kvs, withWait)
+
+ override def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
+ if (stepInnerResult.isEmpty) Future.successful(true)
+ else {
+ val head = stepInnerResult.edgeWithScores.head
+ val zkQuorum = head.edge.innerLabel.hbaseZkAddr
+ val futures = for {
+ edgeWithScore <- stepInnerResult.edgeWithScores
+ } yield {
+ val edge = edgeWithScore.edge
+
+ val edgeSnapshot = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+
+ val edgeForward = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ /* reverted direction */
+ val edgeRevert = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
+
+ writeToStorage(zkQuorum, mutations, withWait = true)
+ }
+
+ Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
+ }
+ }
+
+ override def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] = {
+ val mutations = _edges.flatMap { edge =>
+ val (_, edgeUpdate) =
+ if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
+ else S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
+ io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+ }
+
+ writeToStorage(zkQuorum, mutations, withWait).map { ret =>
+ _edges.zipWithIndex.map { case (edge, idx) =>
+ idx -> ret.isSuccess
+ }
+ }
+ }
+
+ override def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ def mutateEdgesInner(edges: Seq[S2EdgeLike],
+ checkConsistency: Boolean,
+ withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ assert(edges.nonEmpty)
+ // TODO:: remove after code review: unreachable code
+ if (!checkConsistency) {
+
+ val futures = edges.map { edge =>
+ val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+ val mutations =
+ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
+
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
+ } else {
+ optimisticEdgeFetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
+ conflictResolver.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
+ }
+ }
+ }
+
+ val edgeWithIdxs = _edges.zipWithIndex
+ val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
+ (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
+ } toSeq
+
+ val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
+ val edges = edgeGroup.map(_._1)
+ val idxs = edgeGroup.map(_._2)
+ // After deleteAll, process others
+ val mutateEdgeFutures = edges.toList match {
+ case head :: tail =>
+ val edgeFuture = mutateEdgesInner(edges, checkConsistency = true, withWait)
+
+ //TODO: decide what we will do on failure on vertex put
+ val puts = io.buildVertexPutsAsync(head)
+ val vertexFuture = writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
+ Seq(edgeFuture, vertexFuture)
+ case Nil => Nil
+ }
+
+ val composed = for {
+ // deleteRet <- Future.sequence(deleteAllFutures)
+ mutateRet <- Future.sequence(mutateEdgeFutures)
+ } yield mutateRet
+
+ composed.map(_.forall(_.isSuccess)).map { ret => idxs.map(idx => idx -> ret) }
+ }
+
+ Future.sequence(mutateEdges).map { squashedRets =>
+ squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
+ }
+ }
+
+ override def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] = {
+ val futures = for {
+ edge <- edges
+ } yield {
+ val kvs = for {
+ relEdge <- edge.relatedEdges
+ edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
+ } yield {
+ val countWithTs = edge.propertyValueInner(LabelMeta.count)
+ val countVal = countWithTs.innerVal.toString().toLong
+ io.buildIncrementsCountAsync(edgeWithIndex, countVal).head
+ }
+ writeToStorage(zkQuorum, kvs, withWait = withWait)
+ }
+
+ Future.sequence(futures)
+ }
+
+ override def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ val kvs = io.buildDegreePuts(edge, degreeVal)
+
+ writeToStorage(zkQuorum, kvs, withWait = true)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
deleted file mode 100644
index 82cc27a..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticMutator.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.s2graph.core.storage
-
-import org.apache.s2graph.core._
-import org.apache.s2graph.core.schema.LabelMeta
-import org.apache.s2graph.core.utils.logger
-
-import scala.concurrent.{ExecutionContext, Future}
-
-class DefaultOptimisticMutator(graph: S2GraphLike,
- serDe: StorageSerDe,
- optimisticEdgeFetcher: OptimisticEdgeFetcher,
- optimisticMutator: OptimisticMutator) extends VertexMutator with EdgeMutator {
-
- lazy val io: StorageIO = new StorageIO(graph, serDe)
-
- lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, optimisticMutator, optimisticEdgeFetcher)
-
- private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
- optimisticMutator.writeToStorage(cluster, kvs, withWait)
-
- def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
- requestTs: Long,
- retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
- if (stepInnerResult.isEmpty) Future.successful(true)
- else {
- val head = stepInnerResult.edgeWithScores.head
- val zkQuorum = head.edge.innerLabel.hbaseZkAddr
- val futures = for {
- edgeWithScore <- stepInnerResult.edgeWithScores
- } yield {
- val edge = edgeWithScore.edge
-
- val edgeSnapshot = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
-
- val edgeForward = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
- serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- io.buildIncrementsAsync(indexEdge, -1L)
- }
-
- /* reverted direction */
- val edgeRevert = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
- serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- io.buildIncrementsAsync(indexEdge, -1L)
- }
-
- val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
-
- writeToStorage(zkQuorum, mutations, withWait = true)
- }
-
- Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
- }
- }
-
- def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- if (vertex.op == GraphUtil.operations("delete")) {
- writeToStorage(zkQuorum,
- serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
- } else if (vertex.op == GraphUtil.operations("deleteAll")) {
- logger.info(s"deleteAll for vertex is truncated. $vertex")
- Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
- } else {
- writeToStorage(zkQuorum, io.buildPutsAll(vertex), withWait)
- }
- }
-
- def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] = {
- val mutations = _edges.flatMap { edge =>
- val (_, edgeUpdate) =
- if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
- else S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
-
- if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
- io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
- }
-
- writeToStorage(zkQuorum, mutations, withWait).map { ret =>
- _edges.zipWithIndex.map { case (edge, idx) =>
- idx -> ret.isSuccess
- }
- }
- }
-
- def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
- def mutateEdgesInner(edges: Seq[S2EdgeLike],
- checkConsistency: Boolean,
- withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- assert(edges.nonEmpty)
- // TODO:: remove after code review: unreachable code
- if (!checkConsistency) {
-
- val futures = edges.map { edge =>
- val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
- val mutations =
- io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
-
- if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
-
- writeToStorage(zkQuorum, mutations, withWait)
- }
- Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
- } else {
- optimisticEdgeFetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
- conflictResolver.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
- }
- }
- }
-
- val edgeWithIdxs = _edges.zipWithIndex
- val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
- (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
- } toSeq
-
- val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
- val edges = edgeGroup.map(_._1)
- val idxs = edgeGroup.map(_._2)
- // After deleteAll, process others
- val mutateEdgeFutures = edges.toList match {
- case head :: tail =>
- val edgeFuture = mutateEdgesInner(edges, checkConsistency = true, withWait)
-
- //TODO: decide what we will do on failure on vertex put
- val puts = io.buildVertexPutsAsync(head)
- val vertexFuture = writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
- Seq(edgeFuture, vertexFuture)
- case Nil => Nil
- }
-
- val composed = for {
- // deleteRet <- Future.sequence(deleteAllFutures)
- mutateRet <- Future.sequence(mutateEdgeFutures)
- } yield mutateRet
-
- composed.map(_.forall(_.isSuccess)).map { ret => idxs.map(idx => idx -> ret) }
- }
-
- Future.sequence(mutateEdges).map { squashedRets =>
- squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
- }
- }
-
- def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] = {
- val futures = for {
- edge <- edges
- } yield {
- val kvs = for {
- relEdge <- edge.relatedEdges
- edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
- } yield {
- val countWithTs = edge.propertyValueInner(LabelMeta.count)
- val countVal = countWithTs.innerVal.toString().toLong
- io.buildIncrementsCountAsync(edgeWithIndex, countVal).head
- }
- writeToStorage(zkQuorum, kvs, withWait = withWait)
- }
-
- Future.sequence(futures)
- }
-
- def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
- val kvs = io.buildDegreePuts(edge, degreeVal)
-
- writeToStorage(zkQuorum, kvs, withWait = true)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticVertexMutator.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticVertexMutator.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticVertexMutator.scala
new file mode 100644
index 0000000..6be619b
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/DefaultOptimisticVertexMutator.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.core.storage
+
+import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.{GraphUtil, S2GraphLike, S2VertexLike, VertexMutator}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class DefaultOptimisticVertexMutator(graph: S2GraphLike,
+ serDe: StorageSerDe,
+ optimisticEdgeFetcher: OptimisticEdgeFetcher,
+ optimisticMutator: OptimisticMutator,
+ io: StorageIO) extends VertexMutator {
+
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ if (vertex.op == GraphUtil.operations("delete")) {
+ optimisticMutator.writeToStorage(zkQuorum,
+ serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
+ } else if (vertex.op == GraphUtil.operations("deleteAll")) {
+ logger.info(s"deleteAll for vertex is truncated. $vertex")
+ Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
+ } else {
+ optimisticMutator.writeToStorage(zkQuorum, io.buildPutsAll(vertex), withWait)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index 36ecfcb..bf620bf 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -42,12 +42,8 @@ abstract class Storage(val graph: S2GraphLike,
val edgeFetcher: EdgeFetcher
- val edgeBulkFetcher: EdgeBulkFetcher
-
val vertexFetcher: VertexFetcher
- val vertexBulkFetcher: VertexBulkFetcher
-
val edgeMutator: EdgeMutator
val vertexMutator: VertexMutator
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
deleted file mode 100644
index 3d25dd9..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeBulkFetcher.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.hbase
-
-import java.util
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.storage.serde.Serializable
-import org.apache.s2graph.core.{EdgeBulkFetcher, S2EdgeLike, S2Graph, S2GraphLike}
-import org.apache.s2graph.core.storage.{CanSKeyValue, StorageIO, StorageSerDe}
-import org.apache.s2graph.core.types.HBaseType
-import org.apache.s2graph.core.utils.{CanDefer, Extensions}
-import org.hbase.async.{HBaseClient, KeyValue}
-
-import scala.concurrent.{ExecutionContext, Future}
-
-class AsynchbaseEdgeBulkFetcher(val graph: S2GraphLike,
- val config: Config,
- val client: HBaseClient,
- val serDe: StorageSerDe,
- val io: StorageIO) extends EdgeBulkFetcher {
- import Extensions.DeferOps
- import CanDefer._
- import scala.collection.JavaConverters._
- import AsynchbaseStorage._
-
- override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
- val futures = Label.findAll().groupBy(_.hbaseTableName).toSeq.map { case (hTableName, labels) =>
- val distinctLabels = labels.toSet
- val scan = AsynchbasePatcher.newScanner(client, hTableName)
- scan.setFamily(Serializable.edgeCf)
- scan.setMaxVersions(1)
-
- scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
- case null => Seq.empty
- case kvsLs =>
- kvsLs.asScala.flatMap { kvs =>
- kvs.asScala.flatMap { kv =>
- val sKV = implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
-
- serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION)
- .fromKeyValues(Seq(kv), None)
- .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
- }
- }
- }
- }
-
- Future.sequence(futures).map(_.flatten)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
index 4239d15..8eafc68 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseEdgeFetcher.scala
@@ -25,12 +25,14 @@ import com.stumbleupon.async.Deferred
import com.typesafe.config.Config
import org.apache.hadoop.hbase.util.Bytes
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.storage.serde.Serializable
+import org.apache.s2graph.core.storage.{CanSKeyValue, StorageIO, StorageSerDe}
import org.apache.s2graph.core.types.{HBaseType, VertexId}
import org.apache.s2graph.core.utils.{CanDefer, DeferCache, Extensions, logger}
import org.hbase.async._
-import scala.concurrent.ExecutionContext
+import scala.concurrent.{ExecutionContext, Future}
class AsynchbaseEdgeFetcher(val graph: S2GraphLike,
val config: Config,
@@ -64,6 +66,31 @@ class AsynchbaseEdgeFetcher(val graph: S2GraphLike,
}.toFuture(emptyStepResult).map(_.asScala)
}
+ override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
+ val futures = Label.findAll().groupBy(_.hbaseTableName).toSeq.map { case (hTableName, labels) =>
+ val distinctLabels = labels.toSet
+ val scan = AsynchbasePatcher.newScanner(client, hTableName)
+ scan.setFamily(Serializable.edgeCf)
+ scan.setMaxVersions(1)
+
+ scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
+ case null => Seq.empty
+ case kvsLs =>
+ kvsLs.asScala.flatMap { kvs =>
+ kvs.asScala.flatMap { kv =>
+ val sKV = implicitly[CanSKeyValue[KeyValue]].toSKeyValue(kv)
+
+ serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION)
+ .fromKeyValues(Seq(kv), None)
+ .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
+ }
+ }
+ }
+ }
+
+ Future.sequence(futures).map(_.flatten)
+ }
+
/**
* we are using future cache to squash requests into same key on storage.
*
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index f65ee20..89303e6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -323,17 +323,14 @@ class AsynchbaseStorage(override val graph: S2GraphLike,
private lazy val optimisticEdgeFetcher = new AsynchbaseOptimisticEdgeFetcher(client, serDe, io)
private lazy val optimisticMutator = new AsynchbaseOptimisticMutator(graph, serDe, optimisticEdgeFetcher, client, clientWithFlush)
- private lazy val _mutator = new DefaultOptimisticMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator)
override val management: StorageManagement = new AsynchbaseStorageManagement(config, clients)
override val serDe: StorageSerDe = new AsynchbaseStorageSerDe(graph)
override val edgeFetcher: EdgeFetcher = new AsynchbaseEdgeFetcher(graph, config, client, serDe, io)
- override val edgeBulkFetcher: EdgeBulkFetcher = new AsynchbaseEdgeBulkFetcher(graph, config, client, serDe, io)
override val vertexFetcher: VertexFetcher = new AsynchbaseVertexFetcher(graph, config, client, serDe, io)
- override val vertexBulkFetcher: VertexBulkFetcher = new AsynchbaseVertexBulkFetcher(graph, config, client, serDe, io)
- override val edgeMutator: EdgeMutator = _mutator
- override val vertexMutator: VertexMutator = _mutator
+ override val edgeMutator: EdgeMutator = new DefaultOptimisticEdgeMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator, io)
+ override val vertexMutator: VertexMutator = new DefaultOptimisticVertexMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator, io)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
deleted file mode 100644
index e6bf4e6..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexBulkFetcher.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.hbase
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.schema.ServiceColumn
-import org.apache.s2graph.core.storage.serde.Serializable
-import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
-import org.apache.s2graph.core.types.HBaseType
-import org.apache.s2graph.core.utils.Extensions
-import org.apache.s2graph.core.{S2Graph, S2GraphLike, VertexBulkFetcher}
-import org.hbase.async.HBaseClient
-
-import scala.concurrent.{ExecutionContext, Future}
-
-class AsynchbaseVertexBulkFetcher(val graph: S2GraphLike,
- val config: Config,
- val client: HBaseClient,
- val serDe: StorageSerDe,
- val io: StorageIO) extends VertexBulkFetcher {
-
- import AsynchbaseStorage._
- import Extensions.DeferOps
-
- import scala.collection.JavaConverters._
-
- override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
- val futures = ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.map { case (hTableName, columns) =>
- val distinctColumns = columns.toSet
- val scan = AsynchbasePatcher.newScanner(client, hTableName)
- scan.setFamily(Serializable.vertexCf)
- scan.setMaxVersions(1)
-
- scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
- case null => Seq.empty
- case kvsLs =>
- kvsLs.asScala.flatMap { kvs =>
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(kvs.asScala, None)
- .filter(v => distinctColumns(v.serviceColumn))
- }
- }
- }
- Future.sequence(futures).map(_.flatten)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
index 560dd2b..f16c8e9 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseVertexFetcher.scala
@@ -21,7 +21,11 @@ package org.apache.s2graph.core.storage.hbase
import com.typesafe.config.Config
import org.apache.s2graph.core._
+import org.apache.s2graph.core.schema.ServiceColumn
+import org.apache.s2graph.core.storage.serde.Serializable
import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.s2graph.core.utils.Extensions
import org.hbase.async.HBaseClient
import scala.concurrent.{ExecutionContext, Future}
@@ -32,6 +36,9 @@ class AsynchbaseVertexFetcher(val graph: S2GraphLike,
val serDe: StorageSerDe,
val io: StorageIO) extends VertexFetcher {
import AsynchbaseStorage._
+ import Extensions.DeferOps
+ import scala.collection.JavaConverters._
+
private def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]] = {
val rpc = buildRequest(serDe, queryRequest, vertex)
@@ -58,4 +65,23 @@ class AsynchbaseVertexFetcher(val graph: S2GraphLike,
Future.sequence(futures).map(_.flatten)
}
+
+ override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
+ val futures = ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.map { case (hTableName, columns) =>
+ val distinctColumns = columns.toSet
+ val scan = AsynchbasePatcher.newScanner(client, hTableName)
+ scan.setFamily(Serializable.vertexCf)
+ scan.setMaxVersions(1)
+
+ scan.nextRows(S2Graph.FetchAllLimit).toFuture(emptyKeyValuesLs).map {
+ case null => Seq.empty
+ case kvsLs =>
+ kvsLs.asScala.flatMap { kvs =>
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(kvs.asScala, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ }
+ }
+ }
+ Future.sequence(futures).map(_.flatten)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
deleted file mode 100644
index 2ca4b35..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeBulkFetcher.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.rocks
-
-import com.typesafe.config.Config
-import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.{EdgeBulkFetcher, S2EdgeLike, S2GraphLike}
-import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
-import org.apache.s2graph.core.types.HBaseType
-import org.rocksdb.RocksDB
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-
-class RocksEdgeBulkFetcher(val graph: S2GraphLike,
- val config: Config,
- val db: RocksDB,
- val vdb: RocksDB,
- val serDe: StorageSerDe,
- val io: StorageIO) extends EdgeBulkFetcher {
- import RocksStorage._
-
- override def fetchEdgesAll()(implicit ec: ExecutionContext) = {
- val edges = new ArrayBuffer[S2EdgeLike]()
- Label.findAll().groupBy(_.hbaseTableName).toSeq.foreach { case (hTableName, labels) =>
- val distinctLabels = labels.toSet
-
- val iter = db.newIterator()
- try {
- iter.seekToFirst()
- while (iter.isValid) {
- val kv = SKeyValue(table, iter.key(), SKeyValue.EdgeCf, qualifier, iter.value, System.currentTimeMillis())
-
- serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(Seq(kv), None)
- .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
- .foreach { edge =>
- edges += edge
- }
-
-
- iter.next()
- }
-
- } finally {
- iter.close()
- }
- }
-
- Future.successful(edges)
- }
-}
[05/11] incubator-s2graph git commit: Start implement ResourceManager.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
index 628c5e1..796e52e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksEdgeFetcher.scala
@@ -21,10 +21,12 @@ package org.apache.s2graph.core.storage.rocks
import com.typesafe.config.Config
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe}
-import org.apache.s2graph.core.types.VertexId
+import org.apache.s2graph.core.schema.Label
+import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.{HBaseType, VertexId}
import org.rocksdb.RocksDB
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{ExecutionContext, Future}
class RocksEdgeFetcher(val graph: S2GraphLike,
@@ -57,4 +59,33 @@ class RocksEdgeFetcher(val graph: S2GraphLike,
Future.sequence(futures)
}
+
+ override def fetchEdgesAll()(implicit ec: ExecutionContext) = {
+ val edges = new ArrayBuffer[S2EdgeLike]()
+ Label.findAll().groupBy(_.hbaseTableName).toSeq.foreach { case (hTableName, labels) =>
+ val distinctLabels = labels.toSet
+
+ val iter = db.newIterator()
+ try {
+ iter.seekToFirst()
+ while (iter.isValid) {
+ val kv = SKeyValue(table, iter.key(), SKeyValue.EdgeCf, qualifier, iter.value, System.currentTimeMillis())
+
+ serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(Seq(kv), None)
+ .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
+ .foreach { edge =>
+ edges += edge
+ }
+
+
+ iter.next()
+ }
+
+ } finally {
+ iter.close()
+ }
+ }
+
+ Future.successful(edges)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
index 8948e13..aaf9086 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksStorage.scala
@@ -238,16 +238,13 @@ class RocksStorage(override val graph: S2GraphLike,
private lazy val optimisticEdgeFetcher = new RocksOptimisticEdgeFetcher(graph, config, db, vdb, serDe, io)
private lazy val optimisticMutator = new RocksOptimisticMutator(graph, serDe, optimisticEdgeFetcher, db, vdb, lockMap)
- private lazy val _mutator = new DefaultOptimisticMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator)
override val management: StorageManagement = new RocksStorageManagement(config, vdb, db)
override val serDe: StorageSerDe = new RocksStorageSerDe(graph)
override val edgeFetcher: EdgeFetcher = new RocksEdgeFetcher(graph, config, db, vdb, serDe, io)
- override val edgeBulkFetcher: EdgeBulkFetcher = new RocksEdgeBulkFetcher(graph, config, db, vdb, serDe, io)
override val vertexFetcher: VertexFetcher = new RocksVertexFetcher(graph, config, db, vdb, serDe, io)
- override val vertexBulkFetcher: VertexBulkFetcher = new RocksVertexBulkFetcher(graph, config, db, vdb, serDe, io)
- override val edgeMutator: EdgeMutator = _mutator
- override val vertexMutator: VertexMutator = _mutator
+ override val edgeMutator: EdgeMutator = new DefaultOptimisticEdgeMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator, io)
+ override val vertexMutator: VertexMutator = new DefaultOptimisticVertexMutator(graph, serDe, optimisticEdgeFetcher, optimisticMutator, io)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
deleted file mode 100644
index 20acfaa..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexBulkFetcher.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.core.storage.rocks
-
-import com.typesafe.config.Config
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.schema.ServiceColumn
-import org.apache.s2graph.core.{S2GraphLike, S2VertexLike, VertexBulkFetcher}
-import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
-import org.apache.s2graph.core.types.HBaseType
-import org.rocksdb.RocksDB
-
-import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{ExecutionContext, Future}
-
-class RocksVertexBulkFetcher(val graph: S2GraphLike,
- val config: Config,
- val db: RocksDB,
- val vdb: RocksDB,
- val serDe: StorageSerDe,
- val io: StorageIO) extends VertexBulkFetcher {
- import RocksStorage._
-
- override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
- import scala.collection.mutable
-
- val vertices = new ArrayBuffer[S2VertexLike]()
- ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.foreach { case (hTableName, columns) =>
- val distinctColumns = columns.toSet
-
- val iter = vdb.newIterator()
- val buffer = mutable.ListBuffer.empty[SKeyValue]
- var oldVertexIdBytes = Array.empty[Byte]
- var minusPos = 0
-
- try {
- iter.seekToFirst()
- while (iter.isValid) {
- val row = iter.key()
- if (!Bytes.equals(oldVertexIdBytes, 0, oldVertexIdBytes.length - minusPos, row, 0, row.length - 1)) {
- if (buffer.nonEmpty)
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
- .filter(v => distinctColumns(v.serviceColumn))
- .foreach { vertex =>
- vertices += vertex
- }
-
- oldVertexIdBytes = row
- minusPos = 1
- buffer.clear()
- }
- val kv = SKeyValue(table, iter.key(), SKeyValue.VertexCf, qualifier, iter.value(), System.currentTimeMillis())
- buffer += kv
-
- iter.next()
- }
- if (buffer.nonEmpty)
- serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
- .filter(v => distinctColumns(v.serviceColumn))
- .foreach { vertex =>
- vertices += vertex
- }
-
- } finally {
- iter.close()
- }
- }
-
- Future.successful(vertices)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
index 6becd98..2d3880c 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/rocks/RocksVertexFetcher.scala
@@ -20,10 +20,15 @@
package org.apache.s2graph.core.storage.rocks
import com.typesafe.config.Config
+import org.apache.hadoop.hbase.util.Bytes
import org.apache.s2graph.core._
+import org.apache.s2graph.core.schema.ServiceColumn
+import org.apache.s2graph.core.storage.rocks.RocksStorage.{qualifier, table}
import org.apache.s2graph.core.storage.{SKeyValue, StorageIO, StorageSerDe}
+import org.apache.s2graph.core.types.HBaseType
import org.rocksdb.RocksDB
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{ExecutionContext, Future}
class RocksVertexFetcher(val graph: S2GraphLike,
@@ -58,4 +63,52 @@ class RocksVertexFetcher(val graph: S2GraphLike,
Future.sequence(futures).map(_.flatten)
}
+
+ override def fetchVerticesAll()(implicit ec: ExecutionContext) = {
+ import scala.collection.mutable
+
+ val vertices = new ArrayBuffer[S2VertexLike]()
+ ServiceColumn.findAll().groupBy(_.service.hTableName).toSeq.foreach { case (hTableName, columns) =>
+ val distinctColumns = columns.toSet
+
+ val iter = vdb.newIterator()
+ val buffer = mutable.ListBuffer.empty[SKeyValue]
+ var oldVertexIdBytes = Array.empty[Byte]
+ var minusPos = 0
+
+ try {
+ iter.seekToFirst()
+ while (iter.isValid) {
+ val row = iter.key()
+ if (!Bytes.equals(oldVertexIdBytes, 0, oldVertexIdBytes.length - minusPos, row, 0, row.length - 1)) {
+ if (buffer.nonEmpty)
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ .foreach { vertex =>
+ vertices += vertex
+ }
+
+ oldVertexIdBytes = row
+ minusPos = 1
+ buffer.clear()
+ }
+ val kv = SKeyValue(table, iter.key(), SKeyValue.VertexCf, qualifier, iter.value(), System.currentTimeMillis())
+ buffer += kv
+
+ iter.next()
+ }
+ if (buffer.nonEmpty)
+ serDe.vertexDeserializer(schemaVer = HBaseType.DEFAULT_VERSION).fromKeyValues(buffer, None)
+ .filter(v => distinctColumns(v.serviceColumn))
+ .foreach { vertex =>
+ vertices += vertex
+ }
+
+ } finally {
+ iter.close()
+ }
+ }
+
+ Future.successful(vertices)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
index 7d15078..51e45c2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/utils/SafeUpdateCache.scala
@@ -114,7 +114,9 @@ class SafeUpdateCache(val config: Config)
cache.invalidate(cacheKey)
}
- def withCache[T <: AnyRef](key: String, broadcast: Boolean)(op: => T): T = {
+ def withCache[T <: AnyRef](key: String,
+ broadcast: Boolean,
+ forceUpdate: Boolean = false)(op: => T): T = {
val cacheKey = toCacheKey(key)
val cachedValWithTs = cache.getIfPresent(cacheKey)
@@ -127,7 +129,7 @@ class SafeUpdateCache(val config: Config)
val (_cachedVal, updatedAt, isUpdating) = cachedValWithTs
val cachedVal = _cachedVal.asInstanceOf[T]
- if (toTs() < updatedAt + ttl) cachedVal // in cache TTL
+ if (!forceUpdate && toTs() < updatedAt + ttl) cachedVal // in cache TTL
else {
val running = isUpdating.getAndSet(true)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
index 6d95c93..930c517 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/fetcher/EdgeFetcherTest.scala
@@ -23,7 +23,7 @@ import com.typesafe.config.ConfigFactory
import org.apache.s2graph.core.Integrate.IntegrateCommon
import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
import org.apache.s2graph.core.schema.Label
-import org.apache.s2graph.core.{Query, QueryParam}
+import org.apache.s2graph.core.{Query, QueryParam, ResourceManager}
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext}
@@ -44,10 +44,10 @@ class EdgeFetcherTest extends IntegrateCommon {
s"""{
|
| "importer": {
- | "${FetcherManager.ClassNameKey}": "org.apache.s2graph.core.utils.IdentityImporter"
+ | "${ResourceManager.ClassNameKey}": "org.apache.s2graph.core.utils.IdentityImporter"
| },
| "fetcher": {
- | "${FetcherManager.ClassNameKey}": "org.apache.s2graph.core.fetcher.MemoryModelFetcher"
+ | "${ResourceManager.ClassNameKey}": "org.apache.s2graph.core.fetcher.MemoryModelEdgeFetcher"
| }
|}""".stripMargin
@@ -68,11 +68,9 @@ class EdgeFetcherTest extends IntegrateCommon {
"gz",
options
)
- val config = ConfigFactory.parseString(options)
- val importerFuture = graph.modelManager.importModel(label, config)(ExecutionContext.Implicits.global)
- Await.ready(importerFuture, Duration("60 seconds"))
- Thread.sleep(1000)
+ graph.management.updateEdgeFetcher(label, options)
+
val vertex = graph.elementBuilder.toVertex(service.serviceName, serviceColumn.columnName, "daewon")
val queryParam = QueryParam(labelName = labelName)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
index 0bf62d9..eee7c93 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
@@ -40,7 +40,7 @@ import sangria.schema.Schema
import spray.json.{JsBoolean, JsObject, JsString}
import scala.collection.JavaConverters._
-import scala.concurrent.ExecutionContext
+import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
object GraphQLServer {
@@ -63,17 +63,17 @@ object GraphQLServer {
val schemaCache = new SafeUpdateCache(schemaConfig)
- def importModel(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = {
+ def updateEdgeFetcher(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Route = {
val ret = Try {
val spray.json.JsObject(fields) = requestJSON
val spray.json.JsString(labelName) = fields("label")
val jsOptions = fields("options")
- s2graph.management.importModel(labelName, jsOptions.compactPrint)
+ s2graph.management.updateEdgeFetcher(labelName, jsOptions.compactPrint)
}
ret match {
- case Success(f) => complete(f.map(i => OK -> JsString("start")))
+ case Success(f) => complete(OK -> JsString("start"))
case Failure(e) => complete(InternalServerError -> spray.json.JsObject("message" -> JsString(e.toString)))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/be83d07c/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
index 38cdce3..6f57cc4 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
@@ -44,8 +44,8 @@ object Server extends App {
val route: Flow[HttpRequest, HttpResponse, Any] = (post & path("graphql")) {
entity(as[spray.json.JsValue])(GraphQLServer.endpoint)
- } ~ (post & path("importModel")) {
- entity(as[spray.json.JsValue])(GraphQLServer.importModel)
+ } ~ (post & path("updateEdgeFetcher")) {
+ entity(as[spray.json.JsValue])(GraphQLServer.updateEdgeFetcher)
} ~ {
getFromResource("assets/graphiql.html")
}
[08/11] incubator-s2graph git commit: bug fix.
Posted by st...@apache.org.
bug fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/9132f748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/9132f748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/9132f748
Branch: refs/heads/master
Commit: 9132f7480022f653013006d0451e5cef01760ecb
Parents: f6b0740
Author: DO YUNG YOON <st...@apache.org>
Authored: Wed May 9 23:46:54 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Wed May 9 23:46:54 2018 +0900
----------------------------------------------------------------------
s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/9132f748/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 99423d6..fb229ec 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -209,7 +209,7 @@ trait S2GraphLike extends Graph {
if (ids.isEmpty) {
//TODO: default storage need to be fixed.
- val futures = getAllVertexFetchers.map { vertexFetcher =>
+ val futures = (defaultStorage.vertexFetcher +: getAllVertexFetchers).map { vertexFetcher =>
vertexFetcher.fetchVerticesAll()
}
@@ -240,7 +240,7 @@ trait S2GraphLike extends Graph {
def edges(edgeIds: AnyRef*): util.Iterator[structure.Edge] = {
if (edgeIds.isEmpty) {
// FIXME
- val futures = getAllEdgeFetchers().map { edgeFetcher =>
+ val futures = (defaultStorage.edgeFetcher +: getAllEdgeFetchers()).map { edgeFetcher =>
edgeFetcher.fetchEdgesAll()
}
[11/11] incubator-s2graph git commit: [S2GRAPH-213]: Abstract
Query/Mutation from Storage.
Posted by st...@apache.org.
[S2GRAPH-213]: Abstract Query/Mutation from Storage.
JIRA:
[S2GRAPH-213] https://issues.apache.org/jira/browse/S2GRAPH-213
Pull Request:
Closes #165
Author
DO YUNG YOON <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/33f4d055
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/33f4d055
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/33f4d055
Branch: refs/heads/master
Commit: 33f4d055092dd5df5fcdff5181b5b1b19fa32484
Parents: 16feda8
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri May 11 12:05:18 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri May 11 12:05:18 2018 +0900
----------------------------------------------------------------------
CHANGES | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/33f4d055/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 78dbac5..a2454a1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -76,6 +76,7 @@ Release Notes - S2Graph - Version 0.2.0
* [S2GRAPH-193] - Add README for S2Jobs sub-project
* [S2GRAPH-209] - GlobalIndex supports field data types such as Numeric to enable Range Query
* [S2GRAPH-210] - Rename package `mysqls` to `schema`
+ * [S2GRAPH-213] - Abstract Query/Mutation from Storage.
** New Feature
* [S2GRAPH-123] - Support different index on out/in direction.