You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2017/11/19 02:28:44 UTC
[01/23] incubator-s2graph git commit: add S2VertexLike.
Repository: incubator-s2graph
Updated Branches:
refs/heads/master 79e1ffa68 -> 16f18ffe0
add S2VertexLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/fac47d1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/fac47d1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/fac47d1f
Branch: refs/heads/master
Commit: fac47d1f4f25dc91b5c0fa83fa3ee9d261798c90
Parents: 128d67c
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 10:48:24 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 10:48:24 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/S2Vertex.scala | 231 +------------------
.../org/apache/s2graph/core/S2VertexLike.scala | 212 +++++++++++++++++
2 files changed, 219 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fac47d1f/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
index 177d859..62efa14 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
@@ -19,53 +19,22 @@
package org.apache.s2graph.core
-import java.util
-import java.util.function.{BiConsumer, Consumer}
+import java.util.function.BiConsumer
-import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.S2Vertex.Props
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types._
-import org.apache.s2graph.core.utils.logger
-import org.apache.tinkerpop.gremlin.structure.Edge.Exceptions
+import org.apache.tinkerpop.gremlin.structure.Vertex
import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
-import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Graph, Property, T, Vertex, VertexProperty}
-import play.api.libs.json.Json
import scala.collection.JavaConverters._
-import scala.concurrent.{Await, Future}
case class S2Vertex(graph: S2Graph,
- id: VertexId,
- ts: Long = System.currentTimeMillis(),
- props: Props = S2Vertex.EmptyProps,
- op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty) extends GraphElement with Vertex {
-
- val innerId = id.innerId
-
- val innerIdVal = innerId.value
-
- lazy val properties = for {
- (k, v) <- props.asScala
- } yield v.columnMeta.name -> v.value
-
- def schemaVer = serviceColumn.schemaVersion
-
- def serviceColumn = ServiceColumn.findById(id.colId)
-
- def columnName = serviceColumn.columnName
-
- lazy val service = Service.findById(serviceColumn.serviceId)
-
- lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
-
- def defaultProps = {
- val default = S2Vertex.EmptyProps
- val newProps = new S2VertexProperty(this, ColumnMeta.lastModifiedAtColumn, ColumnMeta.lastModifiedAtColumn.name, ts)
- default.put(ColumnMeta.lastModifiedAtColumn.name, newProps)
- default
- }
+ id: VertexId,
+ ts: Long = System.currentTimeMillis(),
+ props: Props = S2Vertex.EmptyProps,
+ op: Byte = 0,
+ belongLabelIds: Seq[Int] = Seq.empty) extends S2VertexLike {
// lazy val kvs = Graph.client.vertexSerializer(this).toKeyValues
@@ -78,10 +47,6 @@ case class S2Vertex(graph: S2Graph,
override def queuePartitionKey = id.innerId.toString
- def propsWithName = for {
- (k, v) <- props.asScala
- } yield (v.columnMeta.name -> v.value.toString)
-
override def hashCode() = {
val hash = id.hashCode()
// logger.debug(s"Vertex.hashCode: $this -> $hash")
@@ -103,188 +68,6 @@ case class S2Vertex(graph: S2Graph,
// v[VertexId(1, 1481694411514)]
s"v[${id}]"
}
-
- def toLogString(): String = {
- val (serviceName, columnName) =
- if (!id.storeColId) ("", "")
- else (serviceColumn.service.serviceName, serviceColumn.columnName)
-
- if (propsWithName.nonEmpty)
- Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName, Json.toJson(propsWithName)).mkString("\t")
- else
- Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
- }
-
- def copyVertexWithState(props: Props): S2Vertex = {
- val newVertex = copy(props = S2Vertex.EmptyProps)
- S2Vertex.fillPropsWithTs(newVertex, props)
- newVertex
- }
-
- override def vertices(direction: Direction, edgeLabels: String*): util.Iterator[Vertex] = {
- val arr = new util.ArrayList[Vertex]()
- edges(direction, edgeLabels: _*).forEachRemaining(new Consumer[Edge] {
- override def accept(edge: Edge): Unit = {
- val s2Edge = edge.asInstanceOf[S2Edge]
-
- s2Edge.direction match {
- case "out" => arr.add(edge.inVertex())
- case "in" => arr.add(edge.outVertex())
- case _ => throw new IllegalStateException("only out/in direction can be found in S2Edge")
- }
-// direction match {
-// case Direction.OUT => arr.add(edge.inVertex())
-// case Direction.IN => arr.add(edge.outVertex())
-// case _ =>
-// val inV = edge.inVertex()
-// val outV = edge.outVertex()
-// if (id != inV.id()) arr.add(inV)
-// if (id != outV.id()) arr.add(outV)
-// }
- }
- })
- arr.iterator()
- }
-
- override def edges(direction: Direction, labelNames: String*): util.Iterator[Edge] = {
- val labelNameWithDirs =
- if (labelNames.isEmpty) {
- // TODO: Let's clarify direction
- if (direction == Direction.BOTH) {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> Direction.OUT.name) ++
- Label.findByTgtColumnId(id.colId).map(l => l.label -> Direction.IN.name)
- } else if (direction == Direction.IN) {
- Label.findByTgtColumnId(id.colId).map(l => l.label -> direction.name)
- } else {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> direction.name)
- }
- } else {
- direction match {
- case Direction.BOTH =>
- Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
- case _ => labelNames.map(_ -> direction.name())
- }
- }
-
- graph.fetchEdges(this, labelNameWithDirs.distinct)
- }
-
- private def edgesAsync(direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
- val labelNameWithDirs =
- if (labelNames.isEmpty) {
- // TODO: Let's clarify direction
- if (direction == Direction.BOTH) {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> Direction.OUT.name) ++
- Label.findByTgtColumnId(id.colId).map(l => l.label -> Direction.IN.name)
- } else if (direction == Direction.IN) {
- Label.findByTgtColumnId(id.colId).map(l => l.label -> direction.name)
- } else {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> direction.name)
- }
- } else {
- direction match {
- case Direction.BOTH =>
- Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
- case _ => labelNames.map(_ -> direction.name())
- }
- }
-
- graph.fetchEdgesAsync(this, labelNameWithDirs.distinct)
- }
-
- // do no save to storage
- def propertyInner[V](cardinality: Cardinality, key: String, value: V, objects: AnyRef*): VertexProperty[V] = {
- S2Property.assertValidProp(key, value)
-
- cardinality match {
- case Cardinality.single =>
- val columnMeta = serviceColumn.metasInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Vertex."))
- val newProps = new S2VertexProperty[V](this, columnMeta, key, value)
- props.put(key, newProps)
- newProps
- case _ => throw new RuntimeException("only single cardinality is supported.")
- }
- }
-
- override def property[V](cardinality: Cardinality, key: String, value: V, objects: AnyRef*): VertexProperty[V] = {
- S2Property.assertValidProp(key, value)
-
- cardinality match {
- case Cardinality.single =>
- val columnMeta = serviceColumn.metasInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Vertex."))
- val newProps = new S2VertexProperty[V](this, columnMeta, key, value)
- props.put(key, newProps)
-
- // FIXME: save to persistent for tp test
- graph.addVertexInner(this)
- newProps
- case _ => throw new RuntimeException("only single cardinality is supported.")
- }
- }
-
- override def addEdge(labelName: String, vertex: Vertex, kvs: AnyRef*): Edge = {
- graph.addEdge(this, labelName, vertex, kvs: _*)
- }
-
- override def property[V](key: String): VertexProperty[V] = {
- if (props.containsKey(key)) {
- props.get(key).asInstanceOf[S2VertexProperty[V]]
- } else {
- VertexProperty.empty()
- }
- }
-
- override def properties[V](keys: String*): util.Iterator[VertexProperty[V]] = {
- val ls = new util.ArrayList[VertexProperty[V]]()
- if (keys.isEmpty) {
- props.forEach(new BiConsumer[String, VertexProperty[_]] {
- override def accept(key: String, property: VertexProperty[_]): Unit = {
- if (!ColumnMeta.reservedMetaNamesSet(key) && property.isPresent && key != T.id.name)
- ls.add(property.asInstanceOf[VertexProperty[V]])
- }
- })
- } else {
- keys.foreach { key =>
- val prop = property[V](key)
- if (prop.isPresent) ls.add(prop)
- }
- }
- ls.iterator
- }
-
- override def label(): String = {
- serviceColumn.columnName
-// if (serviceColumn.columnName == Vertex.DEFAULT_LABEL) Vertex.DEFAULT_LABEL // TP3 default vertex label name
-// else {
-// service.serviceName + S2Vertex.VertexLabelDelimiter + serviceColumn.columnName
-// }
- }
-
- override def remove(): Unit = {
- if (graph.features().vertex().supportsRemoveVertices()) {
- // remove edge
- // TODO: remove related edges also.
- implicit val ec = graph.ec
-
- val verticesToDelete = Seq(this.copy(op = GraphUtil.operations("delete")))
-
- val vertexFuture = graph.mutateVertices(verticesToDelete, withWait = true)
-
- val future = for {
- vertexSuccess <- vertexFuture
- edges <- edgesAsync(Direction.BOTH)
- } yield {
- edges.asScala.toSeq.foreach { edge => edge.remove() }
- if (!vertexSuccess.forall(_.isSuccess)) throw new RuntimeException("Vertex.remove vertex delete failed.")
-
- true
- }
- Await.result(future, graph.WaitTimeout)
-
- } else {
- throw Vertex.Exceptions.vertexRemovalNotSupported()
- }
- }
}
object S2Vertex {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fac47d1f/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
new file mode 100644
index 0000000..c2bc40c
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -0,0 +1,212 @@
+package org.apache.s2graph.core
+import java.util
+import java.util.function.{BiConsumer, Consumer}
+
+import org.apache.s2graph.core.S2Vertex.Props
+import org.apache.s2graph.core.mysqls.{ColumnMeta, Label, Service, ServiceColumn}
+import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
+import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, T, Vertex, VertexProperty}
+import play.api.libs.json.Json
+
+import scala.concurrent.{Await, Future}
+import scala.collection.JavaConverters._
+
+trait S2VertexLike extends Vertex with GraphElement {
+ this: S2Vertex =>
+
+ val innerId = id.innerId
+ val innerIdVal = innerId.value
+
+ lazy val properties = for {
+ (k, v) <- props.asScala
+ } yield v.columnMeta.name -> v.value
+
+ def schemaVer = serviceColumn.schemaVersion
+
+ def serviceColumn = ServiceColumn.findById(id.colId)
+
+ def columnName = serviceColumn.columnName
+
+ lazy val service = Service.findById(serviceColumn.serviceId)
+ lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
+
+ def defaultProps = {
+ val default = S2Vertex.EmptyProps
+ val newProps = new S2VertexProperty(this, ColumnMeta.lastModifiedAtColumn, ColumnMeta.lastModifiedAtColumn.name, ts)
+ default.put(ColumnMeta.lastModifiedAtColumn.name, newProps)
+ default
+ }
+
+ def propsWithName = for {
+ (k, v) <- props.asScala
+ } yield (v.columnMeta.name -> v.value.toString)
+
+ def toLogString(): String = {
+ val (serviceName, columnName) =
+ if (!id.storeColId) ("", "")
+ else (serviceColumn.service.serviceName, serviceColumn.columnName)
+
+ if (propsWithName.nonEmpty)
+ Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName, Json.toJson(propsWithName)).mkString("\t")
+ else
+ Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
+ }
+
+ def copyVertexWithState(props: Props): S2Vertex = {
+ val newVertex = copy(props = S2Vertex.EmptyProps)
+ S2Vertex.fillPropsWithTs(newVertex, props)
+ newVertex
+ }
+
+ def vertices(direction: Direction, edgeLabels: String*): util.Iterator[Vertex] = {
+ val arr = new util.ArrayList[Vertex]()
+ edges(direction, edgeLabels: _*).forEachRemaining(new Consumer[Edge] {
+ override def accept(edge: Edge): Unit = {
+ val s2Edge = edge.asInstanceOf[S2Edge]
+
+ s2Edge.direction match {
+ case "out" => arr.add(edge.inVertex())
+ case "in" => arr.add(edge.outVertex())
+ case _ => throw new IllegalStateException("only out/in direction can be found in S2Edge")
+ }
+ }
+ })
+ arr.iterator()
+ }
+
+ def edges(direction: Direction, labelNames: String*): util.Iterator[Edge] = {
+ val labelNameWithDirs =
+ if (labelNames.isEmpty) {
+ // TODO: Let's clarify direction
+ if (direction == Direction.BOTH) {
+ Label.findBySrcColumnId(id.colId).map(l => l.label -> Direction.OUT.name) ++
+ Label.findByTgtColumnId(id.colId).map(l => l.label -> Direction.IN.name)
+ } else if (direction == Direction.IN) {
+ Label.findByTgtColumnId(id.colId).map(l => l.label -> direction.name)
+ } else {
+ Label.findBySrcColumnId(id.colId).map(l => l.label -> direction.name)
+ }
+ } else {
+ direction match {
+ case Direction.BOTH =>
+ Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
+ case _ => labelNames.map(_ -> direction.name())
+ }
+ }
+
+ graph.fetchEdges(this, labelNameWithDirs.distinct)
+ }
+
+ // do no save to storage
+ def propertyInner[V](cardinality: Cardinality, key: String, value: V, objects: AnyRef*): VertexProperty[V] = {
+ S2Property.assertValidProp(key, value)
+
+ cardinality match {
+ case Cardinality.single =>
+ val columnMeta = serviceColumn.metasInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Vertex."))
+ val newProps = new S2VertexProperty[V](this, columnMeta, key, value)
+ props.put(key, newProps)
+ newProps
+ case _ => throw new RuntimeException("only single cardinality is supported.")
+ }
+ }
+
+ def property[V](cardinality: Cardinality, key: String, value: V, objects: AnyRef*): VertexProperty[V] = {
+ S2Property.assertValidProp(key, value)
+
+ cardinality match {
+ case Cardinality.single =>
+ val columnMeta = serviceColumn.metasInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Vertex."))
+ val newProps = new S2VertexProperty[V](this, columnMeta, key, value)
+ props.put(key, newProps)
+
+ // FIXME: save to persistent for tp test
+ graph.addVertexInner(this)
+ newProps
+ case _ => throw new RuntimeException("only single cardinality is supported.")
+ }
+ }
+
+ def addEdge(labelName: String, vertex: Vertex, kvs: AnyRef*): Edge = {
+ graph.addEdge(this, labelName, vertex, kvs: _*)
+ }
+
+ def property[V](key: String): VertexProperty[V] = {
+ if (props.containsKey(key)) {
+ props.get(key).asInstanceOf[S2VertexProperty[V]]
+ } else {
+ VertexProperty.empty()
+ }
+ }
+
+ def properties[V](keys: String*): util.Iterator[VertexProperty[V]] = {
+ val ls = new util.ArrayList[VertexProperty[V]]()
+ if (keys.isEmpty) {
+ props.forEach(new BiConsumer[String, VertexProperty[_]] {
+ override def accept(key: String, property: VertexProperty[_]): Unit = {
+ if (!ColumnMeta.reservedMetaNamesSet(key) && property.isPresent && key != T.id.name)
+ ls.add(property.asInstanceOf[VertexProperty[V]])
+ }
+ })
+ } else {
+ keys.foreach { key =>
+ val prop = property[V](key)
+ if (prop.isPresent) ls.add(prop)
+ }
+ }
+ ls.iterator
+ }
+
+ def label(): String = {
+ serviceColumn.columnName
+ }
+
+ def remove(): Unit = {
+ if (graph.features().vertex().supportsRemoveVertices()) {
+ // remove edge
+ // TODO: remove related edges also.
+ implicit val ec = graph.ec
+
+ val verticesToDelete = Seq(this.copy(op = GraphUtil.operations("delete")))
+
+ val vertexFuture = graph.mutateVertices(verticesToDelete, withWait = true)
+
+ val future = for {
+ vertexSuccess <- vertexFuture
+ edges <- edgesAsync(Direction.BOTH)
+ } yield {
+ edges.asScala.toSeq.foreach { edge => edge.remove() }
+ if (!vertexSuccess.forall(_.isSuccess)) throw new RuntimeException("Vertex.remove vertex delete failed.")
+
+ true
+ }
+ Await.result(future, graph.WaitTimeout)
+
+ } else {
+ throw Vertex.Exceptions.vertexRemovalNotSupported()
+ }
+ }
+
+ private def edgesAsync(direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
+ val labelNameWithDirs =
+ if (labelNames.isEmpty) {
+ // TODO: Let's clarify direction
+ if (direction == Direction.BOTH) {
+ Label.findBySrcColumnId(id.colId).map(l => l.label -> Direction.OUT.name) ++
+ Label.findByTgtColumnId(id.colId).map(l => l.label -> Direction.IN.name)
+ } else if (direction == Direction.IN) {
+ Label.findByTgtColumnId(id.colId).map(l => l.label -> direction.name)
+ } else {
+ Label.findBySrcColumnId(id.colId).map(l => l.label -> direction.name)
+ }
+ } else {
+ direction match {
+ case Direction.BOTH =>
+ Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
+ case _ => labelNames.map(_ -> direction.name())
+ }
+ }
+
+ graph.fetchEdgesAsync(this, labelNameWithDirs.distinct)
+ }
+}
[16/23] incubator-s2graph git commit: add S2GraphLike.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
index 8e3fdf3..6ebc90c 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
@@ -28,9 +28,9 @@ import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types._
object IndexEdgeDeserializable{
- def getNewInstance(graph: S2Graph) = new IndexEdgeDeserializable(graph)
+ def getNewInstance(graph: S2GraphLike) = new IndexEdgeDeserializable(graph)
}
-class IndexEdgeDeserializable(graph: S2Graph,
+class IndexEdgeDeserializable(graph: S2GraphLike,
bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong,
tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4)) extends Deserializable[S2EdgeLike] {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
index f5d9af9..79e3f2e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
@@ -27,7 +27,7 @@ import org.apache.s2graph.core.storage._
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.types._
-class IndexEdgeDeserializable(graph: S2Graph,
+class IndexEdgeDeserializable(graph: S2GraphLike,
bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[S2EdgeLike] {
type QualifierRaw = (Array[(LabelMeta, InnerValLike)], VertexId, Byte, Boolean, Int)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
index 408822a..4c97f6e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
@@ -27,7 +27,7 @@ import org.apache.s2graph.core.types._
import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.serde.Deserializable
-class SnapshotEdgeDeserializable(graph: S2Graph) extends Deserializable[SnapshotEdge] {
+class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[SnapshotEdge] {
def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
val statusCode = byte >> 4
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
index ff8eb80..a6a64a2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
@@ -27,7 +27,7 @@ import org.apache.s2graph.core.types.{HBaseType, LabelWithDirection, SourceVerte
import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.serde.Deserializable
-class SnapshotEdgeDeserializable(graph: S2Graph) extends Deserializable[SnapshotEdge] {
+class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[SnapshotEdge] {
def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
val statusCode = byte >> 4
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
deleted file mode 100644
index 87f0947..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexDeserializable.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-//
-//package org.apache.s2graph.core.storage.serde.vertex
-//
-//import org.apache.s2graph.core.mysqls.ColumnMeta
-//import org.apache.s2graph.core.storage.serde.StorageDeserializable._
-//import org.apache.s2graph.core.storage.CanSKeyValue
-//import org.apache.s2graph.core.storage.serde.Deserializable
-//import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, VertexId}
-//import org.apache.s2graph.core.{S2Graph, S2Vertex}
-//
-//import scala.collection.mutable.ListBuffer
-//
-//class VertexDeserializable(graph: S2Graph,
-// bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2Vertex] {
-// def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
-// cacheElementOpt: Option[S2Vertex]): Option[S2Vertex] = {
-// try {
-// val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
-// val kv = kvs.head
-// val version = HBaseType.DEFAULT_VERSION
-// val (vertexId, _) = VertexId.fromBytes(kv.row, 0, kv.row.length, version)
-//
-// var maxTs = Long.MinValue
-// val propsMap = new collection.mutable.HashMap[ColumnMeta, InnerValLike]
-// val belongLabelIds = new ListBuffer[Int]
-//
-// for {
-// kv <- kvs
-// } {
-// val propKey =
-// if (kv.qualifier.length == 1) kv.qualifier.head.toInt
-// else bytesToInt(kv.qualifier, 0)
-//
-// val ts = kv.timestamp
-// if (ts > maxTs) maxTs = ts
-//
-// if (S2Vertex.isLabelId(propKey)) {
-// belongLabelIds += S2Vertex.toLabelId(propKey)
-// } else {
-// val v = kv.value
-// val (value, _) = InnerVal.fromBytes(v, 0, v.length, version)
-// val columnMeta = vertexId.column.metasMap(propKey)
-// propsMap += (columnMeta -> value)
-// }
-// }
-// assert(maxTs != Long.MinValue)
-// val vertex = graph.newVertex(vertexId, maxTs, S2Vertex.EmptyProps, belongLabelIds = belongLabelIds)
-// S2Vertex.fillPropsWithTs(vertex, propsMap.toMap)
-//
-// Option(vertex)
-// } catch {
-// case e: Exception => None
-// }
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
deleted file mode 100644
index aa85574..0000000
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/VertexSerializable.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-//
-//package org.apache.s2graph.core.storage.serde.vertex
-//
-//import org.apache.s2graph.core.S2Vertex
-//import org.apache.s2graph.core.storage.serde.StorageSerializable._
-//import org.apache.s2graph.core.storage.SKeyValue
-//import org.apache.s2graph.core.storage.serde.Serializable
-//
-//import scala.collection.JavaConverters._
-//
-//case class VertexSerializable(vertex: S2Vertex, intToBytes: Int => Array[Byte] = intToBytes) extends Serializable[S2Vertex] {
-//
-// override val table = vertex.hbaseTableName.getBytes
-// override val ts = vertex.ts
-// override val cf = Serializable.vertexCf
-//
-// override def toRowKey: Array[Byte] = vertex.id.bytes
-//
-// override def toQualifier: Array[Byte] = Array.empty[Byte]
-//
-// override def toValue: Array[Byte] = {
-// val props = for ((k, v) <- vertex.props.asScala ++ vertex.defaultProps.asScala) yield {
-// v.columnMeta -> v.innerVal.bytes
-// }
-// vertexPropsToBytes(props.toSeq)
-// }
-//
-// /** vertex override toKeyValues since vertex expect to produce multiple sKeyValues */
-// override def toKeyValues: Seq[SKeyValue] = {
-// val row = toRowKey
-// // serializer all props into value.
-// Seq(
-// SKeyValue(vertex.hbaseTableName.getBytes, row, cf, toQualifier, toValue, vertex.ts)
-// )
-//// val base = for ((k, v) <- vertex.props.asScala ++ vertex.defaultProps.asScala) yield {
-//// val columnMeta = v.columnMeta
-//// intToBytes(columnMeta.seq) -> v.innerVal.bytes
-//// }
-//// val belongsTo = vertex.belongLabelIds.map { labelId => intToBytes(S2Vertex.toPropKey(labelId)) -> Array.empty[Byte] }
-//// (base ++ belongsTo).map { case (qualifier, value) =>
-//// SKeyValue(vertex.hbaseTableName.getBytes, row, cf, qualifier, value, vertex.ts)
-//// }.toSeq
-// }
-//}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
index 019394e..70160a8 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
@@ -24,9 +24,9 @@ import org.apache.s2graph.core.storage.CanSKeyValue
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types.{HBaseType, InnerValLike, VertexId}
-import org.apache.s2graph.core.{S2Graph, S2Vertex, S2VertexLike}
+import org.apache.s2graph.core.{S2Graph, S2GraphLike, S2Vertex, S2VertexLike}
-class VertexDeserializable(graph: S2Graph,
+class VertexDeserializable(graph: S2GraphLike,
bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
index ddb90ac..002f577 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
@@ -24,11 +24,11 @@ import org.apache.s2graph.core.storage.CanSKeyValue
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, VertexId}
-import org.apache.s2graph.core.{S2Graph, S2Vertex, S2VertexLike}
+import org.apache.s2graph.core.{S2Graph, S2GraphLike, S2Vertex, S2VertexLike}
import scala.collection.mutable.ListBuffer
-class VertexDeserializable(graph: S2Graph,
+class VertexDeserializable(graph: S2GraphLike,
bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
index c7d474b..3a73d20 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
@@ -86,7 +86,7 @@ class S2GraphProvider extends AbstractGraphProvider {
override def loadGraphData(graph: Graph, loadGraphWith: LoadGraphWith, testClass: Class[_], testName: String): Unit = {
val s2Graph = graph.asInstanceOf[S2Graph]
- val mnt = s2Graph.getManagement()
+ val mnt = s2Graph.management
S2GraphFactory.cleanupDefaultSchema
initTestSchema(testClass, testName)
[10/23] incubator-s2graph git commit: add TraversalHelper.
Posted by st...@apache.org.
add TraversalHelper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/7d082255
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/7d082255
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/7d082255
Branch: refs/heads/master
Commit: 7d082255944228a34e8ea55d9609202ab65362b2
Parents: aa66822
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Nov 4 07:19:27 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 4 07:19:27 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/PostProcess.scala | 380 ----------------
.../scala/org/apache/s2graph/core/S2Graph.scala | 58 +--
.../apache/s2graph/core/TraversalHelper.scala | 442 +++++++++++++++++++
.../apache/s2graph/core/storage/StorageIO.scala | 2 +-
4 files changed, 450 insertions(+), 432 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7d082255/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 4549d84..2d2e183 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -281,384 +281,4 @@ object PostProcess {
withOptionalFields(queryOption, results.size, degrees, results, stepResult.failCount, cursorJson, nextQuery)
}
}
-
- /** Global helper functions */
- @tailrec
- final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (range < sampleNumber || set.size == sampleNumber) set
- else randomInt(sampleNumber, range, set + Random.nextInt(range))
- }
-
- def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
- if (edges.size <= n) {
- edges
- } else {
- val plainEdges = if (queryRequest.queryParam.offset == 0) {
- edges.tail
- } else edges
-
- val randoms = randomInt(n, plainEdges.size)
- var samples = List.empty[EdgeWithScore]
- var idx = 0
- plainEdges.foreach { e =>
- if (randoms.contains(idx)) samples = e :: samples
- idx += 1
- }
- samples
- }
- }
-
- def normalize(edgeWithScores: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
- val sum = edgeWithScores.foldLeft(0.0) { case (acc, cur) => acc + cur.score }
- edgeWithScores.map { edgeWithScore =>
- edgeWithScore.copy(score = edgeWithScore.score / sum)
- }
- }
-
- def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
- val vertices = for {
- edgeWithScore <- edgeWithScoreLs
- edge = edgeWithScore.edge
- vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
- } yield (edge.labelWithDir, vertex) -> true
-
- vertices.toMap
- }
-
- /** common methods for filter out, transform, aggregate queryResult */
- def convertEdges(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
- for {
- convertedEdge <- queryParam.edgeTransformer.transform(queryParam, edge, nextStepOpt) if !edge.isDegree
- } yield convertedEdge
- }
-
- def processTimeDecay(queryParam: QueryParam, edge: S2EdgeLike) = {
- /* process time decay */
- val tsVal = queryParam.timeDecay match {
- case None => 1.0
- case Some(timeDecay) =>
- val tsVal = try {
- val innerValWithTsOpt = edge.propertyValue(timeDecay.labelMeta.name)
- innerValWithTsOpt.map { innerValWithTs =>
- val innerVal = innerValWithTs.innerVal
- timeDecay.labelMeta.dataType match {
- case InnerVal.LONG => innerVal.value match {
- case n: BigDecimal => n.bigDecimal.longValue()
- case _ => innerVal.toString().toLong
- }
- case _ => innerVal.toString().toLong
- }
- } getOrElse (edge.ts)
- } catch {
- case e: Exception =>
- logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
- edge.ts
- }
- val timeDiff = queryParam.timestamp - tsVal
- timeDecay.decay(timeDiff)
- }
-
- tsVal
- }
-
- def processDuplicates[R](queryParam: QueryParam,
- duplicates: Seq[(FilterHashKey, R)])(implicit ev: WithScore[R]): Seq[(FilterHashKey, R)] = {
-
- if (queryParam.label.consistencyLevel != "strong") {
- //TODO:
- queryParam.duplicatePolicy match {
- case DuplicatePolicy.First => Seq(duplicates.head)
- case DuplicatePolicy.Raw => duplicates
- case DuplicatePolicy.CountSum =>
- val countSum = duplicates.size
- val (headFilterHashKey, headEdgeWithScore) = duplicates.head
- Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, countSum))
- case _ =>
- val scoreSum = duplicates.foldLeft(0.0) { case (prev, current) => prev + ev.score(current._2) }
- val (headFilterHashKey, headEdgeWithScore) = duplicates.head
- Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, scoreSum))
- }
- } else {
- duplicates
- }
- }
-
- def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
- val src = edge.srcVertex.innerId.hashCode()
- val tgt = edge.tgtVertex.innerId.hashCode()
- val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
- val filterHashKey = (src, tgt)
-
- (hashKey, filterHashKey)
- }
-
- def filterEdges(q: Query,
- stepIdx: Int,
- queryRequests: Seq[QueryRequest],
- queryResultLsFuture: Future[Seq[StepResult]],
- queryParams: Seq[QueryParam],
- alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
- buildLastStepInnerResult: Boolean = true,
- parentEdges: Map[VertexId, Seq[EdgeWithScore]])
- (implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
-
- queryResultLsFuture.map { queryRequestWithResultLs =>
- val (cursors, failCount) = {
- val _cursors = ArrayBuffer.empty[Array[Byte]]
- var _failCount = 0
-
- queryRequestWithResultLs.foreach { stepResult =>
- _cursors.append(stepResult.cursors: _*)
- _failCount += stepResult.failCount
- }
-
- _cursors -> _failCount
- }
-
-
- if (queryRequestWithResultLs.isEmpty) StepResult.Empty.copy(failCount = failCount)
- else {
- val isLastStep = stepIdx == q.steps.size - 1
- val queryOption = q.queryOption
- val step = q.steps(stepIdx)
-
- val currentStepResults = queryRequests.view.zip(queryRequestWithResultLs)
- val shouldBuildInnerResults = !isLastStep || buildLastStepInnerResult
- val degrees = queryRequestWithResultLs.flatMap(_.degreeEdges)
-
- if (shouldBuildInnerResults) {
- val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
- edgeWithScore
- }
-
- /* process step group by */
- val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
- StepResult(edgeWithScores = results, grouped = Nil, degreeEdges = degrees, cursors = cursors, failCount = failCount)
-
- } else {
- val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
- val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val label = edgeWithScore.label
-
- /* Select */
- val mergedPropsWithTs = edge.propertyValuesInner(propsSelectColumns)
-
- // val newEdge = edge.copy(propsWithTs = mergedPropsWithTs)
- val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
-
- val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
- /* OrderBy */
- val orderByValues =
- if (queryOption.orderByKeys.isEmpty) (score, edge.tsInnerVal, None, None)
- else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
-
- /* StepGroupBy */
- val stepGroupByValues = newEdgeWithScore.toValues(step.groupBy.keys)
-
- /* GroupBy */
- val groupByValues = newEdgeWithScore.toValues(queryOption.groupBy.keys)
-
- /* FilterOut */
- val filterOutValues = newEdgeWithScore.toValues(queryOption.filterOutFields)
-
- newEdgeWithScore.copy(orderByValues = orderByValues,
- stepGroupByValues = stepGroupByValues,
- groupByValues = groupByValues,
- filterOutValues = filterOutValues)
- }
-
- /* process step group by */
- val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
-
- /* process ordered list */
- val ordered = if (queryOption.groupBy.keys.isEmpty) StepResult.orderBy(queryOption, results) else Nil
-
- /* process grouped list */
- val grouped =
- if (queryOption.groupBy.keys.isEmpty) Nil
- else {
- val agg = new scala.collection.mutable.HashMap[StepResult.GroupByKey, (Double, StepResult.Values)]()
- results.groupBy { edgeWithScore =>
- // edgeWithScore.groupByValues.map(_.map(_.toString))
- edgeWithScore.groupByValues
- }.foreach { case (k, ls) =>
- val (scoreSum, merged) = StepResult.mergeOrdered(ls, Nil, queryOption)
-
- val newScoreSum = scoreSum
-
- /*
- * watch out here. by calling toString on Any, we lose type information which will be used
- * later for toJson.
- */
- if (merged.nonEmpty) {
- val newKey = merged.head.groupByValues
- agg += ((newKey, (newScoreSum, merged)))
- }
- }
- agg.toSeq.sortBy(_._2._1 * -1)
- }
-
- StepResult(edgeWithScores = ordered, grouped = grouped, degreeEdges = degrees, cursors = cursors, failCount = failCount)
- }
- }
- }
- }
-
- def toEdgeWithScores(queryRequest: QueryRequest,
- stepResult: StepResult,
- parentEdges: Map[VertexId, Seq[EdgeWithScore]]): Seq[EdgeWithScore] = {
- val queryOption = queryRequest.query.queryOption
- val queryParam = queryRequest.queryParam
- val prevScore = queryRequest.prevStepScore
- val labelWeight = queryRequest.labelWeight
- val edgeWithScores = stepResult.edgeWithScores
-
- val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
- val parents = if (shouldBuildParents) {
- parentEdges.getOrElse(queryRequest.vertex.id, Nil).map { edgeWithScore =>
- val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val label = edgeWithScore.label
-
- /* Select */
- val mergedPropsWithTs =
- if (queryOption.selectColumns.isEmpty) {
- edge.propertyValuesInner()
- } else {
- val initial = Map(LabelMeta.timestamp -> edge.propertyValueInner(LabelMeta.timestamp))
- edge.propertyValues(queryOption.selectColumns) ++ initial
- }
-
- val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
- edgeWithScore.copy(edge = newEdge)
- }
- } else Nil
-
- // skip
- if (queryOption.ignorePrevStepCache) stepResult.edgeWithScores
- else {
- val degreeScore = 0.0
-
- val sampled =
- if (queryRequest.queryParam.sample >= 0) sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
- else edgeWithScores
-
- val withScores = for {
- edgeWithScore <- sampled
- } yield {
- val edge = edgeWithScore.edge
- val edgeScore = edgeWithScore.score
- val score = queryParam.scorePropagateOp match {
- case "plus" => edgeScore + prevScore
- case "divide" =>
- if ((prevScore + queryParam.scorePropagateShrinkage) == 0) 0
- else edgeScore / (prevScore + queryParam.scorePropagateShrinkage)
- case _ => edgeScore * prevScore
- }
-
- val tsVal = processTimeDecay(queryParam, edge)
- val newScore = degreeScore + score
- // val newEdge = if (queryOption.returnTree) edge.copy(parentEdges = parents) else edge
- val newEdge = edge.copyParentEdges(parents)
- edgeWithScore.copy(edge = newEdge, score = newScore * labelWeight * tsVal)
- }
-
- val normalized =
- if (queryParam.shouldNormalize) normalize(withScores)
- else withScores
-
- normalized
- }
- }
-
- def buildResult[R](query: Query,
- stepIdx: Int,
- stepResultLs: Seq[(QueryRequest, StepResult)],
- parentEdges: Map[VertexId, Seq[EdgeWithScore]])
- (createFunc: (EdgeWithScore, Seq[LabelMeta]) => R)
- (implicit ev: WithScore[R]): ListBuffer[R] = {
- import scala.collection._
-
- val results = ListBuffer.empty[R]
- val sequentialLs: ListBuffer[(HashKey, FilterHashKey, R, QueryParam)] = ListBuffer.empty
- val duplicates: mutable.HashMap[HashKey, ListBuffer[(FilterHashKey, R)]] = mutable.HashMap.empty
- val edgesToExclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
- val edgesToInclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
-
- var numOfDuplicates = 0
- val queryOption = query.queryOption
- val step = query.steps(stepIdx)
- val excludeLabelWithDirSet = step.queryParams.filter(_.exclude).map(l => l.labelWithDir).toSet
- val includeLabelWithDirSet = step.queryParams.filter(_.include).map(l => l.labelWithDir).toSet
-
- stepResultLs.foreach { case (queryRequest, stepInnerResult) =>
- val queryParam = queryRequest.queryParam
- val label = queryParam.label
- val shouldBeExcluded = excludeLabelWithDirSet.contains(queryParam.labelWithDir)
- val shouldBeIncluded = includeLabelWithDirSet.contains(queryParam.labelWithDir)
-
- val propsSelectColumns = (for {
- column <- queryOption.propsSelectColumns
- labelMeta <- label.metaPropsInvMap.get(column)
- } yield labelMeta)
-
- for {
- edgeWithScore <- toEdgeWithScores(queryRequest, stepInnerResult, parentEdges)
- } {
- val edge = edgeWithScore.edge
- val (hashKey, filterHashKey) = toHashKey(queryParam, edge, isDegree = false)
- // params += (hashKey -> queryParam) //
-
- /* check if this edge should be exlcuded. */
- if (shouldBeExcluded) {
- edgesToExclude.add(filterHashKey)
- } else {
- if (shouldBeIncluded) {
- edgesToInclude.add(filterHashKey)
- }
- val newEdgeWithScore = createFunc(edgeWithScore, propsSelectColumns)
-
- sequentialLs += ((hashKey, filterHashKey, newEdgeWithScore, queryParam))
- duplicates.get(hashKey) match {
- case None =>
- val newLs = ListBuffer.empty[(FilterHashKey, R)]
- newLs += (filterHashKey -> newEdgeWithScore)
- duplicates += (hashKey -> newLs) //
- case Some(old) =>
- numOfDuplicates += 1
- old += (filterHashKey -> newEdgeWithScore) //
- }
- }
- }
- }
-
-
- if (numOfDuplicates == 0) {
- // no duplicates at all.
- for {
- (hashKey, filterHashKey, edgeWithScore, _) <- sequentialLs
- if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
- } {
- results += edgeWithScore
- }
- } else {
- // need to resolve duplicates.
- val seen = new mutable.HashSet[HashKey]()
- for {
- (hashKey, filterHashKey, edgeWithScore, queryParam) <- sequentialLs
- if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
- if !seen.contains(hashKey)
- } {
- // val queryParam = params(hashKey)
- processDuplicates(queryParam, duplicates(hashKey)).foreach { case (_, duplicate) =>
- if (ev.score(duplicate) >= queryParam.threshold) {
- seen += hashKey
- results += duplicate
- }
- }
- }
- }
- results
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7d082255/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 5e23f9b..f061160 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -25,25 +25,15 @@ import java.util.concurrent.{Executors, TimeUnit}
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
-import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
-import org.apache.s2graph.core.JSONParser._
-import org.apache.s2graph.core.features.S2GraphVariables
import org.apache.s2graph.core.index.IndexProvider
import org.apache.s2graph.core.io.tinkerpop.optimize.S2GraphStepStrategy
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
-import org.apache.s2graph.core.storage.{MutateResponse, SKeyValue, Storage}
+import org.apache.s2graph.core.storage.{MutateResponse, Storage}
import org.apache.s2graph.core.types._
-import org.apache.s2graph.core.PostProcess._
import org.apache.s2graph.core.utils.{DeferCache, Extensions, logger}
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies
-import org.apache.tinkerpop.gremlin.structure
-import org.apache.tinkerpop.gremlin.structure.Edge.Exceptions
-import org.apache.tinkerpop.gremlin.structure.Graph.{Features, Variables}
-import org.apache.tinkerpop.gremlin.structure.io.{GraphReader, GraphWriter, Io, Mapper}
-import org.apache.tinkerpop.gremlin.structure.{Edge, Element, Graph, T, Transaction, Vertex}
-import play.api.libs.json.{JsObject, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, Graph}
import scala.collection.JavaConversions._
import scala.collection.mutable
@@ -577,6 +567,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val elementBuilder = new GraphElementBuilder(this)
+ val traversalHelper = new TraversalHelper(this)
+
def getStorage(service: Service): Storage = {
storagePool.getOrElse(s"service:${service.serviceName}", defaultStorage)
}
@@ -671,48 +663,12 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
else {
- val edgeWithScoreLs = stepInnerResult.edgeWithScores
-
- val q = orgQuery
- val queryOption = orgQuery.queryOption
- val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
- val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
- val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
- val step = q.steps(stepIdx)
-
- val alreadyVisited =
- if (stepIdx == 0) Map.empty[(LabelWithDirection, S2VertexLike), Boolean]
- else alreadyVisitedVertices(stepInnerResult.edgeWithScores)
-
- val initial = (Map.empty[S2VertexLike, Double], Map.empty[S2VertexLike, ArrayBuffer[EdgeWithScore]])
- val (sums, grouped) = edgeWithScoreLs.foldLeft(initial) { case ((sum, group), edgeWithScore) =>
- val key = edgeWithScore.edge.tgtVertex
- val newScore = sum.getOrElse(key, 0.0) + edgeWithScore.score
- val buffer = group.getOrElse(key, ArrayBuffer.empty[EdgeWithScore])
- buffer += edgeWithScore
- (sum + (key -> newScore), group + (key -> buffer))
- }
- val groupedByFiltered = sums.filter(t => t._2 >= prevStepThreshold)
- val prevStepTgtVertexIdEdges = grouped.map(t => t._1.id -> t._2)
-
- val nextStepSrcVertices = if (prevStepLimit >= 0) {
- groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
- } else {
- groupedByFiltered.toSeq
- }
-
- val queryRequests = for {
- (vertex, prevStepScore) <- nextStepSrcVertices
- queryParam <- step.queryParams
- } yield {
- val labelWeight = step.labelWeights.getOrElse(queryParam.labelWithDir.labelId, 1.0)
- val newPrevStepScore = if (queryOption.shouldPropagateScore) prevStepScore else 1.0
- QueryRequest(q, stepIdx, vertex, queryParam, newPrevStepScore, labelWeight)
- }
+ val (alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean], prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
+ traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
- filterEdges(orgQuery, stepIdx, queryRequests,
+ traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
fetchedLs, orgQuery.steps(stepIdx).queryParams, alreadyVisited, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7d082255/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
new file mode 100644
index 0000000..58da145
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -0,0 +1,442 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.S2Graph.{FilterHashKey, HashKey}
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection, VertexId}
+import org.apache.s2graph.core.utils.logger
+
+import scala.annotation.tailrec
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import scala.concurrent.Future
+import scala.util.Random
+
+object TraversalHelper {
+ @tailrec
+ final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (range < sampleNumber || set.size == sampleNumber) set
+ else randomInt(sampleNumber, range, set + Random.nextInt(range))
+ }
+
+ def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
+ if (edges.size <= n) {
+ edges
+ } else {
+ val plainEdges = if (queryRequest.queryParam.offset == 0) {
+ edges.tail
+ } else edges
+
+ val randoms = randomInt(n, plainEdges.size)
+ var samples = List.empty[EdgeWithScore]
+ var idx = 0
+ plainEdges.foreach { e =>
+ if (randoms.contains(idx)) samples = e :: samples
+ idx += 1
+ }
+ samples
+ }
+ }
+
+ def normalize(edgeWithScores: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
+ val sum = edgeWithScores.foldLeft(0.0) { case (acc, cur) => acc + cur.score }
+ edgeWithScores.map { edgeWithScore =>
+ edgeWithScore.copy(score = edgeWithScore.score / sum)
+ }
+ }
+
+ def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
+ val vertices = for {
+ edgeWithScore <- edgeWithScoreLs
+ edge = edgeWithScore.edge
+ vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
+ } yield (edge.labelWithDir, vertex) -> true
+
+ vertices.toMap
+ }
+
+ /** common methods for filter out, transform, aggregate queryResult */
+ def convertEdges(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
+ for {
+ convertedEdge <- queryParam.edgeTransformer.transform(queryParam, edge, nextStepOpt) if !edge.isDegree
+ } yield convertedEdge
+ }
+
+ def processTimeDecay(queryParam: QueryParam, edge: S2EdgeLike) = {
+ /* process time decay */
+ val tsVal = queryParam.timeDecay match {
+ case None => 1.0
+ case Some(timeDecay) =>
+ val tsVal = try {
+ val innerValWithTsOpt = edge.propertyValue(timeDecay.labelMeta.name)
+ innerValWithTsOpt.map { innerValWithTs =>
+ val innerVal = innerValWithTs.innerVal
+ timeDecay.labelMeta.dataType match {
+ case InnerVal.LONG => innerVal.value match {
+ case n: BigDecimal => n.bigDecimal.longValue()
+ case _ => innerVal.toString().toLong
+ }
+ case _ => innerVal.toString().toLong
+ }
+ } getOrElse (edge.ts)
+ } catch {
+ case e: Exception =>
+ logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
+ edge.ts
+ }
+ val timeDiff = queryParam.timestamp - tsVal
+ timeDecay.decay(timeDiff)
+ }
+
+ tsVal
+ }
+
+ def processDuplicates[R](queryParam: QueryParam,
+ duplicates: Seq[(FilterHashKey, R)])(implicit ev: WithScore[R]): Seq[(FilterHashKey, R)] = {
+
+ if (queryParam.label.consistencyLevel != "strong") {
+ //TODO:
+ queryParam.duplicatePolicy match {
+ case DuplicatePolicy.First => Seq(duplicates.head)
+ case DuplicatePolicy.Raw => duplicates
+ case DuplicatePolicy.CountSum =>
+ val countSum = duplicates.size
+ val (headFilterHashKey, headEdgeWithScore) = duplicates.head
+ Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, countSum))
+ case _ =>
+ val scoreSum = duplicates.foldLeft(0.0) { case (prev, current) => prev + ev.score(current._2) }
+ val (headFilterHashKey, headEdgeWithScore) = duplicates.head
+ Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, scoreSum))
+ }
+ } else {
+ duplicates
+ }
+ }
+
+ def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
+ val src = edge.srcVertex.innerId.hashCode()
+ val tgt = edge.tgtVertex.innerId.hashCode()
+ val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
+ val filterHashKey = (src, tgt)
+
+ (hashKey, filterHashKey)
+ }
+}
+
+
+class TraversalHelper(graph: S2GraphLike) {
+ import TraversalHelper._
+
+ def buildNextStepQueryRequests(orgQuery: Query, stepIdx: Int, stepInnerResult: StepResult) = {
+ val edgeWithScoreLs = stepInnerResult.edgeWithScores
+
+ val q = orgQuery
+ val queryOption = orgQuery.queryOption
+ val prevStepOpt = if (stepIdx > 0) Option(q.steps(stepIdx - 1)) else None
+ val prevStepThreshold = prevStepOpt.map(_.nextStepScoreThreshold).getOrElse(QueryParam.DefaultThreshold)
+ val prevStepLimit = prevStepOpt.map(_.nextStepLimit).getOrElse(-1)
+ val step = q.steps(stepIdx)
+
+ val alreadyVisited =
+ if (stepIdx == 0) Map.empty[(LabelWithDirection, S2VertexLike), Boolean]
+ else alreadyVisitedVertices(stepInnerResult.edgeWithScores)
+
+ val initial = (Map.empty[S2VertexLike, Double], Map.empty[S2VertexLike, ArrayBuffer[EdgeWithScore]])
+ val (sums, grouped) = edgeWithScoreLs.foldLeft(initial) { case ((sum, group), edgeWithScore) =>
+ val key = edgeWithScore.edge.tgtVertex
+ val newScore = sum.getOrElse(key, 0.0) + edgeWithScore.score
+ val buffer = group.getOrElse(key, ArrayBuffer.empty[EdgeWithScore])
+ buffer += edgeWithScore
+ (sum + (key -> newScore), group + (key -> buffer))
+ }
+ val groupedByFiltered = sums.filter(t => t._2 >= prevStepThreshold)
+ val prevStepTgtVertexIdEdges = grouped.map(t => t._1.id -> t._2)
+
+ val nextStepSrcVertices = if (prevStepLimit >= 0) {
+ groupedByFiltered.toSeq.sortBy(-1 * _._2).take(prevStepLimit)
+ } else {
+ groupedByFiltered.toSeq
+ }
+
+ val queryRequests = for {
+ (vertex, prevStepScore) <- nextStepSrcVertices
+ queryParam <- step.queryParams
+ } yield {
+ val labelWeight = step.labelWeights.getOrElse(queryParam.labelWithDir.labelId, 1.0)
+ val newPrevStepScore = if (queryOption.shouldPropagateScore) prevStepScore else 1.0
+ QueryRequest(q, stepIdx, vertex, queryParam, newPrevStepScore, labelWeight)
+ }
+ (alreadyVisited, prevStepTgtVertexIdEdges, queryRequests)
+ }
+
+ def filterEdges(q: Query,
+ stepIdx: Int,
+ queryRequests: Seq[QueryRequest],
+ queryResultLsFuture: Future[Seq[StepResult]],
+ queryParams: Seq[QueryParam],
+ alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
+ buildLastStepInnerResult: Boolean = true,
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]])
+ (implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
+
+ queryResultLsFuture.map { queryRequestWithResultLs =>
+ val (cursors, failCount) = {
+ val _cursors = ArrayBuffer.empty[Array[Byte]]
+ var _failCount = 0
+
+ queryRequestWithResultLs.foreach { stepResult =>
+ _cursors.append(stepResult.cursors: _*)
+ _failCount += stepResult.failCount
+ }
+
+ _cursors -> _failCount
+ }
+
+
+ if (queryRequestWithResultLs.isEmpty) StepResult.Empty.copy(failCount = failCount)
+ else {
+ val isLastStep = stepIdx == q.steps.size - 1
+ val queryOption = q.queryOption
+ val step = q.steps(stepIdx)
+
+ val currentStepResults = queryRequests.view.zip(queryRequestWithResultLs)
+ val shouldBuildInnerResults = !isLastStep || buildLastStepInnerResult
+ val degrees = queryRequestWithResultLs.flatMap(_.degreeEdges)
+
+ if (shouldBuildInnerResults) {
+ val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
+ edgeWithScore
+ }
+
+ /* process step group by */
+ val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
+ StepResult(edgeWithScores = results, grouped = Nil, degreeEdges = degrees, cursors = cursors, failCount = failCount)
+
+ } else {
+ val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
+ val edge = edgeWithScore.edge
+ val score = edgeWithScore.score
+ val label = edgeWithScore.label
+
+ /* Select */
+ val mergedPropsWithTs = edge.propertyValuesInner(propsSelectColumns)
+
+ // val newEdge = edge.copy(propsWithTs = mergedPropsWithTs)
+ val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
+
+ val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
+ /* OrderBy */
+ val orderByValues =
+ if (queryOption.orderByKeys.isEmpty) (score, edge.tsInnerVal, None, None)
+ else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
+
+ /* StepGroupBy */
+ val stepGroupByValues = newEdgeWithScore.toValues(step.groupBy.keys)
+
+ /* GroupBy */
+ val groupByValues = newEdgeWithScore.toValues(queryOption.groupBy.keys)
+
+ /* FilterOut */
+ val filterOutValues = newEdgeWithScore.toValues(queryOption.filterOutFields)
+
+ newEdgeWithScore.copy(orderByValues = orderByValues,
+ stepGroupByValues = stepGroupByValues,
+ groupByValues = groupByValues,
+ filterOutValues = filterOutValues)
+ }
+
+ /* process step group by */
+ val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
+
+ /* process ordered list */
+ val ordered = if (queryOption.groupBy.keys.isEmpty) StepResult.orderBy(queryOption, results) else Nil
+
+ /* process grouped list */
+ val grouped =
+ if (queryOption.groupBy.keys.isEmpty) Nil
+ else {
+ val agg = new scala.collection.mutable.HashMap[StepResult.GroupByKey, (Double, StepResult.Values)]()
+ results.groupBy { edgeWithScore =>
+ // edgeWithScore.groupByValues.map(_.map(_.toString))
+ edgeWithScore.groupByValues
+ }.foreach { case (k, ls) =>
+ val (scoreSum, merged) = StepResult.mergeOrdered(ls, Nil, queryOption)
+
+ val newScoreSum = scoreSum
+
+ /*
+ * watch out here. by calling toString on Any, we lose type information which will be used
+ * later for toJson.
+ */
+ if (merged.nonEmpty) {
+ val newKey = merged.head.groupByValues
+ agg += ((newKey, (newScoreSum, merged)))
+ }
+ }
+ agg.toSeq.sortBy(_._2._1 * -1)
+ }
+
+ StepResult(edgeWithScores = ordered, grouped = grouped, degreeEdges = degrees, cursors = cursors, failCount = failCount)
+ }
+ }
+ }
+ }
+
+
+
+
+ private def toEdgeWithScores(queryRequest: QueryRequest,
+ stepResult: StepResult,
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]]): Seq[EdgeWithScore] = {
+ val queryOption = queryRequest.query.queryOption
+ val queryParam = queryRequest.queryParam
+ val prevScore = queryRequest.prevStepScore
+ val labelWeight = queryRequest.labelWeight
+ val edgeWithScores = stepResult.edgeWithScores
+
+ val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
+ val parents = if (shouldBuildParents) {
+ parentEdges.getOrElse(queryRequest.vertex.id, Nil).map { edgeWithScore =>
+ val edge = edgeWithScore.edge
+ val score = edgeWithScore.score
+ val label = edgeWithScore.label
+
+ /* Select */
+ val mergedPropsWithTs =
+ if (queryOption.selectColumns.isEmpty) {
+ edge.propertyValuesInner()
+ } else {
+ val initial = Map(LabelMeta.timestamp -> edge.propertyValueInner(LabelMeta.timestamp))
+ edge.propertyValues(queryOption.selectColumns) ++ initial
+ }
+
+ val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
+ edgeWithScore.copy(edge = newEdge)
+ }
+ } else Nil
+
+ // skip
+ if (queryOption.ignorePrevStepCache) stepResult.edgeWithScores
+ else {
+ val degreeScore = 0.0
+
+ val sampled =
+ if (queryRequest.queryParam.sample >= 0) sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
+ else edgeWithScores
+
+ val withScores = for {
+ edgeWithScore <- sampled
+ } yield {
+ val edge = edgeWithScore.edge
+ val edgeScore = edgeWithScore.score
+ val score = queryParam.scorePropagateOp match {
+ case "plus" => edgeScore + prevScore
+ case "divide" =>
+ if ((prevScore + queryParam.scorePropagateShrinkage) == 0) 0
+ else edgeScore / (prevScore + queryParam.scorePropagateShrinkage)
+ case _ => edgeScore * prevScore
+ }
+
+ val tsVal = processTimeDecay(queryParam, edge)
+ val newScore = degreeScore + score
+ // val newEdge = if (queryOption.returnTree) edge.copy(parentEdges = parents) else edge
+ val newEdge = edge.copyParentEdges(parents)
+ edgeWithScore.copy(edge = newEdge, score = newScore * labelWeight * tsVal)
+ }
+
+ val normalized =
+ if (queryParam.shouldNormalize) normalize(withScores)
+ else withScores
+
+ normalized
+ }
+ }
+
+ private def buildResult[R](query: Query,
+ stepIdx: Int,
+ stepResultLs: Seq[(QueryRequest, StepResult)],
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]])
+ (createFunc: (EdgeWithScore, Seq[LabelMeta]) => R)
+ (implicit ev: WithScore[R]): ListBuffer[R] = {
+ import scala.collection._
+
+ val results = ListBuffer.empty[R]
+ val sequentialLs: ListBuffer[(HashKey, FilterHashKey, R, QueryParam)] = ListBuffer.empty
+ val duplicates: mutable.HashMap[HashKey, ListBuffer[(FilterHashKey, R)]] = mutable.HashMap.empty
+ val edgesToExclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
+ val edgesToInclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
+
+ var numOfDuplicates = 0
+ val queryOption = query.queryOption
+ val step = query.steps(stepIdx)
+ val excludeLabelWithDirSet = step.queryParams.filter(_.exclude).map(l => l.labelWithDir).toSet
+ val includeLabelWithDirSet = step.queryParams.filter(_.include).map(l => l.labelWithDir).toSet
+
+ stepResultLs.foreach { case (queryRequest, stepInnerResult) =>
+ val queryParam = queryRequest.queryParam
+ val label = queryParam.label
+ val shouldBeExcluded = excludeLabelWithDirSet.contains(queryParam.labelWithDir)
+ val shouldBeIncluded = includeLabelWithDirSet.contains(queryParam.labelWithDir)
+
+ val propsSelectColumns = (for {
+ column <- queryOption.propsSelectColumns
+ labelMeta <- label.metaPropsInvMap.get(column)
+ } yield labelMeta)
+
+ for {
+ edgeWithScore <- toEdgeWithScores(queryRequest, stepInnerResult, parentEdges)
+ } {
+ val edge = edgeWithScore.edge
+ val (hashKey, filterHashKey) = toHashKey(queryParam, edge, isDegree = false)
+ // params += (hashKey -> queryParam) //
+
+ /* check if this edge should be exlcuded. */
+ if (shouldBeExcluded) {
+ edgesToExclude.add(filterHashKey)
+ } else {
+ if (shouldBeIncluded) {
+ edgesToInclude.add(filterHashKey)
+ }
+ val newEdgeWithScore = createFunc(edgeWithScore, propsSelectColumns)
+
+ sequentialLs += ((hashKey, filterHashKey, newEdgeWithScore, queryParam))
+ duplicates.get(hashKey) match {
+ case None =>
+ val newLs = ListBuffer.empty[(FilterHashKey, R)]
+ newLs += (filterHashKey -> newEdgeWithScore)
+ duplicates += (hashKey -> newLs) //
+ case Some(old) =>
+ numOfDuplicates += 1
+ old += (filterHashKey -> newEdgeWithScore) //
+ }
+ }
+ }
+ }
+
+
+ if (numOfDuplicates == 0) {
+ // no duplicates at all.
+ for {
+ (hashKey, filterHashKey, edgeWithScore, _) <- sequentialLs
+ if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
+ } {
+ results += edgeWithScore
+ }
+ } else {
+ // need to resolve duplicates.
+ val seen = new mutable.HashSet[HashKey]()
+ for {
+ (hashKey, filterHashKey, edgeWithScore, queryParam) <- sequentialLs
+ if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
+ if !seen.contains(hashKey)
+ } {
+ // val queryParam = params(hashKey)
+ processDuplicates(queryParam, duplicates(hashKey)).foreach { case (_, duplicate) =>
+ if (ev.score(duplicate) >= queryParam.threshold) {
+ seen += hashKey
+ results += duplicate
+ }
+ }
+ }
+ }
+ results
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7d082255/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index 3074b4e..28b15d0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.storage
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.PostProcess._
+import org.apache.s2graph.core.TraversalHelper._
import org.apache.s2graph.core._
import org.apache.s2graph.core.mysqls.LabelMeta
import org.apache.s2graph.core.parsers.WhereParser
[07/23] incubator-s2graph git commit: add GraphElementBuilder.
Posted by st...@apache.org.
add GraphElementBuilder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/bc26642b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/bc26642b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/bc26642b
Branch: refs/heads/master
Commit: bc26642bc8c3c738205c8bc6c8239f7c0c78254b
Parents: 42b7702
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 21:11:13 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 21:11:13 2017 +0900
----------------------------------------------------------------------
.../s2graph/core/GraphElementBuilder.scala | 294 +++++++++++++++++++
1 file changed, 294 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bc26642b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
new file mode 100644
index 0000000..21179aa
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -0,0 +1,294 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
+import org.apache.s2graph.core.JSONParser.{fromJsonToProperties, toInnerVal}
+import org.apache.s2graph.core.mysqls._
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.utils.logger
+import play.api.libs.json.{JsObject, Json}
+
+import scala.util.Try
+
+class GraphElementBuilder(graph: S2Graph) {
+
+ def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] = Try {
+ val parts = GraphUtil.split(s)
+ val logType = parts(2)
+ val element = if (logType == "edge" | logType == "e") {
+ /* current only edge is considered to be bulk loaded */
+ labelMapping.get(parts(5)) match {
+ case None =>
+ case Some(toReplace) =>
+ parts(5) = toReplace
+ }
+ toEdge(parts)
+ } else if (logType == "vertex" | logType == "v") {
+ toVertex(parts)
+ } else {
+ throw new GraphExceptions.JsonParseException("log type is not exist in log.")
+ }
+
+ element
+ } recover {
+ case e: Exception =>
+ logger.error(s"[toElement]: $e", e)
+ None
+ } get
+
+
+ def toVertex(s: String): Option[S2VertexLike] = {
+ toVertex(GraphUtil.split(s))
+ }
+
+ def toEdge(s: String): Option[S2EdgeLike] = {
+ toEdge(GraphUtil.split(s))
+ }
+
+ def toEdge(parts: Array[String]): Option[S2EdgeLike] = Try {
+ val (ts, operation, logType, srcId, tgtId, label) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
+ val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
+ val tempDirection = if (parts.length >= 8) parts(7) else "out"
+ val direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
+ val edge = toEdge(srcId, tgtId, label, direction, props, ts.toLong, operation)
+ Option(edge)
+ } recover {
+ case e: Exception =>
+ logger.error(s"[toEdge]: $e", e)
+ throw e
+ } get
+
+ def toVertex(parts: Array[String]): Option[S2VertexLike] = Try {
+ val (ts, operation, logType, srcId, serviceName, colName) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
+ val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
+ val vertex = toVertex(serviceName, colName, srcId, props, ts.toLong, operation)
+ Option(vertex)
+ } recover {
+ case e: Throwable =>
+ logger.error(s"[toVertex]: $e", e)
+ throw e
+ } get
+
+ def toEdge(srcId: Any,
+ tgtId: Any,
+ labelName: String,
+ direction: String,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2EdgeLike = {
+ val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
+
+ val srcColumn = if (direction == "out") label.srcColumn else label.tgtColumn
+ val tgtColumn = if (direction == "out") label.tgtColumn else label.srcColumn
+
+ val srcVertexIdInnerVal = toInnerVal(srcId, srcColumn.columnType, label.schemaVersion)
+ val tgtVertexIdInnerVal = toInnerVal(tgtId, tgtColumn.columnType, label.schemaVersion)
+
+ val srcVertex = newVertex(new SourceVertexId(srcColumn, srcVertexIdInnerVal), System.currentTimeMillis())
+ val tgtVertex = newVertex(new TargetVertexId(tgtColumn, tgtVertexIdInnerVal), System.currentTimeMillis())
+ val dir = GraphUtil.toDir(direction).getOrElse(throw new RuntimeException(s"$direction is not supported."))
+
+ val propsPlusTs = props ++ Map(LabelMeta.timestamp.name -> ts)
+ val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts)
+ val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
+
+ new S2Edge(graph, srcVertex, tgtVertex, label, dir, op = op, version = ts).copyEdgeWithState(propsWithTs)
+ }
+
+ def toVertex(serviceName: String,
+ columnName: String,
+ id: Any,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2VertexLike = {
+
+ val service = Service.findByName(serviceName).getOrElse(throw new java.lang.IllegalArgumentException(s"$serviceName is not found."))
+ val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new java.lang.IllegalArgumentException(s"$columnName is not found."))
+ val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
+
+ val srcVertexId = id match {
+ case vid: VertexId => id.asInstanceOf[VertexId]
+ case _ => new VertexId(column, toInnerVal(id, column.columnType, column.schemaVersion))
+ }
+
+ val propsInner = column.propsToInnerVals(props) ++
+ Map(ColumnMeta.timestamp -> InnerVal.withLong(ts, column.schemaVersion))
+
+ val vertex = new S2Vertex(graph, srcVertexId, ts, S2Vertex.EmptyProps, op)
+ S2Vertex.fillPropsWithTs(vertex, propsInner)
+ vertex
+ }
+
+
+ /**
+ * helper to create new Edge instance from given parameters on memory(not actually stored in storage).
+ *
+ * Since we are using mutable map for property value(propsWithTs),
+ * we should make sure that reference for mutable map never be shared between multiple Edge instances.
+ * To guarantee this, we never create Edge directly, but rather use this helper which is available on S2Graph.
+ *
+ * Note that we are using following convention
+ * 1. `add*` for method that actually store instance into storage,
+ * 2. `new*` for method that only create instance on memory, but not store it into storage.
+ *
+ * @param srcVertex
+ * @param tgtVertex
+ * @param innerLabel
+ * @param dir
+ * @param op
+ * @param version
+ * @param propsWithTs
+ * @param parentEdges
+ * @param originalEdgeOpt
+ * @param pendingEdgeOpt
+ * @param statusCode
+ * @param lockTs
+ * @param tsInnerValOpt
+ * @return
+ */
+ def newEdge(srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
+ innerLabel: Label,
+ dir: Int,
+ op: Byte = GraphUtil.defaultOpByte,
+ version: Long = System.currentTimeMillis(),
+ propsWithTs: S2Edge.State,
+ parentEdges: Seq[EdgeWithScore] = Nil,
+ originalEdgeOpt: Option[S2EdgeLike] = None,
+ pendingEdgeOpt: Option[S2EdgeLike] = None,
+ statusCode: Byte = 0,
+ lockTs: Option[Long] = None,
+ tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike = {
+ val edge = S2Edge(
+ graph,
+ srcVertex,
+ tgtVertex,
+ innerLabel,
+ dir,
+ op,
+ version,
+ S2Edge.EmptyProps,
+ parentEdges,
+ originalEdgeOpt,
+ pendingEdgeOpt,
+ statusCode,
+ lockTs,
+ tsInnerValOpt)
+ S2Edge.fillPropsWithTs(edge, propsWithTs)
+ edge
+ }
+
+ /**
+ * helper to create new SnapshotEdge instance from given parameters on memory(not actually stored in storage).
+ *
+ * Note that this is only available to S2Graph, not structure.Graph so only internal code should use this method.
+ * @param srcVertex
+ * @param tgtVertex
+ * @param label
+ * @param dir
+ * @param op
+ * @param version
+ * @param propsWithTs
+ * @param pendingEdgeOpt
+ * @param statusCode
+ * @param lockTs
+ * @param tsInnerValOpt
+ * @return
+ */
+ private[core] def newSnapshotEdge(srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
+ label: Label,
+ dir: Int,
+ op: Byte,
+ version: Long,
+ propsWithTs: S2Edge.State,
+ pendingEdgeOpt: Option[S2EdgeLike],
+ statusCode: Byte = 0,
+ lockTs: Option[Long],
+ tsInnerValOpt: Option[InnerValLike] = None): SnapshotEdge = {
+ val snapshotEdge = new SnapshotEdge(graph, srcVertex, tgtVertex, label, dir, op, version, S2Edge.EmptyProps,
+ pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
+ S2Edge.fillPropsWithTs(snapshotEdge, propsWithTs)
+ snapshotEdge
+ }
+
+ def newVertexId(serviceName: String)(columnName: String)(id: Any): VertexId = {
+ val service = Service.findByName(serviceName).getOrElse(throw new RuntimeException(s"$serviceName is not found."))
+ val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new RuntimeException(s"$columnName is not found."))
+ newVertexId(service, column, id)
+ }
+
+ /**
+ * helper to create S2Graph's internal S2VertexId instance with given parameters.
+ * @param service
+ * @param column
+ * @param id
+ * @return
+ */
+ def newVertexId(service: Service,
+ column: ServiceColumn,
+ id: Any): VertexId = {
+ val innerVal = CanInnerValLike.anyToInnerValLike.toInnerVal(id)(column.schemaVersion)
+ new VertexId(column, innerVal)
+ }
+
+ def newVertex(id: VertexId,
+ ts: Long = System.currentTimeMillis(),
+ props: S2Vertex.Props = S2Vertex.EmptyProps,
+ op: Byte = 0,
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
+ val vertex = new S2Vertex(graph, id, ts, S2Vertex.EmptyProps, op, belongLabelIds)
+ S2Vertex.fillPropsWithTs(vertex, props)
+ vertex
+ }
+
+ def toRequestEdge(queryRequest: QueryRequest, parentEdges: Seq[EdgeWithScore]): S2EdgeLike = {
+ val srcVertex = queryRequest.vertex
+ val queryParam = queryRequest.queryParam
+ val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
+ val label = queryParam.label
+ val labelWithDir = queryParam.labelWithDir
+ val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
+ val propsWithTs = label.EmptyPropsWithTs
+
+ tgtVertexIdOpt match {
+ case Some(tgtVertexId) => // _to is given.
+ /* we use toSnapshotEdge so dont need to swap src, tgt */
+ val src = srcVertex.innerId
+ val tgt = tgtVertexId
+ val (srcVId, tgtVId) = (new SourceVertexId(srcColumn, src), new TargetVertexId(tgtColumn, tgt))
+ val (srcV, tgtV) = (newVertex(srcVId), newVertex(tgtVId))
+
+ newEdge(srcV, tgtV, label, labelWithDir.dir, propsWithTs = propsWithTs)
+ case None =>
+ val src = srcVertex.innerId
+ val srcVId = new SourceVertexId(srcColumn, src)
+ val srcV = newVertex(srcVId)
+
+ newEdge(srcV, srcV, label, labelWithDir.dir, propsWithTs = propsWithTs, parentEdges = parentEdges)
+ }
+ }
+
+ def buildEdgesToDelete(edgeWithScoreLs: Seq[EdgeWithScore], requestTs: Long): Seq[EdgeWithScore] = {
+ if (edgeWithScoreLs.isEmpty) Nil
+ else {
+ val head = edgeWithScoreLs.head
+ val label = head.edge.innerLabel
+
+ //Degree edge?
+ edgeWithScoreLs.map { case edgeWithScore =>
+ val edge = edgeWithScore.edge
+ val copiedEdge = label.consistencyLevel match {
+ case "strong" =>
+ edge.copyEdge(op = GraphUtil.operations("delete"),
+ version = requestTs, propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
+ case _ =>
+ edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
+ }
+
+ val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
+ // logger.debug(s"delete edge from deleteAll: ${edgeToDelete.edge.toLogString}")
+ edgeToDelete
+ }
+ }
+ }
+}
[05/23] incubator-s2graph git commit: remove null.
Posted by st...@apache.org.
remove null.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/cdfa0c3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/cdfa0c3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/cdfa0c3d
Branch: refs/heads/master
Commit: cdfa0c3d27d646481955c22f3b389e6cf5980a2a
Parents: 87394b9
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 16:48:34 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 16:48:34 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/S2EdgeLike.scala | 1 +
.../tall/IndexEdgeDeserializable.scala | 27 ++++++++++++--------
.../wide/IndexEdgeDeserializable.scala | 26 +++++++++++--------
3 files changed, 34 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/cdfa0c3d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index bb58554..f823d60 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -48,6 +48,7 @@ trait S2EdgeLike extends Edge with GraphElement {
lazy val labelName = innerLabel.label
lazy val direction = GraphUtil.fromDirection(dir)
+ def setTgtVertex(v: S2VertexLike): Unit = tgtVertex = v
def getOp(): Byte = op
def setOp(newOp: Byte): Unit = op = newOp
def getVersion(): Long = version
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/cdfa0c3d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
index 01f268b..8e3fdf3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
@@ -62,9 +62,6 @@ class IndexEdgeDeserializable(graph: S2Graph,
val label = Label.findById(labelWithDir.labelId)
val schemaVer = label.schemaVersion
val srcVertex = graph.newVertex(srcVertexId, version)
- //TODO:
- val edge = graph.newEdge(srcVertex, null,
- label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
var tsVal = version
val isTallSchema = tallSchemaVersions(label.schemaVersion)
val isDegree = if (isTallSchema) pos == kv.row.length else kv.qualifier.isEmpty
@@ -74,12 +71,17 @@ class IndexEdgeDeserializable(graph: S2Graph,
// val degreeVal = Bytes.toLong(kv.value)
val degreeVal = bytesToLongFunc(kv.value, 0)
val tgtVertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("0", schemaVer))
+ val tgtVertex = graph.newVertex(tgtVertexId, version)
+ val edge = graph.newEdge(srcVertex, tgtVertex,
+ label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
edge.setOp(GraphUtil.defaultOpByte)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
+
+ Option(edge)
} else {
// not degree edge
val (idxPropsRaw, endAt) =
@@ -111,6 +113,10 @@ class IndexEdgeDeserializable(graph: S2Graph,
else kv.qualifier(kv.qualifier.length - 1)
}
+ val tgtVertex = graph.newVertex(tgtVertexIdRaw, version)
+ val edge = graph.newEdge(srcVertex, tgtVertex,
+ label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
+
val index = label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}"))
/* process indexProps */
val size = idxPropsRaw.length
@@ -141,18 +147,19 @@ class IndexEdgeDeserializable(graph: S2Graph,
}
/* process tgtVertexId */
- val tgtVertexId =
- if (edge.checkProperty(LabelMeta.to.name)) {
- val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
- TargetVertexId(ServiceColumn.Default, vId.innerVal)
- } else tgtVertexIdRaw
+
+ if (edge.checkProperty(LabelMeta.to.name)) {
+ val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
+ val tgtVertex = graph.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
+ edge.setTgtVertex(tgtVertex)
+ }
edge.propertyInner(LabelMeta.timestamp.name, tsVal, version)
- edge.tgtVertex = graph.newVertex(tgtVertexId, version)
edge.setOp(op)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
+
+ Option(edge)
}
- Option(edge)
}
} catch {
case e: Exception =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/cdfa0c3d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
index a7fe8a1..f5d9af9 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
@@ -57,19 +57,22 @@ class IndexEdgeDeserializable(graph: S2Graph,
val schemaVer = label.schemaVersion
val srcVertex = graph.newVertex(srcVertexId, version)
//TODO:
- val edge = graph.newEdge(srcVertex, null,
- label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
var tsVal = version
if (kv.qualifier.isEmpty) {
val degreeVal = bytesToLongFunc(kv.value, 0)
val tgtVertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("0", schemaVer))
+ val tgtVertex = graph.newVertex(tgtVertexId, version)
+ val edge = graph.newEdge(srcVertex, tgtVertex,
+ label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
edge.setOp(GraphUtil.defaultOpByte)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
+
+ Option(edge)
} else {
pos = 0
val (idxPropsRaw, endAt) = bytesToProps(kv.qualifier, pos, schemaVer)
@@ -85,6 +88,10 @@ class IndexEdgeDeserializable(graph: S2Graph,
if (kv.qualifier.length == pos) GraphUtil.defaultOpByte
else kv.qualifier(kv.qualifier.length-1)
+ val tgtVertex = graph.newVertex(tgtVertexIdRaw, version)
+ val edge = graph.newEdge(srcVertex, tgtVertex,
+ label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
+
val index = label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}"))
/* process indexProps */
@@ -115,19 +122,18 @@ class IndexEdgeDeserializable(graph: S2Graph,
}
}
/* process tgtVertexId */
- val tgtVertexId =
- if (edge.checkProperty(LabelMeta.to.name)) {
- val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
- TargetVertexId(ServiceColumn.Default, vId.innerVal)
- } else tgtVertexIdRaw
+ if (edge.checkProperty(LabelMeta.to.name)) {
+ val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
+ val tgtVertex = graph.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
+ edge.setTgtVertex(tgtVertex)
+ }
edge.propertyInner(LabelMeta.timestamp.name, tsVal, version)
- edge.tgtVertex = graph.newVertex(tgtVertexId, version)
edge.setOp(op)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
- }
- Option(edge)
+ Option(edge)
+ }
}
} catch {
case e: Exception => None
[12/23] incubator-s2graph git commit: add S2EdgePropertyHelper.
Posted by st...@apache.org.
add S2EdgePropertyHelper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d3a2e75d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d3a2e75d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d3a2e75d
Branch: refs/heads/master
Commit: d3a2e75dfd40de0c2dc4878f14b30cfd179344c9
Parents: 2b5df1d
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Nov 7 08:31:45 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 7 09:00:19 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/PostProcess.scala | 28 +++----
.../org/apache/s2graph/core/QueryResult.scala | 4 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 36 ++-------
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 11 +--
.../org/apache/s2graph/core/S2EdgeLike.scala | 81 ++++++++------------
.../s2graph/core/S2EdgePropertyHelper.scala | 70 +++++++++++++++++
.../scala/org/apache/s2graph/core/S2Graph.scala | 6 +-
.../org/apache/s2graph/core/S2VertexLike.scala | 2 +-
.../apache/s2graph/core/TraversalHelper.scala | 16 ++--
.../s2graph/core/parsers/WhereParser.scala | 2 +-
.../s2graph/core/storage/StorageReadable.scala | 2 +-
.../hbase/AsynchbaseStorageReadable.scala | 2 +-
.../tall/SnapshotEdgeSerializable.scala | 4 +-
.../wide/SnapshotEdgeSerializable.scala | 4 +-
.../rest/play/controllers/EdgeController.scala | 2 +-
15 files changed, 146 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 2d2e183..5118600 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -84,17 +84,17 @@ object PostProcess {
val score = edgeWithScore.score
val label = edgeWithScore.label
if (isDegree) {
- builder += ("from" -> anyValToJsValue(s2Edge.srcId).get)
+ builder += ("from" -> anyValToJsValue(s2Edge.srcVertex.innerIdVal).get)
builder += ("label" -> anyValToJsValue(label.label).get)
- builder += ("direction" -> anyValToJsValue(s2Edge.direction).get)
+ builder += ("direction" -> anyValToJsValue(s2Edge.getDirection()).get)
builder += (LabelMeta.degree.name -> anyValToJsValue(s2Edge.propertyValueInner(LabelMeta.degree).innerVal.value).get)
JsObject(builder)
} else {
if (queryOption.withScore) builder += ("score" -> anyValToJsValue(score).get)
if (queryOption.selectColumns.isEmpty) {
- builder += ("from" -> anyValToJsValue(s2Edge.srcId).get)
- builder += ("to" -> anyValToJsValue(s2Edge.tgtId).get)
+ builder += ("from" -> anyValToJsValue(s2Edge.srcVertex.innerIdVal).get)
+ builder += ("to" -> anyValToJsValue(s2Edge.tgtVertex.innerIdVal).get)
builder += ("label" -> anyValToJsValue(label.label).get)
val innerProps = ArrayBuffer.empty[(String, JsValue)]
@@ -107,23 +107,23 @@ object PostProcess {
builder += ("props" -> JsObject(innerProps))
- builder += ("direction" -> anyValToJsValue(s2Edge.direction).get)
- builder += ("timestamp" -> anyValToJsValue(s2Edge.tsInnerVal).get)
- builder += ("_timestamp" -> anyValToJsValue(s2Edge.tsInnerVal).get) // backward compatibility
+ builder += ("direction" -> anyValToJsValue(s2Edge.getDirection()).get)
+ builder += ("timestamp" -> anyValToJsValue(s2Edge.getTsInnerValValue()).get)
+ builder += ("_timestamp" -> anyValToJsValue(s2Edge.getTsInnerValValue()).get) // backward compatibility
if (parents != JsNull) builder += ("parents" -> parents)
// Json.toJson(builder.result())
JsObject(builder)
} else {
queryOption.selectColumnsMap.foreach { case (columnName, _) =>
columnName match {
- case "from" => builder += ("from" -> anyValToJsValue(s2Edge.srcId).get)
- case "_from" => builder += ("_from" -> anyValToJsValue(s2Edge.srcId).get)
- case "to" => builder += ("to" -> anyValToJsValue(s2Edge.tgtId).get)
- case "_to" => builder += ("_to" -> anyValToJsValue(s2Edge.tgtId).get)
+ case "from" => builder += ("from" -> anyValToJsValue(s2Edge.srcVertex.innerIdVal).get)
+ case "_from" => builder += ("_from" -> anyValToJsValue(s2Edge.srcVertex.innerIdVal).get)
+ case "to" => builder += ("to" -> anyValToJsValue(s2Edge.tgtVertex.innerIdVal).get)
+ case "_to" => builder += ("_to" -> anyValToJsValue(s2Edge.tgtVertex.innerIdVal).get)
case "label" => builder += ("label" -> anyValToJsValue(label.label).get)
- case "direction" => builder += ("direction" -> anyValToJsValue(s2Edge.direction).get)
- case "timestamp" => builder += ("timestamp" -> anyValToJsValue(s2Edge.tsInnerVal).get)
- case "_timestamp" => builder += ("_timestamp" -> anyValToJsValue(s2Edge.tsInnerVal).get)
+ case "direction" => builder += ("direction" -> anyValToJsValue(s2Edge.getDirection()).get)
+ case "timestamp" => builder += ("timestamp" -> anyValToJsValue(s2Edge.getTsInnerValValue()).get)
+ case "_timestamp" => builder += ("_timestamp" -> anyValToJsValue(s2Edge.getTsInnerValValue()).get)
case _ => // should not happen
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index 9bd3cdb..b654e71 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -237,7 +237,7 @@ object StepResult {
// val newOrderByValues = updateScoreOnOrderByValues(globalQueryOption.scoreFieldIdx, t.orderByValues, newScore)
val newOrderByValues =
- if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.tsInnerVal, None, None)
+ if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTsInnerValValue(), None, None)
else toTuple4(newT.toValues(globalQueryOption.orderByKeys))
val newGroupByValues = newT.toValues(globalQueryOption.groupBy.keys)
@@ -262,7 +262,7 @@ object StepResult {
// val newOrderByValues = updateScoreOnOrderByValues(globalQueryOption.scoreFieldIdx, t.orderByValues, newScore)
val newOrderByValues =
- if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.tsInnerVal, None, None)
+ if (globalQueryOption.orderByKeys.isEmpty) (newScore, t.edge.getTsInnerValValue(), None, None)
else toTuple4(newT.toValues(globalQueryOption.orderByKeys))
val newGroupByValues = newT.toValues(globalQueryOption.groupBy.keys)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 03678c8..9f5093c 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -592,7 +592,7 @@ object S2Edge {
for {
(requestEdge, func) <- requestWithFuncs
} {
- val (_newPropsWithTs, _) = func((prevPropsWithTs, propsToState(requestEdge.propsWithTs), requestEdge.ts, requestEdge.schemaVer))
+ val (_newPropsWithTs, _) = func((prevPropsWithTs, propsToState(requestEdge.propsWithTs), requestEdge.ts, requestEdge.innerLabel.schemaVersion))
prevPropsWithTs = _newPropsWithTs
// logger.debug(s"${requestEdge.toLogString}\n$oldPropsWithTs\n$prevPropsWithTs\n")
}
@@ -814,8 +814,8 @@ object S2Edge {
if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
def srcForVertex(e: S2EdgeLike): S2VertexLike = {
- val belongLabelIds = Seq(e.labelWithDir.labelId)
- if (e.labelWithDir.dir == GraphUtil.directions("in")) {
+ val belongLabelIds = Seq(e.getLabelId())
+ if (e.getDir() == GraphUtil.directions("in")) {
val tgtColumn = getServiceColumn(e.tgtVertex, e.innerLabel.tgtColumn)
e.innerGraph.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
} else {
@@ -825,8 +825,8 @@ object S2Edge {
}
def tgtForVertex(e: S2EdgeLike): S2VertexLike = {
- val belongLabelIds = Seq(e.labelWithDir.labelId)
- if (e.labelWithDir.dir == GraphUtil.directions("in")) {
+ val belongLabelIds = Seq(e.getLabelId())
+ if (e.getDir() == GraphUtil.directions("in")) {
val srcColumn = getServiceColumn(e.srcVertex, e.innerLabel.srcColumn)
e.innerGraph.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
} else {
@@ -835,28 +835,6 @@ object S2Edge {
}
}
- def updatePropsWithTs(e: S2EdgeLike, others: Props = S2Edge.EmptyProps): Props = {
- val emptyProp = S2Edge.EmptyProps
-
- e.getPropsWithTs().forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
-
- others.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
-
- emptyProp
- }
-
- def propertyValue(e: S2EdgeLike, key: String): Option[InnerValLikeWithTs] = {
- key match {
- case "from" | "_from" => Option(InnerValLikeWithTs(e.srcVertex.innerId, e.ts))
- case "to" | "_to" => Option(InnerValLikeWithTs(e.tgtVertex.innerId, e.ts))
- case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(e.innerLabel.label, e.innerLabel.schemaVersion), e.ts))
- case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(e.direction, e.innerLabel.schemaVersion), e.ts))
- case _ =>
- e.innerLabel.metaPropsInvMap.get(key).map(labelMeta => e.propertyValueInner(labelMeta))
- }
- }
+ def serializePropsWithTs(edge: S2EdgeLike): Array[Byte] =
+ HBaseSerializable.propsToKeyValuesWithTs(edge.getPropsWithTs().asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
index ea9598e..2ea1504 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
@@ -22,7 +22,7 @@ class S2EdgeBuilder(edge: S2EdgeLike) {
def propsPlusTsValid = edge.getPropsWithTs().asScala.filter(kv => LabelMeta.isValidSeq(kv._2.labelMeta.seq)).asJava
- def labelOrders = LabelIndex.findByLabelIdAll(edge.labelWithDir.labelId)
+ def labelOrders = LabelIndex.findByLabelIdAll(edge.getLabelId())
def edgesWithIndex = for (labelOrder <- labelOrders) yield {
IndexEdge(edge.innerGraph, edge.srcVertex, edge.tgtVertex, edge.innerLabel, edge.getDir(), edge.getOp(),
@@ -35,7 +35,7 @@ class S2EdgeBuilder(edge: S2EdgeLike) {
}
def relatedEdges: Seq[S2EdgeLike] = {
- if (edge.labelWithDir.isDirected) {
+ if (edge.isDirected()) {
val skipReverse = edge.innerLabel.extraOptions.get("skipReverse").map(_.as[Boolean]).getOrElse(false)
if (skipReverse) Seq(edge) else Seq(edge, duplicateEdge)
} else {
@@ -93,11 +93,4 @@ class S2EdgeBuilder(edge: S2EdgeLike) {
else
EdgeId(VertexId(tgtColumn, edge.tgtVertex.id.innerId), VertexId(srcColumn, edge.srcVertex.id.innerId), edge.label(), "out", timestamp)
}
-
- def propertyInner[V](key: String, value: V, ts: Long): Property[V] = {
- val labelMeta = edge.innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
- val newProp = new S2Property[V](edge, labelMeta, key, value, ts)
- edge.getPropsWithTs().put(key, newProp)
- newProp
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index 9963be7..33e7e83 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -19,6 +19,7 @@ trait S2EdgeLike extends Edge with GraphElement {
val builder: S2EdgeBuilder = new S2EdgeBuilder(this)
+
val innerGraph: S2Graph
val srcVertex: S2VertexLike
var tgtVertex: S2VertexLike
@@ -35,20 +36,35 @@ trait S2EdgeLike extends Edge with GraphElement {
val lockTs: Option[Long] = None
// var tsInnerValOpt: Option[InnerValLike] = None
- lazy val labelWithDir = LabelWithDirection(innerLabel.id.get, dir)
- lazy val schemaVer = innerLabel.schemaVersion
lazy val ts = propsWithTs.get(LabelMeta.timestamp.name).innerVal.value match {
case b: BigDecimal => b.longValue()
case l: Long => l
case i: Int => i.toLong
case _ => throw new RuntimeException("ts should be in [BigDecimal/Long/Int].")
}
- lazy val operation = GraphUtil.fromOp(op)
- lazy val tsInnerVal = tsInnerValOpt.get.value
- lazy val srcId = srcVertex.innerIdVal
- lazy val tgtId = tgtVertex.innerIdVal
- lazy val labelName = innerLabel.label
- lazy val direction = GraphUtil.fromDirection(dir)
+
+ private lazy val operation = GraphUtil.fromOp(op)
+ private lazy val direction = GraphUtil.fromDirection(dir)
+ private lazy val tsInnerVal = tsInnerValOpt.get.value
+
+ def graph(): Graph = innerGraph
+
+ lazy val edgeId: EdgeId = builder.edgeId
+
+ def id(): AnyRef = edgeId
+
+ def label(): String = innerLabel.label
+
+ def getLabelId(): Int = innerLabel.id.get
+
+ def getDirection(): String = direction
+
+ def getOperation(): String = operation
+
+ def getTsInnerValValue(): Any = tsInnerVal
+
+ def isDirected(): Boolean =
+ getDir() == 0 || getDir() == 1
def getTs(): Long = ts
def getOriginalEdgeOpt(): Option[S2EdgeLike] = originalEdgeOpt
@@ -68,43 +84,20 @@ trait S2EdgeLike extends Edge with GraphElement {
def toIndexEdge(labelIndexSeq: Byte): IndexEdge = IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelIndexSeq, propsWithTs)
- def serializePropsWithTs(): Array[Byte] = HBaseSerializable.propsToKeyValuesWithTs(propsWithTs.asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
-
def updatePropsWithTs(others: Props = S2Edge.EmptyProps): Props =
- S2Edge.updatePropsWithTs(this, others)
+ S2EdgePropertyHelper.updatePropsWithTs(this, others)
- def propertyValue(key: String): Option[InnerValLikeWithTs] = S2Edge.propertyValue(this, key)
+ def propertyValue(key: String): Option[InnerValLikeWithTs] = S2EdgePropertyHelper.propertyValue(this, key)
- def propertyValueInner(labelMeta: LabelMeta): InnerValLikeWithTs = {
- // propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse()
- if (propsWithTs.containsKey(labelMeta.name)) {
- propsWithTs.get(labelMeta.name).innerValWithTs
- } else {
- innerLabel.metaPropsDefaultMapInner(labelMeta)
- }
- }
+ def propertyValueInner(labelMeta: LabelMeta): InnerValLikeWithTs =
+ S2EdgePropertyHelper.propertyValueInner(this, labelMeta)
def propertyValues(keys: Seq[String] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
- val labelMetas = for {
- key <- keys
- labelMeta <- innerLabel.metaPropsInvMap.get(key)
- } yield labelMeta
-
- propertyValuesInner(labelMetas)
+ S2EdgePropertyHelper.propertyValuesInner(this, S2EdgePropertyHelper.toLabelMetas(this, keys))
}
- def propertyValuesInner(labelMetas: Seq[LabelMeta] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
- if (labelMetas.isEmpty) {
- innerLabel.metaPropsDefaultMapInner.map { case (labelMeta, defaultVal) =>
- labelMeta -> propertyValueInner(labelMeta)
- }
- } else {
- // This is important since timestamp is required for all edges.
- (LabelMeta.timestamp +: labelMetas).map { labelMeta =>
- labelMeta -> propertyValueInner(labelMeta)
- }.toMap
- }
- }
+ def propertyValuesInner(labelMetas: Seq[LabelMeta] = Nil): Map[LabelMeta, InnerValLikeWithTs] =
+ S2EdgePropertyHelper.propertyValuesInner(this, labelMetas)
def relatedEdges = builder.relatedEdges
@@ -114,7 +107,6 @@ trait S2EdgeLike extends Edge with GraphElement {
def duplicateEdge = builder.duplicateEdge
- // def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
def reverseDirEdge = builder.reverseDirEdge
def reverseSrcTgtEdge = builder.reverseSrcTgtEdge
@@ -215,7 +207,8 @@ trait S2EdgeLike extends Edge with GraphElement {
v
}
- def propertyInner[V](key: String, value: V, ts: Long): Property[V] = builder.propertyInner(key, value, ts)
+ def propertyInner[V](key: String, value: V, ts: Long): Property[V] =
+ S2EdgePropertyHelper.propertyInner(this, key, value, ts)
def remove(): Unit = {
if (graph.features().edge().supportsRemoveEdges()) {
@@ -231,14 +224,6 @@ trait S2EdgeLike extends Edge with GraphElement {
}
}
- def graph(): Graph = innerGraph
-
- lazy val edgeId: EdgeId = builder.edgeId
-
- def id(): AnyRef = edgeId
-
- def label(): String = innerLabel.label
-
def toLogString: String = {
// val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
new file mode 100644
index 0000000..2d24a2e
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
@@ -0,0 +1,70 @@
+package org.apache.s2graph.core
+
+import java.util.function.BiConsumer
+
+import org.apache.s2graph.core.S2Edge.Props
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs}
+import org.apache.tinkerpop.gremlin.structure.Property
+
+object S2EdgePropertyHelper {
+ def propertyInner[V](edge: S2EdgeLike, key: String, value: V, ts: Long): Property[V] = {
+ val labelMeta = edge.innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
+ val newProp = new S2Property[V](edge, labelMeta, key, value, ts)
+ edge.getPropsWithTs().put(key, newProp)
+ newProp
+ }
+ def updatePropsWithTs(edge: S2EdgeLike, others: Props = S2Edge.EmptyProps): Props = {
+ val emptyProp = S2Edge.EmptyProps
+
+ edge.getPropsWithTs().forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ others.forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ emptyProp
+ }
+
+ def propertyValue(e: S2EdgeLike, key: String): Option[InnerValLikeWithTs] = {
+ key match {
+ case "from" | "_from" => Option(InnerValLikeWithTs(e.srcVertex.innerId, e.ts))
+ case "to" | "_to" => Option(InnerValLikeWithTs(e.tgtVertex.innerId, e.ts))
+ case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(e.innerLabel.label, e.innerLabel.schemaVersion), e.ts))
+ case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(e.getDirection(), e.innerLabel.schemaVersion), e.ts))
+ case _ =>
+ e.innerLabel.metaPropsInvMap.get(key).map(labelMeta => e.propertyValueInner(labelMeta))
+ }
+ }
+
+ def propertyValuesInner(edge: S2EdgeLike, labelMetas: Seq[LabelMeta] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
+ if (labelMetas.isEmpty) {
+ edge.innerLabel.metaPropsDefaultMapInner.map { case (labelMeta, defaultVal) =>
+ labelMeta -> edge.propertyValueInner(labelMeta)
+ }
+ } else {
+ // This is important since timestamp is required for all edges.
+ (LabelMeta.timestamp +: labelMetas).map { labelMeta =>
+ labelMeta -> propertyValueInner(edge, labelMeta)
+ }.toMap
+ }
+ }
+
+ def propertyValueInner(edge: S2EdgeLike, labelMeta: LabelMeta): InnerValLikeWithTs = {
+ // propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse()
+ if (edge.getPropsWithTs().containsKey(labelMeta.name)) {
+ edge.getPropsWithTs().get(labelMeta.name).innerValWithTs
+ } else {
+ edge.innerLabel.metaPropsDefaultMapInner(labelMeta)
+ }
+ }
+
+ def toLabelMetas(edge: S2EdgeLike, keys: Seq[String]): Seq[LabelMeta] = {
+ for {
+ key <- keys
+ labelMeta <- edge.innerLabel.metaPropsInvMap.get(key)
+ } yield labelMeta
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index f061160..3270e84 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -663,13 +663,13 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
else {
- val (alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean], prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
+ val (_, prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
- fetchedLs, orgQuery.steps(stepIdx).queryParams, alreadyVisited, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
+ fetchedLs, orgQuery.steps(stepIdx).queryParams, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
}
}
@@ -883,7 +883,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val (strongDeleteAll, strongEdgesAll) = strongEdges.partition { case (edge, idx) => edge.getOp() == GraphUtil.operations("deleteAll") }
val deleteAllFutures = strongDeleteAll.map { case (edge, idx) =>
- deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.innerLabel), edge.labelWithDir.dir, edge.ts).map(idx -> _)
+ deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.innerLabel), edge.getDir(), edge.ts).map(idx -> _)
}
val strongEdgesFutures = strongEdgesAll.groupBy { case (edge, idx) => edge.innerLabel }.map { case (label, edgeGroup) =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index b88c18d..9ec2ab0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -73,7 +73,7 @@ trait S2VertexLike extends Vertex with GraphElement {
override def accept(edge: Edge): Unit = {
val s2Edge = edge.asInstanceOf[S2EdgeLike]
- s2Edge.direction match {
+ s2Edge.getDirection() match {
case "out" => arr.add(edge.inVertex())
case "in" => arr.add(edge.outVertex())
case _ => throw new IllegalStateException("only out/in direction can be found in S2Edge")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
index 58da145..25b909e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -43,12 +43,12 @@ object TraversalHelper {
}
}
- def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
+ def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(Int, Int, S2VertexLike), Boolean] = {
val vertices = for {
edgeWithScore <- edgeWithScoreLs
edge = edgeWithScore.edge
- vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
- } yield (edge.labelWithDir, vertex) -> true
+ vertex = if (edge.getDir() == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
+ } yield (edge.getLabelId(), edge.getDir(), vertex) -> true
vertices.toMap
}
@@ -114,7 +114,7 @@ object TraversalHelper {
def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
val src = edge.srcVertex.innerId.hashCode()
val tgt = edge.tgtVertex.innerId.hashCode()
- val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
+ val hashKey = (src, edge.getLabelId(), edge.getDir(), tgt, isDegree)
val filterHashKey = (src, tgt)
(hashKey, filterHashKey)
@@ -136,7 +136,7 @@ class TraversalHelper(graph: S2GraphLike) {
val step = q.steps(stepIdx)
val alreadyVisited =
- if (stepIdx == 0) Map.empty[(LabelWithDirection, S2VertexLike), Boolean]
+ if (stepIdx == 0) Map.empty[(Int, Int, S2VertexLike), Boolean]
else alreadyVisitedVertices(stepInnerResult.edgeWithScores)
val initial = (Map.empty[S2VertexLike, Double], Map.empty[S2VertexLike, ArrayBuffer[EdgeWithScore]])
@@ -172,7 +172,6 @@ class TraversalHelper(graph: S2GraphLike) {
queryRequests: Seq[QueryRequest],
queryResultLsFuture: Future[Seq[StepResult]],
queryParams: Seq[QueryParam],
- alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
buildLastStepInnerResult: Boolean = true,
parentEdges: Map[VertexId, Seq[EdgeWithScore]])
(implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
@@ -214,7 +213,6 @@ class TraversalHelper(graph: S2GraphLike) {
val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
val edge = edgeWithScore.edge
val score = edgeWithScore.score
- val label = edgeWithScore.label
/* Select */
val mergedPropsWithTs = edge.propertyValuesInner(propsSelectColumns)
@@ -225,7 +223,7 @@ class TraversalHelper(graph: S2GraphLike) {
val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
/* OrderBy */
val orderByValues =
- if (queryOption.orderByKeys.isEmpty) (score, edge.tsInnerVal, None, None)
+ if (queryOption.orderByKeys.isEmpty) (score, edge.getTsInnerValValue(), None, None)
else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
/* StepGroupBy */
@@ -296,8 +294,6 @@ class TraversalHelper(graph: S2GraphLike) {
val parents = if (shouldBuildParents) {
parentEdges.getOrElse(queryRequest.vertex.id, Nil).map { edgeWithScore =>
val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val label = edgeWithScore.label
/* Select */
val mergedPropsWithTs =
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
index a0d56b2..d947066 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
@@ -54,7 +54,7 @@ trait ExtractValue {
val (propKey, _) = findParentEdge(edge, key)
val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
- val (srcColumn, tgtColumn) = label.srcTgtColumn(edge.labelWithDir.dir)
+ val (srcColumn, tgtColumn) = label.srcTgtColumn(edge.getDir())
val dataType = propKey match {
case "_to" | "to" => tgtColumn.columnType
case "_from" | "from" => srcColumn.columnType
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
index 44bd4dc..79ca8aa 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
@@ -50,7 +50,7 @@ trait StorageReadable {
def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] = {
val queryParam = QueryParam(labelName = edge.innerLabel.label,
- direction = GraphUtil.fromDirection(edge.labelWithDir.dir),
+ direction = GraphUtil.fromDirection(edge.getDir()),
tgtVertexIdOpt = Option(edge.tgtVertex.innerIdVal),
cacheTTLInMillis = -1)
val q = Query.toQuery(Seq(edge.srcVertex), Seq(queryParam))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
index bdb6e99..5f54e47 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
@@ -237,7 +237,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
serDe.indexEdgeDeserializer(schemaVer = HBaseType.DEFAULT_VERSION)
.fromKeyValues(Seq(kv), None)
- .filter(e => distinctLabels(e.innerLabel) && e.direction == "out" && !e.isDegree)
+ .filter(e => distinctLabels(e.innerLabel) && e.getDirection() == "out" && !e.isDegree)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
index 24775e7..12edf54 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.storage.serde.snapshotedge.tall
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.SnapshotEdge
+import org.apache.s2graph.core.{S2Edge, SnapshotEdge}
import org.apache.s2graph.core.mysqls.LabelIndex
import org.apache.s2graph.core.storage.serde._
import org.apache.s2graph.core.storage.serde.StorageSerializable._
@@ -54,7 +54,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
case Some(pendingEdge) =>
val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
- val propsBytes = pendingEdge.serializePropsWithTs()
+ val propsBytes = S2Edge.serializePropsWithTs(pendingEdge)
val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
index 44f2596..02a72b1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.storage.serde.snapshotedge.wide
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.SnapshotEdge
+import org.apache.s2graph.core.{S2Edge, SnapshotEdge}
import org.apache.s2graph.core.mysqls.LabelIndex
import org.apache.s2graph.core.storage.serde.Serializable
import org.apache.s2graph.core.storage.serde.StorageSerializable._
@@ -61,7 +61,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
case Some(pendingEdge) =>
val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
- val propsBytes = pendingEdge.serializePropsWithTs()
+ val propsBytes = S2Edge.serializePropsWithTs(pendingEdge)
val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d3a2e75d/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
index 2b31fcd..69878f8 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
@@ -95,7 +95,7 @@ object EdgeController extends Controller {
results.get.zip(elements).map {
case (r: MutateResponse, (e: S2EdgeLike, tsv: String)) if !r.isSuccess =>
val kafkaMessages = if(e.getOp() == GraphUtil.operations("deleteAll")){
- toDeleteAllFailMessages(Seq(e.srcVertex), Seq(e.innerLabel), e.labelWithDir.dir, e.ts)
+ toDeleteAllFailMessages(Seq(e.srcVertex), Seq(e.innerLabel), e.getDir(), e.ts)
} else{
Seq(ExceptionHandler.toKafkaMessage(Config.KAFKA_MUTATE_FAIL_TOPIC, e, Some(tsv)))
}
[08/23] incubator-s2graph git commit: add MutationHelper.
Posted by st...@apache.org.
add MutationHelper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/937b55a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/937b55a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/937b55a7
Branch: refs/heads/master
Commit: 937b55a7405899d38fee230a35b54e63b47b1677
Parents: bc26642
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 21:14:04 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 21:14:04 2017 +0900
----------------------------------------------------------------------
.../core/storage/serde/MutationHelper.scala | 172 +++++++++++++++++++
1 file changed, 172 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/937b55a7/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
new file mode 100644
index 0000000..e2621af
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
@@ -0,0 +1,172 @@
+package org.apache.s2graph.core.storage.serde
+
+import org.apache.s2graph.core.mysqls.LabelMeta
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.storage._
+import org.apache.s2graph.core.utils.logger
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class MutationHelper(storage: Storage) {
+ val serDe = storage.serDe
+ val io = storage.io
+ val fetcher = storage.fetcher
+ val mutator = storage.mutator
+ val conflictResolver = storage.conflictResolver
+
+ private def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
+ mutator.writeToStorage(cluster, kvs, withWait)
+
+ def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
+ if (stepInnerResult.isEmpty) Future.successful(true)
+ else {
+ val head = stepInnerResult.edgeWithScores.head
+ val zkQuorum = head.edge.innerLabel.hbaseZkAddr
+ val futures = for {
+ edgeWithScore <- stepInnerResult.edgeWithScores
+ } yield {
+ val edge = edgeWithScore.edge
+ val score = edgeWithScore.score
+
+ val edgeSnapshot = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
+
+ val edgeForward = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ /* reverted direction */
+ val edgeRevert = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
+ serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
+ io.buildIncrementsAsync(indexEdge, -1L)
+ }
+
+ val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
+
+ writeToStorage(zkQuorum, mutations, withWait = true)
+ }
+
+ Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
+ }
+ }
+
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ if (vertex.op == GraphUtil.operations("delete")) {
+ writeToStorage(zkQuorum,
+ serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
+ } else if (vertex.op == GraphUtil.operations("deleteAll")) {
+ logger.info(s"deleteAll for vertex is truncated. $vertex")
+ Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
+ } else {
+ writeToStorage(zkQuorum, io.buildPutsAll(vertex), withWait)
+ }
+ }
+
+ def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] = {
+ val mutations = _edges.flatMap { edge =>
+ val (_, edgeUpdate) =
+ if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
+ else S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+
+ if (bufferIncr.nonEmpty) storage.writeToStorage(zkQuorum, bufferIncr, withWait = false)
+ io.buildVertexPutsAsync(edge) ++ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+ }
+
+ writeToStorage(zkQuorum, mutations, withWait).map { ret =>
+ _edges.zipWithIndex.map { case (edge, idx) =>
+ idx -> ret.isSuccess
+ }
+ }
+ }
+
+ def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ def mutateEdgesInner(edges: Seq[S2EdgeLike],
+ checkConsistency: Boolean,
+ withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ assert(edges.nonEmpty)
+ // TODO:: remove after code review: unreachable code
+ if (!checkConsistency) {
+
+ val futures = edges.map { edge =>
+ val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
+
+ val (bufferIncr, nonBufferIncr) = io.increments(edgeUpdate.deepCopy)
+ val mutations =
+ io.indexedEdgeMutations(edgeUpdate.deepCopy) ++ io.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
+
+ if (bufferIncr.nonEmpty) writeToStorage(zkQuorum, bufferIncr, withWait = false)
+
+ writeToStorage(zkQuorum, mutations, withWait)
+ }
+ Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
+ } else {
+ fetcher.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
+ conflictResolver.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
+ }
+ }
+ }
+
+ val edgeWithIdxs = _edges.zipWithIndex
+ val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
+ (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
+ } toSeq
+
+ val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
+ val edges = edgeGroup.map(_._1)
+ val idxs = edgeGroup.map(_._2)
+ // After deleteAll, process others
+ val mutateEdgeFutures = edges.toList match {
+ case head :: tail =>
+ val edgeFuture = mutateEdgesInner(edges, checkConsistency = true, withWait)
+
+ //TODO: decide what we will do on failure on vertex put
+ val puts = io.buildVertexPutsAsync(head)
+ val vertexFuture = writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
+ Seq(edgeFuture, vertexFuture)
+ case Nil => Nil
+ }
+
+ val composed = for {
+ // deleteRet <- Future.sequence(deleteAllFutures)
+ mutateRet <- Future.sequence(mutateEdgeFutures)
+ } yield mutateRet
+
+ composed.map(_.forall(_.isSuccess)).map { ret => idxs.map(idx => idx -> ret) }
+ }
+
+ Future.sequence(mutateEdges).map { squashedRets =>
+ squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
+ }
+ }
+
+ def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] = {
+ val futures = for {
+ edge <- edges
+ } yield {
+ val kvs = for {
+ relEdge <- edge.relatedEdges
+ edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
+ } yield {
+ val countWithTs = edge.propertyValueInner(LabelMeta.count)
+ val countVal = countWithTs.innerVal.toString().toLong
+ io.buildIncrementsCountAsync(edgeWithIndex, countVal).head
+ }
+ writeToStorage(zkQuorum, kvs, withWait = withWait)
+ }
+
+ Future.sequence(futures)
+ }
+
+ def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] = {
+ val kvs = io.buildDegreePuts(edge, degreeVal)
+
+ mutator.writeToStorage(zkQuorum, kvs, withWait = true)
+ }
+}
[22/23] incubator-s2graph git commit: run apache-rat.
Posted by st...@apache.org.
run apache-rat.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/41885a84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/41885a84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/41885a84
Branch: refs/heads/master
Commit: 41885a849dc79eed767758e7e811bc1cb6757598
Parents: 97bd7e2
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Nov 18 20:08:53 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 18 20:08:53 2017 +0900
----------------------------------------------------------------------
.../s2graph/core/GraphElementBuilder.scala | 19 +++++++++++++++++++
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 19 +++++++++++++++++++
.../org/apache/s2graph/core/S2EdgeLike.scala | 19 +++++++++++++++++++
.../s2graph/core/S2EdgePropertyHelper.scala | 19 +++++++++++++++++++
.../org/apache/s2graph/core/S2GraphLike.scala | 19 +++++++++++++++++++
.../org/apache/s2graph/core/S2GraphTp.scala | 19 +++++++++++++++++++
.../apache/s2graph/core/S2VertexBuilder.scala | 19 +++++++++++++++++++
.../org/apache/s2graph/core/S2VertexLike.scala | 19 +++++++++++++++++++
.../apache/s2graph/core/TraversalHelper.scala | 19 +++++++++++++++++++
.../s2graph/core/features/S2Features.scala | 19 +++++++++++++++++++
.../core/storage/serde/MutationHelper.scala | 19 +++++++++++++++++++
11 files changed, 209 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index aed84c7..c8c25b3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
index dc0baa2..85321d3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import org.apache.s2graph.core.S2Edge.State
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index 0165439..f2ea4ad 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util
import java.util.function.BiConsumer
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
index 2d24a2e..0efb851 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgePropertyHelper.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util.function.BiConsumer
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 9772470..bb36a33 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
index 5ce4086..50dad3e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import com.typesafe.config.Config
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
index 399b142..50d6526 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index 97e3095..0d898d1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util
import java.util.function.{BiConsumer, Consumer}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
index 7a8e63e..c40078e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core
import java.util
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala b/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
index 36c9ecc..caff843 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core.features
import org.apache.tinkerpop.gremlin.structure.Graph.Features
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/41885a84/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
index 9312181..9871b36 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.s2graph.core.storage.serde
import org.apache.s2graph.core.mysqls.LabelMeta
[21/23] incubator-s2graph git commit: add java helpers on S2GraphLike.
Posted by st...@apache.org.
add java helpers on S2GraphLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/97bd7e2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/97bd7e2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/97bd7e2c
Branch: refs/heads/master
Commit: 97bd7e2cc07e800ad31c3ba075808e399e52c62c
Parents: 6c61087
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Nov 18 09:56:53 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 18 10:34:19 2017 +0900
----------------------------------------------------------------------
s2core/build.sbt | 3 +-
.../s2graph/core/GraphElementBuilder.scala | 18 ++++++
.../org/apache/s2graph/core/GraphUtil.scala | 11 +++-
.../org/apache/s2graph/core/S2GraphLike.scala | 66 ++++++++++++++------
4 files changed, 77 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/97bd7e2c/s2core/build.sbt
----------------------------------------------------------------------
diff --git a/s2core/build.sbt b/s2core/build.sbt
index 8033581..b4273ba 100644
--- a/s2core/build.sbt
+++ b/s2core/build.sbt
@@ -48,7 +48,8 @@ libraryDependencies ++= Seq(
"org.specs2" %% "specs2-core" % specs2Version % "test",
"org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion ,
"org.apache.lucene" % "lucene-core" % "6.6.0",
- "org.apache.lucene" % "lucene-queryparser" % "6.6.0"
+ "org.apache.lucene" % "lucene-queryparser" % "6.6.0",
+ "org.scala-lang.modules" %% "scala-java8-compat" % "0.8.0"
)
libraryDependencies := {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/97bd7e2c/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index 08da355..aed84c7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -2,9 +2,11 @@ package org.apache.s2graph.core
import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.JSONParser.{fromJsonToProperties, toInnerVal}
+import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types._
import org.apache.s2graph.core.utils.logger
+import org.apache.tinkerpop.gremlin.structure.T
import play.api.libs.json.{JsObject, Json}
import scala.util.Try
@@ -241,6 +243,22 @@ class GraphElementBuilder(graph: S2GraphLike) {
vertex
}
+ def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2VertexLike = {
+ idValue match {
+ case vId: VertexId =>
+ toVertex(vId.column.service.serviceName, vId.column.columnName, vId, kvsMap)
+ case _ =>
+ val serviceColumnNames = kvsMap.getOrElse(T.label.toString, DefaultColumnName).toString
+
+ val names = serviceColumnNames.split(S2Vertex.VertexLabelDelimiter)
+ val (serviceName, columnName) =
+ if (names.length == 1) (DefaultServiceName, names(0))
+ else throw new RuntimeException("malformed data on vertex label.")
+
+ toVertex(serviceName, columnName, idValue, kvsMap)
+ }
+ }
+
def toRequestEdge(queryRequest: QueryRequest, parentEdges: Seq[EdgeWithScore]): S2EdgeLike = {
val srcVertex = queryRequest.vertex
val queryParam = queryRequest.queryParam
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/97bd7e2c/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
index 939b596..a4f6bde 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphUtil.scala
@@ -21,6 +21,7 @@ package org.apache.s2graph.core
import java.util.regex.Pattern
+import org.apache.tinkerpop.gremlin.structure.Direction
import play.api.libs.json.Json
import scala.util.hashing.MurmurHash3
@@ -67,7 +68,15 @@ object GraphUtil {
}
}
- def fromDirection(direction: Int) = {
+ def toDirection(direction: Direction): Int = {
+ direction.name() match {
+ case "out" => 0
+ case "in" => 1
+ case "both" => throw new IllegalArgumentException("only in/out direction is supported.")
+ }
+ }
+
+ def fromDirection(direction: Int): String = {
direction match {
case 0 => "out"
case 1 => "in"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/97bd7e2c/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 1b80cb4..9772470 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -1,8 +1,10 @@
package org.apache.s2graph.core
import java.util
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{CompletableFuture, TimeUnit}
import java.util.concurrent.atomic.AtomicLong
+import java.lang.{Boolean => JBoolean, Long => JLong}
+import java.util.Optional
import com.typesafe.config.Config
import org.apache.commons.configuration.Configuration
@@ -21,8 +23,11 @@ import org.apache.tinkerpop.gremlin.structure.io.{GraphReader, GraphWriter, Io,
import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Element, Graph, T, Transaction, Vertex}
import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.compat.java8.FutureConverters._
+import scala.compat.java8.OptionConverters._
trait S2GraphLike extends Graph {
@@ -72,34 +77,73 @@ trait S2GraphLike extends Graph {
def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]]
+ def getVerticesJava(vertices: util.List[S2VertexLike]): CompletableFuture[util.List[S2VertexLike]] =
+ getVertices(vertices.toSeq).map(_.asJava).toJava.toCompletableFuture
+
def checkEdges(edges: Seq[S2EdgeLike]): Future[StepResult]
+ def checkEdgesJava(edges: util.List[S2EdgeLike]): CompletableFuture[StepResult] =
+ checkEdges(edges.asScala).toJava.toCompletableFuture
+
def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]]
+ def mutateVerticesJava(vertices: util.List[S2VertexLike], withWait: JBoolean): CompletableFuture[util.List[MutateResponse]] =
+ mutateVertices(vertices.asScala, withWait.booleanValue()).map(_.asJava).toJava.toCompletableFuture
+
def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]]
+ def mutateEdgesJava(edges: util.List[S2EdgeLike], withWait: JBoolean): CompletableFuture[util.List[MutateResponse]] =
+ mutateEdges(edges.asScala, withWait.booleanValue()).map(_.asJava).toJava.toCompletableFuture
+
def mutateElements(elements: Seq[GraphElement],
withWait: Boolean = false): Future[Seq[MutateResponse]]
+ def mutateElementsJava(elements: util.List[GraphElement], withWait: JBoolean): CompletableFuture[util.List[MutateResponse]] =
+ mutateElements(elements.asScala, withWait.booleanValue()).map(_.asJava).toJava.toCompletableFuture
+
def getEdges(q: Query): Future[StepResult]
+ def getEdgesJava(q: Query): CompletableFuture[StepResult] =
+ getEdges(q).toJava.toCompletableFuture
+
def getEdgesMultiQuery(mq: MultiQuery): Future[StepResult]
+ def getEdgesMultiQueryJava(mq: MultiQuery): CompletableFuture[StepResult] =
+ getEdgesMultiQuery(mq).toJava.toCompletableFuture
+
def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
labels: Seq[Label],
dir: Int,
ts: Long): Future[Boolean]
+ def deleteAllAdjacentEdgesJava(srcVertices: util.List[S2VertexLike], labels: util.List[Label], direction: Direction): CompletableFuture[JBoolean] =
+ deleteAllAdjacentEdges(srcVertices.asScala, labels.asScala, GraphUtil.toDirection(direction), System.currentTimeMillis()).map(JBoolean.valueOf(_)).toJava.toCompletableFuture
+
def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]]
+ def incrementCountsJava(edges: util.List[S2EdgeLike], withWait: JBoolean): CompletableFuture[util.List[MutateResponse]] =
+ incrementCounts(edges.asScala, withWait.booleanValue()).map(_.asJava).toJava.toCompletableFuture
+
def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse]
+ def updateDegreeJava(edge: S2EdgeLike, degreeVal: JLong): CompletableFuture[MutateResponse] =
+ updateDegree(edge, degreeVal.longValue()).toJava.toCompletableFuture
+
def getVertex(vertexId: VertexId): Option[S2VertexLike]
+ def getVertexJava(vertexId: VertexId): Optional[S2VertexLike] =
+ getVertex(vertexId).asJava
+
def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge]
+ def fetchEdgesJava(vertex: S2VertexLike, labelNameWithDirs: util.List[(String, String)]): util.Iterator[Edge] =
+ fetchEdges(vertex, labelNameWithDirs.asScala)
+
def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]]
+ def edgesAsyncJava(vertex: S2VertexLike, direction: Direction, labelNames: String*): CompletableFuture[util.Iterator[Edge]] =
+ edgesAsync(vertex, direction, labelNames: _*).toJava.toCompletableFuture
+
/** Convert to Graph Element **/
def toVertex(serviceName: String,
columnName: String,
@@ -196,22 +240,6 @@ trait S2GraphLike extends Graph {
addVertex(Seq(T.label, label): _*)
}
- def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2VertexLike = {
- idValue match {
- case vId: VertexId =>
- elementBuilder.toVertex(vId.column.service.serviceName, vId.column.columnName, vId, kvsMap)
- case _ =>
- val serviceColumnNames = kvsMap.getOrElse(T.label.toString, DefaultColumnName).toString
-
- val names = serviceColumnNames.split(S2Vertex.VertexLabelDelimiter)
- val (serviceName, columnName) =
- if (names.length == 1) (DefaultServiceName, names(0))
- else throw new RuntimeException("malformed data on vertex label.")
-
- elementBuilder.toVertex(serviceName, columnName, idValue, kvsMap)
- }
- }
-
def addVertex(kvs: AnyRef*): structure.Vertex = {
if (!features().vertex().supportsUserSuppliedIds() && kvs.contains(T.id)) {
throw Vertex.Exceptions.userSuppliedIdsNotSupported
@@ -232,9 +260,9 @@ trait S2GraphLike extends Graph {
val vertex = kvsMap.get(T.id.name()) match {
case None => // do nothing
val id = localLongId.getAndIncrement()
- makeVertex(Long.box(id), kvsMap)
+ elementBuilder.makeVertex(Long.box(id), kvsMap)
case Some(idValue) if S2Property.validType(idValue) =>
- makeVertex(idValue, kvsMap)
+ elementBuilder.makeVertex(idValue, kvsMap)
case _ =>
throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported
}
[18/23] incubator-s2graph git commit: remove
newVertex/newEdge/newVertexId on S2GraphLike.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
index 55658b9..66e44ec 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
@@ -67,8 +67,8 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
if (version == VERSION2) TargetVertexId(ServiceColumn.Default, InnerVal.withStr("2", version))
else TargetVertexId(ServiceColumn.Default, InnerVal.withLong(2, version))
- val srcVertex = graph.newVertex(srcId, ts)
- val tgtVertex = graph.newVertex(tgtId, ts)
+ val srcVertex = builder.newVertex(srcId, ts)
+ val tgtVertex = builder.newVertex(tgtId, ts)
val (_label, dir) = if (version == VERSION2) (labelV2, labelWithDirV2.dir) else (label, labelWithDir.dir)
(srcVertex, tgtVertex, _label, dir)
@@ -81,7 +81,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
/** test for each version */
val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "phone_number" -> "1234")
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val edge = graph.newEdge(srcVertex, tgtVertex, label, dir, 0.toByte, ts, propsInner)
+ val edge = builder.newEdge(srcVertex, tgtVertex, label, dir, 0.toByte, ts, propsInner)
val f = validate(label)(edge) _
@@ -101,21 +101,21 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
/** test for each version */
var js = Json.obj("phone_number" -> "")
var propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- var edge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
+ var edge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
var f = validate(label)(edge) _
f(s"phone_number = '' ")(true)
js = Json.obj("phone_number" -> "010 3167 1897")
propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- edge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
+ edge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
f = validate(label)(edge) _
f(s"phone_number = '010 3167 1897' ")(true)
js = Json.obj("phone_number" -> "010' 3167 1897")
propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- edge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
+ edge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
f = validate(label)(edge) _
f(s"phone_number = '010\\' 3167 1897' ")(true)
@@ -129,7 +129,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
/** test for each version */
val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val edge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
+ val edge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner)
val f = validate(label)(edge) _
@@ -159,7 +159,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
/** test for each version */
val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val edge = graph.newEdge(srcVertex, tgtVertex, label, dir, 0.toByte, ts, propsInner)
+ val edge = builder.newEdge(srcVertex, tgtVertex, label, dir, 0.toByte, ts, propsInner)
val f = validate(label)(edge) _
@@ -183,12 +183,12 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
val parentPropsInner = Management.toProps(label, parentJs.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
- val grandParentEdge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, parentPropsInner)
+ val grandParentEdge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, parentPropsInner)
- val parentEdge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, parentPropsInner,
+ val parentEdge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, parentPropsInner,
parentEdges = Seq(EdgeWithScore(grandParentEdge, 1.0, grandParentEdge.innerLabel)))
- val edge = graph.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner,
+ val edge = builder.newEdge(srcVertex, tgtVertex, label, labelWithDir.dir, 0.toByte, ts, propsInner,
parentEdges = Seq(EdgeWithScore(parentEdge, 1.0, grandParentEdge.innerLabel)))
println(edge.toString)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
index 0cd975c..f1331e3 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
@@ -46,8 +46,8 @@ class StorageIOTest extends FunSuite with Matchers with TestCommonWithModels {
throw new IllegalStateException("column not found.")
}
- val vertexId = graph.newVertexId(service, column, 1L)
- val vertex = graph.newVertex(vertexId)
+ val vertexId = builder.newVertexId(service, column, 1L)
+ val vertex = builder.newVertex(vertexId)
check(vertex, serDe.vertexSerializer, serDe.vertexDeserializer(vertex.serviceColumn.schemaVersion))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
index 24e98a1..4f38e18 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
@@ -40,11 +40,11 @@ class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels {
val from = InnerVal.withLong(1, l.schemaVersion)
val vertexId = SourceVertexId(ServiceColumn.Default, from)
val tgtVertexId = TargetVertexId(ServiceColumn.Default, to)
- val vertex = graph.newVertex(vertexId, ts)
- val tgtVertex = graph.newVertex(tgtVertexId, ts)
+ val vertex = builder.newVertex(vertexId, ts)
+ val tgtVertex = builder.newVertex(tgtVertexId, ts)
val labelWithDir = LabelWithDirection(l.id.get, 0)
val labelOpt = Option(l)
- val edge = graph.newEdge(vertex, tgtVertex, l, labelWithDir.dir, 0, ts, props, tsInnerValOpt = Option(InnerVal.withLong(ts, l.schemaVersion)))
+ val edge = builder.newEdge(vertex, tgtVertex, l, labelWithDir.dir, 0, ts, props, tsInnerValOpt = Option(InnerVal.withLong(ts, l.schemaVersion)))
val indexEdge = edge.edgesWithIndex.find(_.labelIndexSeq == LabelIndex.DefaultSeq).head
val kvs = graph.getStorage(l).serDe.indexEdgeSerializer(indexEdge).toKeyValues
val _indexEdgeOpt = graph.getStorage(l).serDe.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(kvs, None)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/process/S2GraphProcessStandardTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/process/S2GraphProcessStandardTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/process/S2GraphProcessStandardTest.scala
index 495f161..37be246 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/process/S2GraphProcessStandardTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/process/S2GraphProcessStandardTest.scala
@@ -19,7 +19,7 @@
package org.apache.s2graph.core.tinkerpop.process
-import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.core.{S2Graph, S2GraphTp}
import org.apache.s2graph.core.tinkerpop.S2GraphProvider
import org.apache.tinkerpop.gremlin.GraphProviderClass
import org.apache.tinkerpop.gremlin.process.ProcessStandardSuite
@@ -29,7 +29,7 @@ import org.junit.runners.MethodSorters
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@RunWith(classOf[ProcessStandardSuite])
-@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2Graph])
+@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2GraphTp])
class S2GraphProcessStandardTest {
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureIntegrateTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureIntegrateTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureIntegrateTest.scala
index 4b88714..dd72ba6 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureIntegrateTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureIntegrateTest.scala
@@ -19,14 +19,14 @@
package org.apache.s2graph.core.tinkerpop.structure
-import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.core.{S2Graph, S2GraphTp}
import org.apache.s2graph.core.tinkerpop.S2GraphProvider
import org.apache.tinkerpop.gremlin.GraphProviderClass
import org.apache.tinkerpop.gremlin.structure.{StructureIntegrateSuite, StructureStandardSuite}
import org.junit.runner.RunWith
@RunWith(classOf[StructureIntegrateSuite])
-@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2Graph])
+@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2GraphTp])
class S2GraphStructureIntegrateTest {
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureStandardTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureStandardTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureStandardTest.scala
index ed9ab8d..3d08224 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureStandardTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphStructureStandardTest.scala
@@ -19,7 +19,7 @@
package org.apache.s2graph.core.tinkerpop.structure
-import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.core.{S2Graph, S2GraphTp}
import org.apache.s2graph.core.tinkerpop.S2GraphProvider
import org.apache.tinkerpop.gremlin.structure.StructureStandardSuite
import org.apache.tinkerpop.gremlin.{GraphProviderClass, LoadGraphWith}
@@ -29,7 +29,7 @@ import org.junit.runners.MethodSorters
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@RunWith(classOf[StructureStandardSuite])
-@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2Graph])
+@GraphProviderClass(provider = classOf[S2GraphProvider], graph = classOf[S2GraphTp])
class S2GraphStructureStandardTest {
}
[04/23] incubator-s2graph git commit: passed s2tests.
Posted by st...@apache.org.
passed s2tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/87394b9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/87394b9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/87394b9f
Branch: refs/heads/master
Commit: 87394b9f7b7e241642201460a460ba2403a0fb99
Parents: 7413aad
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 15:28:08 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 15:28:08 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/QueryParam.scala | 10 +--
.../org/apache/s2graph/core/QueryResult.scala | 2 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 68 +++++++++-----------
.../org/apache/s2graph/core/S2EdgeLike.scala | 54 +++++++++++++---
.../scala/org/apache/s2graph/core/S2Graph.scala | 50 +++++++-------
.../org/apache/s2graph/core/S2Property.scala | 2 +-
.../org/apache/s2graph/core/S2VertexLike.scala | 2 +-
.../s2graph/core/index/IndexProvider.scala | 12 ++--
.../s2graph/core/parsers/WhereParser.scala | 34 +++++-----
.../s2graph/core/rest/RequestParser.scala | 4 +-
.../apache/s2graph/core/storage/Storage.scala | 10 +--
.../apache/s2graph/core/storage/StorageIO.scala | 20 +++---
.../s2graph/core/storage/StorageReadable.scala | 6 +-
.../storage/WriteWriteConflictResolver.scala | 34 +++++-----
.../hbase/AsynchbaseStorageReadable.scala | 6 +-
.../tall/IndexEdgeDeserializable.scala | 17 +++--
.../wide/IndexEdgeDeserializable.scala | 12 ++--
.../tall/SnapshotEdgeSerializable.scala | 2 +-
.../wide/SnapshotEdgeSerializable.scala | 2 +-
.../s2graph/core/parsers/WhereParserTest.scala | 2 +-
.../core/tinkerpop/S2GraphProvider.scala | 1 +
.../rest/play/controllers/EdgeController.scala | 6 +-
22 files changed, 195 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
index 2e8d1f4..e98ef37 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
@@ -163,7 +163,7 @@ case class EdgeTransformer(jsValue: JsValue) {
}
}
- def toInnerValOpt(queryParam: QueryParam, edge: S2Edge, fieldName: String): Option[InnerValLike] = {
+ def toInnerValOpt(queryParam: QueryParam, edge: S2EdgeLike, fieldName: String): Option[InnerValLike] = {
fieldName match {
case LabelMeta.to.name => Option(edge.tgtVertex.innerId)
case LabelMeta.from.name => Option(edge.srcVertex.innerId)
@@ -171,7 +171,7 @@ case class EdgeTransformer(jsValue: JsValue) {
}
}
- def transform(queryParam: QueryParam, edge: S2Edge, nextStepOpt: Option[Step]): Seq[S2Edge] = {
+ def transform(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
if (isDefault) Seq(edge)
else {
val edges = for {
@@ -311,7 +311,7 @@ case class QueryParam(labelName: String,
CanInnerValLike.anyToInnerValLike.toInnerVal(id)(label.tgtColumnWithDir(dir).schemaVersion)
}
- def buildInterval(edgeOpt: Option[S2Edge]) = intervalOpt match {
+ def buildInterval(edgeOpt: Option[S2EdgeLike]) = intervalOpt match {
case None => Array.empty[Byte] -> Array.empty[Byte]
case Some(interval) =>
val (froms, tos) = interval
@@ -359,7 +359,7 @@ case class QueryParam(labelName: String,
Bytes.add(bytes, optionalCacheKey)
}
- private def convertToInner(kvs: Seq[(String, JsValue)], edgeOpt: Option[S2Edge]): Seq[(LabelMeta, InnerValLike)] = {
+ private def convertToInner(kvs: Seq[(String, JsValue)], edgeOpt: Option[S2EdgeLike]): Seq[(LabelMeta, InnerValLike)] = {
kvs.map { case (propKey, propValJs) =>
propValJs match {
case JsString(in) if edgeOpt.isDefined && in.contains("_parent.") =>
@@ -392,7 +392,7 @@ case class QueryParam(labelName: String,
}
}
- def paddingInterval(len: Byte, froms: Seq[(String, JsValue)], tos: Seq[(String, JsValue)], edgeOpt: Option[S2Edge] = None) = {
+ def paddingInterval(len: Byte, froms: Seq[(String, JsValue)], tos: Seq[(String, JsValue)], edgeOpt: Option[S2EdgeLike] = None) = {
val fromInnerVal = convertToInner(froms, edgeOpt)
val toInnerVal = convertToInner(tos, edgeOpt)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index 7506b40..9bd3cdb 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -91,7 +91,7 @@ object WithScore {
}
}
-case class EdgeWithScore(edge: S2Edge,
+case class EdgeWithScore(edge: S2EdgeLike,
score: Double,
label: Label,
orderByValues: (Any, Any, Any, Any) = StepResult.EmptyOrderByValues,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 97abd26..3529991 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -71,7 +71,7 @@ case class SnapshotEdge(graph: S2Graph,
op: Byte,
version: Long,
private val propsWithTs: Props,
- pendingEdgeOpt: Option[S2Edge],
+ pendingEdgeOpt: Option[S2EdgeLike],
statusCode: Byte = 0,
lockTs: Option[Long],
tsInnerValOpt: Option[InnerValLike] = None) {
@@ -89,7 +89,7 @@ case class SnapshotEdge(graph: S2Graph,
def allPropsDeleted = S2Edge.allPropsDeleted(propsWithTs)
- def toEdge: S2Edge = {
+ def toEdge: S2EdgeLike = {
S2Edge(graph, srcVertex, tgtVertex, label, dir, op,
version, propsWithTs, pendingEdgeOpt = pendingEdgeOpt,
statusCode = statusCode, lockTs = lockTs, tsInnerValOpt = tsInnerValOpt)
@@ -243,7 +243,7 @@ case class IndexEdge(graph: S2Graph,
} yield meta.name -> jsValue
- def toEdge: S2Edge = S2Edge(graph, srcVertex, tgtVertex, label, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
+ def toEdge: S2EdgeLike = S2Edge(graph, srcVertex, tgtVertex, label, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
// only for debug
def toLogString() = {
@@ -310,11 +310,11 @@ case class S2Edge(override val innerGraph: S2Graph,
var version: Long = System.currentTimeMillis(),
override val propsWithTs: Props = S2Edge.EmptyProps,
override val parentEdges: Seq[EdgeWithScore] = Nil,
- override val originalEdgeOpt: Option[S2Edge] = None,
- override val pendingEdgeOpt: Option[S2Edge] = None,
+ override val originalEdgeOpt: Option[S2EdgeLike] = None,
+ override val pendingEdgeOpt: Option[S2EdgeLike] = None,
override val statusCode: Byte = 0,
override val lockTs: Option[Long] = None,
- var tsInnerValOpt: Option[InnerValLike] = None) extends S2EdgeLike with GraphElement {
+ var tsInnerValOpt: Option[InnerValLike] = None) extends S2EdgeLike {
// if (!props.contains(LabelMeta.timestamp)) throw new Exception("Timestamp is required.")
// assert(propsWithTs.contains(LabelMeta.timeStampSeq))
@@ -335,10 +335,10 @@ case class S2Edge(override val innerGraph: S2Graph,
// }
- def toLogString: String = {
- // val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
- List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
- }
+// def toLogString: String = {
+// // val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
+// List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
+// }
override def hashCode(): Int = {
id().hashCode()
@@ -455,7 +455,7 @@ object S2Edge {
type State = Map[LabelMeta, InnerValLikeWithTs]
type PropsPairWithTs = (State, State, Long, String)
type MergeState = PropsPairWithTs => (State, Boolean)
- type UpdateFunc = (Option[S2Edge], S2Edge, MergeState)
+ type UpdateFunc = (Option[S2EdgeLike], S2EdgeLike, MergeState)
def EmptyProps = new java.util.HashMap[String, S2Property[_]]
def EmptyState = Map.empty[LabelMeta, InnerValLikeWithTs]
@@ -490,7 +490,7 @@ object S2Edge {
def fillPropsWithTs(indexEdge: IndexEdge, state: State): Unit = {
state.foreach { case (k, v) => indexEdge.property(k.name, v.innerVal.value, v.ts) }
}
- def fillPropsWithTs(edge: S2Edge, state: State): Unit = {
+ def fillPropsWithTs(edge: S2EdgeLike, state: State): Unit = {
state.foreach { case (k, v) => edge.propertyInner(k.name, v.innerVal.value, v.ts) }
}
@@ -500,7 +500,7 @@ object S2Edge {
}.toMap
}
- def stateToProps(edge: S2Edge, state: State): Props = {
+ def stateToProps(edge: S2EdgeLike, state: State): Props = {
state.foreach { case (k, v) =>
edge.propertyInner(k.name, v.innerVal.value, v.ts)
}
@@ -533,7 +533,7 @@ object S2Edge {
ret
}
- def buildDeleteBulk(invertedEdge: Option[S2Edge], requestEdge: S2Edge): (S2Edge, EdgeMutate) = {
+ def buildDeleteBulk(invertedEdge: Option[S2EdgeLike], requestEdge: S2EdgeLike): (S2EdgeLike, EdgeMutate) = {
// assert(invertedEdge.isEmpty)
// assert(requestEdge.op == GraphUtil.operations("delete"))
@@ -543,7 +543,7 @@ object S2Edge {
(requestEdge, EdgeMutate(edgesToDelete, edgesToInsert = Nil, newSnapshotEdge = edgeInverted))
}
- def buildOperation(invertedEdge: Option[S2Edge], requestEdges: Seq[S2Edge]): (S2Edge, EdgeMutate) = {
+ def buildOperation(invertedEdge: Option[S2EdgeLike], requestEdges: Seq[S2EdgeLike]): (S2EdgeLike, EdgeMutate) = {
// logger.debug(s"oldEdge: ${invertedEdge.map(_.toStringRaw)}")
// logger.debug(s"requestEdge: ${requestEdge.toStringRaw}")
val oldPropsWithTs =
@@ -551,21 +551,21 @@ object S2Edge {
else propsToState(invertedEdge.get.propsWithTs)
val funcs = requestEdges.map { edge =>
- if (edge.op == GraphUtil.operations("insert")) {
+ if (edge.getOp() == GraphUtil.operations("insert")) {
edge.innerLabel.consistencyLevel match {
case "strong" => S2Edge.mergeUpsert _
case _ => S2Edge.mergeInsertBulk _
}
- } else if (edge.op == GraphUtil.operations("insertBulk")) {
+ } else if (edge.getOp() == GraphUtil.operations("insertBulk")) {
S2Edge.mergeInsertBulk _
- } else if (edge.op == GraphUtil.operations("delete")) {
+ } else if (edge.getOp() == GraphUtil.operations("delete")) {
edge.innerLabel.consistencyLevel match {
case "strong" => S2Edge.mergeDelete _
case _ => throw new RuntimeException("not supported")
}
}
- else if (edge.op == GraphUtil.operations("update")) S2Edge.mergeUpdate _
- else if (edge.op == GraphUtil.operations("increment")) S2Edge.mergeIncrement _
+ else if (edge.getOp() == GraphUtil.operations("update")) S2Edge.mergeUpdate _
+ else if (edge.getOp() == GraphUtil.operations("increment")) S2Edge.mergeIncrement _
else throw new RuntimeException(s"not supported operation on edge: $edge")
}
@@ -587,7 +587,7 @@ object S2Edge {
}
val requestTs = requestEdge.ts
/* version should be monotoniously increasing so our RPC mutation should be applied safely */
- val newVersion = invertedEdge.map(e => e.version + incrementVersion).getOrElse(requestTs)
+ val newVersion = invertedEdge.map(e => e.getVersion() + incrementVersion).getOrElse(requestTs)
val maxTs = prevPropsWithTs.map(_._2.ts).max
val newTs = if (maxTs > requestTs) maxTs else requestTs
val propsWithTs = prevPropsWithTs ++
@@ -597,14 +597,14 @@ object S2Edge {
// logger.debug(s"${edgeMutate.toLogString}\n${propsWithTs}")
// logger.error(s"$propsWithTs")
- val newEdge = requestEdge.copy(propsWithTs = EmptyProps)
- fillPropsWithTs(newEdge, propsWithTs)
+ val newEdge =requestEdge.copyEdgeWithState(propsWithTs)
+
(newEdge, edgeMutate)
}
}
- def buildMutation(snapshotEdgeOpt: Option[S2Edge],
- requestEdge: S2Edge,
+ def buildMutation(snapshotEdgeOpt: Option[S2EdgeLike],
+ requestEdge: S2EdgeLike,
newVersion: Long,
oldPropsWithTs: Map[LabelMeta, InnerValLikeWithTs],
newPropsWithTs: Map[LabelMeta, InnerValLikeWithTs]): EdgeMutate = {
@@ -615,14 +615,14 @@ object S2Edge {
} else {
val withOutDeletedAt = newPropsWithTs.filter(kv => kv._1 != LabelMeta.lastDeletedAtSeq)
val newOp = snapshotEdgeOpt match {
- case None => requestEdge.op
+ case None => requestEdge.getOp()
case Some(old) =>
val oldMaxTs = old.propsWithTs.asScala.map(_._2.ts).max
- if (oldMaxTs > requestEdge.ts) old.op
- else requestEdge.op
+ if (oldMaxTs > requestEdge.ts) old.getOp()
+ else requestEdge.getOp()
}
- val newSnapshotEdge = requestEdge.copy(op = newOp, version = newVersion).copyEdgeWithState(newPropsWithTs)
+ val newSnapshotEdge = requestEdge.copyOp(newOp).copyVersion(newVersion).copyEdgeWithState(newPropsWithTs)
val newSnapshotEdgeOpt = Option(newSnapshotEdge.toSnapshotEdge)
// delete request must always update snapshot.
@@ -631,8 +631,8 @@ object S2Edge {
EdgeMutate(edgesToDelete = Nil, edgesToInsert = Nil, newSnapshotEdge = newSnapshotEdgeOpt)
} else {
val edgesToDelete = snapshotEdgeOpt match {
- case Some(snapshotEdge) if snapshotEdge.op != GraphUtil.operations("delete") =>
- snapshotEdge.copy(op = GraphUtil.defaultOpByte)
+ case Some(snapshotEdge) if snapshotEdge.getOp() != GraphUtil.operations("delete") =>
+ snapshotEdge.copyOp(GraphUtil.defaultOpByte)
.relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
case _ => Nil
}
@@ -640,11 +640,7 @@ object S2Edge {
val edgesToInsert =
if (newPropsWithTs.isEmpty || allPropsDeleted(newPropsWithTs)) Nil
else {
- val newEdge = requestEdge.copy(
- version = newVersion,
- propsWithTs = S2Edge.EmptyProps,
- op = GraphUtil.defaultOpByte
- )
+ val newEdge = requestEdge.copyOp(GraphUtil.defaultOpByte).copyVersion(newVersion).copyEdgeWithState(S2Edge.EmptyState)
newPropsWithTs.foreach { case (k, v) => newEdge.propertyInner(k.name, v.innerVal.value, v.ts) }
newEdge.relatedEdges.flatMap { relEdge => relEdge.edgesWithIndexValid }
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index 04ed7ab..bb58554 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -14,7 +14,7 @@ import play.api.libs.json.Json
import scala.concurrent.Await
import scala.collection.JavaConverters._
-trait S2EdgeLike extends Edge {
+trait S2EdgeLike extends Edge with GraphElement {
this: S2Edge =>
val innerGraph: S2Graph
@@ -27,8 +27,8 @@ trait S2EdgeLike extends Edge {
// var version: Long = System.currentTimeMillis()
val propsWithTs: Props = S2Edge.EmptyProps
val parentEdges: Seq[EdgeWithScore] = Nil
- val originalEdgeOpt: Option[S2Edge] = None
- val pendingEdgeOpt: Option[S2Edge] = None
+ val originalEdgeOpt: Option[S2EdgeLike] = None
+ val pendingEdgeOpt: Option[S2EdgeLike] = None
val statusCode: Byte = 0
val lockTs: Option[Long] = None
// var tsInnerValOpt: Option[InnerValLike] = None
@@ -48,6 +48,13 @@ trait S2EdgeLike extends Edge {
lazy val labelName = innerLabel.label
lazy val direction = GraphUtil.fromDirection(dir)
+ def getOp(): Byte = op
+ def setOp(newOp: Byte): Unit = op = newOp
+ def getVersion(): Long = version
+ def setVersion(newVersion: Long): Unit = version = newVersion
+ def getTsInnerValOpt(): Option[InnerValLike] = tsInnerValOpt
+ def setTsInnerValOpt(newTsInnerValOpt: Option[InnerValLike]): Unit = tsInnerValOpt = newTsInnerValOpt
+
def toIndexEdge(labelIndexSeq: Byte): IndexEdge = IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelIndexSeq, propsWithTs)
def serializePropsWithTs(): Array[Byte] = HBaseSerializable.propsToKeyValuesWithTs(propsWithTs.asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
@@ -196,6 +203,11 @@ trait S2EdgeLike extends Edge {
S2Edge(innerGraph, srcVertex, newTgtVertex, innerLabel, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
}
+ def updateOriginalEdgeOpt(newOriginalEdgeOpt: S2EdgeLike): S2EdgeLike = {
+ S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs, parentEdges,
+ Option(newOriginalEdgeOpt), pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
+ }
+
def rank(r: RankParam): Double =
if (r.keySeqAndWeights.size <= 0) 1.0f
else {
@@ -225,12 +237,12 @@ trait S2EdgeLike extends Edge {
version: Long = version,
propsWithTs: State = S2Edge.propsToState(this.propsWithTs),
parentEdges: Seq[EdgeWithScore] = parentEdges,
- originalEdgeOpt: Option[S2Edge] = originalEdgeOpt,
- pendingEdgeOpt: Option[S2Edge] = pendingEdgeOpt,
+ originalEdgeOpt: Option[S2EdgeLike] = originalEdgeOpt,
+ pendingEdgeOpt: Option[S2EdgeLike] = pendingEdgeOpt,
statusCode: Byte = statusCode,
lockTs: Option[Long] = lockTs,
tsInnerValOpt: Option[InnerValLike] = tsInnerValOpt,
- ts: Long = ts): S2Edge = {
+ ts: Long = ts): S2EdgeLike = {
val edge = new S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, S2Edge.EmptyProps,
parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
S2Edge.fillPropsWithTs(edge, propsWithTs)
@@ -238,19 +250,39 @@ trait S2EdgeLike extends Edge {
edge
}
- def copyEdgeWithState(state: State, ts: Long): S2Edge = {
+ def copyEdgeWithState(state: State, ts: Long): S2EdgeLike = {
val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
S2Edge.fillPropsWithTs(newEdge, state)
newEdge.propertyInner(LabelMeta.timestamp.name, ts, ts)
newEdge
}
- def copyEdgeWithState(state: State): S2Edge = {
+ def copyEdgeWithState(state: State): S2EdgeLike = {
val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
S2Edge.fillPropsWithTs(newEdge, state)
newEdge
}
+ def copyOp(newOp: Byte): S2EdgeLike = {
+ copy(op = newOp)
+ }
+
+ def copyVersion(newVersion: Long): S2EdgeLike = {
+ copy(version = newVersion)
+ }
+
+ def copyParentEdges(parents: Seq[EdgeWithScore]): S2EdgeLike = {
+ copy(parentEdges = parents)
+ }
+
+ def copyStatusCode(newStatusCode: Byte): S2EdgeLike = {
+ copy(statusCode = newStatusCode)
+ }
+
+ def copyLockTs(newLockTs: Option[Long]): S2EdgeLike = {
+ copy(lockTs = newLockTs)
+ }
+
def vertices(direction: Direction): util.Iterator[structure.Vertex] = {
val arr = new util.ArrayList[Vertex]()
@@ -373,7 +405,13 @@ trait S2EdgeLike extends Edge {
}
}
+ def toLogString: String = {
+ // val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
+ List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
+ }
+
private def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 90190cf..8bb95fc 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -201,13 +201,13 @@ object S2Graph {
}
/** common methods for filter out, transform, aggregate queryResult */
- def convertEdges(queryParam: QueryParam, edge: S2Edge, nextStepOpt: Option[Step]): Seq[S2Edge] = {
+ def convertEdges(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
for {
convertedEdge <- queryParam.edgeTransformer.transform(queryParam, edge, nextStepOpt) if !edge.isDegree
} yield convertedEdge
}
- def processTimeDecay(queryParam: QueryParam, edge: S2Edge) = {
+ def processTimeDecay(queryParam: QueryParam, edge: S2EdgeLike) = {
/* process time decay */
val tsVal = queryParam.timeDecay match {
case None => 1.0
@@ -258,7 +258,7 @@ object S2Graph {
}
}
- def toHashKey(queryParam: QueryParam, edge: S2Edge, isDegree: Boolean): (HashKey, FilterHashKey) = {
+ def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
val src = edge.srcVertex.innerId.hashCode()
val tgt = edge.tgtVertex.innerId.hashCode()
val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
@@ -435,7 +435,7 @@ object S2Graph {
val tsVal = processTimeDecay(queryParam, edge)
val newScore = degreeScore + score
// val newEdge = if (queryOption.returnTree) edge.copy(parentEdges = parents) else edge
- val newEdge = edge.copy(parentEdges = parents)
+ val newEdge = edge.copyParentEdges(parents)
edgeWithScore.copy(edge = newEdge, score = newScore * labelWeight * tsVal)
}
@@ -971,7 +971,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
def fallback = Future.successful(StepResult.Empty)
- def checkEdges(edges: Seq[S2Edge]): Future[StepResult] = {
+ def checkEdges(edges: Seq[S2EdgeLike]): Future[StepResult] = {
val futures = for {
edge <- edges
} yield {
@@ -1319,11 +1319,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
def mutateElements(elements: Seq[GraphElement],
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- val edgeBuffer = ArrayBuffer[(S2Edge, Int)]()
+ val edgeBuffer = ArrayBuffer[(S2EdgeLike, Int)]()
val vertexBuffer = ArrayBuffer[(S2VertexLike, Int)]()
elements.zipWithIndex.foreach {
- case (e: S2Edge, idx: Int) => edgeBuffer.append((e, idx))
+ case (e: S2EdgeLike, idx: Int) => edgeBuffer.append((e, idx))
case (v: S2VertexLike, idx: Int) => vertexBuffer.append((v, idx))
case any@_ => logger.error(s"Unknown type: ${any}")
}
@@ -1347,13 +1347,13 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
// def mutateEdges(edges: Seq[Edge], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateEdges(edges, withWait)
- def mutateEdges(edges: Seq[S2Edge], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val edgeWithIdxs = edges.zipWithIndex
val (strongEdges, weakEdges) =
edgeWithIdxs.partition { case (edge, idx) =>
val e = edge
- e.innerLabel.consistencyLevel == "strong" && e.op != GraphUtil.operations("insertBulk")
+ e.innerLabel.consistencyLevel == "strong" && e.getOp() != GraphUtil.operations("insertBulk")
}
val weakEdgesFutures = weakEdges.groupBy { case (edge, idx) => edge.innerLabel.hbaseZkAddr }.map { case (zkQuorum, edgeWithIdxs) =>
@@ -1365,7 +1365,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
/* multiple edges with weak consistency level will be processed as batch */
val mutations = edges.flatMap { edge =>
val (_, edgeUpdate) =
- if (edge.op == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
+ if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
else S2Edge.buildOperation(None, Seq(edge))
val (bufferIncr, nonBufferIncr) = storage.increments(edgeUpdate.deepCopy)
@@ -1380,7 +1380,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
Future.sequence(futures)
}
- val (strongDeleteAll, strongEdgesAll) = strongEdges.partition { case (edge, idx) => edge.op == GraphUtil.operations("deleteAll") }
+ val (strongDeleteAll, strongEdgesAll) = strongEdges.partition { case (edge, idx) => edge.getOp() == GraphUtil.operations("deleteAll") }
val deleteAllFutures = strongDeleteAll.map { case (edge, idx) =>
deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.innerLabel), edge.labelWithDir.dir, edge.ts).map(idx -> _)
@@ -1404,7 +1404,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- private def mutateStrongEdges(storage: Storage)(_edges: Seq[S2Edge], withWait: Boolean): Future[Seq[Boolean]] = {
+ private def mutateStrongEdges(storage: Storage)(_edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[Boolean]] = {
val edgeWithIdxs = _edges.zipWithIndex
val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
@@ -1440,7 +1440,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
- private def mutateEdgesInner(storage: Storage)(edges: Seq[S2Edge],
+ private def mutateEdgesInner(storage: Storage)(edges: Seq[S2EdgeLike],
checkConsistency: Boolean,
withWait: Boolean): Future[MutateResponse] = {
assert(edges.nonEmpty)
@@ -1495,8 +1495,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
- def incrementCounts(edges: Seq[S2Edge], withWait: Boolean): Future[Seq[MutateResponse]] = {
- def incrementCounts(storage: Storage)(edges: Seq[S2Edge], withWait: Boolean): Future[Seq[MutateResponse]] = {
+ def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
+ def incrementCounts(storage: Storage)(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
val futures = for {
edge <- edges
} yield {
@@ -1523,7 +1523,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- def updateDegree(edge: S2Edge, degreeVal: Long = 0): Future[MutateResponse] = {
+ def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
val label = edge.innerLabel
val storage = getStorage(label)
@@ -1571,11 +1571,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
toVertex(GraphUtil.split(s))
}
- def toEdge(s: String): Option[S2Edge] = {
+ def toEdge(s: String): Option[S2EdgeLike] = {
toEdge(GraphUtil.split(s))
}
- def toEdge(parts: Array[String]): Option[S2Edge] = Try {
+ def toEdge(parts: Array[String]): Option[S2EdgeLike] = Try {
val (ts, operation, logType, srcId, tgtId, label) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
val tempDirection = if (parts.length >= 8) parts(7) else "out"
@@ -1605,7 +1605,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
direction: String,
props: Map[String, Any] = Map.empty,
ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2Edge = {
+ operation: String = "insert"): S2EdgeLike = {
val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
val srcColumn = if (direction == "out") label.srcColumn else label.tgtColumn
@@ -1649,7 +1649,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
vertex
}
- def toRequestEdge(queryRequest: QueryRequest, parentEdges: Seq[EdgeWithScore]): S2Edge = {
+ def toRequestEdge(queryRequest: QueryRequest, parentEdges: Seq[EdgeWithScore]): S2EdgeLike = {
val srcVertex = queryRequest.vertex
val queryParam = queryRequest.queryParam
val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
@@ -1710,11 +1710,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
version: Long = System.currentTimeMillis(),
propsWithTs: S2Edge.State,
parentEdges: Seq[EdgeWithScore] = Nil,
- originalEdgeOpt: Option[S2Edge] = None,
- pendingEdgeOpt: Option[S2Edge] = None,
+ originalEdgeOpt: Option[S2EdgeLike] = None,
+ pendingEdgeOpt: Option[S2EdgeLike] = None,
statusCode: Byte = 0,
lockTs: Option[Long] = None,
- tsInnerValOpt: Option[InnerValLike] = None): S2Edge = {
+ tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike = {
val edge = S2Edge(
this,
srcVertex,
@@ -1758,7 +1758,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
op: Byte,
version: Long,
propsWithTs: S2Edge.State,
- pendingEdgeOpt: Option[S2Edge],
+ pendingEdgeOpt: Option[S2EdgeLike],
statusCode: Byte = 0,
lockTs: Option[Long],
tsInnerValOpt: Option[InnerValLike] = None): SnapshotEdge = {
@@ -1874,7 +1874,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
def edgesAsync(edgeIds: AnyRef*): Future[util.Iterator[structure.Edge]] = {
val s2EdgeIds = edgeIds.collect {
- case s2Edge: S2Edge => s2Edge.id().asInstanceOf[EdgeId]
+ case s2Edge: S2EdgeLike => s2Edge.id().asInstanceOf[EdgeId]
case id: EdgeId => id
case s: String => EdgeId.fromString(s)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/S2Property.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Property.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Property.scala
index e86c17f..50b94de 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Property.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Property.scala
@@ -102,7 +102,7 @@ object S2Property {
}
}
-case class S2Property[V](element: S2Edge,
+case class S2Property[V](element: S2EdgeLike,
labelMeta: LabelMeta,
key: String,
v: V,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index 5a8f722..b88c18d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -71,7 +71,7 @@ trait S2VertexLike extends Vertex with GraphElement {
val arr = new util.ArrayList[Vertex]()
edges(direction, edgeLabels: _*).forEachRemaining(new Consumer[Edge] {
override def accept(edge: Edge): Unit = {
- val s2Edge = edge.asInstanceOf[S2Edge]
+ val s2Edge = edge.asInstanceOf[S2EdgeLike]
s2Edge.direction match {
case "out" => arr.add(edge.inVertex())
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index 098d0b4..e5005b7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -29,7 +29,7 @@ import org.apache.lucene.queryparser.classic.{ParseException, QueryParser}
import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.store.{BaseDirectory, RAMDirectory}
import org.apache.s2graph.core.io.Conversions
-import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex, S2VertexLike}
+import org.apache.s2graph.core._
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types.{InnerValLike, VertexId}
import org.apache.s2graph.core.utils.logger
@@ -130,8 +130,8 @@ trait IndexProvider {
def mutateVertices(vertices: Seq[S2VertexLike]): Seq[Boolean]
def mutateVerticesAsync(vertices: Seq[S2VertexLike]): Future[Seq[Boolean]]
- def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean]
- def mutateEdgesAsync(edges: Seq[S2Edge]): Future[Seq[Boolean]]
+ def mutateEdges(edges: Seq[S2EdgeLike]): Seq[Boolean]
+ def mutateEdgesAsync(edges: Seq[S2EdgeLike]): Future[Seq[Boolean]]
def shutdown(): Unit
}
@@ -179,7 +179,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
}
}
- private def toDocument(globalIndex: GlobalIndex, edge: S2Edge): Option[Document] = {
+ private def toDocument(globalIndex: GlobalIndex, edge: S2EdgeLike): Option[Document] = {
val props = edge.propsWithTs.asScala
val exist = props.exists(t => globalIndex.propNamesSet(t._1))
if (!exist) None
@@ -222,7 +222,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
vertices.map(_ => true)
}
- override def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] = {
+ override def mutateEdges(edges: Seq[S2EdgeLike]): Seq[Boolean] = {
val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.EdgeType)
globalIndexOptions.map { globalIndex =>
@@ -316,5 +316,5 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
override def mutateVerticesAsync(vertices: Seq[S2VertexLike]): Future[Seq[Boolean]] = Future.successful(mutateVertices(vertices))
- override def mutateEdgesAsync(edges: Seq[S2Edge]): Future[Seq[Boolean]] = Future.successful(mutateEdges(edges))
+ override def mutateEdgesAsync(edges: Seq[S2EdgeLike]): Future[Seq[Boolean]] = Future.successful(mutateEdges(edges))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
index d754bb7..a0d56b2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
@@ -22,7 +22,7 @@ package org.apache.s2graph.core.parsers
import org.apache.s2graph.core.GraphExceptions.{LabelNotExistException, WhereParserException}
import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
import org.apache.s2graph.core.types.InnerValLike
-import org.apache.s2graph.core.{S2Edge, GraphUtil}
+import org.apache.s2graph.core.{GraphUtil, S2Edge, S2EdgeLike}
import org.apache.s2graph.core.JSONParser._
import org.apache.s2graph.core.utils.logger
@@ -33,7 +33,7 @@ import scala.util.parsing.combinator.JavaTokenParsers
trait ExtractValue {
val parent = "_parent."
- def propToInnerVal(edge: S2Edge, key: String) = {
+ def propToInnerVal(edge: S2EdgeLike, key: String) = {
val (propKey, parentEdge) = findParentEdge(edge, key)
val label = parentEdge.innerLabel
@@ -47,7 +47,7 @@ trait ExtractValue {
}
}
- def valueToCompare(edge: S2Edge, key: String, value: String) = {
+ def valueToCompare(edge: S2EdgeLike, key: String, value: String) = {
val label = edge.innerLabel
if (value.startsWith(parent) || label.metaPropsInvMap.contains(value)) propToInnerVal(edge, value)
else {
@@ -65,11 +65,11 @@ trait ExtractValue {
}
@tailrec
- private def findParent(edge: S2Edge, depth: Int): S2Edge =
+ private def findParent(edge: S2EdgeLike, depth: Int): S2EdgeLike =
if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1)
else edge
- private def findParentEdge(edge: S2Edge, key: String): (String, S2Edge) = {
+ private def findParentEdge(edge: S2EdgeLike, key: String): (String, S2EdgeLike) = {
if (!key.startsWith(parent)) (key, edge)
else {
val split = key.split(parent)
@@ -88,9 +88,9 @@ trait Clause extends ExtractValue {
def or(otherField: Clause): Clause = Or(this, otherField)
- def filter(edge: S2Edge): Boolean
+ def filter(edge: S2EdgeLike): Boolean
- def binaryOp(binOp: (InnerValLike, InnerValLike) => Boolean)(propKey: String, value: String)(edge: S2Edge): Boolean = {
+ def binaryOp(binOp: (InnerValLike, InnerValLike) => Boolean)(propKey: String, value: String)(edge: S2EdgeLike): Boolean = {
val propValue = propToInnerVal(edge, propKey)
val compValue = valueToCompare(edge, propKey, value)
@@ -105,20 +105,20 @@ object Where {
}
}
case class Where(clauses: Seq[Clause] = Seq.empty[Clause]) {
- def filter(edge: S2Edge) =
+ def filter(edge: S2EdgeLike) =
if (clauses.isEmpty) true else clauses.map(_.filter(edge)).forall(identity)
}
case class Gt(propKey: String, value: String) extends Clause {
- override def filter(edge: S2Edge): Boolean = binaryOp(_ > _)(propKey, value)(edge)
+ override def filter(edge: S2EdgeLike): Boolean = binaryOp(_ > _)(propKey, value)(edge)
}
case class Lt(propKey: String, value: String) extends Clause {
- override def filter(edge: S2Edge): Boolean = binaryOp(_ < _)(propKey, value)(edge)
+ override def filter(edge: S2EdgeLike): Boolean = binaryOp(_ < _)(propKey, value)(edge)
}
case class Eq(propKey: String, value: String) extends Clause {
- override def filter(edge: S2Edge): Boolean = binaryOp(_ == _)(propKey, value)(edge)
+ override def filter(edge: S2EdgeLike): Boolean = binaryOp(_ == _)(propKey, value)(edge)
}
case class InWithoutParent(label: Label, propKey: String, values: Set[String]) extends Clause {
@@ -144,7 +144,7 @@ case class InWithoutParent(label: Label, propKey: String, values: Set[String]) e
toInnerVal(value, dataType, label.schemaVersion)
}
- override def filter(edge: S2Edge): Boolean = {
+ override def filter(edge: S2EdgeLike): Boolean = {
if (edge.dir == GraphUtil.directions("in")) {
val propVal = propToInnerVal(edge, propKey)
innerValLikeLsIn.contains(propVal)
@@ -156,7 +156,7 @@ case class InWithoutParent(label: Label, propKey: String, values: Set[String]) e
}
case class IN(propKey: String, values: Set[String]) extends Clause {
- override def filter(edge: S2Edge): Boolean = {
+ override def filter(edge: S2EdgeLike): Boolean = {
val propVal = propToInnerVal(edge, propKey)
values.exists { value =>
valueToCompare(edge, propKey, value) == propVal
@@ -165,7 +165,7 @@ case class IN(propKey: String, values: Set[String]) extends Clause {
}
case class Between(propKey: String, minValue: String, maxValue: String) extends Clause {
- override def filter(edge: S2Edge): Boolean = {
+ override def filter(edge: S2EdgeLike): Boolean = {
val propVal = propToInnerVal(edge, propKey)
val minVal = valueToCompare(edge, propKey, minValue)
val maxVal = valueToCompare(edge, propKey, maxValue)
@@ -175,15 +175,15 @@ case class Between(propKey: String, minValue: String, maxValue: String) extends
}
case class Not(self: Clause) extends Clause {
- override def filter(edge: S2Edge) = !self.filter(edge)
+ override def filter(edge: S2EdgeLike) = !self.filter(edge)
}
case class And(left: Clause, right: Clause) extends Clause {
- override def filter(edge: S2Edge) = left.filter(edge) && right.filter(edge)
+ override def filter(edge: S2EdgeLike) = left.filter(edge) && right.filter(edge)
}
case class Or(left: Clause, right: Clause) extends Clause {
- override def filter(edge: S2Edge) = left.filter(edge) || right.filter(edge)
+ override def filter(edge: S2EdgeLike) = left.filter(edge) || right.filter(edge)
}
object WhereParser {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index 55b6e12..6afbd87 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -548,12 +548,12 @@ class RequestParser(graph: S2Graph) {
elementsWithTsv
}
- def parseJsonFormat(jsValue: JsValue, operation: String): Seq[(S2Edge, String)] = {
+ def parseJsonFormat(jsValue: JsValue, operation: String): Seq[(S2EdgeLike, String)] = {
val jsValues = toJsValues(jsValue)
jsValues.flatMap(toEdgeWithTsv(_, operation))
}
- private def toEdgeWithTsv(jsValue: JsValue, operation: String): Seq[(S2Edge, String)] = {
+ private def toEdgeWithTsv(jsValue: JsValue, operation: String): Seq[(S2EdgeLike, String)] = {
val srcIds = (jsValue \ "from").asOpt[JsValue].map(Seq(_)).getOrElse(Nil) ++ (jsValue \ "froms").asOpt[Seq[JsValue]].getOrElse(Nil)
val tgtIds = (jsValue \ "to").asOpt[JsValue].map(Seq(_)).getOrElse(Nil) ++ (jsValue \ "tos").asOpt[Seq[JsValue]].getOrElse(Nil)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index 01dd128..e6075ec 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -93,13 +93,13 @@ abstract class Storage(val graph: S2Graph,
def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
io.buildIncrementsCountAsync(indexedEdge, amount)
- def buildVertexPutsAsync(edge: S2Edge): Seq[SKeyValue] =
+ def buildVertexPutsAsync(edge: S2EdgeLike): Seq[SKeyValue] =
io.buildVertexPutsAsync(edge)
def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
io.snapshotEdgeMutations(edgeMutate)
- def buildDegreePuts(edge: S2Edge, degreeVal: Long): Seq[SKeyValue] =
+ def buildDegreePuts(edge: S2EdgeLike, degreeVal: Long): Seq[SKeyValue] =
io.buildDegreePuts(edge, degreeVal)
def buildPutsAll(vertex: S2VertexLike): Seq[SKeyValue] =
@@ -121,15 +121,15 @@ abstract class Storage(val graph: S2Graph,
def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] =
fetcher.fetchVertices(vertices)
- def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2Edge]] = fetcher.fetchEdgesAll()
+ def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = fetcher.fetchEdgesAll()
def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = fetcher.fetchVerticesAll()
- def fetchSnapshotEdgeInner(edge: S2Edge)(implicit ec: ExecutionContext): Future[(Option[S2Edge], Option[SKeyValue])] =
+ def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] =
fetcher.fetchSnapshotEdgeInner(edge)
/** Conflict Resolver **/
- def retry(tryNum: Int)(edges: Seq[S2Edge], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2Edge])(implicit ec: ExecutionContext): Future[Boolean] =
+ def retry(tryNum: Int)(edges: Seq[S2EdgeLike], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2EdgeLike])(implicit ec: ExecutionContext): Future[Boolean] =
conflictResolver.retry(tryNum)(edges, statusCode, fetchedSnapshotEdgeOpt)
/** Management **/
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index 4014b6d..d0a59b2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -32,8 +32,8 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
/** Parsing Logic: parse from kv from Storage into Edge */
def toEdge[K: CanSKeyValue](kv: K,
queryRequest: QueryRequest,
- cacheElementOpt: Option[S2Edge],
- parentEdges: Seq[EdgeWithScore]): Option[S2Edge] = {
+ cacheElementOpt: Option[S2EdgeLike],
+ parentEdges: Seq[EdgeWithScore]): Option[S2EdgeLike] = {
logger.debug(s"toEdge: $kv")
try {
@@ -41,7 +41,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
val queryParam = queryRequest.queryParam
val schemaVer = queryParam.label.schemaVersion
val indexEdgeOpt = serDe.indexEdgeDeserializer(schemaVer).fromKeyValues(Seq(kv), cacheElementOpt)
- if (!queryOption.returnTree) indexEdgeOpt.map(indexEdge => indexEdge.copy(parentEdges = parentEdges))
+ if (!queryOption.returnTree) indexEdgeOpt.map(indexEdge => indexEdge.copyParentEdges(parentEdges))
else indexEdgeOpt
} catch {
case ex: Exception =>
@@ -54,7 +54,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
queryRequest: QueryRequest,
cacheElementOpt: Option[SnapshotEdge] = None,
isInnerCall: Boolean,
- parentEdges: Seq[EdgeWithScore]): Option[S2Edge] = {
+ parentEdges: Seq[EdgeWithScore]): Option[S2EdgeLike] = {
// logger.debug(s"SnapshottoEdge: $kv")
val queryParam = queryRequest.queryParam
val schemaVer = queryParam.label.schemaVersion
@@ -62,7 +62,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
if (isInnerCall) {
snapshotEdgeOpt.flatMap { snapshotEdge =>
- val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
+ val edge = snapshotEdge.toEdge.copyParentEdges(parentEdges)
if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
else None
}
@@ -70,7 +70,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
snapshotEdgeOpt.flatMap { snapshotEdge =>
if (snapshotEdge.allPropsDeleted) None
else {
- val edge = snapshotEdge.toEdge.copy(parentEdges = parentEdges)
+ val edge = snapshotEdge.toEdge.copyParentEdges(parentEdges)
if (queryParam.where.map(_.filter(edge)).getOrElse(true)) Option(edge)
else None
}
@@ -144,7 +144,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
}
val tsVal = processTimeDecay(queryParam, edge)
val newScore = degreeScore + score
- EdgeWithScore(convertedEdge.copy(parentEdges = parentEdges), score = newScore * labelWeight * tsVal, label = label)
+ EdgeWithScore(convertedEdge.copyParentEdges(parentEdges), score = newScore * labelWeight * tsVal, label = label)
}
val sampled =
@@ -229,11 +229,11 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
}
}
- def buildVertexPutsAsync(edge: S2Edge): Seq[SKeyValue] = {
+ def buildVertexPutsAsync(edge: S2EdgeLike): Seq[SKeyValue] = {
val storeVertex = edge.innerLabel.extraOptions.get("storeVertex").map(_.as[Boolean]).getOrElse(false)
if (storeVertex) {
- if (edge.op == GraphUtil.operations("delete"))
+ if (edge.getOp() == GraphUtil.operations("delete"))
buildDeleteBelongsToId(edge.srcForVertex) ++ buildDeleteBelongsToId(edge.tgtForVertex)
else
serDe.vertexSerializer(edge.srcForVertex).toKeyValues ++ serDe.vertexSerializer(edge.tgtForVertex).toKeyValues
@@ -242,7 +242,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
}
}
- def buildDegreePuts(edge: S2Edge, degreeVal: Long): Seq[SKeyValue] = {
+ def buildDegreePuts(edge: S2EdgeLike, degreeVal: Long): Seq[SKeyValue] = {
edge.propertyInner(LabelMeta.degree.name, degreeVal, edge.ts)
val kvs = edge.edgesWithIndexValid.flatMap { indexEdge =>
serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put, durability = indexEdge.label.durability))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
index c3b38e8..44bd4dc 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
@@ -39,16 +39,16 @@ trait StorageReadable {
def fetches(queryRequests: Seq[QueryRequest],
prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]]
- def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2Edge]]
+ def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]]
def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
- protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2Edge)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
+ protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
protected def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
- def fetchSnapshotEdgeInner(edge: S2Edge)(implicit ec: ExecutionContext): Future[(Option[S2Edge], Option[SKeyValue])] = {
+ def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] = {
val queryParam = QueryParam(labelName = edge.innerLabel.label,
direction = GraphUtil.fromDirection(edge.labelWithDir.dir),
tgtVertexIdOpt = Option(edge.tgtVertex.innerIdVal),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
index 227cfa7..854fc18 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
@@ -52,7 +52,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
Random.nextInt(Math.min(BackoffTimeout, slot * Math.pow(2, tryNum)).toInt)
}
- def retry(tryNum: Int)(edges: Seq[S2Edge], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2Edge])(implicit ec: ExecutionContext): Future[Boolean] = {
+ def retry(tryNum: Int)(edges: Seq[S2EdgeLike], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2EdgeLike])(implicit ec: ExecutionContext): Future[Boolean] = {
if (tryNum >= MaxRetryNum) {
edges.foreach { edge =>
logger.error(s"commit failed after $MaxRetryNum\n${edge.toLogString}")
@@ -112,9 +112,9 @@ class WriteWriteConflictResolver(graph: S2Graph,
}
}
- protected def commitUpdate(edges: Seq[S2Edge],
+ protected def commitUpdate(edges: Seq[S2EdgeLike],
statusCode: Byte,
- fetchedSnapshotEdgeOpt: Option[S2Edge])(implicit ec: ExecutionContext): Future[Boolean] = {
+ fetchedSnapshotEdgeOpt: Option[S2EdgeLike])(implicit ec: ExecutionContext): Future[Boolean] = {
// Future.failed(new PartialFailureException(edges.head, 0, "ahahah"))
assert(edges.nonEmpty)
// assert(statusCode == 0 || fetchedSnapshotEdgeOpt.isDefined)
@@ -135,7 +135,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
assert(edgeMutate.newSnapshotEdge.isDefined)
val lockTs = Option(System.currentTimeMillis())
- val pendingEdge = squashedEdge.copy(statusCode = 1, lockTs = lockTs, version = squashedEdge.ts + 1)
+ val pendingEdge = squashedEdge.copyStatusCode(1).copyLockTs(lockTs).copyVersion(squashedEdge.ts + 1)
val lockSnapshotEdge = squashedEdge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
val releaseLockSnapshotEdge = edgeMutate.newSnapshotEdge.get.copy(statusCode = 0,
pendingEdgeOpt = None, version = lockSnapshotEdge.version + 1)
@@ -158,7 +158,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
Future.successful(true)
} else {
val lockTs = Option(System.currentTimeMillis())
- val pendingEdge = squashedEdge.copy(statusCode = 1, lockTs = lockTs, version = snapshotEdge.version + 1)
+ val pendingEdge = squashedEdge.copyStatusCode(1).copyLockTs(lockTs).copyVersion(snapshotEdge.getVersion() + 1)
val lockSnapshotEdge = snapshotEdge.toSnapshotEdge.copy(pendingEdgeOpt = Option(pendingEdge))
val releaseLockSnapshotEdge = edgeMutate.newSnapshotEdge.get.copy(statusCode = 0,
pendingEdgeOpt = None, version = lockSnapshotEdge.version + 1)
@@ -182,7 +182,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
else S2Edge.buildOperation(fetchedSnapshotEdgeOpt, pendingEdge +: edges)
val lockTs = Option(System.currentTimeMillis())
- val newPendingEdge = squashedEdge.copy(statusCode = 1, lockTs = lockTs, version = snapshotEdge.version + 1)
+ val newPendingEdge = squashedEdge.copyStatusCode(1).copyLockTs(lockTs).copyVersion(snapshotEdge.getVersion() + 1)
val lockSnapshotEdge = snapshotEdge.toSnapshotEdge.copy(pendingEdgeOpt = Option(newPendingEdge))
val releaseLockSnapshotEdge = edgeMutate.newSnapshotEdge.get.copy(statusCode = 0,
pendingEdgeOpt = None, version = lockSnapshotEdge.version + 1)
@@ -222,7 +222,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
if (fetchedSnapshotEdgeOpt.isDefined && fetchedSnapshotEdgeOpt.get.pendingEdgeOpt.isDefined) fetchedSnapshotEdgeOpt.get.pendingEdgeOpt.get +: edges
else edges
val (squashedEdge, edgeMutate) = S2Edge.buildOperation(fetchedSnapshotEdgeOpt, _edges)
- val newVersion = fetchedSnapshotEdgeOpt.map(_.version).getOrElse(squashedEdge.ts) + 2
+ val newVersion = fetchedSnapshotEdgeOpt.map(_.getVersion()).getOrElse(squashedEdge.ts) + 2
val releaseLockSnapshotEdge = edgeMutate.newSnapshotEdge match {
case None => squashedEdge.toSnapshotEdge.copy(statusCode = 0, pendingEdgeOpt = None, version = newVersion)
case Some(newSnapshotEdge) => newSnapshotEdge.copy(statusCode = 0, pendingEdgeOpt = None, version = newVersion)
@@ -246,8 +246,8 @@ class WriteWriteConflictResolver(graph: S2Graph,
* @return
*/
protected def commitProcess(statusCode: Byte,
- squashedEdge: S2Edge,
- fetchedSnapshotEdgeOpt: Option[S2Edge],
+ squashedEdge: S2EdgeLike,
+ fetchedSnapshotEdgeOpt: Option[S2EdgeLike],
lockSnapshotEdge: SnapshotEdge,
releaseLockSnapshotEdge: SnapshotEdge,
edgeMutate: EdgeMutate)(implicit ec: ExecutionContext): Future[Boolean] = {
@@ -259,7 +259,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
} yield lockReleased
}
- case class PartialFailureException(edge: S2Edge, statusCode: Byte, failReason: String) extends NoStackException(failReason)
+ case class PartialFailureException(edge: S2EdgeLike, statusCode: Byte, failReason: String) extends NoStackException(failReason)
protected def debug(ret: Boolean, phase: String, snapshotEdge: SnapshotEdge) = {
val msg = Seq(s"[$ret] [$phase]", s"${snapshotEdge.toLogString()}").mkString("\n")
@@ -282,8 +282,8 @@ class WriteWriteConflictResolver(graph: S2Graph,
* @return
*/
protected def acquireLock(statusCode: Byte,
- squashedEdge: S2Edge,
- fetchedSnapshotEdgeOpt: Option[S2Edge],
+ squashedEdge: S2EdgeLike,
+ fetchedSnapshotEdgeOpt: Option[S2EdgeLike],
lockEdge: SnapshotEdge)(implicit ec: ExecutionContext): Future[Boolean] = {
if (statusCode >= 1) {
logger.debug(s"skip acquireLock: [$statusCode]\n${squashedEdge.toLogString}")
@@ -334,7 +334,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
*/
protected def releaseLock(predicate: Boolean,
statusCode: Byte,
- squashedEdge: S2Edge,
+ squashedEdge: S2EdgeLike,
releaseLockEdge: SnapshotEdge)(implicit ec: ExecutionContext): Future[Boolean] = {
if (!predicate) {
Future.failed(new PartialFailureException(squashedEdge, 3, "predicate failed."))
@@ -379,7 +379,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
*/
protected def commitIndexEdgeMutations(predicate: Boolean,
statusCode: Byte,
- squashedEdge: S2Edge,
+ squashedEdge: S2EdgeLike,
edgeMutate: EdgeMutate)(implicit ec: ExecutionContext): Future[Boolean] = {
if (!predicate) Future.failed(new PartialFailureException(squashedEdge, 1, "predicate failed."))
else {
@@ -413,7 +413,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
*/
protected def commitIndexEdgeDegreeMutations(predicate: Boolean,
statusCode: Byte,
- squashedEdge: S2Edge,
+ squashedEdge: S2EdgeLike,
edgeMutate: EdgeMutate)(implicit ec: ExecutionContext): Future[Boolean] = {
def _write(kvs: Seq[SKeyValue], withWait: Boolean): Future[Boolean] = {
@@ -445,8 +445,8 @@ class WriteWriteConflictResolver(graph: S2Graph,
/** end of methods for consistency */
- def mutateLog(snapshotEdgeOpt: Option[S2Edge], edges: Seq[S2Edge],
- newEdge: S2Edge, edgeMutate: EdgeMutate) =
+ def mutateLog(snapshotEdgeOpt: Option[S2EdgeLike], edges: Seq[S2EdgeLike],
+ newEdge: S2EdgeLike, edgeMutate: EdgeMutate) =
Seq("----------------------------------------------",
s"SnapshotEdge: ${snapshotEdgeOpt.map(_.toLogString)}",
s"requestEdges: ${edges.map(_.toLogString).mkString("\n")}",
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
index 92130f5..6be5f60 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
@@ -67,7 +67,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
* @param queryRequest
* @return
*/
- private def buildRequest(queryRequest: QueryRequest, edge: S2Edge) = {
+ private def buildRequest(queryRequest: QueryRequest, edge: S2EdgeLike) = {
import Serializable._
val queryParam = queryRequest.queryParam
val label = queryParam.label
@@ -178,7 +178,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
Left(get)
}
- override def fetchKeyValues(queryRequest: QueryRequest, edge: S2Edge)(implicit ec: ExecutionContext) = {
+ override def fetchKeyValues(queryRequest: QueryRequest, edge: S2EdgeLike)(implicit ec: ExecutionContext) = {
val rpc = buildRequest(queryRequest, edge)
fetchKeyValues(rpc)
}
@@ -221,7 +221,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
}
}
- override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2Edge]] = {
+ override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] = {
val futures = Label.findAll().groupBy(_.hbaseTableName).toSeq.map { case (hTableName, labels) =>
val distinctLabels = labels.toSet
val scan = AsynchbasePatcher.newScanner(client, hTableName)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
index 2501ed9..01f268b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
@@ -22,10 +22,9 @@ package org.apache.s2graph.core.storage.serde.indexedge.tall
import org.apache.hadoop.hbase.util.Bytes
import org.apache.s2graph.core._
import org.apache.s2graph.core.mysqls.{Label, LabelMeta, ServiceColumn}
-import org.apache.s2graph.core.storage.serde._
-import org.apache.s2graph.core.storage.serde.StorageDeserializable._
-import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.CanSKeyValue
+import org.apache.s2graph.core.storage.serde.Deserializable
+import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types._
object IndexEdgeDeserializable{
@@ -33,13 +32,13 @@ object IndexEdgeDeserializable{
}
class IndexEdgeDeserializable(graph: S2Graph,
bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong,
- tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4)) extends Deserializable[S2Edge] {
+ tallSchemaVersions: Set[String] = Set(HBaseType.VERSION4)) extends Deserializable[S2EdgeLike] {
type QualifierRaw = (Array[(LabelMeta, InnerValLike)], VertexId, Byte, Boolean, Int)
type ValueRaw = (Array[(LabelMeta, InnerValLike)], Int)
override def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
- cacheElementOpt: Option[S2Edge]): Option[S2Edge] = {
+ cacheElementOpt: Option[S2EdgeLike]): Option[S2EdgeLike] = {
try {
assert(_kvs.size == 1)
@@ -79,8 +78,8 @@ class IndexEdgeDeserializable(graph: S2Graph,
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
- edge.op = GraphUtil.defaultOpByte
- edge.tsInnerValOpt = Option(InnerVal.withLong(tsVal, schemaVer))
+ edge.setOp(GraphUtil.defaultOpByte)
+ edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
} else {
// not degree edge
val (idxPropsRaw, endAt) =
@@ -150,8 +149,8 @@ class IndexEdgeDeserializable(graph: S2Graph,
edge.propertyInner(LabelMeta.timestamp.name, tsVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
- edge.op = op
- edge.tsInnerValOpt = Option(InnerVal.withLong(tsVal, schemaVer))
+ edge.setOp(op)
+ edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
}
Option(edge)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
index 68732ce..a7fe8a1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
@@ -28,13 +28,13 @@ import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.types._
class IndexEdgeDeserializable(graph: S2Graph,
- bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[S2Edge] {
+ bytesToLongFunc: (Array[Byte], Int) => Long = bytesToLong) extends Deserializable[S2EdgeLike] {
type QualifierRaw = (Array[(LabelMeta, InnerValLike)], VertexId, Byte, Boolean, Int)
type ValueRaw = (Array[(LabelMeta, InnerValLike)], Int)
override def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
- cacheElementOpt: Option[S2Edge]): Option[S2Edge] = {
+ cacheElementOpt: Option[S2EdgeLike]): Option[S2EdgeLike] = {
try {
assert(_kvs.size == 1)
@@ -68,8 +68,8 @@ class IndexEdgeDeserializable(graph: S2Graph,
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
- edge.op = GraphUtil.defaultOpByte
- edge.tsInnerValOpt = Option(InnerVal.withLong(tsVal, schemaVer))
+ edge.setOp(GraphUtil.defaultOpByte)
+ edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
} else {
pos = 0
val (idxPropsRaw, endAt) = bytesToProps(kv.qualifier, pos, schemaVer)
@@ -123,8 +123,8 @@ class IndexEdgeDeserializable(graph: S2Graph,
edge.propertyInner(LabelMeta.timestamp.name, tsVal, version)
edge.tgtVertex = graph.newVertex(tgtVertexId, version)
- edge.op = op
- edge.tsInnerValOpt = Option(InnerVal.withLong(tsVal, schemaVer))
+ edge.setOp(op)
+ edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
}
Option(edge)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
index 5f00b48..24775e7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
@@ -52,7 +52,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
snapshotEdge.pendingEdgeOpt match {
case None => valueBytes()
case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
val propsBytes = pendingEdge.serializePropsWithTs()
val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
index df84e86..44f2596 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
@@ -59,7 +59,7 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
snapshotEdge.pendingEdgeOpt match {
case None => valueBytes()
case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.op)
+ val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
val propsBytes = pendingEdge.serializePropsWithTs()
val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
index ad9299c..55658b9 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/parsers/WhereParserTest.scala
@@ -37,7 +37,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val ts = System.currentTimeMillis()
val dummyTs = LabelMeta.timestamp -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion)
- def validate(label: Label)(edge: S2Edge)(sql: String)(expected: Boolean) = {
+ def validate(label: Label)(edge: S2EdgeLike)(sql: String)(expected: Boolean) = {
def debug(whereOpt: Try[Where]) = {
println("==================")
println(s"$whereOpt")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
index d8b2cfa..c7d474b 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
@@ -37,6 +37,7 @@ import scala.collection.JavaConverters._
object S2GraphProvider {
val Implementation: Set[Class[_]] = Set(
+ classOf[S2EdgeLike],
classOf[S2Edge],
classOf[S2Vertex],
classOf[S2VertexLike],
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/87394b9f/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
index 101b331..2b31fcd 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
@@ -54,7 +54,7 @@ object EdgeController extends Controller {
graphElem match {
case v: S2VertexLike =>
enqueue(kafkaTopic, graphElem, tsv)
- case e: S2Edge =>
+ case e: S2EdgeLike =>
e.innerLabel.extraOptions.get("walLog") match {
case None =>
enqueue(kafkaTopic, e, tsv)
@@ -93,8 +93,8 @@ object EdgeController extends Controller {
val result = s2.mutateElements(elements.map(_._1), true)
result onComplete { results =>
results.get.zip(elements).map {
- case (r: MutateResponse, (e: S2Edge, tsv: String)) if !r.isSuccess =>
- val kafkaMessages = if(e.op == GraphUtil.operations("deleteAll")){
+ case (r: MutateResponse, (e: S2EdgeLike, tsv: String)) if !r.isSuccess =>
+ val kafkaMessages = if(e.getOp() == GraphUtil.operations("deleteAll")){
toDeleteAllFailMessages(Seq(e.srcVertex), Seq(e.innerLabel), e.labelWithDir.dir, e.ts)
} else{
Seq(ExceptionHandler.toKafkaMessage(Config.KAFKA_MUTATE_FAIL_TOPIC, e, Some(tsv)))
[13/23] incubator-s2graph git commit: remove SelfType bidirectional
dependencies on S2EdgeLike.
Posted by st...@apache.org.
remove SelfType bidirectional dependencies on S2EdgeLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/3514060a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/3514060a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/3514060a
Branch: refs/heads/master
Commit: 3514060ab112303efb0782929e4333dd5ee1f2cd
Parents: d3a2e75
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Nov 7 10:38:44 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 7 11:13:27 2017 +0900
----------------------------------------------------------------------
.../s2graph/core/GraphElementBuilder.scala | 11 +++++++---
.../org/apache/s2graph/core/PostProcess.scala | 8 ++++----
.../org/apache/s2graph/core/QueryParam.scala | 4 ++--
.../scala/org/apache/s2graph/core/S2Edge.scala | 8 ++++----
.../org/apache/s2graph/core/S2EdgeLike.scala | 21 ++++++++++++--------
.../s2graph/core/index/IndexProvider.scala | 2 +-
.../s2graph/core/parsers/WhereParser.scala | 4 ++--
.../storage/WriteWriteConflictResolver.scala | 6 +++---
.../core/storage/serde/MutationHelper.scala | 7 +++----
.../tall/SnapshotEdgeSerializable.scala | 4 ++--
.../wide/SnapshotEdgeSerializable.scala | 4 ++--
11 files changed, 44 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index c9133b1..3ef7ba3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -279,10 +279,15 @@ class GraphElementBuilder(graph: S2Graph) {
val edge = edgeWithScore.edge
val copiedEdge = label.consistencyLevel match {
case "strong" =>
- edge.builder.copyEdge(op = GraphUtil.operations("delete"),
- version = requestTs, propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
+ edge
+ .copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ .copyTs(requestTs)
+ .copyOp(GraphUtil.operations("delete"))
+ .copyVersion(requestTs)
case _ =>
- edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
+ edge
+ .copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
+ .copyTs(requestTs)
}
val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 5118600..900bbbd 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -64,9 +64,9 @@ object PostProcess {
else {
val ancestors = for {
current <- parentEdges
- parents = s2EdgeParent(graph, queryOption, current.edge.parentEdges) if parents != JsNull
+ parents = s2EdgeParent(graph, queryOption, current.edge.getParentEdges()) if parents != JsNull
} yield {
- val s2Edge = current.edge.originalEdgeOpt.getOrElse(current.edge)
+ val s2Edge = current.edge.getOriginalEdgeOpt().getOrElse(current.edge)
s2EdgeToJsValue(queryOption, current.copy(edge = s2Edge), false, parents = parents, checkSelectColumns = true)
}
Json.toJson(ancestors)
@@ -240,7 +240,7 @@ object PostProcess {
// no group by specified on query.
val results = if (limitOpt.isDefined) stepResult.edgeWithScores.take(limitOpt.get) else stepResult.edgeWithScores
val ls = results.map { t =>
- val parents = if (queryOption.returnTree) s2EdgeParent(graph, queryOption, t.edge.parentEdges) else JsNull
+ val parents = if (queryOption.returnTree) s2EdgeParent(graph, queryOption, t.edge.getParentEdges()) else JsNull
s2EdgeToJsValue(queryOption, t, false, parents)
}
@@ -266,7 +266,7 @@ object PostProcess {
)
} else {
val agg = edges.map { t =>
- val parents = if (queryOption.returnTree) s2EdgeParent(graph, queryOption, t.edge.parentEdges) else JsNull
+ val parents = if (queryOption.returnTree) s2EdgeParent(graph, queryOption, t.edge.getParentEdges()) else JsNull
s2EdgeToJsValue(queryOption, t, false, parents)
}
val aggJson = Json.toJson(agg)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
index 3d0d076..748e8c5 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
@@ -185,7 +185,7 @@ case class EdgeTransformer(jsValue: JsValue) {
replace(queryParam, fmt, fieldNames.flatMap(fieldName => toInnerValOpt(queryParam, edge, fieldName)), nextStepOpt)
}
}
- } yield edge.builder.updateTgtVertex(innerVal).copyOriginalEdgeOpt(Option(edge))
+ } yield edge.updateTgtVertex(innerVal).copyOriginalEdgeOpt(Option(edge))
edges
@@ -383,7 +383,7 @@ case class QueryParam(labelName: String,
propValJs match {
case JsString(in) if edgeOpt.isDefined && in.contains("_parent.") =>
val parentLen = in.split("_parent.").length - 1
- val edge = (0 until parentLen).foldLeft(edgeOpt.get) { case (acc, _) => acc.parentEdges.head.edge }
+ val edge = (0 until parentLen).foldLeft(edgeOpt.get) { case (acc, _) => acc.getParentEdges().head.edge }
val timePivot = edge.ts
val replaced = TemplateHelper.replaceVariable(timePivot, in).trim
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 9f5093c..db530ac 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -515,7 +515,7 @@ object S2Edge {
state.foreach { case (k, v) =>
edge.propertyInner(k.name, v.innerVal.value, v.ts)
}
- edge.propsWithTs
+ edge.getPropsWithTs()
}
def allPropsDeleted(props: Map[LabelMeta, InnerValLikeWithTs]): Boolean =
@@ -559,7 +559,7 @@ object S2Edge {
// logger.debug(s"requestEdge: ${requestEdge.toStringRaw}")
val oldPropsWithTs =
if (invertedEdge.isEmpty) Map.empty[LabelMeta, InnerValLikeWithTs]
- else propsToState(invertedEdge.get.propsWithTs)
+ else propsToState(invertedEdge.get.getPropsWithTs())
val funcs = requestEdges.map { edge =>
if (edge.getOp() == GraphUtil.operations("insert")) {
@@ -592,7 +592,7 @@ object S2Edge {
for {
(requestEdge, func) <- requestWithFuncs
} {
- val (_newPropsWithTs, _) = func((prevPropsWithTs, propsToState(requestEdge.propsWithTs), requestEdge.ts, requestEdge.innerLabel.schemaVersion))
+ val (_newPropsWithTs, _) = func((prevPropsWithTs, propsToState(requestEdge.getPropsWithTs()), requestEdge.ts, requestEdge.innerLabel.schemaVersion))
prevPropsWithTs = _newPropsWithTs
// logger.debug(s"${requestEdge.toLogString}\n$oldPropsWithTs\n$prevPropsWithTs\n")
}
@@ -628,7 +628,7 @@ object S2Edge {
val newOp = snapshotEdgeOpt match {
case None => requestEdge.getOp()
case Some(old) =>
- val oldMaxTs = old.propsWithTs.asScala.map(_._2.ts).max
+ val oldMaxTs = old.getPropsWithTs().asScala.map(_._2.ts).max
if (oldMaxTs > requestEdge.ts) old.getOp()
else requestEdge.getOp()
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index 33e7e83..0b40cf5 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -15,26 +15,25 @@ import scala.concurrent.Await
import scala.collection.JavaConverters._
trait S2EdgeLike extends Edge with GraphElement {
- this: S2Edge =>
-
- val builder: S2EdgeBuilder = new S2EdgeBuilder(this)
-
-
val innerGraph: S2Graph
val srcVertex: S2VertexLike
var tgtVertex: S2VertexLike
val innerLabel: Label
val dir: Int
-// var op: Byte = GraphUtil.defaultOpByte
-// var version: Long = System.currentTimeMillis()
+ val builder: S2EdgeBuilder = new S2EdgeBuilder(this)
+
+ var op: Byte
+ var version: Long
+ var tsInnerValOpt: Option[InnerValLike]
+
val propsWithTs: Props = S2Edge.EmptyProps
+
val parentEdges: Seq[EdgeWithScore] = Nil
val originalEdgeOpt: Option[S2EdgeLike] = None
val pendingEdgeOpt: Option[S2EdgeLike] = None
val statusCode: Byte = 0
val lockTs: Option[Long] = None
-// var tsInnerValOpt: Option[InnerValLike] = None
lazy val ts = propsWithTs.get(LabelMeta.timestamp.name).innerVal.value match {
case b: BigDecimal => b.longValue()
@@ -149,6 +148,12 @@ trait S2EdgeLike extends Edge with GraphElement {
builder.copyEdge(lockTs = newLockTs)
}
+ def copyTs(newTs: Long): S2EdgeLike =
+ builder.copyEdge(ts = newTs)
+
+ def updateTgtVertex(id: InnerValLike): S2EdgeLike =
+ builder.updateTgtVertex(id)
+
def vertices(direction: Direction): util.Iterator[structure.Vertex] = {
val arr = new util.ArrayList[Vertex]()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index e5005b7..2411e65 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -180,7 +180,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
}
private def toDocument(globalIndex: GlobalIndex, edge: S2EdgeLike): Option[Document] = {
- val props = edge.propsWithTs.asScala
+ val props = edge.getPropsWithTs().asScala
val exist = props.exists(t => globalIndex.propNamesSet(t._1))
if (!exist) None
else {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
index d947066..d9e6a7b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/parsers/WhereParser.scala
@@ -66,7 +66,7 @@ trait ExtractValue {
@tailrec
private def findParent(edge: S2EdgeLike, depth: Int): S2EdgeLike =
- if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1)
+ if (depth > 0) findParent(edge.getParentEdges().head.edge, depth - 1)
else edge
private def findParentEdge(edge: S2EdgeLike, key: String): (String, S2EdgeLike) = {
@@ -145,7 +145,7 @@ case class InWithoutParent(label: Label, propKey: String, values: Set[String]) e
}
override def filter(edge: S2EdgeLike): Boolean = {
- if (edge.dir == GraphUtil.directions("in")) {
+ if (edge.getDir() == GraphUtil.directions("in")) {
val propVal = propToInnerVal(edge, propKey)
innerValLikeLsIn.contains(propVal)
} else {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
index 854fc18..af0d53d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
@@ -143,7 +143,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
commitProcess(statusCode, squashedEdge, fetchedSnapshotEdgeOpt, lockSnapshotEdge, releaseLockSnapshotEdge, edgeMutate)
case Some(snapshotEdge) =>
- snapshotEdge.pendingEdgeOpt match {
+ snapshotEdge.getPendingEdgeOpt() match {
case None =>
/*
* others finished commit on this SN. but there is no contention.
@@ -165,7 +165,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
commitProcess(statusCode, squashedEdge, fetchedSnapshotEdgeOpt, lockSnapshotEdge, releaseLockSnapshotEdge, edgeMutate)
}
case Some(pendingEdge) =>
- val isLockExpired = pendingEdge.lockTs.get + LockExpireDuration < System.currentTimeMillis()
+ val isLockExpired = pendingEdge.getLockTs().get + LockExpireDuration < System.currentTimeMillis()
if (isLockExpired) {
/*
* if pendingEdge.ts == snapshotEdge.ts =>
@@ -219,7 +219,7 @@ class WriteWriteConflictResolver(graph: S2Graph,
* releaseLock = (edgeMutate.newSnapshotEdge, None)
*/
val _edges =
- if (fetchedSnapshotEdgeOpt.isDefined && fetchedSnapshotEdgeOpt.get.pendingEdgeOpt.isDefined) fetchedSnapshotEdgeOpt.get.pendingEdgeOpt.get +: edges
+ if (fetchedSnapshotEdgeOpt.isDefined && fetchedSnapshotEdgeOpt.get.getPendingEdgeOpt().isDefined) fetchedSnapshotEdgeOpt.get.pendingEdgeOpt.get +: edges
else edges
val (squashedEdge, edgeMutate) = S2Edge.buildOperation(fetchedSnapshotEdgeOpt, _edges)
val newVersion = fetchedSnapshotEdgeOpt.map(_.getVersion()).getOrElse(squashedEdge.ts) + 2
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
index 79c9dc3..9312181 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
@@ -28,19 +28,18 @@ class MutationHelper(storage: Storage) {
edgeWithScore <- stepInnerResult.edgeWithScores
} yield {
val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val edgeSnapshot = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeSnapshot = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
- val edgeForward = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeForward = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
io.buildIncrementsAsync(indexEdge, -1L)
}
/* reverted direction */
- val edgeRevert = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeRevert = edge.copyEdgeWithState(S2Edge.propsToState(edge.updatePropsWithTs()))
val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
io.buildIncrementsAsync(indexEdge, -1L)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
index 12edf54..02b2977 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeSerializable.scala
@@ -52,10 +52,10 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
snapshotEdge.pendingEdgeOpt match {
case None => valueBytes()
case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
+ val opBytes = statusCodeWithOp(pendingEdge.getStatusCode(), pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
val propsBytes = S2Edge.serializePropsWithTs(pendingEdge)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+ val lockBytes = Bytes.toBytes(pendingEdge.getLockTs().get)
Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3514060a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
index 02a72b1..44d4a2a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeSerializable.scala
@@ -59,10 +59,10 @@ class SnapshotEdgeSerializable(snapshotEdge: SnapshotEdge) extends Serializable[
snapshotEdge.pendingEdgeOpt match {
case None => valueBytes()
case Some(pendingEdge) =>
- val opBytes = statusCodeWithOp(pendingEdge.statusCode, pendingEdge.getOp())
+ val opBytes = statusCodeWithOp(pendingEdge.getStatusCode(), pendingEdge.getOp())
val versionBytes = Array.empty[Byte]
val propsBytes = S2Edge.serializePropsWithTs(pendingEdge)
- val lockBytes = Bytes.toBytes(pendingEdge.lockTs.get)
+ val lockBytes = Bytes.toBytes(pendingEdge.getLockTs().get)
Bytes.add(Bytes.add(valueBytes(), opBytes, versionBytes), Bytes.add(propsBytes, lockBytes))
}
[17/23] incubator-s2graph git commit: add S2GraphLike.
Posted by st...@apache.org.
add S2GraphLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6403bc9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6403bc9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6403bc9d
Branch: refs/heads/master
Commit: 6403bc9d69a1ec19824a9ee2f9779cb7c973b1dc
Parents: e51289f
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 10 13:16:47 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 10 15:42:25 2017 +0900
----------------------------------------------------------------------
.../s2graph/core/GraphElementBuilder.scala | 2 +-
.../org/apache/s2graph/core/Management.scala | 2 +-
.../org/apache/s2graph/core/PostProcess.scala | 4 +-
.../org/apache/s2graph/core/QueryResult.scala | 6 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 6 +-
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 2 +-
.../org/apache/s2graph/core/S2EdgeLike.scala | 2 +-
.../scala/org/apache/s2graph/core/S2Graph.scala | 853 ++++++-------------
.../org/apache/s2graph/core/S2GraphLike.scala | 151 +++-
.../org/apache/s2graph/core/S2Vertex.scala | 2 +-
.../apache/s2graph/core/S2VertexBuilder.scala | 2 +-
.../org/apache/s2graph/core/S2VertexLike.scala | 2 +-
.../s2graph/core/rest/RequestParser.scala | 2 +-
.../apache/s2graph/core/rest/RestHandler.scala | 4 +-
.../apache/s2graph/core/storage/Storage.scala | 2 +-
.../apache/s2graph/core/storage/StorageIO.scala | 2 +-
.../storage/WriteWriteConflictResolver.scala | 3 +-
.../core/storage/hbase/AsynchbaseStorage.scala | 2 +-
.../hbase/AsynchbaseStorageReadable.scala | 2 +-
.../storage/hbase/AsynchbaseStorageSerDe.scala | 2 +-
.../tall/IndexEdgeDeserializable.scala | 4 +-
.../wide/IndexEdgeDeserializable.scala | 2 +-
.../tall/SnapshotEdgeDeserializable.scala | 2 +-
.../wide/SnapshotEdgeDeserializable.scala | 2 +-
.../serde/vertex/VertexDeserializable.scala | 73 --
.../serde/vertex/VertexSerializable.scala | 62 --
.../vertex/tall/VertexDeserializable.scala | 4 +-
.../vertex/wide/VertexDeserializable.scala | 4 +-
.../core/tinkerpop/S2GraphProvider.scala | 2 +-
29 files changed, 438 insertions(+), 770 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index 3ef7ba3..08da355 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -9,7 +9,7 @@ import play.api.libs.json.{JsObject, Json}
import scala.util.Try
-class GraphElementBuilder(graph: S2Graph) {
+class GraphElementBuilder(graph: S2GraphLike) {
def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] = Try {
val parts = GraphUtil.split(s)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index 49d3c0e..8d2d62a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -293,7 +293,7 @@ object Management {
}
}
-class Management(graph: S2Graph) {
+class Management(graph: S2GraphLike) {
import Management._
import scala.collection.JavaConversions._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 7047214..462c1e4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -57,7 +57,7 @@ object PostProcess {
case _ => Json.obj("message" -> ex.getMessage)
}
- def s2EdgeParent(graph: S2Graph,
+ def s2EdgeParent(graph: S2GraphLike,
queryOption: QueryOption,
parentEdges: Seq[EdgeWithScore]): JsValue = {
if (parentEdges.isEmpty) JsNull
@@ -193,7 +193,7 @@ object PostProcess {
case _ => js
}
- def toJson(orgQuery: Option[JsValue])(graph: S2Graph,
+ def toJson(orgQuery: Option[JsValue])(graph: S2GraphLike,
queryOption: QueryOption,
stepResult: StepResult): JsValue = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index b654e71..7d187c6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.collection.{Seq, mutable}
object QueryResult {
- def fromVertices(graph: S2Graph, vertices: Seq[S2VertexLike], queryParams: Seq[QueryParam]): StepResult = {
+ def fromVertices(graph: S2GraphLike, vertices: Seq[S2VertexLike], queryParams: Seq[QueryParam]): StepResult = {
val edgeWithScores = vertices.flatMap { vertex =>
queryParams.map { queryParam =>
val label = queryParam.label
@@ -44,7 +44,7 @@ object QueryResult {
StepResult(edgeWithScores = edgeWithScores, grouped = Nil, degreeEdges = Nil, false)
}
- def fromVertices(graph: S2Graph,
+ def fromVertices(graph: S2GraphLike,
query: Query): StepResult = {
if (query.steps.isEmpty || query.steps.head.queryParams.isEmpty) {
StepResult.Empty
@@ -301,7 +301,7 @@ object StepResult {
}
//TODO: Optimize this.
- def filterOut(graph: S2Graph,
+ def filterOut(graph: S2GraphLike,
queryOption: QueryOption,
baseStepResult: StepResult,
filterOutStepResult: StepResult): StepResult = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index db530ac..3e33ed4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -74,7 +74,7 @@ object SnapshotEdge {
}
}
-case class SnapshotEdge(graph: S2Graph,
+case class SnapshotEdge(graph: S2GraphLike,
srcVertex: S2VertexLike,
tgtVertex: S2VertexLike,
label: Label,
@@ -178,7 +178,7 @@ object IndexEdge {
}
}
-case class IndexEdge(graph: S2Graph,
+case class IndexEdge(graph: S2GraphLike,
srcVertex: S2VertexLike,
tgtVertex: S2VertexLike,
label: Label,
@@ -312,7 +312,7 @@ case class IndexEdge(graph: S2Graph,
}
}
-case class S2Edge(override val innerGraph: S2Graph,
+case class S2Edge(override val innerGraph: S2GraphLike,
override val srcVertex: S2VertexLike,
override var tgtVertex: S2VertexLike,
override val innerLabel: Label,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
index 2ea1504..4004a13 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
@@ -46,7 +46,7 @@ class S2EdgeBuilder(edge: S2EdgeLike) {
}
}
- def copyEdge(innerGraph: S2Graph = edge.innerGraph,
+ def copyEdge(innerGraph: S2GraphLike = edge.innerGraph,
srcVertex: S2VertexLike = edge.srcVertex,
tgtVertex: S2VertexLike = edge.tgtVertex,
innerLabel: Label = edge.innerLabel,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index 0b40cf5..0165439 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -15,7 +15,7 @@ import scala.concurrent.Await
import scala.collection.JavaConverters._
trait S2EdgeLike extends Edge with GraphElement {
- val innerGraph: S2Graph
+ val innerGraph: S2GraphLike
val srcVertex: S2VertexLike
var tgtVertex: S2VertexLike
val innerLabel: Label
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index fc1205d..f80d5c2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -121,11 +121,11 @@ object S2Graph {
configuration
}
- def open(configuration: Configuration): S2Graph = {
+ def open(configuration: Configuration): S2GraphLike = {
new S2Graph(configuration)(ec)
}
- def initStorage(graph: S2Graph, config: Config)(ec: ExecutionContext): Storage = {
+ def initStorage(graph: S2GraphLike, config: Config)(ec: ExecutionContext): Storage = {
val storageBackend = config.getString("s2graph.storage.backend")
logger.info(s"[InitStorage]: $storageBackend")
@@ -158,130 +158,13 @@ object S2Graph {
@Graph.OptOuts(value = Array(
/* Process */
/* branch: passed all. */
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.BranchTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.ChooseTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.OptionalTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.LocalTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-
/* filter */
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.AndTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CoinTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropTest$Traversals", method = "g_V_properties_drop", reason = "please find bug on this case."),
-// passed: all, failed: g_V_properties_drop
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.OrTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SampleTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.TailTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest$Traversals", method = "*", reason = "no"),
-// passed: all,
/* map */
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.AddEdgeTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CoalesceTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ConstantTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_both_both_count", reason = "count differ very little. fix this."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX3X_count", reason = "count differ very little. fix this."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX8X_count", reason = "count differ very litter. fix this."),
-// passed: all, failed: g_V_both_both_count, g_V_repeatXoutX_timesX3X_count, g_V_repeatXoutX_timesX8X_count
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.FlatMapTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.LoopsTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MapTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MapKeysTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MapValuesTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$CountMatchTraversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$GreedyMatchTraversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MaxTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MeanTest$Traversals", method = "*", reason = "no"),
-// failed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MinTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest$Traversals", method = "*", reason = "no"),
-// failed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest$Traversals", method = "*", reason = "no"),
-// passed: all
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallback", reason = "NullPointerException. fix this."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "g_V_whereXinXcreatedX_count_isX1XX_name_profile", reason = "java.lang.AssertionError: There should be 3 top-level metrics. fix this."),
@@ -291,71 +174,15 @@ object S2Graph {
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profileXmetricsX", reason = "expected 2, actual 6. fix this."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profile", reason = "expected 2, actual 6. fix this."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallbackSideEffect", reason = "NullPointerException. fix this."),
-// failed: grateful_V_out_out_profileXmetricsX, g_V_repeat_both_profileXmetricsX, grateful_V_out_out_profile, g_V_repeat_both_profile
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest$Traversals", method = "*", reason = "no"),
-// passed: all
/* sideEffect */
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTestV3d0$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.InjectTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
- // new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SackTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SideEffectCapTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SideEffectTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StoreTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup", reason = "Expected 5, Actual 6."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_outEXknowsX_subgraphXsgX_name_capXsgX", reason = "Expected 3, Actual 6"),
-// passed: all, failed: g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup, g_V_withSideEffectXsgX_outEXknowsX_subgraphXsgX_name_capXsgX
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TreeTest$Traversals", method = "*", reason = "no"),
-// passed: all
-
/* compliance */
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.CoreTraversalTest", method = "shouldThrowExceptionWhenIdsMixed", reason = "VertexId is not Element."),
-// passed: all
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionTest", method = "*", reason = "not supported yet."),
-// failed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.TranslationStrategyProcessTest", method = "*", reason = "no"),
-// passed: all
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
@@ -363,120 +190,56 @@ object S2Graph {
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedDefaultId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
-// failed: shouldGenerateDefaultIdOnAddVWithSpecifiedId, shouldGenerateDefaultIdOnAddVWithGeneratedCustomId, shouldGenerateDefaultIdOnGraphAddVWithGeneratedDefaultId,
-// shouldGenerateDefaultIdOnAddVWithGeneratedDefaultId, shouldGenerateDefaultIdOnGraphAddVWithGeneratedCustomId, shouldGenerateDefaultIdOnGraphAddVWithSpecifiedId
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategyProcessTest", method = "*", reason = "not supported yet."),
-// failed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategyProcessTest", method = "*", reason = "no"),
-// passed: all
new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategyProcessTest", method = "*", reason = "not supported yet."),
-// failed: all
-
-// new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategyProcessTest", method = "*", reason = "no"),
-// passed: all
/* Structure */
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method="shouldValidateIdEquality", reason="reference equals on EdgeId is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method="shouldValidateEquality", reason="reference equals on EdgeId is not supported."),
- // passed: all, failed: none
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.GraphConstructionTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateIdEquality", reason = "reference equals on EdgeId is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateEquality", reason = "reference equals on EdgeId is not supported."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.PropertyTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.VertexTest$BasicVertexTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnEdge", reason = "S2Vertex.addEdge behave as upsert."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.VertexPropertyTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "reference equals is not supported."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.FeatureSupportTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPropertyTest", method = "shouldNotBeEqualPropertiesAsThereIsDifferentKey", reason = "reference equals is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.VertexTest$BasicVertexTest", method="shouldHaveExceptionConsistencyWhenAssigningSameIdOnEdge", reason="S2Vertex.addEdge behave as upsert."),
- // passed: , failed: shouldHaveExceptionConsistencyWhenAssigningSameIdOnEdge
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveVertices", reason = "random label creation is not supported. all label need to be pre-configured."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnVertex", reason = "Assigning the same ID to an Element update instead of throwing exception."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveEdges", reason = "random label creation is not supported. all label need to be pre-configured."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdgeTest", method="shouldNotEvaluateToEqualDifferentId", reason="reference equals is not supported."),
- // passed: all, failed: shouldNotEvaluateToEqualDifferentId
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "Assigning the same ID to an Element update instead of throwing exception."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.detached.DetachedGraphTest", method="*", reason="no"),
- // passed: all, failed: none, all ignored
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.DistributionGeneratorTest", method = "*", reason = "non-deterministic test."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPropertyTest", method="shouldNotBeEqualPropertiesAsThereIsDifferentKey", reason="reference equals is not supported."),
-// // passed: , failed: shouldNotBeEqualPropertiesAsThereIsDifferentKey
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GryoTest", method = "shouldSerializeTree", reason = "order of children is reversed. not sure why."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GraphSONTest", method = "shouldSerializeTraversalMetrics", reason = "expected 2, actual 3."),
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexPropertyTest", method="*", reason="no"),
- // passed: all, failed: none
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithBOTHEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithINEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexAsReferenceNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithOUTEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.GraphTest", method="shouldRemoveVertices", reason="random label creation is not supported. all label need to be pre-configured."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.GraphTest", method="shouldHaveExceptionConsistencyWhenAssigningSameIdOnVertex", reason="Assigning the same ID to an Element update instead of throwing exception."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.GraphTest", method="shouldRemoveEdges", reason="random label creation is not supported. all label need to be pre-configured."),
- // passed: , failed:
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdgeAsReference", specific = "graphson-v2-embedded", reason = "no"),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteEdge", specific = "graphson-v2-embedded", reason = "no"),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdge", specific = "graphson-v2-embedded", reason = "no"),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdgeTest", method="shouldNotEvaluateToEqualDifferentId", reason="Assigning the same ID to an Element update instead of throwing exception."),
- // passed: all, skip: shouldNotEvaluateToEqualDifferentId
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoGraphTest", method = "*", reason = "no"), // all failed.
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexPropertyTest", method="*", reason="no"),
- // passed: all, failed: none
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceGraphTest", method="*", reason="no"),
- // passed: all, failed: none, all ignored
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexTest", method="*", reason="no"),
- // passed: all, failed: none, all ignored
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.util.star.StarGraphTest", method="*", reason="no"),
- // passed: all,
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.1)", reason="graphson-v2-embedded is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.5)", reason="graphson-v2-embedded is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.5)", reason="graphson-v2-embedded is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.1)", reason="graphson-v2-embedded is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(PowerLawDistribution{gamma=2.3, multiplier=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.25)", reason="graphson-v2-embedded is not supported."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method="shouldGenerateDifferentGraph", specific="test(PowerLawDistribution{gamma=2.3, multiplier=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.25)", reason="graphson-v2-embedded is not supported."),
- // passed: all, except shouldGenerateDifferentGraph method.
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.algorithm.generator.DistributionGeneratorTest", method="*", reason="non-deterministic test."),
- // all failed.
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.SerializationTest$GryoTest", method="shouldSerializeTree", reason="order of children is reversed. not sure why."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.SerializationTest$GraphSONTest", method="shouldSerializeTraversalMetrics", reason="expected 2, actual 3."),
- // passed: all, failed: $GryoTest.shouldSerializeTree
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoCustomTest", method="*", reason="no"),
- // all ignored.
-
-// new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoPropertyTest", method="*", reason="no"),
- // all passed.
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteVertexWithBOTHEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteVertexWithINEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteDetachedVertexAsReferenceNoEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteVertexNoEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteVertexWithOUTEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method="shouldReadWriteDetachedVertexNoEdges", specific="graphson-v2-embedded", reason="Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- // passed: all, except graphson-v2-embedded.
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method="shouldReadWriteDetachedEdgeAsReference", specific="graphson-v2-embedded", reason="no"),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method="shouldReadWriteEdge", specific="graphson-v2-embedded", reason="no"),
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method="shouldReadWriteDetachedEdge", specific="graphson-v2-embedded", reason="no"),
- // passed: all, except graphson-v2-embedded.
-
- // TODO:
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoGraphTest", method="*", reason="no"), // all failed.
-
- new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoTest", method="*", reason="no")
- // all failed.
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoTest", method = "*", reason = "no")
))
class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2GraphLike {
- import S2Graph._
-
var apacheConfiguration: Configuration = _
def dbSession() = scalikejdbc.AutoSession
@@ -488,35 +251,30 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
private val running = new AtomicBoolean(true)
- val config = _config.withFallback(S2Graph.DefaultConfig)
+ override val config = _config.withFallback(S2Graph.DefaultConfig)
Model.apply(config)
Model.loadCache()
- val MaxRetryNum = config.getInt("max.retry.number")
- val MaxBackOff = config.getInt("max.back.off")
- val BackoffTimeout = config.getInt("back.off.timeout")
- val DeleteAllFetchCount = config.getInt("delete.all.fetch.count")
- val DeleteAllFetchSize = config.getInt("delete.all.fetch.size")
- val FailProb = config.getDouble("hbase.fail.prob")
- val LockExpireDuration = config.getInt("lock.expire.time")
- val MaxSize = config.getInt("future.cache.max.size")
- val ExpireAfterWrite = config.getInt("future.cache.expire.after.write")
- val ExpireAfterAccess = config.getInt("future.cache.expire.after.access")
- val WaitTimeout = Duration(600, TimeUnit.SECONDS)
-
- val management = new Management(this)
+ override val management = new Management(this)
- def getManagement() = management
+ override val indexProvider = IndexProvider.apply(config)
- private val localLongId = new AtomicLong()
+ override val elementBuilder = new GraphElementBuilder(this)
- def nextLocalLongId = localLongId.getAndIncrement()
+ override val traversalHelper = new TraversalHelper(this)
private def confWithFallback(conf: Config): Config = {
conf.withFallback(config)
}
+ val defaultStorage: Storage = S2Graph.initStorage(this, config)(ec)
+
+ for {
+ entry <- config.entrySet() if S2Graph.DefaultConfigs.contains(entry.getKey)
+ (k, v) = (entry.getKey, entry.getValue)
+ } logger.info(s"[Initialized]: $k, ${this.config.getAnyRef(k)}")
+
/**
* TODO: we need to some way to handle malformed configuration for storage.
*/
@@ -553,31 +311,15 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
m
}
- val defaultStorage: Storage = S2Graph.initStorage(this, config)(ec)
-
- /** QueryLevel FutureCache */
- val queryFutureCache = new DeferCache[StepResult, Promise, Future](parseCacheConfig(config, "query."), empty = StepResult.Empty)
-
- for {
- entry <- config.entrySet() if S2Graph.DefaultConfigs.contains(entry.getKey)
- (k, v) = (entry.getKey, entry.getValue)
- } logger.info(s"[Initialized]: $k, ${this.config.getAnyRef(k)}")
-
- val indexProvider = IndexProvider.apply(config)
-
- val elementBuilder = new GraphElementBuilder(this)
-
- val traversalHelper = new TraversalHelper(this)
-
- def getStorage(service: Service): Storage = {
+ override def getStorage(service: Service): Storage = {
storagePool.getOrElse(s"service:${service.serviceName}", defaultStorage)
}
- def getStorage(label: Label): Storage = {
+ override def getStorage(label: Label): Storage = {
storagePool.getOrElse(s"label:${label.label}", defaultStorage)
}
- def flushStorage(): Unit = {
+ override def flushStorage(): Unit = {
storagePool.foreach { case (_, storage) =>
/* flush is blocking */
@@ -585,9 +327,26 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
}
- def fallback = Future.successful(StepResult.Empty)
+ override def shutdown(modelDataDelete: Boolean = false): Unit =
+ if (running.compareAndSet(true, false)) {
+ flushStorage()
+ Model.shutdown(modelDataDelete)
+ defaultStorage.shutdown()
+ localLongId.set(0l)
+ }
+
+ override def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]] = {
+ val verticesWithIdx = vertices.zipWithIndex
+ val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
+ getStorage(service).fetchVertices(vertices).map(_.zip(vertexGroup.map(_._2)))
+ }
+
+ Future.sequence(futures).map { ls =>
+ ls.flatten.toSeq.sortBy(_._2).map(_._1)
+ }
+ }
- def checkEdges(edges: Seq[S2EdgeLike]): Future[StepResult] = {
+ override def checkEdges(edges: Seq[S2EdgeLike]): Future[StepResult] = {
val futures = for {
edge <- edges
} yield {
@@ -602,9 +361,95 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
}
- // def checkEdges(edges: Seq[Edge]): Future[StepResult] = storage.checkEdges(edges)
+ override def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ def mutateVertices(storage: Storage)(zkQuorum: String, vertices: Seq[S2VertexLike],
+ withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ val futures = vertices.map { vertex => storage.mutateVertex(zkQuorum, vertex, withWait) }
+ Future.sequence(futures)
+ }
+
+ val verticesWithIdx = vertices.zipWithIndex
+ val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
+ mutateVertices(getStorage(service))(service.cluster, vertexGroup.map(_._1), withWait).map(_.zip(vertexGroup.map(_._2)))
+ }
+ Future.sequence(futures).map { ls => ls.flatten.toSeq.sortBy(_._2).map(_._1) }
+ }
+
+ override def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ val edgeWithIdxs = edges.zipWithIndex
+
+ val (strongEdges, weakEdges) =
+ edgeWithIdxs.partition { case (edge, idx) =>
+ val e = edge
+ e.innerLabel.consistencyLevel == "strong" && e.getOp() != GraphUtil.operations("insertBulk")
+ }
+
+ val weakEdgesFutures = weakEdges.groupBy { case (edge, idx) => edge.innerLabel.hbaseZkAddr }.map { case (zkQuorum, edgeWithIdxs) =>
+ val futures = edgeWithIdxs.groupBy(_._1.innerLabel).map { case (label, edgeGroup) =>
+ val storage = getStorage(label)
+ val edges = edgeGroup.map(_._1)
+ val idxs = edgeGroup.map(_._2)
+
+ /* multiple edges with weak consistency level will be processed as batch */
+ storage.mutateWeakEdges(zkQuorum, edges, withWait)
+ }
+ Future.sequence(futures)
+ }
+ val (strongDeleteAll, strongEdgesAll) = strongEdges.partition { case (edge, idx) => edge.getOp() == GraphUtil.operations("deleteAll") }
+
+ val deleteAllFutures = strongDeleteAll.map { case (edge, idx) =>
+ deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.innerLabel), edge.getDir(), edge.ts).map(idx -> _)
+ }
+
+ val strongEdgesFutures = strongEdgesAll.groupBy { case (edge, idx) => edge.innerLabel }.map { case (label, edgeGroup) =>
+ val edges = edgeGroup.map(_._1)
+ val idxs = edgeGroup.map(_._2)
+ val storage = getStorage(label)
+ val zkQuorum = label.hbaseZkAddr
+ storage.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
+ idxs.zip(rets)
+ }
+ }
+
+ for {
+ weak <- Future.sequence(weakEdgesFutures)
+ deleteAll <- Future.sequence(deleteAllFutures)
+ strong <- Future.sequence(strongEdgesFutures)
+ } yield {
+ (deleteAll ++ weak.flatten.flatten ++ strong.flatten).sortBy(_._1).map(r => new MutateResponse(r._2))
+ }
+ }
+
+ override def mutateElements(elements: Seq[GraphElement],
+ withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+
+ val edgeBuffer = ArrayBuffer[(S2EdgeLike, Int)]()
+ val vertexBuffer = ArrayBuffer[(S2VertexLike, Int)]()
+
+ elements.zipWithIndex.foreach {
+ case (e: S2EdgeLike, idx: Int) => edgeBuffer.append((e, idx))
+ case (v: S2VertexLike, idx: Int) => vertexBuffer.append((v, idx))
+ case any@_ => logger.error(s"Unknown type: ${any}")
+ }
+
+ val edgeFutureWithIdx = mutateEdges(edgeBuffer.map(_._1), withWait).map { result =>
+ edgeBuffer.map(_._2).zip(result)
+ }
+
+ val vertexFutureWithIdx = mutateVertices(vertexBuffer.map(_._1), withWait).map { result =>
+ vertexBuffer.map(_._2).zip(result)
+ }
+
+ val graphFuture = for {
+ edgesMutated <- edgeFutureWithIdx
+ verticesMutated <- vertexFutureWithIdx
+ } yield (edgesMutated ++ verticesMutated).sortBy(_._1).map(_._2)
- def getEdges(q: Query): Future[StepResult] = {
+ graphFuture
+
+ }
+
+ override def getEdges(q: Query): Future[StepResult] = {
Try {
if (q.steps.isEmpty) {
// TODO: this should be get vertex query.
@@ -629,76 +474,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
} get
}
- def getEdgesStepInner(q: Query, buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
- Try {
- if (q.steps.isEmpty) fallback
- else {
- def fetch: Future[StepResult] = {
- val startStepInnerResult = QueryResult.fromVertices(this, q)
- q.steps.zipWithIndex.foldLeft(Future.successful(startStepInnerResult)) { case (prevStepInnerResultFuture, (step, stepIdx)) =>
- for {
- prevStepInnerResult <- prevStepInnerResultFuture
- currentStepInnerResult <- fetchStep(q, stepIdx, prevStepInnerResult, buildLastStepInnerResult)
- } yield {
- currentStepInnerResult.copy(
- accumulatedCursors = prevStepInnerResult.accumulatedCursors :+ currentStepInnerResult.cursors,
- failCount = currentStepInnerResult.failCount + prevStepInnerResult.failCount
- )
- }
- }
- }
-
- fetch
- }
- } recover {
- case e: Exception =>
- logger.error(s"getEdgesAsync: $e", e)
- fallback
- } get
- }
-
- def fetchStep(orgQuery: Query,
- stepIdx: Int,
- stepInnerResult: StepResult,
- buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
- if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
- else {
- val (_, prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
- traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
-
- val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
-
- traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
- fetchedLs, orgQuery.steps(stepIdx).queryParams, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
- }
- }
-
-
- /**
- * responsible to fire parallel fetch call into storage and create future that will return merged result.
- *
- * @param queryRequests
- * @param prevStepEdges
- * @return
- */
- def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[StepResult]] = {
-
- val reqWithIdxs = queryRequests.zipWithIndex
- val requestsPerLabel = reqWithIdxs.groupBy(t => t._1.queryParam.label)
- val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
- for {
- prev <- prevFuture
- cur <- getStorage(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
- } yield {
- prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
- }
- }
- aggFuture.map { agg => agg.toSeq.sortBy(_._1).map(_._2) }
- }
-
-
- def getEdgesMultiQuery(mq: MultiQuery): Future[StepResult] = {
+ override def getEdgesMultiQuery(mq: MultiQuery): Future[StepResult] = {
Try {
if (mq.queries.isEmpty) fallback
else {
@@ -722,18 +498,63 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
} get
}
- def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]] = {
- val verticesWithIdx = vertices.zipWithIndex
- val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
- getStorage(service).fetchVertices(vertices).map(_.zip(vertexGroup.map(_._2)))
+ override def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
+ labels: Seq[Label],
+ dir: Int,
+ ts: Long): Future[Boolean] = {
+ val requestTs = ts
+ val vertices = srcVertices
+ /* create query per label */
+ val queries = for {
+ label <- labels
+ } yield {
+ val queryParam = QueryParam(labelName = label.label, direction = GraphUtil.fromDirection(dir),
+ offset = 0, limit = DeleteAllFetchSize, duplicatePolicy = DuplicatePolicy.Raw)
+ val step = Step(List(queryParam))
+ Query(vertices, Vector(step))
+ }
+
+ val retryFuture = Extensions.retryOnSuccess(DeleteAllFetchCount) {
+ fetchAndDeleteAll(queries, requestTs)
+ } { case (allDeleted, deleteSuccess) =>
+ allDeleted && deleteSuccess
+ }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
+
+ retryFuture onFailure {
+ case ex =>
+ logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
}
+ retryFuture
+ }
+
+ override def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
+ val edgesWithIdx = edges.zipWithIndex
+ val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
+ getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
+ }
Future.sequence(futures).map { ls =>
ls.flatten.toSeq.sortBy(_._2).map(_._1)
}
}
- def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
+ override def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
+ val label = edge.innerLabel
+ val storage = getStorage(label)
+
+ storage.updateDegree(label.hbaseZkAddr, edge, degreeVal)
+ }
+
+ override def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
+ val v = newVertex(vertexId)
+ Await.result(getVertices(Seq(v)).map { vertices => vertices.headOption }, WaitTimeout)
+ }
+
+ override def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge] = {
+ Await.result(fetchEdgesAsync(vertex, labelNameWithDirs), WaitTimeout)
+ }
+
+ override def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
val labelNameWithDirs =
if (labelNames.isEmpty) {
// TODO: Let's clarify direction
@@ -756,39 +577,71 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
fetchEdgesAsync(vertex, labelNameWithDirs.distinct)
}
- /** mutate */
- def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
- labels: Seq[Label],
- dir: Int,
- ts: Long): Future[Boolean] = {
+ def isRunning(): Boolean = running.get()
- val requestTs = ts
- val vertices = srcVertices
- /* create query per label */
- val queries = for {
- label <- labels
- } yield {
- val queryParam = QueryParam(labelName = label.label, direction = GraphUtil.fromDirection(dir),
- offset = 0, limit = DeleteAllFetchSize, duplicatePolicy = DuplicatePolicy.Raw)
- val step = Step(List(queryParam))
- Query(vertices, Vector(step))
- }
+ /** Private **/
- val retryFuture = Extensions.retryOnSuccess(DeleteAllFetchCount) {
- fetchAndDeleteAll(queries, requestTs)
- } { case (allDeleted, deleteSuccess) =>
- allDeleted && deleteSuccess
- }.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
+ private def getEdgesStepInner(q: Query, buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
+ Try {
+ if (q.steps.isEmpty) fallback
+ else {
+ def fetch: Future[StepResult] = {
+ val startStepInnerResult = QueryResult.fromVertices(this, q)
+ q.steps.zipWithIndex.foldLeft(Future.successful(startStepInnerResult)) { case (prevStepInnerResultFuture, (step, stepIdx)) =>
+ for {
+ prevStepInnerResult <- prevStepInnerResultFuture
+ currentStepInnerResult <- fetchStep(q, stepIdx, prevStepInnerResult, buildLastStepInnerResult)
+ } yield {
+ currentStepInnerResult.copy(
+ accumulatedCursors = prevStepInnerResult.accumulatedCursors :+ currentStepInnerResult.cursors,
+ failCount = currentStepInnerResult.failCount + prevStepInnerResult.failCount
+ )
+ }
+ }
+ }
- retryFuture onFailure {
- case ex =>
- logger.error(s"[Error]: deleteAllAdjacentEdges failed.")
+ fetch
+ }
+ } recover {
+ case e: Exception =>
+ logger.error(s"getEdgesAsync: $e", e)
+ fallback
+ } get
+ }
+
+ private def fetchStep(orgQuery: Query,
+ stepIdx: Int,
+ stepInnerResult: StepResult,
+ buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
+ if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
+ else {
+ val (_, prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
+ traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
+
+ val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
+
+ traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
+ fetchedLs, orgQuery.steps(stepIdx).queryParams, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
}
+ }
- retryFuture
+ private def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[StepResult]] = {
+
+ val reqWithIdxs = queryRequests.zipWithIndex
+ val requestsPerLabel = reqWithIdxs.groupBy(t => t._1.queryParam.label)
+ val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
+ for {
+ prev <- prevFuture
+ cur <- getStorage(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
+ } yield {
+ prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
+ }
+ }
+ aggFuture.map { agg => agg.toSeq.sortBy(_._1).map(_._2) }
}
- def fetchAndDeleteAll(queries: Seq[Query], requestTs: Long): Future[(Boolean, Boolean)] = {
+ private def fetchAndDeleteAll(queries: Seq[Query], requestTs: Long): Future[(Boolean, Boolean)] = {
val futures = queries.map(getEdgesStepInner(_, true))
val future = for {
stepInnerResultLs <- Future.sequence(futures)
@@ -807,8 +660,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
- def deleteAllFetchedEdgesLs(stepInnerResultLs: Seq[StepResult],
- requestTs: Long): Future[(Boolean, Boolean)] = {
+ private def deleteAllFetchedEdgesLs(stepInnerResultLs: Seq[StepResult],
+ requestTs: Long): Future[(Boolean, Boolean)] = {
stepInnerResultLs.foreach { stepInnerResult =>
if (stepInnerResult.isFailure) throw new RuntimeException("fetched result is fallback.")
}
@@ -853,162 +706,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
}
-
- def mutateElements(elements: Seq[GraphElement],
- withWait: Boolean = false): Future[Seq[MutateResponse]] = {
-
- val edgeBuffer = ArrayBuffer[(S2EdgeLike, Int)]()
- val vertexBuffer = ArrayBuffer[(S2VertexLike, Int)]()
-
- elements.zipWithIndex.foreach {
- case (e: S2EdgeLike, idx: Int) => edgeBuffer.append((e, idx))
- case (v: S2VertexLike, idx: Int) => vertexBuffer.append((v, idx))
- case any@_ => logger.error(s"Unknown type: ${any}")
- }
-
- val edgeFutureWithIdx = mutateEdges(edgeBuffer.map(_._1), withWait).map { result =>
- edgeBuffer.map(_._2).zip(result)
- }
-
- val vertexFutureWithIdx = mutateVertices(vertexBuffer.map(_._1), withWait).map { result =>
- vertexBuffer.map(_._2).zip(result)
- }
-
- val graphFuture = for {
- edgesMutated <- edgeFutureWithIdx
- verticesMutated <- vertexFutureWithIdx
- } yield (edgesMutated ++ verticesMutated).sortBy(_._1).map(_._2)
-
- graphFuture
-
- }
-
- def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- val edgeWithIdxs = edges.zipWithIndex
-
- val (strongEdges, weakEdges) =
- edgeWithIdxs.partition { case (edge, idx) =>
- val e = edge
- e.innerLabel.consistencyLevel == "strong" && e.getOp() != GraphUtil.operations("insertBulk")
- }
-
- val weakEdgesFutures = weakEdges.groupBy { case (edge, idx) => edge.innerLabel.hbaseZkAddr }.map { case (zkQuorum, edgeWithIdxs) =>
- val futures = edgeWithIdxs.groupBy(_._1.innerLabel).map { case (label, edgeGroup) =>
- val storage = getStorage(label)
- val edges = edgeGroup.map(_._1)
- val idxs = edgeGroup.map(_._2)
-
- /* multiple edges with weak consistency level will be processed as batch */
- storage.mutateWeakEdges(zkQuorum, edges, withWait)
- }
- Future.sequence(futures)
- }
- val (strongDeleteAll, strongEdgesAll) = strongEdges.partition { case (edge, idx) => edge.getOp() == GraphUtil.operations("deleteAll") }
-
- val deleteAllFutures = strongDeleteAll.map { case (edge, idx) =>
- deleteAllAdjacentEdges(Seq(edge.srcVertex), Seq(edge.innerLabel), edge.getDir(), edge.ts).map(idx -> _)
- }
-
- val strongEdgesFutures = strongEdgesAll.groupBy { case (edge, idx) => edge.innerLabel }.map { case (label, edgeGroup) =>
- val edges = edgeGroup.map(_._1)
- val idxs = edgeGroup.map(_._2)
- val storage = getStorage(label)
- val zkQuorum = label.hbaseZkAddr
- storage.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
- idxs.zip(rets)
- }
- }
-
- for {
- weak <- Future.sequence(weakEdgesFutures)
- deleteAll <- Future.sequence(deleteAllFutures)
- strong <- Future.sequence(strongEdgesFutures)
- } yield {
- (deleteAll ++ weak.flatten.flatten ++ strong.flatten).sortBy(_._1).map(r => new MutateResponse(r._2))
- }
- }
-
- def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- def mutateVertices(storage: Storage)(zkQuorum: String, vertices: Seq[S2VertexLike],
- withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- val futures = vertices.map { vertex => storage.mutateVertex(zkQuorum, vertex, withWait) }
- Future.sequence(futures)
- }
-
- val verticesWithIdx = vertices.zipWithIndex
- val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
- mutateVertices(getStorage(service))(service.cluster, vertexGroup.map(_._1), withWait).map(_.zip(vertexGroup.map(_._2)))
- }
- Future.sequence(futures).map { ls => ls.flatten.toSeq.sortBy(_._2).map(_._1) }
- }
-
- def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
- val edgesWithIdx = edges.zipWithIndex
- val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
- getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
- }
- Future.sequence(futures).map { ls =>
- ls.flatten.toSeq.sortBy(_._2).map(_._1)
- }
- }
-
- def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
- val label = edge.innerLabel
- val storage = getStorage(label)
-
- storage.updateDegree(label.hbaseZkAddr, edge, degreeVal)
- }
-
- def isRunning(): Boolean = running.get()
-
- def shutdown(modelDataDelete: Boolean = false): Unit =
- if (running.compareAndSet(true, false)) {
- flushStorage()
- Model.shutdown(modelDataDelete)
- defaultStorage.shutdown()
- localLongId.set(0l)
- }
-
-
- def newEdge(srcVertex: S2VertexLike,
- tgtVertex: S2VertexLike,
- innerLabel: Label,
- dir: Int,
- op: Byte = GraphUtil.defaultOpByte,
- version: Long = System.currentTimeMillis(),
- propsWithTs: S2Edge.State,
- parentEdges: Seq[EdgeWithScore] = Nil,
- originalEdgeOpt: Option[S2EdgeLike] = None,
- pendingEdgeOpt: Option[S2EdgeLike] = None,
- statusCode: Byte = 0,
- lockTs: Option[Long] = None,
- tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike =
- elementBuilder.newEdge(srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs,
- parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
-
- def newVertexId(service: Service,
- column: ServiceColumn,
- id: Any): VertexId =
- elementBuilder.newVertexId(service, column, id)
-
- def newVertex(id: VertexId,
- ts: Long = System.currentTimeMillis(),
- props: S2Vertex.Props = S2Vertex.EmptyProps,
- op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike =
- elementBuilder.newVertex(id, ts, props, op, belongLabelIds)
-
-
- def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
- val v = newVertex(vertexId)
- Await.result(getVertices(Seq(v)).map { vertices => vertices.headOption }, WaitTimeout)
- }
-
- def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge] = {
- Await.result(fetchEdgesAsync(vertex, labelNameWithDirs), WaitTimeout)
- }
-
- def fetchEdgesAsync(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
+ private def fetchEdgesAsync(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
val queryParams = labelNameWithDirs.map { case (l, direction) =>
QueryParam(labelName = l, direction = direction.toLowerCase)
}
@@ -1021,25 +719,4 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
ls.iterator()
}
}
-
- def toVertex(serviceName: String,
- columnName: String,
- id: Any,
- props: Map[String, Any] = Map.empty,
- ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2VertexLike =
- elementBuilder.toVertex(serviceName, columnName, id, props, ts, operation)
-
- def toEdge(srcId: Any,
- tgtId: Any,
- labelName: String,
- direction: String,
- props: Map[String, Any] = Map.empty,
- ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2EdgeLike =
- elementBuilder.toEdge(srcId, tgtId, labelName, direction, props, ts, operation)
-
- def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] =
- elementBuilder.toGraphElement(s, labelMapping)
-
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index 03a92c6..a58f1e0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -1,31 +1,157 @@
package org.apache.s2graph.core
+
import java.util
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicLong
+import com.typesafe.config.Config
import org.apache.commons.configuration.Configuration
import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core.features.{S2Features, S2GraphVariables}
-import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
-import org.apache.s2graph.core.types.VertexId
+import org.apache.s2graph.core.index.IndexProvider
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta, Service, ServiceColumn}
+import org.apache.s2graph.core.storage.{MutateResponse, Storage}
+import org.apache.s2graph.core.types.{InnerValLike, VertexId}
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
import org.apache.tinkerpop.gremlin.structure
import org.apache.tinkerpop.gremlin.structure.Edge.Exceptions
-import org.apache.tinkerpop.gremlin.structure.Graph.{Features, Variables}
+import org.apache.tinkerpop.gremlin.structure.Graph.Variables
import org.apache.tinkerpop.gremlin.structure.io.{GraphReader, GraphWriter, Io, Mapper}
-import org.apache.tinkerpop.gremlin.structure.{Edge, Element, Graph, T, Transaction, Vertex}
+import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Element, Graph, T, Transaction, Vertex}
-import scala.concurrent.{Await, Future}
import scala.collection.JavaConversions._
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext, Future}
+
trait S2GraphLike extends Graph {
- this: S2Graph =>
+ implicit val ec: ExecutionContext
var apacheConfiguration: Configuration
- private val s2Features = new S2Features
+ protected val localLongId = new AtomicLong()
+
+ protected val s2Features = new S2Features
+
+ val config: Config
+
+ val management: Management
+
+ val indexProvider: IndexProvider
+
+ val elementBuilder: GraphElementBuilder
+
+ val traversalHelper: TraversalHelper
+
+ lazy val MaxRetryNum: Int = config.getInt("max.retry.number")
+ lazy val MaxBackOff: Int = config.getInt("max.back.off")
+ lazy val BackoffTimeout: Int = config.getInt("back.off.timeout")
+ lazy val DeleteAllFetchCount: Int = config.getInt("delete.all.fetch.count")
+ lazy val DeleteAllFetchSize: Int = config.getInt("delete.all.fetch.size")
+ lazy val FailProb: Double = config.getDouble("hbase.fail.prob")
+ lazy val LockExpireDuration: Int = config.getInt("lock.expire.time")
+ lazy val MaxSize: Int = config.getInt("future.cache.max.size")
+ lazy val ExpireAfterWrite: Int = config.getInt("future.cache.expire.after.write")
+ lazy val ExpireAfterAccess: Int = config.getInt("future.cache.expire.after.access")
+ lazy val WaitTimeout: Duration = Duration(600, TimeUnit.SECONDS)
override def features() = s2Features
+ def nextLocalLongId = localLongId.getAndIncrement()
+
+ def fallback = Future.successful(StepResult.Empty)
+
+ def defaultStorage: Storage
+
+ def getStorage(service: Service): Storage
+
+ def getStorage(label: Label): Storage
+
+ def flushStorage(): Unit
+
+ def shutdown(modelDataDelete: Boolean = false): Unit
+
+ def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]]
+
+ def checkEdges(edges: Seq[S2EdgeLike]): Future[StepResult]
+
+ def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]]
+
+ def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]]
+
+ def mutateElements(elements: Seq[GraphElement],
+ withWait: Boolean = false): Future[Seq[MutateResponse]]
+
+ def getEdges(q: Query): Future[StepResult]
+
+ def getEdgesMultiQuery(mq: MultiQuery): Future[StepResult]
+
+ def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
+ labels: Seq[Label],
+ dir: Int,
+ ts: Long): Future[Boolean]
+
+ def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]]
+
+ def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse]
+
+ def getVertex(vertexId: VertexId): Option[S2VertexLike]
+
+ def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge]
+
+ def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]]
+
+ /** Convert to Graph Element **/
+ def newEdge(srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
+ innerLabel: Label,
+ dir: Int,
+ op: Byte = GraphUtil.defaultOpByte,
+ version: Long = System.currentTimeMillis(),
+ propsWithTs: S2Edge.State,
+ parentEdges: Seq[EdgeWithScore] = Nil,
+ originalEdgeOpt: Option[S2EdgeLike] = None,
+ pendingEdgeOpt: Option[S2EdgeLike] = None,
+ statusCode: Byte = 0,
+ lockTs: Option[Long] = None,
+ tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike =
+ elementBuilder.newEdge(srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs,
+ parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
+
+ def newVertexId(service: Service,
+ column: ServiceColumn,
+ id: Any): VertexId =
+ elementBuilder.newVertexId(service, column, id)
+
+ def newVertex(id: VertexId,
+ ts: Long = System.currentTimeMillis(),
+ props: S2Vertex.Props = S2Vertex.EmptyProps,
+ op: Byte = 0,
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike =
+ elementBuilder.newVertex(id, ts, props, op, belongLabelIds)
+
+ def toVertex(serviceName: String,
+ columnName: String,
+ id: Any,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2VertexLike =
+ elementBuilder.toVertex(serviceName, columnName, id, props, ts, operation)
+
+ def toEdge(srcId: Any,
+ tgtId: Any,
+ labelName: String,
+ direction: String,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2EdgeLike =
+ elementBuilder.toEdge(srcId, tgtId, labelName, direction, props, ts, operation)
+
+ def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] =
+ elementBuilder.toGraphElement(s, labelMapping)
+
+ /** TinkerPop Interfaces **/
def vertices(ids: AnyRef*): util.Iterator[structure.Vertex] = {
val fetchVertices = ids.lastOption.map { lastParam =>
if (lastParam.isInstanceOf[Boolean]) lastParam.asInstanceOf[Boolean]
@@ -38,9 +164,9 @@ trait S2GraphLike extends Graph {
} else {
val vertices = ids.collect {
case s2Vertex: S2VertexLike => s2Vertex
- case vId: VertexId => newVertex(vId)
- case vertex: Vertex => newVertex(vertex.id().asInstanceOf[VertexId])
- case other@_ => newVertex(VertexId.fromString(other.toString))
+ case vId: VertexId => elementBuilder.newVertex(vId)
+ case vertex: Vertex => elementBuilder.newVertex(vertex.id().asInstanceOf[VertexId])
+ case other@_ => elementBuilder.newVertex(VertexId.fromString(other.toString))
}
if (fetchVertices) {
@@ -153,7 +279,7 @@ trait S2GraphLike extends Graph {
props: S2Vertex.Props = S2Vertex.EmptyProps,
op: Byte = 0,
belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
- val vertex = newVertex(id, ts, props, op, belongLabelIds)
+ val vertex = elementBuilder.newVertex(id, ts, props, op, belongLabelIds)
val future = mutateVertices(Seq(vertex), withWait = true).map { rets =>
if (rets.forall(_.isSuccess)) vertex
@@ -201,7 +327,7 @@ trait S2GraphLike extends Graph {
val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts)
val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
- val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs)
+ val edge = elementBuilder.newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs)
val future = mutateEdges(Seq(edge), withWait = true).flatMap { rets =>
indexProvider.mutateEdgesAsync(Seq(edge))
@@ -221,6 +347,7 @@ trait S2GraphLike extends Graph {
shutdown()
}
+
def compute[C <: GraphComputer](aClass: Class[C]): C = ???
def compute(): GraphComputer = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
index 96e7afe..954bab0 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
@@ -29,7 +29,7 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
import scala.collection.JavaConverters._
-case class S2Vertex(graph: S2Graph,
+case class S2Vertex(graph: S2GraphLike,
id: VertexId,
ts: Long = System.currentTimeMillis(),
props: Props = S2Vertex.EmptyProps,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
index e0205c5..399b142 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
@@ -15,7 +15,7 @@ class S2VertexBuilder(vertex: S2VertexLike) {
default
}
- def copyVertex(graph: S2Graph = vertex.graph,
+ def copyVertex(graph: S2GraphLike = vertex.graph,
id: VertexId = vertex.id,
ts: Long = vertex.ts,
props: Props = vertex.props,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index 29db49d..ad35efd 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -13,7 +13,7 @@ import scala.collection.JavaConverters._
import scala.concurrent.Await
trait S2VertexLike extends Vertex with GraphElement {
- val graph: S2Graph
+ val graph: S2GraphLike
val id: VertexId
val ts: Long
val props: Props
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index 6afbd87..54b865a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -118,7 +118,7 @@ object RequestParser {
}
-class RequestParser(graph: S2Graph) {
+class RequestParser(graph: S2GraphLike) {
import Management.JsonModel._
import RequestParser._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
index 1a85dba..460c627 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RestHandler.scala
@@ -54,7 +54,7 @@ object RestHandler {
* Public API, only return Future.successful or Future.failed
* Don't throw exception
*/
-class RestHandler(graph: S2Graph)(implicit ec: ExecutionContext) {
+class RestHandler(graph: S2GraphLike)(implicit ec: ExecutionContext) {
import RestHandler._
val requestParser = new RequestParser(graph)
@@ -172,7 +172,7 @@ class RestHandler(graph: S2Graph)(implicit ec: ExecutionContext) {
}
def getEdgesAsync(jsonQuery: JsValue, impIdOpt: Option[String] = None)
- (post: (S2Graph, QueryOption, StepResult) => JsValue): Future[JsValue] = {
+ (post: (S2GraphLike, QueryOption, StepResult) => JsValue): Future[JsValue] = {
def query(obj: JsValue): Future[JsValue] = {
(obj \ "queries").asOpt[JsValue] match {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index 2a8f1e2..18e6fa1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -28,7 +28,7 @@ import org.apache.s2graph.core.types._
import scala.concurrent.{ExecutionContext, Future}
-abstract class Storage(val graph: S2Graph,
+abstract class Storage(val graph: S2GraphLike,
val config: Config) {
/* Storage backend specific resource management */
val management: StorageManagement
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index 1b9c94b..2d74a7c 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -26,7 +26,7 @@ import org.apache.s2graph.core.mysqls.LabelMeta
import org.apache.s2graph.core.parsers.WhereParser
import org.apache.s2graph.core.utils.logger
-class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
+class StorageIO(val graph: S2GraphLike, val serDe: StorageSerDe) {
val dummyCursor: Array[Byte] = Array.empty
/** Parsing Logic: parse from kv from Storage into Edge */
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
index af0d53d..dcef1cc 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/WriteWriteConflictResolver.scala
@@ -28,12 +28,11 @@ import org.apache.s2graph.core.utils.logger
import scala.concurrent.{ExecutionContext, Future, Promise}
import scala.util.Random
-class WriteWriteConflictResolver(graph: S2Graph,
+class WriteWriteConflictResolver(graph: S2GraphLike,
serDe: StorageSerDe,
io: StorageIO,
mutator: StorageWritable,
fetcher: StorageReadable) {
-
val BackoffTimeout = graph.BackoffTimeout
val MaxRetryNum = graph.MaxRetryNum
val MaxBackOff = graph.MaxBackOff
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index ef1350a..8b3d862 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -138,7 +138,7 @@ object AsynchbaseStorage {
}
-class AsynchbaseStorage(override val graph: S2Graph,
+class AsynchbaseStorage(override val graph: S2GraphLike,
override val config: Config) extends Storage(graph, config) {
/**
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
index 5f54e47..af82439 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
@@ -38,7 +38,7 @@ import org.hbase.async._
import scala.collection.JavaConversions._
import scala.concurrent.{ExecutionContext, Future}
-class AsynchbaseStorageReadable(val graph: S2Graph,
+class AsynchbaseStorageReadable(val graph: S2GraphLike,
val config: Config,
val client: HBaseClient,
val serDe: StorageSerDe,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6403bc9d/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
index bb47e3b..c9a7dd3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
@@ -23,7 +23,7 @@ import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe, serde}
-class AsynchbaseStorageSerDe(val graph: S2Graph) extends StorageSerDe {
+class AsynchbaseStorageSerDe(val graph: S2GraphLike) extends StorageSerDe {
import org.apache.s2graph.core.types.HBaseType._
/**
[14/23] incubator-s2graph git commit: remove SelfType bidirectional
dependencies on S2VertexLike.
Posted by st...@apache.org.
remove SelfType bidirectional dependencies on S2VertexLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5f2d4ffc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5f2d4ffc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5f2d4ffc
Branch: refs/heads/master
Commit: 5f2d4ffc161abac372a7a1d4cbadc89cce45a84b
Parents: 3514060
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Nov 7 11:41:42 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 7 11:41:42 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/PostProcess.scala | 6 +-
.../scala/org/apache/s2graph/core/S2Graph.scala | 25 +++++++-
.../apache/s2graph/core/S2VertexBuilder.scala | 35 +++++++++++
.../org/apache/s2graph/core/S2VertexLike.scala | 63 ++++----------------
4 files changed, 74 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5f2d4ffc/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 900bbbd..7047214 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -147,9 +147,9 @@ object PostProcess {
def s2VertexToJson(s2Vertex: S2VertexLike): Option[JsValue] = {
val props = for {
- (k, v) <- s2Vertex.properties
- jsVal <- anyValToJsValue(v)
- } yield k -> jsVal
+ (_, property) <- s2Vertex.props
+ jsVal <- anyValToJsValue(property.value)
+ } yield property.columnMeta.name -> jsVal
for {
id <- anyValToJsValue(s2Vertex.innerIdVal)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5f2d4ffc/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 3270e84..fc1205d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -33,7 +33,7 @@ import org.apache.s2graph.core.storage.{MutateResponse, Storage}
import org.apache.s2graph.core.types._
import org.apache.s2graph.core.utils.{DeferCache, Extensions, logger}
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies
-import org.apache.tinkerpop.gremlin.structure.{Edge, Graph}
+import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Graph}
import scala.collection.JavaConversions._
import scala.collection.mutable
@@ -733,6 +733,29 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
}
+ def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
+ val labelNameWithDirs =
+ if (labelNames.isEmpty) {
+ // TODO: Let's clarify direction
+ if (direction == Direction.BOTH) {
+ Label.findBySrcColumnId(vertex.id.colId).map(l => l.label -> Direction.OUT.name) ++
+ Label.findByTgtColumnId(vertex.id.colId).map(l => l.label -> Direction.IN.name)
+ } else if (direction == Direction.IN) {
+ Label.findByTgtColumnId(vertex.id.colId).map(l => l.label -> direction.name)
+ } else {
+ Label.findBySrcColumnId(vertex.id.colId).map(l => l.label -> direction.name)
+ }
+ } else {
+ direction match {
+ case Direction.BOTH =>
+ Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
+ case _ => labelNames.map(_ -> direction.name())
+ }
+ }
+
+ fetchEdgesAsync(vertex, labelNameWithDirs.distinct)
+ }
+
/** mutate */
def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
labels: Seq[Label],
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5f2d4ffc/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
new file mode 100644
index 0000000..4a75036
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
@@ -0,0 +1,35 @@
+package org.apache.s2graph.core
+
+import java.util
+import java.util.function.BiConsumer
+
+import org.apache.s2graph.core.S2Vertex.Props
+import org.apache.s2graph.core.mysqls.ColumnMeta
+import org.apache.s2graph.core.types.VertexId
+
+class S2VertexBuilder(vertex: S2VertexLike) {
+ def defaultProps: util.HashMap[String, S2VertexProperty[_]] = {
+ val default = S2Vertex.EmptyProps
+ val newProps = new S2VertexProperty(vertex, ColumnMeta.lastModifiedAtColumn, ColumnMeta.lastModifiedAtColumn.name, vertex.ts)
+ default.put(ColumnMeta.lastModifiedAtColumn.name, newProps)
+ default
+ }
+
+ def copyVertex(graph: S2Graph = vertex.graph,
+ id: VertexId = vertex.id,
+ ts: Long = vertex.ts,
+ props: Props = vertex.props,
+ op: Byte = vertex.op,
+ belongLabelIds: Seq[Int] = vertex.belongLabelIds): S2VertexLike = {
+ val newProps = S2Vertex.EmptyProps
+ val v = new S2Vertex(graph, id, ts, newProps, op, belongLabelIds)
+
+ props.forEach(new BiConsumer[String, S2VertexProperty[_]] {
+ override def accept(t: String, u: S2VertexProperty[_]) = {
+ v.property(t, u)
+ }
+ })
+
+ v
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5f2d4ffc/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index 9ec2ab0..29db49d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -9,12 +9,10 @@ import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, T, Vertex, VertexProperty}
import play.api.libs.json.Json
-import scala.concurrent.{Await, Future}
import scala.collection.JavaConverters._
+import scala.concurrent.Await
trait S2VertexLike extends Vertex with GraphElement {
- this: S2Vertex =>
-
val graph: S2Graph
val id: VertexId
val ts: Long
@@ -23,12 +21,12 @@ trait S2VertexLike extends Vertex with GraphElement {
val belongLabelIds: Seq[Int]
val innerId = id.innerId
+
val innerIdVal = innerId.value
+ val builder = new S2VertexBuilder(this)
- lazy val properties = for {
- (k, v) <- props.asScala
- } yield v.columnMeta.name -> v.value
+ def label(): String = serviceColumn.columnName
def schemaVer = serviceColumn.schemaVersion
@@ -37,20 +35,16 @@ trait S2VertexLike extends Vertex with GraphElement {
def columnName = serviceColumn.columnName
lazy val service = Service.findById(serviceColumn.serviceId)
- lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
- def defaultProps = {
- val default = S2Vertex.EmptyProps
- val newProps = new S2VertexProperty(this, ColumnMeta.lastModifiedAtColumn, ColumnMeta.lastModifiedAtColumn.name, ts)
- default.put(ColumnMeta.lastModifiedAtColumn.name, newProps)
- default
- }
+ lazy val (hbaseZkAddr, hbaseTableName) = (service.cluster, service.hTableName)
- def propsWithName = for {
- (k, v) <- props.asScala
- } yield (v.columnMeta.name -> v.value.toString)
+ def defaultProps: util.HashMap[String, S2VertexProperty[_]] = builder.defaultProps
def toLogString(): String = {
+ val propsWithName = for {
+ (k, v) <- props.asScala
+ } yield (v.columnMeta.name -> v.value.toString)
+
val (serviceName, columnName) =
if (!id.storeColId) ("", "")
else (serviceColumn.service.serviceName, serviceColumn.columnName)
@@ -61,12 +55,6 @@ trait S2VertexLike extends Vertex with GraphElement {
Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
}
- def copyVertexWithState(props: Props): S2VertexLike = {
- val newVertex = copy(props = S2Vertex.EmptyProps)
- S2Vertex.fillPropsWithTs(newVertex, props)
- newVertex
- }
-
def vertices(direction: Direction, edgeLabels: String*): util.Iterator[Vertex] = {
val arr = new util.ArrayList[Vertex]()
edges(direction, edgeLabels: _*).forEachRemaining(new Consumer[Edge] {
@@ -166,23 +154,19 @@ trait S2VertexLike extends Vertex with GraphElement {
ls.iterator
}
- def label(): String = {
- serviceColumn.columnName
- }
-
def remove(): Unit = {
if (graph.features().vertex().supportsRemoveVertices()) {
// remove edge
// TODO: remove related edges also.
implicit val ec = graph.ec
- val verticesToDelete = Seq(this.copy(op = GraphUtil.operations("delete")))
+ val verticesToDelete = Seq(builder.copyVertex(op = GraphUtil.operations("delete")))
val vertexFuture = graph.mutateVertices(verticesToDelete, withWait = true)
val future = for {
vertexSuccess <- vertexFuture
- edges <- edgesAsync(Direction.BOTH)
+ edges <- graph.edgesAsync(this, Direction.BOTH)
} yield {
edges.asScala.toSeq.foreach { edge => edge.remove() }
if (!vertexSuccess.forall(_.isSuccess)) throw new RuntimeException("Vertex.remove vertex delete failed.")
@@ -195,27 +179,4 @@ trait S2VertexLike extends Vertex with GraphElement {
throw Vertex.Exceptions.vertexRemovalNotSupported()
}
}
-
- private def edgesAsync(direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
- val labelNameWithDirs =
- if (labelNames.isEmpty) {
- // TODO: Let's clarify direction
- if (direction == Direction.BOTH) {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> Direction.OUT.name) ++
- Label.findByTgtColumnId(id.colId).map(l => l.label -> Direction.IN.name)
- } else if (direction == Direction.IN) {
- Label.findByTgtColumnId(id.colId).map(l => l.label -> direction.name)
- } else {
- Label.findBySrcColumnId(id.colId).map(l => l.label -> direction.name)
- }
- } else {
- direction match {
- case Direction.BOTH =>
- Seq(Direction.OUT, Direction.IN).flatMap { dir => labelNames.map(_ -> dir.name()) }
- case _ => labelNames.map(_ -> direction.name())
- }
- }
-
- graph.fetchEdgesAsync(this, labelNameWithDirs.distinct)
- }
}
[03/23] incubator-s2graph git commit: create S2EdgeLike.
Posted by st...@apache.org.
create S2EdgeLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/7413aad4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/7413aad4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/7413aad4
Branch: refs/heads/master
Commit: 7413aad429b8f91ff223583ecb7e85688408c76d
Parents: f717023
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 13:51:27 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 13:51:27 2017 +0900
----------------------------------------------------------------------
.../scala/org/apache/s2graph/core/S2Edge.scala | 398 ++-----------------
.../org/apache/s2graph/core/S2EdgeLike.scala | 379 ++++++++++++++++++
2 files changed, 406 insertions(+), 371 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7413aad4/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 641db74..97abd26 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -301,167 +301,25 @@ case class IndexEdge(graph: S2Graph,
}
}
-case class S2Edge(innerGraph: S2Graph,
- srcVertex: S2VertexLike,
- var tgtVertex: S2VertexLike,
- innerLabel: Label,
- dir: Int,
- var op: Byte = GraphUtil.defaultOpByte,
- var version: Long = System.currentTimeMillis(),
- propsWithTs: Props = S2Edge.EmptyProps,
- parentEdges: Seq[EdgeWithScore] = Nil,
- originalEdgeOpt: Option[S2Edge] = None,
- pendingEdgeOpt: Option[S2Edge] = None,
- statusCode: Byte = 0,
- lockTs: Option[Long] = None,
- var tsInnerValOpt: Option[InnerValLike] = None) extends GraphElement with Edge {
-
- lazy val labelWithDir = LabelWithDirection(innerLabel.id.get, dir)
- lazy val schemaVer = innerLabel.schemaVersion
- lazy val ts = propsWithTs.get(LabelMeta.timestamp.name).innerVal.value match {
- case b: BigDecimal => b.longValue()
- case l: Long => l
- case i: Int => i.toLong
- case _ => throw new RuntimeException("ts should be in [BigDecimal/Long/Int].")
- }
-
- lazy val operation = GraphUtil.fromOp(op)
- lazy val tsInnerVal = tsInnerValOpt.get.value
- lazy val srcId = srcVertex.innerIdVal
- lazy val tgtId = tgtVertex.innerIdVal
- lazy val labelName = innerLabel.label
- lazy val direction = GraphUtil.fromDirection(dir)
-
- def toIndexEdge(labelIndexSeq: Byte): IndexEdge = IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelIndexSeq, propsWithTs)
-
- def serializePropsWithTs(): Array[Byte] = HBaseSerializable.propsToKeyValuesWithTs(propsWithTs.asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
-
- def updatePropsWithTs(others: Props = S2Edge.EmptyProps): Props = {
- val emptyProp = S2Edge.EmptyProps
-
- propsWithTs.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
-
- others.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
-
- emptyProp
- }
-
- def propertyValue(key: String): Option[InnerValLikeWithTs] = {
- key match {
- case "from" | "_from" => Option(InnerValLikeWithTs(srcVertex.innerId, ts))
- case "to" | "_to" => Option(InnerValLikeWithTs(tgtVertex.innerId, ts))
- case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(innerLabel.label, schemaVer), ts))
- case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(direction, schemaVer), ts))
- case _ =>
- innerLabel.metaPropsInvMap.get(key).map(labelMeta => propertyValueInner(labelMeta))
- }
- }
-
- def propertyValueInner(labelMeta: LabelMeta): InnerValLikeWithTs= {
- // propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse()
- if (propsWithTs.containsKey(labelMeta.name)) {
- propsWithTs.get(labelMeta.name).innerValWithTs
- } else {
- innerLabel.metaPropsDefaultMapInner(labelMeta)
- }
- }
-
- def propertyValues(keys: Seq[String] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
- val labelMetas = for {
- key <- keys
- labelMeta <- innerLabel.metaPropsInvMap.get(key)
- } yield labelMeta
-
- propertyValuesInner(labelMetas)
- }
-
- def propertyValuesInner(labelMetas: Seq[LabelMeta] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
- if (labelMetas.isEmpty) {
- innerLabel.metaPropsDefaultMapInner.map { case (labelMeta, defaultVal) =>
- labelMeta -> propertyValueInner(labelMeta)
- }
- } else {
- // This is important since timestamp is required for all edges.
- (LabelMeta.timestamp +: labelMetas).map { labelMeta =>
- labelMeta -> propertyValueInner(labelMeta)
- }.toMap
- }
- }
-
-// if (!props.contains(LabelMeta.timestamp)) throw new Exception("Timestamp is required.")
+case class S2Edge(override val innerGraph: S2Graph,
+ override val srcVertex: S2VertexLike,
+ override var tgtVertex: S2VertexLike,
+ override val innerLabel: Label,
+ override val dir: Int,
+ var op: Byte = GraphUtil.defaultOpByte,
+ var version: Long = System.currentTimeMillis(),
+ override val propsWithTs: Props = S2Edge.EmptyProps,
+ override val parentEdges: Seq[EdgeWithScore] = Nil,
+ override val originalEdgeOpt: Option[S2Edge] = None,
+ override val pendingEdgeOpt: Option[S2Edge] = None,
+ override val statusCode: Byte = 0,
+ override val lockTs: Option[Long] = None,
+ var tsInnerValOpt: Option[InnerValLike] = None) extends S2EdgeLike with GraphElement {
+
+ // if (!props.contains(LabelMeta.timestamp)) throw new Exception("Timestamp is required.")
// assert(propsWithTs.contains(LabelMeta.timeStampSeq))
- lazy val properties = toProps()
-
- def props = propsWithTs.asScala.mapValues(_.innerVal)
-
-
- private def toProps(): Map[String, Any] = {
- for {
- (labelMeta, defaultVal) <- innerLabel.metaPropsDefaultMapInner
- } yield {
- // labelMeta.name -> propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse(defaultVal).innerVal.value
- val value =
- if (propsWithTs.containsKey(labelMeta.name)) {
- propsWithTs.get(labelMeta.name).value
- } else {
- defaultVal.innerVal.value
- }
- labelMeta.name -> value
- }
- }
-
- def relatedEdges = {
- if (labelWithDir.isDirected) {
- val skipReverse = innerLabel.extraOptions.get("skipReverse").map(_.as[Boolean]).getOrElse(false)
- if (skipReverse) List(this) else List(this, duplicateEdge)
- } else {
-// val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
-// val base = copy(labelWithDir = outDir)
- val base = copy(dir = GraphUtil.directions("out"))
- List(base, base.reverseSrcTgtEdge)
- }
- }
-
- // def relatedEdges = List(this)
-
- private def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
- if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
-
- def srcForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
- innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- } else {
- val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
- innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- }
- }
-
- def tgtForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
- innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- } else {
- val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
- innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- }
- }
-
- def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
-
-// def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
- def reverseDirEdge = copy(dir = GraphUtil.toggleDir(dir))
-
- def reverseSrcTgtEdge = copy(srcVertex = tgtVertex, tgtVertex = srcVertex)
- def labelOrders = LabelIndex.findByLabelIdAll(labelWithDir.labelId)
override def serviceName = innerLabel.serviceName
@@ -471,74 +329,14 @@ case class S2Edge(innerGraph: S2Graph,
override def isAsync = innerLabel.isAsync
- def isDegree = propsWithTs.containsKey(LabelMeta.degree.name)
+ // def propsPlusTs = propsWithTs.get(LabelMeta.timeStampSeq) match {
+ // case Some(_) => props
+ // case None => props ++ Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, schemaVer))
+ // }
-// def propsPlusTs = propsWithTs.get(LabelMeta.timeStampSeq) match {
-// case Some(_) => props
-// case None => props ++ Map(LabelMeta.timeStampSeq -> InnerVal.withLong(ts, schemaVer))
-// }
-
- def propsPlusTsValid = propsWithTs.asScala.filter(kv => LabelMeta.isValidSeq(kv._2.labelMeta.seq)).asJava
-
- def edgesWithIndex = for (labelOrder <- labelOrders) yield {
- IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsWithTs, tsInnerValOpt = tsInnerValOpt)
- }
-
- def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
- IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsPlusTsValid, tsInnerValOpt = tsInnerValOpt)
- }
-
- /** force direction as out on invertedEdge */
- def toSnapshotEdge: SnapshotEdge = {
- val (smaller, larger) = (srcForVertex, tgtForVertex)
-
-// val newLabelWithDir = LabelWithDirection(labelWithDir.labelId, GraphUtil.directions("out"))
-
- propertyInner(LabelMeta.timestamp.name, ts, ts)
- val ret = SnapshotEdge(innerGraph, smaller, larger, innerLabel,
- GraphUtil.directions("out"), op, version, propsWithTs,
- pendingEdgeOpt = pendingEdgeOpt, statusCode = statusCode, lockTs = lockTs, tsInnerValOpt = tsInnerValOpt)
- ret
- }
-
- def defaultPropsWithName = Json.obj("from" -> srcVertex.innerId.toString(), "to" -> tgtVertex.innerId.toString(),
- "label" -> innerLabel.label, "service" -> innerLabel.serviceName)
-
- def propsWithName =
- for {
- (_, v) <- propsWithTs.asScala
- meta = v.labelMeta
- jsValue <- innerValToJsValue(v.innerVal, meta.dataType)
- } yield meta.name -> jsValue
-
-
- def updateTgtVertex(id: InnerValLike) = {
- val newId = TargetVertexId(tgtVertex.id.column, id)
- val newTgtVertex = innerGraph.newVertex(newId, tgtVertex.ts, tgtVertex.props)
- S2Edge(innerGraph, srcVertex, newTgtVertex, innerLabel, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
- }
-
- def rank(r: RankParam): Double =
- if (r.keySeqAndWeights.size <= 0) 1.0f
- else {
- var sum: Double = 0
-
- for ((labelMeta, w) <- r.keySeqAndWeights) {
- if (propsWithTs.containsKey(labelMeta.name)) {
- val innerValWithTs = propsWithTs.get(labelMeta.name)
- val cost = try innerValWithTs.innerVal.toString.toDouble catch {
- case e: Exception =>
- logger.error("toInnerval failed in rank", e)
- 1.0
- }
- sum += w * cost
- }
- }
- sum
- }
def toLogString: String = {
-// val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
+ // val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
}
@@ -551,12 +349,12 @@ case class S2Edge(innerGraph: S2Graph,
case _ => false
}
-// override def toString(): String = {
-// Map("srcVertex" -> srcVertex.toString, "tgtVertex" -> tgtVertex.toString, "label" -> labelName, "direction" -> direction,
-// "operation" -> operation, "version" -> version, "props" -> propsWithTs.asScala.map(kv => kv._1 -> kv._2.value).toString,
-// "parentEdges" -> parentEdges, "originalEdge" -> originalEdgeOpt, "statusCode" -> statusCode, "lockTs" -> lockTs
-// ).toString
-// }
+ // override def toString(): String = {
+ // Map("srcVertex" -> srcVertex.toString, "tgtVertex" -> tgtVertex.toString, "label" -> labelName, "direction" -> direction,
+ // "operation" -> operation, "version" -> version, "props" -> propsWithTs.asScala.map(kv => kv._1 -> kv._2.value).toString,
+ // "parentEdges" -> parentEdges, "originalEdge" -> originalEdgeOpt, "statusCode" -> statusCode, "lockTs" -> lockTs
+ // ).toString
+ // }
override def toString: String = {
// E + L_BRACKET + edge.id() + R_BRACKET + L_BRACKET + edge.outVertex().id() + DASH + edge.label() + ARROW + edge.inVertex().id() + R_BRACKET;
@@ -564,148 +362,6 @@ case class S2Edge(innerGraph: S2Graph,
// s"e[${srcForVertex.id}-${innerLabel.label}->${tgtForVertex.id}]"
}
- def checkProperty(key: String): Boolean = propsWithTs.containsKey(key)
-
- def copyEdge(srcVertex: S2VertexLike = srcVertex,
- tgtVertex: S2VertexLike = tgtVertex,
- innerLabel: Label = innerLabel,
- dir: Int = dir,
- op: Byte = op,
- version: Long = version,
- propsWithTs: State = S2Edge.propsToState(this.propsWithTs),
- parentEdges: Seq[EdgeWithScore] = parentEdges,
- originalEdgeOpt: Option[S2Edge] = originalEdgeOpt,
- pendingEdgeOpt: Option[S2Edge] = pendingEdgeOpt,
- statusCode: Byte = statusCode,
- lockTs: Option[Long] = lockTs,
- tsInnerValOpt: Option[InnerValLike] = tsInnerValOpt,
- ts: Long = ts): S2Edge = {
- val edge = new S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, S2Edge.EmptyProps,
- parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
- S2Edge.fillPropsWithTs(edge, propsWithTs)
- edge.propertyInner(LabelMeta.timestamp.name, ts, ts)
- edge
- }
-
- def copyEdgeWithState(state: State, ts: Long): S2Edge = {
- val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
- S2Edge.fillPropsWithTs(newEdge, state)
- newEdge.propertyInner(LabelMeta.timestamp.name, ts, ts)
- newEdge
- }
-
- def copyEdgeWithState(state: State): S2Edge = {
- val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
- S2Edge.fillPropsWithTs(newEdge, state)
- newEdge
- }
-
- override def vertices(direction: Direction): util.Iterator[structure.Vertex] = {
- val arr = new util.ArrayList[Vertex]()
-
- direction match {
- case Direction.OUT =>
-// val newVertexId = this.direction match {
-// case "out" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
-// case "in" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
-// case _ => throw new IllegalArgumentException("direction can only be out/in.")
-// }
- val newVertexId = edgeId.srcVertexId
- innerGraph.getVertex(newVertexId).foreach(arr.add)
- case Direction.IN =>
-// val newVertexId = this.direction match {
-// case "in" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
-// case "out" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
-// case _ => throw new IllegalArgumentException("direction can only be out/in.")
-// }
- val newVertexId = edgeId.tgtVertexId
- innerGraph.getVertex(newVertexId).foreach(arr.add)
- case _ =>
- import scala.collection.JavaConversions._
- vertices(Direction.OUT).foreach(arr.add)
- vertices(Direction.IN).foreach(arr.add)
- }
- arr.iterator()
- }
-
- override def properties[V](keys: String*): util.Iterator[Property[V]] = {
- val ls = new util.ArrayList[Property[V]]()
- if (keys.isEmpty) {
- propsWithTs.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, property: S2Property[_]): Unit = {
- if (!LabelMeta.reservedMetaNamesSet(key) && property.isPresent && key != T.id.name)
- ls.add(property.asInstanceOf[S2Property[V]])
- }
- })
- } else {
- keys.foreach { key =>
- val prop = property[V](key)
- if (prop.isPresent) ls.add(prop)
- }
- }
- ls.iterator()
- }
-
- override def property[V](key: String): Property[V] = {
- val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new java.lang.IllegalStateException(s"$key is not configured on Edge."))
- if (propsWithTs.containsKey(key)) propsWithTs.get(key).asInstanceOf[Property[V]]
- else {
- Property.empty()
-// val default = innerLabel.metaPropsDefaultMapInner(labelMeta)
-// propertyInner(key, default.innerVal.value, default.ts).asInstanceOf[Property[V]]
- }
- }
-
- // just for tinkerpop: save to storage, do not use for internal
- override def property[V](key: String, value: V): Property[V] = {
- S2Property.assertValidProp(key, value)
-
- val v = propertyInner(key, value, System.currentTimeMillis())
- val newTs = props.get(LabelMeta.timestamp.name).map(_.toString.toLong + 1).getOrElse(System.currentTimeMillis())
- val newEdge = this.copyEdge(ts = newTs)
-
- Await.result(innerGraph.mutateEdges(Seq(newEdge), withWait = true), innerGraph.WaitTimeout)
-
- v
- }
-
- def propertyInner[V](key: String, value: V, ts: Long): Property[V] = {
- val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
- val newProp = new S2Property[V](this, labelMeta, key, value, ts)
- propsWithTs.put(key, newProp)
- newProp
- }
-
- override def remove(): Unit = {
- if (graph.features().edge().supportsRemoveEdges()) {
- val requestTs = System.currentTimeMillis()
- val edgeToDelete = this.copyEdge(op = GraphUtil.operations("delete"),
- version = version + S2Edge.incrementVersion, propsWithTs = S2Edge.propsToState(updatePropsWithTs()), ts = requestTs)
- // should we delete related edges also?
- val future = innerGraph.mutateEdges(Seq(edgeToDelete), withWait = true)
- val mutateSuccess = Await.result(future, innerGraph.WaitTimeout)
- if (!mutateSuccess.forall(_.isSuccess)) throw new RuntimeException("edge remove failed.")
- } else {
- throw Edge.Exceptions.edgeRemovalNotSupported()
- }
- }
-
- override def graph(): Graph = innerGraph
-
- lazy val edgeId: EdgeId = {
- // NOTE: xxxForVertex makes direction to be "out"
- val timestamp = if (this.innerLabel.consistencyLevel == "strong") 0l else ts
-// EdgeId(srcVertex.innerId, tgtVertex.innerId, label(), "out", timestamp)
- val (srcColumn, tgtColumn) = innerLabel.srcTgtColumn(dir)
- if (direction == "out")
- EdgeId(VertexId(srcColumn, srcVertex.id.innerId), VertexId(tgtColumn, tgtVertex.id.innerId), label(), "out", timestamp)
- else
- EdgeId(VertexId(tgtColumn, tgtVertex.id.innerId), VertexId(srcColumn, srcVertex.id.innerId), label(), "out", timestamp)
- }
-
- override def id(): AnyRef = edgeId
-
- override def label(): String = innerLabel.label
}
object EdgeId {
val EdgeIdDelimiter = ","
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7413aad4/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
new file mode 100644
index 0000000..04ed7ab
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -0,0 +1,379 @@
+package org.apache.s2graph.core
+import java.util
+import java.util.function.BiConsumer
+
+import org.apache.s2graph.core.JSONParser.innerValToJsValue
+import org.apache.s2graph.core.S2Edge.{Props, State}
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta, ServiceColumn}
+import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.utils.logger
+import org.apache.tinkerpop.gremlin.structure
+import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, Graph, Property, T, Vertex}
+import play.api.libs.json.Json
+
+import scala.concurrent.Await
+import scala.collection.JavaConverters._
+
+trait S2EdgeLike extends Edge {
+ this: S2Edge =>
+
+ val innerGraph: S2Graph
+ val srcVertex: S2VertexLike
+ var tgtVertex: S2VertexLike
+ val innerLabel: Label
+ val dir: Int
+
+// var op: Byte = GraphUtil.defaultOpByte
+// var version: Long = System.currentTimeMillis()
+ val propsWithTs: Props = S2Edge.EmptyProps
+ val parentEdges: Seq[EdgeWithScore] = Nil
+ val originalEdgeOpt: Option[S2Edge] = None
+ val pendingEdgeOpt: Option[S2Edge] = None
+ val statusCode: Byte = 0
+ val lockTs: Option[Long] = None
+// var tsInnerValOpt: Option[InnerValLike] = None
+
+ lazy val labelWithDir = LabelWithDirection(innerLabel.id.get, dir)
+ lazy val schemaVer = innerLabel.schemaVersion
+ lazy val ts = propsWithTs.get(LabelMeta.timestamp.name).innerVal.value match {
+ case b: BigDecimal => b.longValue()
+ case l: Long => l
+ case i: Int => i.toLong
+ case _ => throw new RuntimeException("ts should be in [BigDecimal/Long/Int].")
+ }
+ lazy val operation = GraphUtil.fromOp(op)
+ lazy val tsInnerVal = tsInnerValOpt.get.value
+ lazy val srcId = srcVertex.innerIdVal
+ lazy val tgtId = tgtVertex.innerIdVal
+ lazy val labelName = innerLabel.label
+ lazy val direction = GraphUtil.fromDirection(dir)
+
+ def toIndexEdge(labelIndexSeq: Byte): IndexEdge = IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelIndexSeq, propsWithTs)
+
+ def serializePropsWithTs(): Array[Byte] = HBaseSerializable.propsToKeyValuesWithTs(propsWithTs.asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
+
+ def updatePropsWithTs(others: Props = S2Edge.EmptyProps): Props = {
+ val emptyProp = S2Edge.EmptyProps
+
+ propsWithTs.forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ others.forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ emptyProp
+ }
+
+ def propertyValue(key: String): Option[InnerValLikeWithTs] = {
+ key match {
+ case "from" | "_from" => Option(InnerValLikeWithTs(srcVertex.innerId, ts))
+ case "to" | "_to" => Option(InnerValLikeWithTs(tgtVertex.innerId, ts))
+ case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(innerLabel.label, schemaVer), ts))
+ case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(direction, schemaVer), ts))
+ case _ =>
+ innerLabel.metaPropsInvMap.get(key).map(labelMeta => propertyValueInner(labelMeta))
+ }
+ }
+
+ def propertyValueInner(labelMeta: LabelMeta): InnerValLikeWithTs = {
+ // propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse()
+ if (propsWithTs.containsKey(labelMeta.name)) {
+ propsWithTs.get(labelMeta.name).innerValWithTs
+ } else {
+ innerLabel.metaPropsDefaultMapInner(labelMeta)
+ }
+ }
+
+ def propertyValues(keys: Seq[String] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
+ val labelMetas = for {
+ key <- keys
+ labelMeta <- innerLabel.metaPropsInvMap.get(key)
+ } yield labelMeta
+
+ propertyValuesInner(labelMetas)
+ }
+
+ def propertyValuesInner(labelMetas: Seq[LabelMeta] = Nil): Map[LabelMeta, InnerValLikeWithTs] = {
+ if (labelMetas.isEmpty) {
+ innerLabel.metaPropsDefaultMapInner.map { case (labelMeta, defaultVal) =>
+ labelMeta -> propertyValueInner(labelMeta)
+ }
+ } else {
+ // This is important since timestamp is required for all edges.
+ (LabelMeta.timestamp +: labelMetas).map { labelMeta =>
+ labelMeta -> propertyValueInner(labelMeta)
+ }.toMap
+ }
+ }
+
+ lazy val properties = toProps()
+
+ def props = propsWithTs.asScala.mapValues(_.innerVal)
+
+ def relatedEdges = {
+ if (labelWithDir.isDirected) {
+ val skipReverse = innerLabel.extraOptions.get("skipReverse").map(_.as[Boolean]).getOrElse(false)
+ if (skipReverse) List(this) else List(this, duplicateEdge)
+ } else {
+ // val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
+ // val base = copy(labelWithDir = outDir)
+ val base = copy(dir = GraphUtil.directions("out"))
+ List(base, base.reverseSrcTgtEdge)
+ }
+ }
+
+ def srcForVertex = {
+ val belongLabelIds = Seq(labelWithDir.labelId)
+ if (labelWithDir.dir == GraphUtil.directions("in")) {
+ val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
+ innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
+ innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+
+ def tgtForVertex = {
+ val belongLabelIds = Seq(labelWithDir.labelId)
+ if (labelWithDir.dir == GraphUtil.directions("in")) {
+ val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
+ innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
+ innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+
+ def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
+
+ // def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
+ def reverseDirEdge = copy(dir = GraphUtil.toggleDir(dir))
+
+ def reverseSrcTgtEdge = copy(srcVertex = tgtVertex, tgtVertex = srcVertex)
+
+ def labelOrders = LabelIndex.findByLabelIdAll(labelWithDir.labelId)
+
+ def isDegree = propsWithTs.containsKey(LabelMeta.degree.name)
+
+ def propsPlusTsValid = propsWithTs.asScala.filter(kv => LabelMeta.isValidSeq(kv._2.labelMeta.seq)).asJava
+
+ def edgesWithIndex = for (labelOrder <- labelOrders) yield {
+ IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsWithTs, tsInnerValOpt = tsInnerValOpt)
+ }
+
+ def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
+ IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsPlusTsValid, tsInnerValOpt = tsInnerValOpt)
+ }
+
+ /** force direction as out on invertedEdge */
+ def toSnapshotEdge: SnapshotEdge = {
+ val (smaller, larger) = (srcForVertex, tgtForVertex)
+
+ // val newLabelWithDir = LabelWithDirection(labelWithDir.labelId, GraphUtil.directions("out"))
+
+ propertyInner(LabelMeta.timestamp.name, ts, ts)
+ val ret = SnapshotEdge(innerGraph, smaller, larger, innerLabel,
+ GraphUtil.directions("out"), op, version, propsWithTs,
+ pendingEdgeOpt = pendingEdgeOpt, statusCode = statusCode, lockTs = lockTs, tsInnerValOpt = tsInnerValOpt)
+ ret
+ }
+
+ def defaultPropsWithName = Json.obj("from" -> srcVertex.innerId.toString(), "to" -> tgtVertex.innerId.toString(),
+ "label" -> innerLabel.label, "service" -> innerLabel.serviceName)
+
+ def propsWithName =
+ for {
+ (_, v) <- propsWithTs.asScala
+ meta = v.labelMeta
+ jsValue <- innerValToJsValue(v.innerVal, meta.dataType)
+ } yield meta.name -> jsValue
+
+ def updateTgtVertex(id: InnerValLike) = {
+ val newId = TargetVertexId(tgtVertex.id.column, id)
+ val newTgtVertex = innerGraph.newVertex(newId, tgtVertex.ts, tgtVertex.props)
+ S2Edge(innerGraph, srcVertex, newTgtVertex, innerLabel, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
+ }
+
+ def rank(r: RankParam): Double =
+ if (r.keySeqAndWeights.size <= 0) 1.0f
+ else {
+ var sum: Double = 0
+
+ for ((labelMeta, w) <- r.keySeqAndWeights) {
+ if (propsWithTs.containsKey(labelMeta.name)) {
+ val innerValWithTs = propsWithTs.get(labelMeta.name)
+ val cost = try innerValWithTs.innerVal.toString.toDouble catch {
+ case e: Exception =>
+ logger.error("toInnerval failed in rank", e)
+ 1.0
+ }
+ sum += w * cost
+ }
+ }
+ sum
+ }
+
+ def checkProperty(key: String): Boolean = propsWithTs.containsKey(key)
+
+ def copyEdge(srcVertex: S2VertexLike = srcVertex,
+ tgtVertex: S2VertexLike = tgtVertex,
+ innerLabel: Label = innerLabel,
+ dir: Int = dir,
+ op: Byte = op,
+ version: Long = version,
+ propsWithTs: State = S2Edge.propsToState(this.propsWithTs),
+ parentEdges: Seq[EdgeWithScore] = parentEdges,
+ originalEdgeOpt: Option[S2Edge] = originalEdgeOpt,
+ pendingEdgeOpt: Option[S2Edge] = pendingEdgeOpt,
+ statusCode: Byte = statusCode,
+ lockTs: Option[Long] = lockTs,
+ tsInnerValOpt: Option[InnerValLike] = tsInnerValOpt,
+ ts: Long = ts): S2Edge = {
+ val edge = new S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, S2Edge.EmptyProps,
+ parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
+ S2Edge.fillPropsWithTs(edge, propsWithTs)
+ edge.propertyInner(LabelMeta.timestamp.name, ts, ts)
+ edge
+ }
+
+ def copyEdgeWithState(state: State, ts: Long): S2Edge = {
+ val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
+ S2Edge.fillPropsWithTs(newEdge, state)
+ newEdge.propertyInner(LabelMeta.timestamp.name, ts, ts)
+ newEdge
+ }
+
+ def copyEdgeWithState(state: State): S2Edge = {
+ val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
+ S2Edge.fillPropsWithTs(newEdge, state)
+ newEdge
+ }
+
+ def vertices(direction: Direction): util.Iterator[structure.Vertex] = {
+ val arr = new util.ArrayList[Vertex]()
+
+ direction match {
+ case Direction.OUT =>
+ // val newVertexId = this.direction match {
+ // case "out" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
+ // case "in" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
+ // case _ => throw new IllegalArgumentException("direction can only be out/in.")
+ // }
+ val newVertexId = edgeId.srcVertexId
+ innerGraph.getVertex(newVertexId).foreach(arr.add)
+ case Direction.IN =>
+ // val newVertexId = this.direction match {
+ // case "in" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
+ // case "out" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
+ // case _ => throw new IllegalArgumentException("direction can only be out/in.")
+ // }
+ val newVertexId = edgeId.tgtVertexId
+ innerGraph.getVertex(newVertexId).foreach(arr.add)
+ case _ =>
+ import scala.collection.JavaConversions._
+ vertices(Direction.OUT).foreach(arr.add)
+ vertices(Direction.IN).foreach(arr.add)
+ }
+ arr.iterator()
+ }
+
+ def properties[V](keys: String*): util.Iterator[Property[V]] = {
+ val ls = new util.ArrayList[Property[V]]()
+ if (keys.isEmpty) {
+ propsWithTs.forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, property: S2Property[_]): Unit = {
+ if (!LabelMeta.reservedMetaNamesSet(key) && property.isPresent && key != T.id.name)
+ ls.add(property.asInstanceOf[S2Property[V]])
+ }
+ })
+ } else {
+ keys.foreach { key =>
+ val prop = property[V](key)
+ if (prop.isPresent) ls.add(prop)
+ }
+ }
+ ls.iterator()
+ }
+
+ def property[V](key: String): Property[V] = {
+ val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new java.lang.IllegalStateException(s"$key is not configured on Edge."))
+ if (propsWithTs.containsKey(key)) propsWithTs.get(key).asInstanceOf[Property[V]]
+ else {
+ Property.empty()
+ // val default = innerLabel.metaPropsDefaultMapInner(labelMeta)
+ // propertyInner(key, default.innerVal.value, default.ts).asInstanceOf[Property[V]]
+ }
+ }
+
+ // just for tinkerpop: save to storage, do not use for internal
+ def property[V](key: String, value: V): Property[V] = {
+ S2Property.assertValidProp(key, value)
+
+ val v = propertyInner(key, value, System.currentTimeMillis())
+ val newTs = props.get(LabelMeta.timestamp.name).map(_.toString.toLong + 1).getOrElse(System.currentTimeMillis())
+ val newEdge = this.copyEdge(ts = newTs)
+
+ Await.result(innerGraph.mutateEdges(Seq(newEdge), withWait = true), innerGraph.WaitTimeout)
+
+ v
+ }
+
+ def propertyInner[V](key: String, value: V, ts: Long): Property[V] = {
+ val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
+ val newProp = new S2Property[V](this, labelMeta, key, value, ts)
+ propsWithTs.put(key, newProp)
+ newProp
+ }
+
+ def remove(): Unit = {
+ if (graph.features().edge().supportsRemoveEdges()) {
+ val requestTs = System.currentTimeMillis()
+ val edgeToDelete = this.copyEdge(op = GraphUtil.operations("delete"),
+ version = version + S2Edge.incrementVersion, propsWithTs = S2Edge.propsToState(updatePropsWithTs()), ts = requestTs)
+ // should we delete related edges also?
+ val future = innerGraph.mutateEdges(Seq(edgeToDelete), withWait = true)
+ val mutateSuccess = Await.result(future, innerGraph.WaitTimeout)
+ if (!mutateSuccess.forall(_.isSuccess)) throw new RuntimeException("edge remove failed.")
+ } else {
+ throw Edge.Exceptions.edgeRemovalNotSupported()
+ }
+ }
+
+ def graph(): Graph = innerGraph
+
+ lazy val edgeId: EdgeId = {
+ // NOTE: xxxForVertex makes direction to be "out"
+ val timestamp = if (this.innerLabel.consistencyLevel == "strong") 0l else ts
+ // EdgeId(srcVertex.innerId, tgtVertex.innerId, label(), "out", timestamp)
+ val (srcColumn, tgtColumn) = innerLabel.srcTgtColumn(dir)
+ if (direction == "out")
+ EdgeId(VertexId(srcColumn, srcVertex.id.innerId), VertexId(tgtColumn, tgtVertex.id.innerId), label(), "out", timestamp)
+ else
+ EdgeId(VertexId(tgtColumn, tgtVertex.id.innerId), VertexId(srcColumn, srcVertex.id.innerId), label(), "out", timestamp)
+ }
+
+ def id(): AnyRef = edgeId
+
+ def label(): String = innerLabel.label
+
+ private def toProps(): Map[String, Any] = {
+ for {
+ (labelMeta, defaultVal) <- innerLabel.metaPropsDefaultMapInner
+ } yield {
+ // labelMeta.name -> propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse(defaultVal).innerVal.value
+ val value =
+ if (propsWithTs.containsKey(labelMeta.name)) {
+ propsWithTs.get(labelMeta.name).value
+ } else {
+ defaultVal.innerVal.value
+ }
+ labelMeta.name -> value
+ }
+ }
+
+ private def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
+ if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
+
+}
[19/23] incubator-s2graph git commit: remove
newVertex/newEdge/newVertexId on S2GraphLike.
Posted by st...@apache.org.
remove newVertex/newEdge/newVertexId on S2GraphLike.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/eabe7570
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/eabe7570
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/eabe7570
Branch: refs/heads/master
Commit: eabe7570a9ef9ec0cb7b36d518d5bef4d1fd53dd
Parents: 6403bc9
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 10 22:57:51 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 11 10:40:19 2017 +0900
----------------------------------------------------------------------
.../loader/subscriber/GraphSubscriber.scala | 67 +----
.../loader/subscriber/TransferToHFile.scala | 4 +-
.../org/apache/s2graph/core/QueryResult.scala | 2 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 8 +-
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 2 +-
.../scala/org/apache/s2graph/core/S2Graph.scala | 256 ++-----------------
.../org/apache/s2graph/core/S2GraphLike.scala | 45 +---
.../org/apache/s2graph/core/S2GraphTp.scala | 98 +++++++
.../org/apache/s2graph/core/S2VertexLike.scala | 3 +-
.../apache/s2graph/core/TraversalHelper.scala | 153 ++++++++++-
.../s2graph/core/rest/RequestParser.scala | 2 +-
.../tall/IndexEdgeDeserializable.scala | 15 +-
.../wide/IndexEdgeDeserializable.scala | 15 +-
.../tall/SnapshotEdgeDeserializable.scala | 12 +-
.../wide/SnapshotEdgeDeserializable.scala | 13 +-
.../vertex/tall/VertexDeserializable.scala | 3 +-
.../vertex/wide/VertexDeserializable.scala | 3 +-
.../org/apache/s2graph/core/S2EdgeTest.scala | 24 +-
.../s2graph/core/TestCommonWithModels.scala | 3 +-
.../s2graph/core/parsers/WhereParserTest.scala | 22 +-
.../s2graph/core/storage/StorageIOTest.scala | 4 +-
.../core/storage/hbase/IndexEdgeTest.scala | 6 +-
.../process/S2GraphProcessStandardTest.scala | 4 +-
.../S2GraphStructureIntegrateTest.scala | 4 +-
.../S2GraphStructureStandardTest.scala | 4 +-
25 files changed, 354 insertions(+), 418 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
index a371b6b..6ecb070 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
@@ -67,6 +67,7 @@ object GraphSubscriberHelper extends WithKafka {
var g: S2Graph = null
var management: Management = null
val conns = new scala.collection.mutable.HashMap[String, Connection]()
+ var builder: GraphElementBuilder = null
def toOption(s: String) = {
s match {
@@ -82,6 +83,7 @@ object GraphSubscriberHelper extends WithKafka {
val ec = ExecutionContext.Implicits.global
g = new S2Graph(config)(ec)
management = new Management(g)
+ builder = g.elementBuilder
}
}
@@ -106,7 +108,7 @@ object GraphSubscriberHelper extends WithKafka {
(statFunc: (String, Int) => Unit): Iterable[GraphElement] = {
(for (msg <- msgs) yield {
statFunc("total", 1)
- g.elementBuilder.toGraphElement(msg, labelMapping) match {
+ builder.toGraphElement(msg, labelMapping) match {
case Some(e) if e.isInstanceOf[S2Edge] =>
statFunc("EdgeParseOk", 1)
e.asInstanceOf[S2Edge]
@@ -122,69 +124,6 @@ object GraphSubscriberHelper extends WithKafka {
}).toList
}
-// private def storeRec(zkQuorum: String, tableName: String, puts: List[Put], elementsSize: Int, tryNum: Int)
-// (statFunc: (String, Int) => Unit, statPrefix: String = "edge"): Unit = {
-// if (tryNum <= 0) {
-// statFunc("errorStore", elementsSize)
-// throw new RuntimeException(s"retry failed after $maxTryNum")
-// }
-// val conn = getConn(zkQuorum)
-// val mutator = conn.getBufferedMutator(TableName.valueOf(tableName))
-// // val table = conn.getTable(TableName.valueOf(tableName))
-// // table.setAutoFlush(false, false)
-//
-// try {
-// puts.foreach { put => put.setDurability(Durability.ASYNC_WAL) }
-// mutator.mutate(puts)
-// // table.put(puts)
-// statFunc(s"$statPrefix:storeOk", elementsSize)
-// } catch {
-// case e: Throwable =>
-// e.printStackTrace()
-// Thread.sleep(sleepPeriod)
-// storeRec(zkQuorum, tableName, puts, elementsSize, tryNum - 1)(statFunc)
-// } finally {
-// mutator.close()
-// // table.close()
-// }
-// }
-//
-// def storeDegreeBulk(zkQuorum: String, tableName: String)
-// (degrees: Iterable[(String, String, String, Int)], labelMapping: Map[String, String] = Map.empty)
-// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
-// val counts = HashMap[String, Long]()
-// val statFunc = storeStat(counts)(mapAccOpt) _
-//
-// for {
-// (vertexId, labelName, direction, degreeVal) <- degrees
-// incrementRequests <- TransferToHFile.buildDegreePutRequests(vertexId, labelName, direction, degreeVal)
-// } {
-// storeRec(zkQuorum, tableName, incrementRequests, degrees.size, maxTryNum)(statFunc, "degree")
-// }
-// counts
-// }
-// def storeBulk(zkQuorum: String, tableName: String)
-// (msgs: Seq[String], labelMapping: Map[String, String] = Map.empty, autoCreateEdge: Boolean = false)
-// (mapAccOpt: Option[HashMapAccumulable]): Iterable[(String, Long)] = {
-//
-// val counts = HashMap[String, Long]()
-// val statFunc = storeStat(counts)(mapAccOpt) _
-// val elements = toGraphElements(msgs, labelMapping)(statFunc)
-//
-// val puts = elements.flatMap { element =>
-// element match {
-// case v: Vertex if v.op == GraphUtil.operations("insert") || v.op == GraphUtil.operations("insertBulk") =>
-// v.buildPuts()
-// case e: Edge if e.op == GraphUtil.operations("insert") || e.op == GraphUtil.operations("insertBulk") =>
-// EdgeWriter(e).insertBulkForLoader(autoCreateEdge)
-// case _ => Nil
-// }
-// } toList
-//
-// storeRec(zkQuorum, tableName, puts, msgs.size, maxTryNum)(statFunc)
-// counts
-// }
-
def storeStat(counts: HashMap[String, Long])(mapAccOpt: Option[HashMapAccumulable])(key: String, value: Int) = {
counts.put(key, counts.getOrElse(key, 0L) + value)
mapAccOpt match {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
index a7b4e00..6aaf6fd 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
@@ -97,11 +97,11 @@ object TransferToHFile extends SparkApp {
val innerVal = JSONParser.jsValueToInnerVal(Json.toJson(vertexId), label.srcColumnWithDir(dir).columnType, label.schemaVersion).getOrElse {
throw new RuntimeException(s"$vertexId can not be converted into innerval")
}
- val vertex = GraphSubscriberHelper.g.newVertex(SourceVertexId(label.srcColumn, innerVal))
+ val vertex = GraphSubscriberHelper.builder.newVertex(SourceVertexId(label.srcColumn, innerVal))
val ts = System.currentTimeMillis()
val propsWithTs = Map(LabelMeta.timestamp -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
- val edge = GraphSubscriberHelper.g.newEdge(vertex, vertex, label, dir, propsWithTs=propsWithTs)
+ val edge = GraphSubscriberHelper.builder.newEdge(vertex, vertex, label, dir, propsWithTs=propsWithTs)
edge.edgesWithIndex.flatMap { indexEdge =>
GraphSubscriberHelper.g.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index 7d187c6..37ddf06 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -35,7 +35,7 @@ object QueryResult {
val propsWithTs = Map(LabelMeta.timestamp ->
InnerValLikeWithTs(InnerVal.withLong(currentTs, label.schemaVersion), currentTs))
- val edge = graph.newEdge(vertex, vertex, label, queryParam.labelWithDir.dir, propsWithTs = propsWithTs)
+ val edge = graph.elementBuilder.newEdge(vertex, vertex, label, queryParam.labelWithDir.dir, propsWithTs = propsWithTs)
val edgeWithScore = EdgeWithScore(edge, S2Graph.DefaultScore, queryParam.label)
edgeWithScore
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 3e33ed4..3eb1fa7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -817,10 +817,10 @@ object S2Edge {
val belongLabelIds = Seq(e.getLabelId())
if (e.getDir() == GraphUtil.directions("in")) {
val tgtColumn = getServiceColumn(e.tgtVertex, e.innerLabel.tgtColumn)
- e.innerGraph.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
+ e.innerGraph.elementBuilder.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
} else {
val srcColumn = getServiceColumn(e.srcVertex, e.innerLabel.srcColumn)
- e.innerGraph.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
+ e.innerGraph.elementBuilder.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
}
}
@@ -828,10 +828,10 @@ object S2Edge {
val belongLabelIds = Seq(e.getLabelId())
if (e.getDir() == GraphUtil.directions("in")) {
val srcColumn = getServiceColumn(e.srcVertex, e.innerLabel.srcColumn)
- e.innerGraph.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
+ e.innerGraph.elementBuilder.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
} else {
val tgtColumn = getServiceColumn(e.tgtVertex, e.innerLabel.tgtColumn)
- e.innerGraph.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
+ e.innerGraph.elementBuilder.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
index 4004a13..dc0baa2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
@@ -80,7 +80,7 @@ class S2EdgeBuilder(edge: S2EdgeLike) {
def updateTgtVertex(id: InnerValLike): S2EdgeLike = {
val newId = TargetVertexId(edge.tgtVertex.id.column, id)
- val newTgtVertex = edge.innerGraph.newVertex(newId, edge.tgtVertex.ts, edge.tgtVertex.props)
+ val newTgtVertex = edge.innerGraph.elementBuilder.newVertex(newId, edge.tgtVertex.ts, edge.tgtVertex.props)
copyEdge(tgtVertex = newTgtVertex)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index f80d5c2..3905442 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -21,7 +21,7 @@ package org.apache.s2graph.core
import java.util
import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
-import java.util.concurrent.{Executors, TimeUnit}
+import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
@@ -96,6 +96,8 @@ object S2Graph {
val DefaultColumnName = "vertex"
val DefaultLabelName = "_s2graph"
+ var hbaseExecutor: ExecutorService = _
+
val graphStrategies: TraversalStrategies =
TraversalStrategies.GlobalCache.getStrategies(classOf[Graph]).addStrategies(S2GraphStepStrategy.instance)
@@ -130,7 +132,14 @@ object S2Graph {
logger.info(s"[InitStorage]: $storageBackend")
storageBackend match {
- case "hbase" => new AsynchbaseStorage(graph, config)
+ case "hbase" =>
+ hbaseExecutor =
+ if (config.getString("hbase.zookeeper.quorum") == "localhost")
+ AsynchbaseStorage.initLocalHBase(config)
+ else
+ null
+
+ new AsynchbaseStorage(graph, config)
case _ => throw new RuntimeException("not supported storage.")
}
}
@@ -150,94 +159,6 @@ object S2Graph {
}
}
-
-@Graph.OptIns(value = Array(
- new Graph.OptIn(value = Graph.OptIn.SUITE_PROCESS_STANDARD),
- new Graph.OptIn(value = Graph.OptIn.SUITE_STRUCTURE_STANDARD)
-))
-@Graph.OptOuts(value = Array(
- /* Process */
- /* branch: passed all. */
- /* filter */
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropTest$Traversals", method = "g_V_properties_drop", reason = "please find bug on this case."),
-
- /* map */
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_both_both_count", reason = "count differ very little. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX3X_count", reason = "count differ very little. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX8X_count", reason = "count differ very litter. fix this."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallback", reason = "NullPointerException. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "g_V_whereXinXcreatedX_count_isX1XX_name_profile", reason = "java.lang.AssertionError: There should be 3 top-level metrics. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "g_V_whereXinXcreatedX_count_isX1XX_name_profileXmetricsX", reason = "expected 2, actual 6. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "grateful_V_out_out_profileXmetricsX", reason = "expected 8049, actual 8046. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "grateful_V_out_out_profile", reason = "expected 8049, actual 8046. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profileXmetricsX", reason = "expected 2, actual 6. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profile", reason = "expected 2, actual 6. fix this."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallbackSideEffect", reason = "NullPointerException. fix this."),
-
- /* sideEffect */
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup", reason = "Expected 5, Actual 6."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_outEXknowsX_subgraphXsgX_name_capXsgX", reason = "Expected 3, Actual 6"),
-
- /* compliance */
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.CoreTraversalTest", method = "shouldThrowExceptionWhenIdsMixed", reason = "VertexId is not Element."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionTest", method = "*", reason = "not supported yet."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithGeneratedDefaultId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedDefaultId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategyProcessTest", method = "*", reason = "not supported yet."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategyProcessTest", method = "*", reason = "not supported yet."),
-
- /* Structure */
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateIdEquality", reason = "reference equals on EdgeId is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateEquality", reason = "reference equals on EdgeId is not supported."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.VertexTest$BasicVertexTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnEdge", reason = "S2Vertex.addEdge behave as upsert."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "reference equals is not supported."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPropertyTest", method = "shouldNotBeEqualPropertiesAsThereIsDifferentKey", reason = "reference equals is not supported."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveVertices", reason = "random label creation is not supported. all label need to be pre-configured."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnVertex", reason = "Assigning the same ID to an Element update instead of throwing exception."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveEdges", reason = "random label creation is not supported. all label need to be pre-configured."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "Assigning the same ID to an Element update instead of throwing exception."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.DistributionGeneratorTest", method = "*", reason = "non-deterministic test."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GryoTest", method = "shouldSerializeTree", reason = "order of children is reversed. not sure why."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GraphSONTest", method = "shouldSerializeTraversalMetrics", reason = "expected 2, actual 3."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithBOTHEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithINEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexAsReferenceNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithOUTEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdgeAsReference", specific = "graphson-v2-embedded", reason = "no"),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteEdge", specific = "graphson-v2-embedded", reason = "no"),
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdge", specific = "graphson-v2-embedded", reason = "no"),
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoGraphTest", method = "*", reason = "no"), // all failed.
-
- new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoTest", method = "*", reason = "no")
-))
class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2GraphLike {
var apacheConfiguration: Configuration = _
@@ -457,10 +378,10 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
} else {
val filterOutFuture = q.queryOption.filterOutQuery match {
case None => fallback
- case Some(filterOutQuery) => getEdgesStepInner(filterOutQuery)
+ case Some(filterOutQuery) => traversalHelper.getEdgesStepInner(filterOutQuery)
}
for {
- stepResult <- getEdgesStepInner(q)
+ stepResult <- traversalHelper.getEdgesStepInner(q)
filterOutInnerResult <- filterOutFuture
} yield {
if (filterOutInnerResult.isEmpty) stepResult
@@ -480,7 +401,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
else {
val filterOutFuture = mq.queryOption.filterOutQuery match {
case None => fallback
- case Some(filterOutQuery) => getEdgesStepInner(filterOutQuery)
+ case Some(filterOutQuery) => traversalHelper.getEdgesStepInner(filterOutQuery)
}
val multiQueryFutures = Future.sequence(mq.queries.map { query => getEdges(query) })
@@ -515,7 +436,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
val retryFuture = Extensions.retryOnSuccess(DeleteAllFetchCount) {
- fetchAndDeleteAll(queries, requestTs)
+ traversalHelper.fetchAndDeleteAll(queries, requestTs)
} { case (allDeleted, deleteSuccess) =>
allDeleted && deleteSuccess
}.map { case (allDeleted, deleteSuccess) => allDeleted && deleteSuccess }
@@ -546,12 +467,12 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
override def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
- val v = newVertex(vertexId)
+ val v = elementBuilder.newVertex(vertexId)
Await.result(getVertices(Seq(v)).map { vertices => vertices.headOption }, WaitTimeout)
}
override def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge] = {
- Await.result(fetchEdgesAsync(vertex, labelNameWithDirs), WaitTimeout)
+ Await.result(traversalHelper.fetchEdgesAsync(vertex, labelNameWithDirs), WaitTimeout)
}
override def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]] = {
@@ -574,149 +495,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
}
}
- fetchEdgesAsync(vertex, labelNameWithDirs.distinct)
+ traversalHelper.fetchEdgesAsync(vertex, labelNameWithDirs.distinct)
}
def isRunning(): Boolean = running.get()
-
- /** Private **/
-
- private def getEdgesStepInner(q: Query, buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
- Try {
- if (q.steps.isEmpty) fallback
- else {
- def fetch: Future[StepResult] = {
- val startStepInnerResult = QueryResult.fromVertices(this, q)
- q.steps.zipWithIndex.foldLeft(Future.successful(startStepInnerResult)) { case (prevStepInnerResultFuture, (step, stepIdx)) =>
- for {
- prevStepInnerResult <- prevStepInnerResultFuture
- currentStepInnerResult <- fetchStep(q, stepIdx, prevStepInnerResult, buildLastStepInnerResult)
- } yield {
- currentStepInnerResult.copy(
- accumulatedCursors = prevStepInnerResult.accumulatedCursors :+ currentStepInnerResult.cursors,
- failCount = currentStepInnerResult.failCount + prevStepInnerResult.failCount
- )
- }
- }
- }
-
- fetch
- }
- } recover {
- case e: Exception =>
- logger.error(s"getEdgesAsync: $e", e)
- fallback
- } get
- }
-
- private def fetchStep(orgQuery: Query,
- stepIdx: Int,
- stepInnerResult: StepResult,
- buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
- if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
- else {
- val (_, prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
- traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
-
- val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
-
- traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
- fetchedLs, orgQuery.steps(stepIdx).queryParams, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(ec)
- }
- }
-
- private def fetches(queryRequests: Seq[QueryRequest],
- prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[StepResult]] = {
-
- val reqWithIdxs = queryRequests.zipWithIndex
- val requestsPerLabel = reqWithIdxs.groupBy(t => t._1.queryParam.label)
- val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
- for {
- prev <- prevFuture
- cur <- getStorage(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
- } yield {
- prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
- }
- }
- aggFuture.map { agg => agg.toSeq.sortBy(_._1).map(_._2) }
- }
-
- private def fetchAndDeleteAll(queries: Seq[Query], requestTs: Long): Future[(Boolean, Boolean)] = {
- val futures = queries.map(getEdgesStepInner(_, true))
- val future = for {
- stepInnerResultLs <- Future.sequence(futures)
- (allDeleted, ret) <- deleteAllFetchedEdgesLs(stepInnerResultLs, requestTs)
- } yield {
- // logger.debug(s"fetchAndDeleteAll: ${allDeleted}, ${ret}")
- (allDeleted, ret)
- }
-
- Extensions.retryOnFailure(MaxRetryNum) {
- future
- } {
- logger.error(s"fetch and deleteAll failed.")
- (true, false)
- }
-
- }
-
- private def deleteAllFetchedEdgesLs(stepInnerResultLs: Seq[StepResult],
- requestTs: Long): Future[(Boolean, Boolean)] = {
- stepInnerResultLs.foreach { stepInnerResult =>
- if (stepInnerResult.isFailure) throw new RuntimeException("fetched result is fallback.")
- }
- val futures = for {
- stepInnerResult <- stepInnerResultLs
- filtered = stepInnerResult.edgeWithScores.filter { edgeWithScore =>
- (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
- }
- edgesToDelete = elementBuilder.buildEdgesToDelete(filtered, requestTs)
- if edgesToDelete.nonEmpty
- } yield {
- val head = edgesToDelete.head
- val label = head.edge.innerLabel
- val stepResult = StepResult(edgesToDelete, Nil, Nil, false)
- val ret = label.schemaVersion match {
- case HBaseType.VERSION3 | HBaseType.VERSION4 =>
- if (label.consistencyLevel == "strong") {
- /*
- * read: snapshotEdge on queryResult = O(N)
- * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
- */
- mutateEdges(edgesToDelete.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
- } else {
- getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
- }
- case _ =>
-
- /*
- * read: x
- * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
- */
- getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
- }
- ret
- }
-
- if (futures.isEmpty) {
- // all deleted.
- Future.successful(true -> true)
- } else {
- Future.sequence(futures).map { rets => false -> rets.forall(identity) }
- }
- }
-
- private def fetchEdgesAsync(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
- val queryParams = labelNameWithDirs.map { case (l, direction) =>
- QueryParam(labelName = l, direction = direction.toLowerCase)
- }
-
- val query = Query.toQuery(Seq(vertex), queryParams)
- val queryRequests = queryParams.map { param => QueryRequest(query, 0, vertex, param) }
- val ls = new util.ArrayList[Edge]()
- fetches(queryRequests, Map.empty).map { stepResultLs =>
- stepResultLs.foreach(_.edgeWithScores.foreach(es => ls.add(es.edge)))
- ls.iterator()
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
index a58f1e0..1b80cb4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -58,8 +58,6 @@ trait S2GraphLike extends Graph {
override def features() = s2Features
- def nextLocalLongId = localLongId.getAndIncrement()
-
def fallback = Future.successful(StepResult.Empty)
def defaultStorage: Storage
@@ -103,34 +101,6 @@ trait S2GraphLike extends Graph {
def edgesAsync(vertex: S2VertexLike, direction: Direction, labelNames: String*): Future[util.Iterator[Edge]]
/** Convert to Graph Element **/
- def newEdge(srcVertex: S2VertexLike,
- tgtVertex: S2VertexLike,
- innerLabel: Label,
- dir: Int,
- op: Byte = GraphUtil.defaultOpByte,
- version: Long = System.currentTimeMillis(),
- propsWithTs: S2Edge.State,
- parentEdges: Seq[EdgeWithScore] = Nil,
- originalEdgeOpt: Option[S2EdgeLike] = None,
- pendingEdgeOpt: Option[S2EdgeLike] = None,
- statusCode: Byte = 0,
- lockTs: Option[Long] = None,
- tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike =
- elementBuilder.newEdge(srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs,
- parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
-
- def newVertexId(service: Service,
- column: ServiceColumn,
- id: Any): VertexId =
- elementBuilder.newVertexId(service, column, id)
-
- def newVertex(id: VertexId,
- ts: Long = System.currentTimeMillis(),
- props: S2Vertex.Props = S2Vertex.EmptyProps,
- op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike =
- elementBuilder.newVertex(id, ts, props, op, belongLabelIds)
-
def toVertex(serviceName: String,
columnName: String,
id: Any,
@@ -261,7 +231,7 @@ trait S2GraphLike extends Graph {
val vertex = kvsMap.get(T.id.name()) match {
case None => // do nothing
- val id = nextLocalLongId
+ val id = localLongId.getAndIncrement()
makeVertex(Long.box(id), kvsMap)
case Some(idValue) if S2Property.validType(idValue) =>
makeVertex(idValue, kvsMap)
@@ -269,7 +239,7 @@ trait S2GraphLike extends Graph {
throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported
}
- addVertexInner(vertex)
+ addVertex(vertex.id, vertex.ts, vertex.props, vertex.op, vertex.belongLabelIds)
vertex
}
@@ -290,17 +260,6 @@ trait S2GraphLike extends Graph {
vertex
}
- def addVertexInner(vertex: S2VertexLike): S2VertexLike = {
- val future = mutateVertices(Seq(vertex), withWait = true).flatMap { rets =>
- if (rets.forall(_.isSuccess)) {
- indexProvider.mutateVerticesAsync(Seq(vertex))
- } else throw new RuntimeException("addVertex failed.")
- }
- Await.ready(future, WaitTimeout)
-
- vertex
- }
-
/* tp3 only */
def addEdge(srcVertex: S2VertexLike, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
val containsId = kvs.contains(T.id)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
new file mode 100644
index 0000000..5ce4086
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphTp.scala
@@ -0,0 +1,98 @@
+package org.apache.s2graph.core
+
+import com.typesafe.config.Config
+import org.apache.tinkerpop.gremlin.structure.Graph
+
+import scala.concurrent.ExecutionContext
+
+
+@Graph.OptIns(value = Array(
+ new Graph.OptIn(value = Graph.OptIn.SUITE_PROCESS_STANDARD),
+ new Graph.OptIn(value = Graph.OptIn.SUITE_STRUCTURE_STANDARD)
+))
+@Graph.OptOuts(value = Array(
+ /* Process */
+ /* branch: passed all. */
+ /* filter */
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropTest$Traversals", method = "g_V_properties_drop", reason = "please find bug on this case."),
+
+ /* map */
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_both_both_count", reason = "count differ very little. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX3X_count", reason = "count differ very little. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_repeatXoutX_timesX8X_count", reason = "count differ very litter. fix this."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallback", reason = "NullPointerException. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "g_V_whereXinXcreatedX_count_isX1XX_name_profile", reason = "java.lang.AssertionError: There should be 3 top-level metrics. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "g_V_whereXinXcreatedX_count_isX1XX_name_profileXmetricsX", reason = "expected 2, actual 6. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "grateful_V_out_out_profileXmetricsX", reason = "expected 8049, actual 8046. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "grateful_V_out_out_profile", reason = "expected 8049, actual 8046. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profileXmetricsX", reason = "expected 2, actual 6. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "modern_V_out_out_profile", reason = "expected 2, actual 6. fix this."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "testProfileStrategyCallbackSideEffect", reason = "NullPointerException. fix this."),
+
+ /* sideEffect */
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_repeatXbothEXcreatedX_subgraphXsgX_outVX_timesX5X_name_dedup", reason = "Expected 5, Actual 6."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest$Traversals", method = "g_V_withSideEffectXsgX_outEXknowsX_subgraphXsgX_name_capXsgX", reason = "Expected 3, Actual 6"),
+
+ /* compliance */
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.CoreTraversalTest", method = "shouldThrowExceptionWhenIdsMixed", reason = "VertexId is not Element."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionTest", method = "*", reason = "not supported yet."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithGeneratedDefaultId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnAddVWithGeneratedDefaultId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithGeneratedCustomId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest", method = "shouldGenerateDefaultIdOnGraphAddVWithSpecifiedId", reason = "GraphStep.processNextStart throw FastNoSuchElementException when GraphStep.start = true and GraphStep.end = true."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategyProcessTest", method = "*", reason = "not supported yet."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategyProcessTest", method = "*", reason = "not supported yet."),
+
+ /* Structure */
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateIdEquality", reason = "reference equals on EdgeId is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$BasicEdgeTest", method = "shouldValidateEquality", reason = "reference equals on EdgeId is not supported."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.VertexTest$BasicVertexTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnEdge", reason = "S2Vertex.addEdge behave as upsert."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "reference equals is not supported."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPropertyTest", method = "shouldNotBeEqualPropertiesAsThereIsDifferentKey", reason = "reference equals is not supported."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveVertices", reason = "random label creation is not supported. all label need to be pre-configured."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldHaveExceptionConsistencyWhenAssigningSameIdOnVertex", reason = "Assigning the same ID to an Element update instead of throwing exception."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.GraphTest", method = "shouldRemoveEdges", reason = "random label creation is not supported. all label need to be pre-configured."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdgeTest", method = "shouldNotEvaluateToEqualDifferentId", reason = "Assigning the same ID to an Element update instead of throwing exception."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.5)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(NormalDistribution{stdDeviation=2.0, mean=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.1)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},PowerLawDistribution{gamma=2.4, multiplier=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.CommunityGeneratorTest$DifferentDistributionsTest", method = "shouldGenerateDifferentGraph", specific = "test(PowerLawDistribution{gamma=2.3, multiplier=0.0},NormalDistribution{stdDeviation=4.0, mean=0.0},0.25)", reason = "graphson-v2-embedded is not supported."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.algorithm.generator.DistributionGeneratorTest", method = "*", reason = "non-deterministic test."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GryoTest", method = "shouldSerializeTree", reason = "order of children is reversed. not sure why."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.SerializationTest$GraphSONTest", method = "shouldSerializeTraversalMetrics", reason = "expected 2, actual 3."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithBOTHEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithINEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexAsReferenceNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteVertexWithOUTEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoVertexTest", method = "shouldReadWriteDetachedVertexNoEdges", specific = "graphson-v2-embedded", reason = "Vertex.id() is deserialized as string, not class in graphson-v2-embedded."),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdgeAsReference", specific = "graphson-v2-embedded", reason = "no"),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteEdge", specific = "graphson-v2-embedded", reason = "no"),
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest", method = "shouldReadWriteDetachedEdge", specific = "graphson-v2-embedded", reason = "no"),
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoGraphTest", method = "*", reason = "no"), // all failed.
+
+ new Graph.OptOut(test = "org.apache.tinkerpop.gremlin.structure.io.IoTest", method = "*", reason = "no")
+))
+class S2GraphTp(config: Config)(override implicit val ec: ExecutionContext) extends S2Graph(config) {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index ad35efd..97e3095 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -118,7 +118,8 @@ trait S2VertexLike extends Vertex with GraphElement {
props.put(key, newProps)
// FIXME: save to persistent for tp test
- graph.addVertexInner(this)
+// graph.addVertexInner(this)
+ graph.addVertex(id, ts, props, op, belongLabelIds)
newProps
case _ => throw new RuntimeException("only single cardinality is supported.")
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
index 25b909e..7a8e63e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/TraversalHelper.scala
@@ -1,14 +1,17 @@
package org.apache.s2graph.core
+import java.util
+
import org.apache.s2graph.core.S2Graph.{FilterHashKey, HashKey}
import org.apache.s2graph.core.mysqls.LabelMeta
-import org.apache.s2graph.core.types.{InnerVal, LabelWithDirection, VertexId}
-import org.apache.s2graph.core.utils.logger
+import org.apache.s2graph.core.types.{HBaseType, InnerVal, LabelWithDirection, VertexId}
+import org.apache.s2graph.core.utils.{Extensions, logger}
+import org.apache.tinkerpop.gremlin.structure.Edge
import scala.annotation.tailrec
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.concurrent.Future
-import scala.util.Random
+import scala.util.{Random, Try}
object TraversalHelper {
@tailrec
@@ -125,6 +128,150 @@ object TraversalHelper {
class TraversalHelper(graph: S2GraphLike) {
import TraversalHelper._
+ implicit val ec = graph.ec
+ val MaxRetryNum = graph.config.getInt("max.retry.number")
+
+ def fallback = Future.successful(StepResult.Empty)
+
+ def getEdgesStepInner(q: Query, buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
+ Try {
+ if (q.steps.isEmpty) fallback
+ else {
+ def fetch: Future[StepResult] = {
+ val startStepInnerResult = QueryResult.fromVertices(graph, q)
+ q.steps.zipWithIndex.foldLeft(Future.successful(startStepInnerResult)) { case (prevStepInnerResultFuture, (step, stepIdx)) =>
+ for {
+ prevStepInnerResult <- prevStepInnerResultFuture
+ currentStepInnerResult <- fetchStep(q, stepIdx, prevStepInnerResult, buildLastStepInnerResult)
+ } yield {
+ currentStepInnerResult.copy(
+ accumulatedCursors = prevStepInnerResult.accumulatedCursors :+ currentStepInnerResult.cursors,
+ failCount = currentStepInnerResult.failCount + prevStepInnerResult.failCount
+ )
+ }
+ }
+ }
+
+ fetch
+ }
+ } recover {
+ case e: Exception =>
+ logger.error(s"getEdgesAsync: $e", e)
+ fallback
+ } get
+ }
+
+ def fetchStep(orgQuery: Query,
+ stepIdx: Int,
+ stepInnerResult: StepResult,
+ buildLastStepInnerResult: Boolean = false): Future[StepResult] = {
+ if (stepInnerResult.isEmpty) Future.successful(StepResult.Empty)
+ else {
+ val (_, prevStepTgtVertexIdEdges: Map[VertexId, ArrayBuffer[EdgeWithScore]], queryRequests: Seq[QueryRequest]) =
+ graph.traversalHelper.buildNextStepQueryRequests(orgQuery, stepIdx, stepInnerResult)
+
+ val fetchedLs = fetches(queryRequests, prevStepTgtVertexIdEdges)
+
+ graph.traversalHelper.filterEdges(orgQuery, stepIdx, queryRequests,
+ fetchedLs, orgQuery.steps(stepIdx).queryParams, buildLastStepInnerResult, prevStepTgtVertexIdEdges)(graph.ec)
+ }
+ }
+
+ def fetches(queryRequests: Seq[QueryRequest],
+ prevStepEdges: Map[VertexId, Seq[EdgeWithScore]]): Future[Seq[StepResult]] = {
+
+ val reqWithIdxs = queryRequests.zipWithIndex
+ val requestsPerLabel = reqWithIdxs.groupBy(t => t._1.queryParam.label)
+ val aggFuture = requestsPerLabel.foldLeft(Future.successful(Map.empty[Int, StepResult])) { case (prevFuture, (label, reqWithIdxs)) =>
+ for {
+ prev <- prevFuture
+ cur <- graph.getStorage(label).fetches(reqWithIdxs.map(_._1), prevStepEdges)
+ } yield {
+ prev ++ reqWithIdxs.map(_._2).zip(cur).toMap
+ }
+ }
+ aggFuture.map { agg => agg.toSeq.sortBy(_._1).map(_._2) }
+ }
+
+ def fetchAndDeleteAll(queries: Seq[Query], requestTs: Long): Future[(Boolean, Boolean)] = {
+ val futures = queries.map(getEdgesStepInner(_, true))
+ val future = for {
+ stepInnerResultLs <- Future.sequence(futures)
+ (allDeleted, ret) <- deleteAllFetchedEdgesLs(stepInnerResultLs, requestTs)
+ } yield {
+ // logger.debug(s"fetchAndDeleteAll: ${allDeleted}, ${ret}")
+ (allDeleted, ret)
+ }
+
+ Extensions.retryOnFailure(MaxRetryNum) {
+ future
+ } {
+ logger.error(s"fetch and deleteAll failed.")
+ (true, false)
+ }
+
+ }
+
+ def deleteAllFetchedEdgesLs(stepInnerResultLs: Seq[StepResult],
+ requestTs: Long): Future[(Boolean, Boolean)] = {
+ stepInnerResultLs.foreach { stepInnerResult =>
+ if (stepInnerResult.isFailure) throw new RuntimeException("fetched result is fallback.")
+ }
+ val futures = for {
+ stepInnerResult <- stepInnerResultLs
+ filtered = stepInnerResult.edgeWithScores.filter { edgeWithScore =>
+ (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
+ }
+ edgesToDelete = graph.elementBuilder.buildEdgesToDelete(filtered, requestTs)
+ if edgesToDelete.nonEmpty
+ } yield {
+ val head = edgesToDelete.head
+ val label = head.edge.innerLabel
+ val stepResult = StepResult(edgesToDelete, Nil, Nil, false)
+ val ret = label.schemaVersion match {
+ case HBaseType.VERSION3 | HBaseType.VERSION4 =>
+ if (label.consistencyLevel == "strong") {
+ /*
+ * read: snapshotEdge on queryResult = O(N)
+ * write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
+ */
+ graph.mutateEdges(edgesToDelete.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
+ } else {
+ graph.getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ }
+ case _ =>
+
+ /*
+ * read: x
+ * write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
+ */
+ graph.getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
+ }
+ ret
+ }
+
+ if (futures.isEmpty) {
+ // all deleted.
+ Future.successful(true -> true)
+ } else {
+ Future.sequence(futures).map { rets => false -> rets.forall(identity) }
+ }
+ }
+
+ def fetchEdgesAsync(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
+ val queryParams = labelNameWithDirs.map { case (l, direction) =>
+ QueryParam(labelName = l, direction = direction.toLowerCase)
+ }
+
+ val query = Query.toQuery(Seq(vertex), queryParams)
+ val queryRequests = queryParams.map { param => QueryRequest(query, 0, vertex, param) }
+ val ls = new util.ArrayList[Edge]()
+ fetches(queryRequests, Map.empty).map { stepResultLs =>
+ stepResultLs.foreach(_.edgeWithScores.foreach(es => ls.add(es.edge)))
+ ls.iterator()
+ }
+ }
+
def buildNextStepQueryRequests(orgQuery: Query, stepIdx: Int, stepInnerResult: StepResult) = {
val edgeWithScoreLs = stepInnerResult.edgeWithScores
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index 54b865a..55a3638 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -269,7 +269,7 @@ class RequestParser(graph: S2GraphLike) {
id <- ids
innerId <- jsValueToInnerVal(id, serviceColumn.columnType, label.schemaVersion)
} yield {
- graph.newVertex(SourceVertexId(serviceColumn, innerId), System.currentTimeMillis())
+ graph.elementBuilder.newVertex(SourceVertexId(serviceColumn, innerId), System.currentTimeMillis())
}
vertices
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
index 6ebc90c..945f246 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/tall/IndexEdgeDeserializable.scala
@@ -36,6 +36,7 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
type QualifierRaw = (Array[(LabelMeta, InnerValLike)], VertexId, Byte, Boolean, Int)
type ValueRaw = (Array[(LabelMeta, InnerValLike)], Int)
+ val builder = graph.elementBuilder
override def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2EdgeLike]): Option[S2EdgeLike] = {
@@ -61,7 +62,7 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
else {
val label = Label.findById(labelWithDir.labelId)
val schemaVer = label.schemaVersion
- val srcVertex = graph.newVertex(srcVertexId, version)
+ val srcVertex = builder.newVertex(srcVertexId, version)
var tsVal = version
val isTallSchema = tallSchemaVersions(label.schemaVersion)
val isDegree = if (isTallSchema) pos == kv.row.length else kv.qualifier.isEmpty
@@ -71,13 +72,13 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
// val degreeVal = Bytes.toLong(kv.value)
val degreeVal = bytesToLongFunc(kv.value, 0)
val tgtVertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("0", schemaVer))
- val tgtVertex = graph.newVertex(tgtVertexId, version)
- val edge = graph.newEdge(srcVertex, tgtVertex,
+ val tgtVertex = builder.newVertex(tgtVertexId, version)
+ val edge = graph.elementBuilder.newEdge(srcVertex, tgtVertex,
label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
- edge.tgtVertex = graph.newVertex(tgtVertexId, version)
+ edge.tgtVertex = builder.newVertex(tgtVertexId, version)
edge.setOp(GraphUtil.defaultOpByte)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
@@ -113,8 +114,8 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
else kv.qualifier(kv.qualifier.length - 1)
}
- val tgtVertex = graph.newVertex(tgtVertexIdRaw, version)
- val edge = graph.newEdge(srcVertex, tgtVertex,
+ val tgtVertex = builder.newVertex(tgtVertexIdRaw, version)
+ val edge = graph.elementBuilder.newEdge(srcVertex, tgtVertex,
label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
val index = label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}"))
@@ -150,7 +151,7 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
if (edge.checkProperty(LabelMeta.to.name)) {
val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
- val tgtVertex = graph.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
+ val tgtVertex = builder.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
edge.setTgtVertex(tgtVertex)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
index 79e3f2e..e533b4b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/indexedge/wide/IndexEdgeDeserializable.scala
@@ -32,6 +32,7 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
type QualifierRaw = (Array[(LabelMeta, InnerValLike)], VertexId, Byte, Boolean, Int)
type ValueRaw = (Array[(LabelMeta, InnerValLike)], Int)
+ val builder = graph.elementBuilder
override def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2EdgeLike]): Option[S2EdgeLike] = {
@@ -55,20 +56,20 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
else {
val label = Label.findById(labelWithDir.labelId)
val schemaVer = label.schemaVersion
- val srcVertex = graph.newVertex(srcVertexId, version)
+ val srcVertex = builder.newVertex(srcVertexId, version)
//TODO:
var tsVal = version
if (kv.qualifier.isEmpty) {
val degreeVal = bytesToLongFunc(kv.value, 0)
val tgtVertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("0", schemaVer))
- val tgtVertex = graph.newVertex(tgtVertexId, version)
- val edge = graph.newEdge(srcVertex, tgtVertex,
+ val tgtVertex = builder.newVertex(tgtVertexId, version)
+ val edge = builder.newEdge(srcVertex, tgtVertex,
label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
edge.propertyInner(LabelMeta.timestamp.name, version, version)
edge.propertyInner(LabelMeta.degree.name, degreeVal, version)
- edge.tgtVertex = graph.newVertex(tgtVertexId, version)
+ edge.tgtVertex = builder.newVertex(tgtVertexId, version)
edge.setOp(GraphUtil.defaultOpByte)
edge.setTsInnerValOpt(Option(InnerVal.withLong(tsVal, schemaVer)))
@@ -88,8 +89,8 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
if (kv.qualifier.length == pos) GraphUtil.defaultOpByte
else kv.qualifier(kv.qualifier.length-1)
- val tgtVertex = graph.newVertex(tgtVertexIdRaw, version)
- val edge = graph.newEdge(srcVertex, tgtVertex,
+ val tgtVertex = builder.newVertex(tgtVertexIdRaw, version)
+ val edge = builder.newEdge(srcVertex, tgtVertex,
label, labelWithDir.dir, GraphUtil.defaultOpByte, version, S2Edge.EmptyState)
val index = label.indicesMap.getOrElse(labelIdxSeq, throw new RuntimeException(s"invalid index seq: ${label.id.get}, ${labelIdxSeq}"))
@@ -124,7 +125,7 @@ class IndexEdgeDeserializable(graph: S2GraphLike,
/* process tgtVertexId */
if (edge.checkProperty(LabelMeta.to.name)) {
val vId = edge.property(LabelMeta.to.name).asInstanceOf[S2Property[_]].innerValWithTs
- val tgtVertex = graph.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
+ val tgtVertex = builder.newVertex(TargetVertexId(ServiceColumn.Default, vId.innerVal), version)
edge.setTgtVertex(tgtVertex)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
index 4c97f6e..b7f5ba1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
@@ -28,7 +28,7 @@ import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.serde.Deserializable
class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[SnapshotEdge] {
-
+ val builder = graph.elementBuilder
def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
val statusCode = byte >> 4
val op = byte & ((1 << 4) - 1)
@@ -89,15 +89,19 @@ class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[Snap
val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
val pendingEdge =
- graph.newEdge(graph.newVertex(srcVertexId, version),
- graph.newVertex(tgtVertexId, version),
+ builder.newEdge(
+ builder.newVertex(srcVertexId, version),
+ builder.newVertex(tgtVertexId, version),
label, labelWithDir.dir, pendingEdgeOp,
version, pendingEdgeProps.toMap,
statusCode = pendingEdgeStatusCode, lockTs = lockTs, tsInnerValOpt = Option(tsInnerVal))
+
Option(pendingEdge)
}
- val snapshotEdge = graph.elementBuilder.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
+ val snapshotEdge = builder.newSnapshotEdge(
+ builder.newVertex(srcVertexId, ts),
+ builder.newVertex(tgtVertexId, ts),
label, labelWithDir.dir, op, version, props.toMap, statusCode = statusCode,
pendingEdgeOpt = _pendingEdgeOpt, lockTs = None, tsInnerValOpt = Option(tsInnerVal))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
index a6a64a2..7dec6d9 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
@@ -28,7 +28,7 @@ import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.serde.Deserializable
class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[SnapshotEdge] {
-
+ val builder = graph.elementBuilder
def statusCodeWithOp(byte: Byte): (Byte, Byte) = {
val statusCode = byte >> 4
val op = byte & ((1 << 4) - 1)
@@ -55,7 +55,7 @@ class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[Snap
else {
val label = Label.findById(labelWithDir.labelId)
val schemaVer = label.schemaVersion
- val srcVertex = graph.newVertex(srcVertexId, version)
+ val srcVertex = builder.newVertex(srcVertexId, version)
val (tgtVertexId, _) = TargetVertexId.fromBytes(kv.qualifier, 0, kv.qualifier.length, schemaVer)
@@ -80,15 +80,18 @@ class SnapshotEdgeDeserializable(graph: S2GraphLike) extends Deserializable[Snap
val lockTs = Option(Bytes.toLong(kv.value, pos, 8))
val pendingEdge =
- graph.newEdge(graph.newVertex(srcVertexId, version),
- graph.newVertex(tgtVertexId, version),
+ builder.newEdge(
+ builder.newVertex(srcVertexId, version),
+ builder.newVertex(tgtVertexId, version),
label, labelWithDir.dir, pendingEdgeOp,
version, pendingEdgeProps.toMap,
statusCode = pendingEdgeStatusCode, lockTs = lockTs, tsInnerValOpt = Option(tsInnerVal))
Option(pendingEdge)
}
- val snapshotEdge = graph.elementBuilder.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
+ val snapshotEdge = builder.newSnapshotEdge(
+ builder.newVertex(srcVertexId, ts),
+ builder.newVertex(tgtVertexId, ts),
label, labelWithDir.dir, op, version, props.toMap, statusCode = statusCode,
pendingEdgeOpt = _pendingEdgeOpt, lockTs = None, tsInnerValOpt = Option(tsInnerVal))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
index 70160a8..5576017 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
@@ -28,6 +28,7 @@ import org.apache.s2graph.core.{S2Graph, S2GraphLike, S2Vertex, S2VertexLike}
class VertexDeserializable(graph: S2GraphLike,
bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
+ val builder = graph.elementBuilder
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
try {
@@ -47,7 +48,7 @@ class VertexDeserializable(graph: S2GraphLike,
propsMap += (columnMeta -> innerVal)
}
- val vertex = graph.newVertex(vertexId, kv.timestamp, S2Vertex.EmptyProps, belongLabelIds = Nil)
+ val vertex = builder.newVertex(vertexId, kv.timestamp, S2Vertex.EmptyProps, belongLabelIds = Nil)
S2Vertex.fillPropsWithTs(vertex, propsMap.toMap)
Option(vertex)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
index 002f577..d1d4d7d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
@@ -30,6 +30,7 @@ import scala.collection.mutable.ListBuffer
class VertexDeserializable(graph: S2GraphLike,
bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
+ val builder = graph.elementBuilder
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
try {
@@ -62,7 +63,7 @@ class VertexDeserializable(graph: S2GraphLike,
}
}
assert(maxTs != Long.MinValue)
- val vertex = graph.newVertex(vertexId, maxTs, S2Vertex.EmptyProps, belongLabelIds = belongLabelIds)
+ val vertex = builder.newVertex(vertexId, maxTs, S2Vertex.EmptyProps, belongLabelIds = belongLabelIds)
S2Vertex.fillPropsWithTs(vertex, propsMap.toMap)
Option(vertex)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/S2EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/S2EdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/S2EdgeTest.scala
index 94883c9..2b439bc 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/S2EdgeTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/S2EdgeTest.scala
@@ -66,14 +66,14 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
test("buildOperation") {
val schemaVersion = "v2"
val vertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = graph.newVertex(vertexId)
+ val srcVertex = builder.newVertex(vertexId)
val tgtVertex = srcVertex
val timestampProp = LabelMeta.timestamp -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
val snapshotEdge = None
val propsWithTs = Map(timestampProp)
- val requestEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
+ val requestEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
val newVersion = 0L
@@ -93,14 +93,14 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
test("buildMutation: snapshotEdge: None with newProps") {
val schemaVersion = "v2"
val vertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = graph.newVertex(vertexId)
+ val srcVertex = builder.newVertex(vertexId)
val tgtVertex = srcVertex
val timestampProp = LabelMeta.timestamp -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
val snapshotEdge = None
val propsWithTs = Map(timestampProp)
- val requestEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
+ val requestEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
val newVersion = 0L
@@ -120,14 +120,14 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
test("buildMutation: oldPropsWithTs == newPropsWithTs, Drop all requests") {
val schemaVersion = "v2"
val vertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = graph.newVertex(vertexId)
+ val srcVertex = builder.newVertex(vertexId)
val tgtVertex = srcVertex
val timestampProp = LabelMeta.timestamp -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
val snapshotEdge = None
val propsWithTs = Map(timestampProp)
- val requestEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
+ val requestEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
val newVersion = 0L
@@ -144,7 +144,7 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
test("buildMutation: All props older than snapshotEdge's LastDeletedAt") {
val schemaVersion = "v2"
val vertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = graph.newVertex(vertexId)
+ val srcVertex = builder.newVertex(vertexId)
val tgtVertex = srcVertex
val timestampProp = LabelMeta.timestamp -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
@@ -159,12 +159,12 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
)
- val _snapshotEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, op = GraphUtil.operations("delete"), propsWithTs = propsWithTs)
+ val _snapshotEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, op = GraphUtil.operations("delete"), propsWithTs = propsWithTs)
val snapshotEdge = Option(_snapshotEdge)
- val requestEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
+ val requestEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
val newVersion = 0L
val edgeMutate = S2Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
@@ -178,7 +178,7 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
test("buildMutation: All props newer than snapshotEdge's LastDeletedAt") {
val schemaVersion = "v2"
val vertexId = VertexId(ServiceColumn.Default, InnerVal.withStr("dummy", schemaVersion))
- val srcVertex = graph.newVertex(vertexId)
+ val srcVertex = builder.newVertex(vertexId)
val tgtVertex = srcVertex
val timestampProp = LabelMeta.timestamp -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 1)
@@ -193,11 +193,11 @@ class S2EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
LabelMeta.lastDeletedAt -> InnerValLikeWithTs(InnerVal.withLong(0, schemaVersion), 3)
)
- val _snapshotEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, op = GraphUtil.operations("delete"), propsWithTs = propsWithTs)
+ val _snapshotEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, op = GraphUtil.operations("delete"), propsWithTs = propsWithTs)
val snapshotEdge = Option(_snapshotEdge)
- val requestEdge = graph.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
+ val requestEdge = builder.newEdge(srcVertex, tgtVertex, labelV2, labelWithDirV2.dir, propsWithTs = propsWithTs)
val newVersion = 0L
val edgeMutate = S2Edge.buildMutation(snapshotEdge, requestEdge, newVersion, oldPropsWithTs, propsWithTs)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/eabe7570/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
index 4614bed..6ac77e4 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/TestCommonWithModels.scala
@@ -35,12 +35,13 @@ trait TestCommonWithModels {
var graph: S2Graph = _
var config: Config = _
var management: Management = _
+ var builder: GraphElementBuilder = _
def initTests() = {
config = ConfigFactory.load()
graph = new S2Graph(config)(ExecutionContext.Implicits.global)
management = new Management(graph)
-
+ builder = graph.elementBuilder
implicit val session = AutoSession
deleteTestLabel()
[15/23] incubator-s2graph git commit: bug fix on copyVertex.
Posted by st...@apache.org.
bug fix on copyVertex.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e51289fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e51289fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e51289fb
Branch: refs/heads/master
Commit: e51289fb53526a57cc40cf5fe22d99914d2ae3e5
Parents: 5f2d4ff
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Nov 7 16:30:10 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 7 16:30:10 2017 +0900
----------------------------------------------------------------------
.../src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e51289fb/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
index 4a75036..e0205c5 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexBuilder.scala
@@ -26,7 +26,7 @@ class S2VertexBuilder(vertex: S2VertexLike) {
props.forEach(new BiConsumer[String, S2VertexProperty[_]] {
override def accept(t: String, u: S2VertexProperty[_]) = {
- v.property(t, u)
+ v.property(t, u.value)
}
})
[02/23] incubator-s2graph git commit: passed s2test.
Posted by st...@apache.org.
passed s2test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f7170235
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f7170235
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f7170235
Branch: refs/heads/master
Commit: f71702357255ba191ee3670978be79cfe8c5b7b0
Parents: fac47d1
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 12:13:03 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 12:13:03 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/PostProcess.scala | 4 +-
.../org/apache/s2graph/core/QueryParam.scala | 6 +--
.../org/apache/s2graph/core/QueryResult.scala | 4 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 18 +++----
.../scala/org/apache/s2graph/core/S2Graph.scala | 56 ++++++++++----------
.../org/apache/s2graph/core/S2Vertex.scala | 6 +--
.../org/apache/s2graph/core/S2VertexLike.scala | 11 +++-
.../apache/s2graph/core/S2VertexProperty.scala | 2 +-
.../s2graph/core/index/IndexProvider.scala | 12 ++---
.../s2graph/core/rest/RequestParser.scala | 4 +-
.../apache/s2graph/core/storage/Storage.scala | 10 ++--
.../apache/s2graph/core/storage/StorageIO.scala | 4 +-
.../s2graph/core/storage/StorageReadable.scala | 8 +--
.../s2graph/core/storage/StorageSerDe.scala | 6 +--
.../hbase/AsynchbaseStorageReadable.scala | 4 +-
.../storage/hbase/AsynchbaseStorageSerDe.scala | 6 +--
.../vertex/tall/VertexDeserializable.scala | 6 +--
.../serde/vertex/tall/VertexSerializable.scala | 4 +-
.../vertex/wide/VertexDeserializable.scala | 6 +--
.../serde/vertex/wide/VertexSerializable.scala | 4 +-
.../s2graph/core/storage/StorageIOTest.scala | 8 +--
.../core/tinkerpop/S2GraphProvider.scala | 1 +
.../rest/play/controllers/EdgeController.scala | 6 +--
23 files changed, 103 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index b22eb65..3017749 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -141,7 +141,7 @@ object PostProcess {
}
}
- def s2VertexToJson(s2Vertex: S2Vertex): Option[JsValue] = {
+ def s2VertexToJson(s2Vertex: S2VertexLike): Option[JsValue] = {
val props = for {
(k, v) <- s2Vertex.properties
jsVal <- anyValToJsValue(v)
@@ -160,7 +160,7 @@ object PostProcess {
}
}
- def verticesToJson(s2Vertices: Seq[S2Vertex]): JsValue =
+ def verticesToJson(s2Vertices: Seq[S2VertexLike]): JsValue =
Json.toJson(s2Vertices.flatMap(s2VertexToJson(_)))
def withOptionalFields(queryOption: QueryOption,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
index 1100f6c..2e8d1f4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
@@ -40,7 +40,7 @@ object Query {
def apply(query: Query): Query = {
Query(query.vertices, query.steps, query.queryOption, query.jsonQuery)
}
- def toQuery(srcVertices: Seq[S2Vertex], queryParams: Seq[QueryParam]) = Query(srcVertices, Vector(Step(queryParams)))
+ def toQuery(srcVertices: Seq[S2VertexLike], queryParams: Seq[QueryParam]) = Query(srcVertices, Vector(Step(queryParams)))
}
@@ -97,7 +97,7 @@ case class QueryOption(removeCycle: Boolean = false,
}
-case class Query(vertices: Seq[S2Vertex] = Seq.empty[S2Vertex],
+case class Query(vertices: Seq[S2VertexLike] = Nil,
steps: IndexedSeq[Step] = Vector.empty[Step],
queryOption: QueryOption = QueryOption(),
jsonQuery: JsValue = JsNull) {
@@ -219,7 +219,7 @@ case class Step(queryParams: Seq[QueryParam],
}
}
-case class VertexParam(vertices: Seq[S2Vertex]) {
+case class VertexParam(vertices: Seq[S2VertexLike]) {
var filters: Option[Map[Byte, InnerValLike]] = None
def has(what: Option[Map[Byte, InnerValLike]]): VertexParam = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
index 3916f39..7506b40 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryResult.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.collection.{Seq, mutable}
object QueryResult {
- def fromVertices(graph: S2Graph, vertices: Seq[S2Vertex], queryParams: Seq[QueryParam]): StepResult = {
+ def fromVertices(graph: S2Graph, vertices: Seq[S2VertexLike], queryParams: Seq[QueryParam]): StepResult = {
val edgeWithScores = vertices.flatMap { vertex =>
queryParams.map { queryParam =>
val label = queryParam.label
@@ -69,7 +69,7 @@ object QueryResult {
case class QueryRequest(query: Query,
stepIdx: Int,
- vertex: S2Vertex,
+ vertex: S2VertexLike,
queryParam: QueryParam,
prevStepScore: Double = 1.0,
labelWeight: Double = 1.0) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 51af831..641db74 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -64,8 +64,8 @@ object SnapshotEdge {
}
case class SnapshotEdge(graph: S2Graph,
- srcVertex: S2Vertex,
- tgtVertex: S2Vertex,
+ srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
label: Label,
dir: Int,
op: Byte,
@@ -168,8 +168,8 @@ object IndexEdge {
}
case class IndexEdge(graph: S2Graph,
- srcVertex: S2Vertex,
- tgtVertex: S2Vertex,
+ srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
label: Label,
dir: Int,
op: Byte,
@@ -302,8 +302,8 @@ case class IndexEdge(graph: S2Graph,
}
case class S2Edge(innerGraph: S2Graph,
- srcVertex: S2Vertex,
- var tgtVertex: S2Vertex,
+ srcVertex: S2VertexLike,
+ var tgtVertex: S2VertexLike,
innerLabel: Label,
dir: Int,
var op: Byte = GraphUtil.defaultOpByte,
@@ -429,7 +429,7 @@ case class S2Edge(innerGraph: S2Graph,
// def relatedEdges = List(this)
- private def getServiceColumn(vertex: S2Vertex, defaultServiceColumn: ServiceColumn) =
+ private def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
def srcForVertex = {
@@ -566,8 +566,8 @@ case class S2Edge(innerGraph: S2Graph,
def checkProperty(key: String): Boolean = propsWithTs.containsKey(key)
- def copyEdge(srcVertex: S2Vertex = srcVertex,
- tgtVertex: S2Vertex = tgtVertex,
+ def copyEdge(srcVertex: S2VertexLike = srcVertex,
+ tgtVertex: S2VertexLike = tgtVertex,
innerLabel: Label = innerLabel,
dir: Int = dir,
op: Byte = op,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 34db9e4..90190cf 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -190,7 +190,7 @@ object S2Graph {
}
}
- def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2Vertex), Boolean] = {
+ def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
val vertices = for {
edgeWithScore <- edgeWithScoreLs
edge = edgeWithScore.edge
@@ -272,7 +272,7 @@ object S2Graph {
queryRequests: Seq[QueryRequest],
queryResultLsFuture: Future[Seq[StepResult]],
queryParams: Seq[QueryParam],
- alreadyVisited: Map[(LabelWithDirection, S2Vertex), Boolean] = Map.empty,
+ alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
buildLastStepInnerResult: Boolean = true,
parentEdges: Map[VertexId, Seq[EdgeWithScore]])
(implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
@@ -1059,10 +1059,10 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
val step = q.steps(stepIdx)
val alreadyVisited =
- if (stepIdx == 0) Map.empty[(LabelWithDirection, S2Vertex), Boolean]
+ if (stepIdx == 0) Map.empty[(LabelWithDirection, S2VertexLike), Boolean]
else alreadyVisitedVertices(stepInnerResult.edgeWithScores)
- val initial = (Map.empty[S2Vertex, Double], Map.empty[S2Vertex, ArrayBuffer[EdgeWithScore]])
+ val initial = (Map.empty[S2VertexLike, Double], Map.empty[S2VertexLike, ArrayBuffer[EdgeWithScore]])
val (sums, grouped) = edgeWithScoreLs.foldLeft(initial) { case ((sum, group), edgeWithScore) =>
val key = edgeWithScore.edge.tgtVertex
val newScore = sum.getOrElse(key, 0.0) + edgeWithScore.score
@@ -1144,7 +1144,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
} get
}
- def getVertices(vertices: Seq[S2Vertex]): Future[Seq[S2Vertex]] = {
+ def getVertices(vertices: Seq[S2VertexLike]): Future[Seq[S2VertexLike]] = {
val verticesWithIdx = vertices.zipWithIndex
val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
getStorage(service).fetchVertices(vertices).map(_.zip(vertexGroup.map(_._2)))
@@ -1156,7 +1156,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
/** mutate */
- def deleteAllAdjacentEdges(srcVertices: Seq[S2Vertex],
+ def deleteAllAdjacentEdges(srcVertices: Seq[S2VertexLike],
labels: Seq[Label],
dir: Int,
ts: Long): Future[Boolean] = {
@@ -1320,11 +1320,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val edgeBuffer = ArrayBuffer[(S2Edge, Int)]()
- val vertexBuffer = ArrayBuffer[(S2Vertex, Int)]()
+ val vertexBuffer = ArrayBuffer[(S2VertexLike, Int)]()
elements.zipWithIndex.foreach {
case (e: S2Edge, idx: Int) => edgeBuffer.append((e, idx))
- case (v: S2Vertex, idx: Int) => vertexBuffer.append((v, idx))
+ case (v: S2VertexLike, idx: Int) => vertexBuffer.append((v, idx))
case any@_ => logger.error(s"Unknown type: ${any}")
}
@@ -1467,8 +1467,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- def mutateVertices(vertices: Seq[S2Vertex], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- def mutateVertex(storage: Storage)(vertex: S2Vertex, withWait: Boolean): Future[MutateResponse] = {
+ def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ def mutateVertex(storage: Storage)(vertex: S2VertexLike, withWait: Boolean): Future[MutateResponse] = {
if (vertex.op == GraphUtil.operations("delete")) {
storage.writeToStorage(vertex.hbaseZkAddr,
storage.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
@@ -1480,7 +1480,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- def mutateVertices(storage: Storage)(vertices: Seq[S2Vertex],
+ def mutateVertices(storage: Storage)(vertices: Seq[S2VertexLike],
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val futures = vertices.map { vertex => mutateVertex(storage)(vertex, withWait) }
Future.sequence(futures)
@@ -1567,7 +1567,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
} get
- def toVertex(s: String): Option[S2Vertex] = {
+ def toVertex(s: String): Option[S2VertexLike] = {
toVertex(GraphUtil.split(s))
}
@@ -1588,7 +1588,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
throw e
} get
- def toVertex(parts: Array[String]): Option[S2Vertex] = Try {
+ def toVertex(parts: Array[String]): Option[S2VertexLike] = Try {
val (ts, operation, logType, srcId, serviceName, colName) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
val vertex = toVertex(serviceName, colName, srcId, props, ts.toLong, operation)
@@ -1630,7 +1630,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
id: Any,
props: Map[String, Any] = Map.empty,
ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2Vertex = {
+ operation: String = "insert"): S2VertexLike = {
val service = Service.findByName(serviceName).getOrElse(throw new java.lang.IllegalArgumentException(s"$serviceName is not found."))
val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new java.lang.IllegalArgumentException(s"$columnName is not found."))
@@ -1702,8 +1702,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
* @param tsInnerValOpt
* @return
*/
- def newEdge(srcVertex: S2Vertex,
- tgtVertex: S2Vertex,
+ def newEdge(srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
innerLabel: Label,
dir: Int,
op: Byte = GraphUtil.defaultOpByte,
@@ -1751,8 +1751,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
* @param tsInnerValOpt
* @return
*/
- private[core] def newSnapshotEdge(srcVertex: S2Vertex,
- tgtVertex: S2Vertex,
+ private[core] def newSnapshotEdge(srcVertex: S2VertexLike,
+ tgtVertex: S2VertexLike,
label: Label,
dir: Int,
op: Byte,
@@ -1792,22 +1792,22 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
ts: Long = System.currentTimeMillis(),
props: S2Vertex.Props = S2Vertex.EmptyProps,
op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2Vertex = {
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
val vertex = new S2Vertex(this, id, ts, S2Vertex.EmptyProps, op, belongLabelIds)
S2Vertex.fillPropsWithTs(vertex, props)
vertex
}
- def getVertex(vertexId: VertexId): Option[S2Vertex] = {
+ def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
val v = newVertex(vertexId)
Await.result(getVertices(Seq(v)).map { vertices => vertices.headOption }, WaitTimeout)
}
- def fetchEdges(vertex: S2Vertex, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge] = {
+ def fetchEdges(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): util.Iterator[Edge] = {
Await.result(fetchEdgesAsync(vertex, labelNameWithDirs), WaitTimeout)
}
- def fetchEdgesAsync(vertex: S2Vertex, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
+ def fetchEdgesAsync(vertex: S2VertexLike, labelNameWithDirs: Seq[(String, String)]): Future[util.Iterator[Edge]] = {
val queryParams = labelNameWithDirs.map { case (l, direction) =>
QueryParam(labelName = l, direction = direction.toLowerCase)
}
@@ -1844,7 +1844,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
Await.result(defaultStorage.fetchVerticesAll(), WaitTimeout).iterator
} else {
val vertices = ids.collect {
- case s2Vertex: S2Vertex => s2Vertex
+ case s2Vertex: S2VertexLike => s2Vertex
case vId: VertexId => newVertex(vId)
case vertex: Vertex => newVertex(vertex.id().asInstanceOf[VertexId])
case other @ _ => newVertex(VertexId.fromString(other.toString))
@@ -1906,7 +1906,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
addVertex(Seq(T.label, label): _*)
}
- def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2Vertex = {
+ def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2VertexLike = {
idValue match {
case vId: VertexId =>
toVertex(vId.column.service.serviceName, vId.column.columnName, vId, kvsMap)
@@ -1958,7 +1958,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
ts: Long = System.currentTimeMillis(),
props: S2Vertex.Props = S2Vertex.EmptyProps,
op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2Vertex = {
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
val vertex = newVertex(id, ts, props, op, belongLabelIds)
val future = mutateVertices(Seq(vertex), withWait = true).map { rets =>
@@ -1970,7 +1970,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
vertex
}
- def addVertexInner(vertex: S2Vertex): S2Vertex = {
+ def addVertexInner(vertex: S2VertexLike): S2VertexLike = {
val future = mutateVertices(Seq(vertex), withWait = true).flatMap { rets =>
if (rets.forall(_.isSuccess)) {
indexProvider.mutateVerticesAsync(Seq(vertex))
@@ -1982,11 +1982,11 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
/* tp3 only */
- def addEdge(srcVertex: S2Vertex, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
+ def addEdge(srcVertex: S2VertexLike, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
val containsId = kvs.contains(T.id)
tgtVertex match {
- case otherV: S2Vertex =>
+ case otherV: S2VertexLike =>
if (!features().edge().supportsUserSuppliedIds() && containsId) {
throw Exceptions.userSuppliedIdsNotSupported()
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
index 62efa14..96e7afe 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Vertex.scala
@@ -85,7 +85,7 @@ object S2Vertex {
def isLabelId(propKey: Int): Boolean = propKey > Byte.MaxValue
- def fillPropsWithTs(vertex: S2Vertex, props: Props): Unit = {
+ def fillPropsWithTs(vertex: S2VertexLike, props: Props): Unit = {
props.forEach(new BiConsumer[String, S2VertexProperty[_]] {
override def accept(key: String, p: S2VertexProperty[_]): Unit = {
// vertex.property(Cardinality.single, key, p.value)
@@ -94,7 +94,7 @@ object S2Vertex {
})
}
- def fillPropsWithTs(vertex: S2Vertex, state: State): Unit = {
+ def fillPropsWithTs(vertex: S2VertexLike, state: State): Unit = {
state.foreach { case (k, v) => vertex.propertyInner(Cardinality.single, k.name, v.value) }
}
@@ -104,7 +104,7 @@ object S2Vertex {
}.toMap
}
- def stateToProps(vertex: S2Vertex, state: State): Props = {
+ def stateToProps(vertex: S2VertexLike, state: State): Props = {
state.foreach { case (k, v) =>
vertex.property(k.name, v.value)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
index c2bc40c..5a8f722 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexLike.scala
@@ -4,6 +4,7 @@ import java.util.function.{BiConsumer, Consumer}
import org.apache.s2graph.core.S2Vertex.Props
import org.apache.s2graph.core.mysqls.{ColumnMeta, Label, Service, ServiceColumn}
+import org.apache.s2graph.core.types.VertexId
import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality
import org.apache.tinkerpop.gremlin.structure.{Direction, Edge, T, Vertex, VertexProperty}
import play.api.libs.json.Json
@@ -14,9 +15,17 @@ import scala.collection.JavaConverters._
trait S2VertexLike extends Vertex with GraphElement {
this: S2Vertex =>
+ val graph: S2Graph
+ val id: VertexId
+ val ts: Long
+ val props: Props
+ val op: Byte
+ val belongLabelIds: Seq[Int]
+
val innerId = id.innerId
val innerIdVal = innerId.value
+
lazy val properties = for {
(k, v) <- props.asScala
} yield v.columnMeta.name -> v.value
@@ -52,7 +61,7 @@ trait S2VertexLike extends Vertex with GraphElement {
Seq(ts, GraphUtil.fromOp(op), "v", id.innerId, serviceName, columnName).mkString("\t")
}
- def copyVertexWithState(props: Props): S2Vertex = {
+ def copyVertexWithState(props: Props): S2VertexLike = {
val newVertex = copy(props = S2Vertex.EmptyProps)
S2Vertex.fillPropsWithTs(newVertex, props)
newVertex
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/S2VertexProperty.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexProperty.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexProperty.scala
index d0b56a0..1fbc894 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2VertexProperty.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2VertexProperty.scala
@@ -37,7 +37,7 @@ case class S2VertexPropertyId(columnMeta: ColumnMeta, value: InnerValLike) {
}
}
-case class S2VertexProperty[V](element: S2Vertex,
+case class S2VertexProperty[V](element: S2VertexLike,
columnMeta: ColumnMeta,
key: String,
v: V) extends VertexProperty[V] {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index b52ffda..098d0b4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -29,7 +29,7 @@ import org.apache.lucene.queryparser.classic.{ParseException, QueryParser}
import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.store.{BaseDirectory, RAMDirectory}
import org.apache.s2graph.core.io.Conversions
-import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex}
+import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex, S2VertexLike}
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types.{InnerValLike, VertexId}
import org.apache.s2graph.core.utils.logger
@@ -127,8 +127,8 @@ trait IndexProvider {
def fetchVertexIds(hasContainers: java.util.List[HasContainer]): java.util.List[VertexId]
def fetchVertexIdsAsync(hasContainers: java.util.List[HasContainer]): Future[java.util.List[VertexId]]
- def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean]
- def mutateVerticesAsync(vertices: Seq[S2Vertex]): Future[Seq[Boolean]]
+ def mutateVertices(vertices: Seq[S2VertexLike]): Seq[Boolean]
+ def mutateVerticesAsync(vertices: Seq[S2VertexLike]): Future[Seq[Boolean]]
def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean]
def mutateEdgesAsync(edges: Seq[S2Edge]): Future[Seq[Boolean]]
@@ -154,7 +154,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
})
}
- private def toDocument(globalIndex: GlobalIndex, vertex: S2Vertex): Option[Document] = {
+ private def toDocument(globalIndex: GlobalIndex, vertex: S2VertexLike): Option[Document] = {
val props = vertex.props.asScala
val exist = props.exists(t => globalIndex.propNamesSet(t._1))
if (!exist) None
@@ -204,7 +204,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
}
}
- override def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] = {
+ override def mutateVertices(vertices: Seq[S2VertexLike]): Seq[Boolean] = {
val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.VertexType)
globalIndexOptions.map { globalIndex =>
@@ -314,7 +314,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
override def fetchVertexIdsAsync(hasContainers: java.util.List[HasContainer]): Future[util.List[VertexId]] = Future.successful(fetchVertexIds(hasContainers))
- override def mutateVerticesAsync(vertices: Seq[S2Vertex]): Future[Seq[Boolean]] = Future.successful(mutateVertices(vertices))
+ override def mutateVerticesAsync(vertices: Seq[S2VertexLike]): Future[Seq[Boolean]] = Future.successful(mutateVertices(vertices))
override def mutateEdgesAsync(edges: Seq[S2Edge]): Future[Seq[Boolean]] = Future.successful(mutateEdges(edges))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
index 4bc9376..55b6e12 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/rest/RequestParser.scala
@@ -262,7 +262,7 @@ class RequestParser(graph: S2Graph) {
case _ => GroupBy.Empty
}.getOrElse(GroupBy.Empty)
- def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[S2Vertex] = {
+ def toVertices(labelName: String, direction: String, ids: Seq[JsValue]): Seq[S2VertexLike] = {
val vertices = for {
label <- Label.findByName(labelName).toSeq
serviceColumn = if (direction == "out") label.srcColumn else label.tgtColumn
@@ -581,7 +581,7 @@ class RequestParser(graph: S2Graph) {
toJsValues(jsValue).map(toVertex(_, operation, serviceName, columnName))
}
- def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): S2Vertex = {
+ def toVertex(jsValue: JsValue, operation: String, serviceName: Option[String] = None, columnName: Option[String] = None): S2VertexLike = {
val id = parse[JsValue](jsValue, "id")
val ts = parseOption[Long](jsValue, "timestamp").getOrElse(System.currentTimeMillis())
val sName = if (serviceName.isEmpty) parse[String](jsValue, "serviceName") else serviceName.get
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index e4eafbf..01dd128 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -68,7 +68,7 @@ abstract class Storage(val graph: S2Graph,
def indexEdgeSerializer(indexEdge: IndexEdge): serde.Serializable[IndexEdge] =
serDe.indexEdgeSerializer(indexEdge)
- def vertexSerializer(vertex: S2Vertex): serde.Serializable[S2Vertex] =
+ def vertexSerializer(vertex: S2VertexLike): serde.Serializable[S2VertexLike] =
serDe.vertexSerializer(vertex)
def snapshotEdgeDeserializer(schemaVer: String): Deserializable[SnapshotEdge] =
@@ -77,7 +77,7 @@ abstract class Storage(val graph: S2Graph,
def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable =
serDe.indexEdgeDeserializer(schemaVer)
- def vertexDeserializer(schemaVer: String): Deserializable[S2Vertex] =
+ def vertexDeserializer(schemaVer: String): Deserializable[S2VertexLike] =
serDe.vertexDeserializer(schemaVer)
/** Mutation Builder */
@@ -102,7 +102,7 @@ abstract class Storage(val graph: S2Graph,
def buildDegreePuts(edge: S2Edge, degreeVal: Long): Seq[SKeyValue] =
io.buildDegreePuts(edge, degreeVal)
- def buildPutsAll(vertex: S2Vertex): Seq[SKeyValue] =
+ def buildPutsAll(vertex: S2VertexLike): Seq[SKeyValue] =
io.buildPutsAll(vertex)
/** Mutation **/
@@ -118,12 +118,12 @@ abstract class Storage(val graph: S2Graph,
prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] =
fetcher.fetches(queryRequests, prevStepEdges)
- def fetchVertices(vertices: Seq[S2Vertex])(implicit ec: ExecutionContext): Future[Seq[S2Vertex]] =
+ def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] =
fetcher.fetchVertices(vertices)
def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2Edge]] = fetcher.fetchEdgesAll()
- def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2Vertex]] = fetcher.fetchVerticesAll()
+ def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = fetcher.fetchVerticesAll()
def fetchSnapshotEdgeInner(edge: S2Edge)(implicit ec: ExecutionContext): Future[(Option[S2Edge], Option[SKeyValue])] =
fetcher.fetchSnapshotEdgeInner(edge)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index 67033f0..4014b6d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -221,7 +221,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
}
//TODO: ServiceColumn do not have durability property yet.
- def buildDeleteBelongsToId(vertex: S2Vertex): Seq[SKeyValue] = {
+ def buildDeleteBelongsToId(vertex: S2VertexLike): Seq[SKeyValue] = {
val kvs = serDe.vertexSerializer(vertex).toKeyValues
val kv = kvs.head
vertex.belongLabelIds.map { id =>
@@ -251,7 +251,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
kvs
}
- def buildPutsAll(vertex: S2Vertex): Seq[SKeyValue] = {
+ def buildPutsAll(vertex: S2VertexLike): Seq[SKeyValue] = {
vertex.op match {
case d: Byte if d == GraphUtil.operations("delete") => serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete))
case _ => serDe.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Put))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
index 052ca69..c3b38e8 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageReadable.scala
@@ -41,11 +41,11 @@ trait StorageReadable {
def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2Edge]]
- def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2Vertex]]
+ def fetchVerticesAll()(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]]
protected def fetchKeyValues(queryRequest: QueryRequest, edge: S2Edge)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
- protected def fetchKeyValues(queryRequest: QueryRequest, vertex: S2Vertex)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
+ protected def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext): Future[Seq[SKeyValue]]
def fetchSnapshotEdgeInner(edge: S2Edge)(implicit ec: ExecutionContext): Future[(Option[S2Edge], Option[SKeyValue])] = {
@@ -72,8 +72,8 @@ trait StorageReadable {
}
}
- def fetchVertices(vertices: Seq[S2Vertex])(implicit ec: ExecutionContext): Future[Seq[S2Vertex]] = {
- def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2Vertex] = {
+ def fetchVertices(vertices: Seq[S2VertexLike])(implicit ec: ExecutionContext): Future[Seq[S2VertexLike]] = {
+ def fromResult(kvs: Seq[SKeyValue], version: String): Seq[S2VertexLike] = {
if (kvs.isEmpty) Nil
else serDe.vertexDeserializer(version).fromKeyValues(kvs, None).toSeq
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
index 15b3576..32d640c 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
@@ -19,7 +19,7 @@
package org.apache.s2graph.core.storage
-import org.apache.s2graph.core.{IndexEdge, S2Graph, S2Vertex, SnapshotEdge}
+import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.indexedge.tall.IndexEdgeDeserializable
@@ -57,7 +57,7 @@ trait StorageSerDe {
* @param vertex : vertex to serialize
* @return serializer implementation
*/
- def vertexSerializer(vertex: S2Vertex): serde.Serializable[S2Vertex]
+ def vertexSerializer(vertex: S2VertexLike): serde.Serializable[S2VertexLike]
/**
* create deserializer that can parse stored CanSKeyValue into snapshotEdge.
@@ -72,7 +72,7 @@ trait StorageSerDe {
def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable
- def vertexDeserializer(schemaVer: String): Deserializable[S2Vertex]
+ def vertexDeserializer(schemaVer: String): Deserializable[S2VertexLike]
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
index 8ff0ee0..92130f5 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
@@ -168,7 +168,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
* @param vertex
* @return
*/
- private def buildRequest(queryRequest: QueryRequest, vertex: S2Vertex) = {
+ private def buildRequest(queryRequest: QueryRequest, vertex: S2VertexLike) = {
val kvs = serDe.vertexSerializer(vertex).toKeyValues
val get = new GetRequest(vertex.hbaseTableName.getBytes, kvs.head.row, Serializable.vertexCf)
// get.setTimeout(this.singleGetTimeout.toShort)
@@ -183,7 +183,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
fetchKeyValues(rpc)
}
- override def fetchKeyValues(queryRequest: QueryRequest, vertex: S2Vertex)(implicit ec: ExecutionContext) = {
+ override def fetchKeyValues(queryRequest: QueryRequest, vertex: S2VertexLike)(implicit ec: ExecutionContext) = {
val rpc = buildRequest(queryRequest, vertex)
fetchKeyValues(rpc)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
index 1bdd74e..bb47e3b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageSerDe.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.storage.hbase
import org.apache.s2graph.core.storage.serde.Deserializable
-import org.apache.s2graph.core.{IndexEdge, S2Graph, S2Vertex, SnapshotEdge}
+import org.apache.s2graph.core._
import org.apache.s2graph.core.storage.{StorageIO, StorageSerDe, serde}
class AsynchbaseStorageSerDe(val graph: S2Graph) extends StorageSerDe {
@@ -63,7 +63,7 @@ class AsynchbaseStorageSerDe(val graph: S2Graph) extends StorageSerDe {
* @param vertex : vertex to serialize
* @return serializer implementation
*/
- override def vertexSerializer(vertex: S2Vertex) = new serde.vertex.wide.VertexSerializable(vertex)
+ override def vertexSerializer(vertex: S2VertexLike) = new serde.vertex.wide.VertexSerializable(vertex)
/**
* create deserializer that can parse stored CanSKeyValue into snapshotEdge.
@@ -83,5 +83,5 @@ class AsynchbaseStorageSerDe(val graph: S2Graph) extends StorageSerDe {
/** create deserializer that can parser stored CanSKeyValue into vertex. */
private val vertexDeserializer = new serde.vertex.wide.VertexDeserializable(graph)
- override def vertexDeserializer(schemaVer: String): Deserializable[S2Vertex] = vertexDeserializer
+ override def vertexDeserializer(schemaVer: String): Deserializable[S2VertexLike] = vertexDeserializer
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
index 648c9df..019394e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexDeserializable.scala
@@ -24,12 +24,12 @@ import org.apache.s2graph.core.storage.CanSKeyValue
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types.{HBaseType, InnerValLike, VertexId}
-import org.apache.s2graph.core.{S2Graph, S2Vertex}
+import org.apache.s2graph.core.{S2Graph, S2Vertex, S2VertexLike}
class VertexDeserializable(graph: S2Graph,
- bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2Vertex] {
+ bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
- cacheElementOpt: Option[S2Vertex]): Option[S2Vertex] = {
+ cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
try {
assert(_kvs.size == 1)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexSerializable.scala
index 87f050d..6e82b87 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/tall/VertexSerializable.scala
@@ -19,14 +19,14 @@
package org.apache.s2graph.core.storage.serde.vertex.tall
-import org.apache.s2graph.core.S2Vertex
+import org.apache.s2graph.core.{S2Vertex, S2VertexLike}
import org.apache.s2graph.core.storage.SKeyValue
import org.apache.s2graph.core.storage.serde.Serializable
import org.apache.s2graph.core.storage.serde.StorageSerializable._
import scala.collection.JavaConverters._
-case class VertexSerializable(vertex: S2Vertex, intToBytes: Int => Array[Byte] = intToBytes) extends Serializable[S2Vertex] {
+case class VertexSerializable(vertex: S2VertexLike, intToBytes: Int => Array[Byte] = intToBytes) extends Serializable[S2VertexLike] {
override val table = vertex.hbaseTableName.getBytes
override val ts = vertex.ts
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
index bae7941..ddb90ac 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexDeserializable.scala
@@ -24,14 +24,14 @@ import org.apache.s2graph.core.storage.CanSKeyValue
import org.apache.s2graph.core.storage.serde.Deserializable
import org.apache.s2graph.core.storage.serde.StorageDeserializable._
import org.apache.s2graph.core.types.{HBaseType, InnerVal, InnerValLike, VertexId}
-import org.apache.s2graph.core.{S2Graph, S2Vertex}
+import org.apache.s2graph.core.{S2Graph, S2Vertex, S2VertexLike}
import scala.collection.mutable.ListBuffer
class VertexDeserializable(graph: S2Graph,
- bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2Vertex] {
+ bytesToInt: (Array[Byte], Int) => Int = bytesToInt) extends Deserializable[S2VertexLike] {
def fromKeyValues[T: CanSKeyValue](_kvs: Seq[T],
- cacheElementOpt: Option[S2Vertex]): Option[S2Vertex] = {
+ cacheElementOpt: Option[S2VertexLike]): Option[S2VertexLike] = {
try {
val kvs = _kvs.map { kv => implicitly[CanSKeyValue[T]].toSKeyValue(kv) }
val kv = kvs.head
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexSerializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexSerializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexSerializable.scala
index 59db0ab..9efd4e6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexSerializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/vertex/wide/VertexSerializable.scala
@@ -19,14 +19,14 @@
package org.apache.s2graph.core.storage.serde.vertex.wide
-import org.apache.s2graph.core.S2Vertex
+import org.apache.s2graph.core.{S2Vertex, S2VertexLike}
import org.apache.s2graph.core.storage.SKeyValue
import org.apache.s2graph.core.storage.serde.Serializable
import org.apache.s2graph.core.storage.serde.StorageSerializable._
import scala.collection.JavaConverters._
-case class VertexSerializable(vertex: S2Vertex, intToBytes: Int => Array[Byte] = intToBytes) extends Serializable[S2Vertex] {
+case class VertexSerializable(vertex: S2VertexLike, intToBytes: Int => Array[Byte] = intToBytes) extends Serializable[S2VertexLike] {
override val table = vertex.hbaseTableName.getBytes
override val ts = vertex.ts
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
index a05be79..0cd975c 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/StorageIOTest.scala
@@ -22,7 +22,7 @@ package org.apache.s2graph.core.storage
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorageSerDe
import org.apache.s2graph.core.storage.serde.{StorageDeserializable, StorageSerializable}
-import org.apache.s2graph.core.{S2Vertex, TestCommonWithModels}
+import org.apache.s2graph.core.{S2Vertex, S2VertexLike, TestCommonWithModels}
import org.scalatest.{FunSuite, Matchers}
class StorageIOTest extends FunSuite with Matchers with TestCommonWithModels {
@@ -30,9 +30,9 @@ class StorageIOTest extends FunSuite with Matchers with TestCommonWithModels {
initTests()
test("AsynchbaseStorageIO: VertexSerializer/Deserializer") {
- def check(vertex: S2Vertex,
- op: S2Vertex => StorageSerializable[S2Vertex],
- deserializer: StorageDeserializable[S2Vertex]): Boolean = {
+ def check(vertex: S2VertexLike,
+ op: S2VertexLike => StorageSerializable[S2VertexLike],
+ deserializer: StorageDeserializable[S2VertexLike]): Boolean = {
val sKeyValues = op(vertex).toKeyValues
val deserialized = deserializer.fromKeyValues(sKeyValues, None)
vertex == deserialized
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
index 52d182e..d8b2cfa 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
@@ -39,6 +39,7 @@ object S2GraphProvider {
val Implementation: Set[Class[_]] = Set(
classOf[S2Edge],
classOf[S2Vertex],
+ classOf[S2VertexLike],
classOf[S2Property[_]],
classOf[S2VertexProperty[_]],
classOf[S2Graph]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f7170235/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
index 9a45bd5..101b331 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/EdgeController.scala
@@ -52,7 +52,7 @@ object EdgeController extends Controller {
val kafkaTopic = toKafkaTopic(graphElem.isAsync)
graphElem match {
- case v: S2Vertex =>
+ case v: S2VertexLike =>
enqueue(kafkaTopic, graphElem, tsv)
case e: S2Edge =>
e.innerLabel.extraOptions.get("walLog") match {
@@ -74,7 +74,7 @@ object EdgeController extends Controller {
}
}
- private def toDeleteAllFailMessages(srcVertices: Seq[S2Vertex], labels: Seq[Label], dir: Int, ts: Long ) = {
+ private def toDeleteAllFailMessages(srcVertices: Seq[S2VertexLike], labels: Seq[Label], dir: Int, ts: Long ) = {
for {
vertex <- srcVertices
id = vertex.id.toString
@@ -268,7 +268,7 @@ object EdgeController extends Controller {
}
def deleteEach(labels: Seq[Label], direction: String, ids: Seq[JsValue],
- ts: Long, vertices: Seq[S2Vertex]) = {
+ ts: Long, vertices: Seq[S2VertexLike]) = {
val future = s2.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
if (withWait) {
[06/23] incubator-s2graph git commit: move S2Graph global helpers
into PostProcess.
Posted by st...@apache.org.
move S2Graph global helpers into PostProcess.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/42b7702e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/42b7702e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/42b7702e
Branch: refs/heads/master
Commit: 42b7702e5d32e32b8888067d21031b9eb13ce1c1
Parents: cdfa0c3
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 20:48:07 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 3 20:48:07 2017 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/PostProcess.scala | 386 +++++++++++++++++-
.../scala/org/apache/s2graph/core/S2Graph.scala | 390 +------------------
.../apache/s2graph/core/storage/StorageIO.scala | 2 +-
3 files changed, 392 insertions(+), 386 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/42b7702e/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
index 3017749..4549d84 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/PostProcess.scala
@@ -24,15 +24,19 @@ import java.util.Base64
import com.google.protobuf.ByteString
import org.apache.s2graph.core.GraphExceptions.{BadQueryException, LabelNotExistException}
import org.apache.s2graph.core.mysqls._
-import org.apache.s2graph.core.types.{InnerVal, InnerValLike, InnerValLikeWithTs}
+import org.apache.s2graph.core.types._
import org.apache.s2graph.core.JSONParser._
+import org.apache.s2graph.core.S2Graph.{FilterHashKey, HashKey}
import org.apache.s2graph.core.rest.RequestParser
import org.apache.s2graph.core.utils.logger
import play.api.libs.json.{Json, _}
+import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.collection.immutable
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import scala.concurrent.Future
+import scala.util.Random
object PostProcess {
@@ -277,4 +281,384 @@ object PostProcess {
withOptionalFields(queryOption, results.size, degrees, results, stepResult.failCount, cursorJson, nextQuery)
}
}
+
+ /** Global helper functions */
+ @tailrec
+ final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
+ if (range < sampleNumber || set.size == sampleNumber) set
+ else randomInt(sampleNumber, range, set + Random.nextInt(range))
+ }
+
+ def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
+ if (edges.size <= n) {
+ edges
+ } else {
+ val plainEdges = if (queryRequest.queryParam.offset == 0) {
+ edges.tail
+ } else edges
+
+ val randoms = randomInt(n, plainEdges.size)
+ var samples = List.empty[EdgeWithScore]
+ var idx = 0
+ plainEdges.foreach { e =>
+ if (randoms.contains(idx)) samples = e :: samples
+ idx += 1
+ }
+ samples
+ }
+ }
+
+ def normalize(edgeWithScores: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
+ val sum = edgeWithScores.foldLeft(0.0) { case (acc, cur) => acc + cur.score }
+ edgeWithScores.map { edgeWithScore =>
+ edgeWithScore.copy(score = edgeWithScore.score / sum)
+ }
+ }
+
+ def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
+ val vertices = for {
+ edgeWithScore <- edgeWithScoreLs
+ edge = edgeWithScore.edge
+ vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
+ } yield (edge.labelWithDir, vertex) -> true
+
+ vertices.toMap
+ }
+
+ /** common methods for filter out, transform, aggregate queryResult */
+ def convertEdges(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
+ for {
+ convertedEdge <- queryParam.edgeTransformer.transform(queryParam, edge, nextStepOpt) if !edge.isDegree
+ } yield convertedEdge
+ }
+
+ def processTimeDecay(queryParam: QueryParam, edge: S2EdgeLike) = {
+ /* process time decay */
+ val tsVal = queryParam.timeDecay match {
+ case None => 1.0
+ case Some(timeDecay) =>
+ val tsVal = try {
+ val innerValWithTsOpt = edge.propertyValue(timeDecay.labelMeta.name)
+ innerValWithTsOpt.map { innerValWithTs =>
+ val innerVal = innerValWithTs.innerVal
+ timeDecay.labelMeta.dataType match {
+ case InnerVal.LONG => innerVal.value match {
+ case n: BigDecimal => n.bigDecimal.longValue()
+ case _ => innerVal.toString().toLong
+ }
+ case _ => innerVal.toString().toLong
+ }
+ } getOrElse (edge.ts)
+ } catch {
+ case e: Exception =>
+ logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
+ edge.ts
+ }
+ val timeDiff = queryParam.timestamp - tsVal
+ timeDecay.decay(timeDiff)
+ }
+
+ tsVal
+ }
+
+ def processDuplicates[R](queryParam: QueryParam,
+ duplicates: Seq[(FilterHashKey, R)])(implicit ev: WithScore[R]): Seq[(FilterHashKey, R)] = {
+
+ if (queryParam.label.consistencyLevel != "strong") {
+ //TODO:
+ queryParam.duplicatePolicy match {
+ case DuplicatePolicy.First => Seq(duplicates.head)
+ case DuplicatePolicy.Raw => duplicates
+ case DuplicatePolicy.CountSum =>
+ val countSum = duplicates.size
+ val (headFilterHashKey, headEdgeWithScore) = duplicates.head
+ Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, countSum))
+ case _ =>
+ val scoreSum = duplicates.foldLeft(0.0) { case (prev, current) => prev + ev.score(current._2) }
+ val (headFilterHashKey, headEdgeWithScore) = duplicates.head
+ Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, scoreSum))
+ }
+ } else {
+ duplicates
+ }
+ }
+
+ def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
+ val src = edge.srcVertex.innerId.hashCode()
+ val tgt = edge.tgtVertex.innerId.hashCode()
+ val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
+ val filterHashKey = (src, tgt)
+
+ (hashKey, filterHashKey)
+ }
+
+ def filterEdges(q: Query,
+ stepIdx: Int,
+ queryRequests: Seq[QueryRequest],
+ queryResultLsFuture: Future[Seq[StepResult]],
+ queryParams: Seq[QueryParam],
+ alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
+ buildLastStepInnerResult: Boolean = true,
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]])
+ (implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
+
+ queryResultLsFuture.map { queryRequestWithResultLs =>
+ val (cursors, failCount) = {
+ val _cursors = ArrayBuffer.empty[Array[Byte]]
+ var _failCount = 0
+
+ queryRequestWithResultLs.foreach { stepResult =>
+ _cursors.append(stepResult.cursors: _*)
+ _failCount += stepResult.failCount
+ }
+
+ _cursors -> _failCount
+ }
+
+
+ if (queryRequestWithResultLs.isEmpty) StepResult.Empty.copy(failCount = failCount)
+ else {
+ val isLastStep = stepIdx == q.steps.size - 1
+ val queryOption = q.queryOption
+ val step = q.steps(stepIdx)
+
+ val currentStepResults = queryRequests.view.zip(queryRequestWithResultLs)
+ val shouldBuildInnerResults = !isLastStep || buildLastStepInnerResult
+ val degrees = queryRequestWithResultLs.flatMap(_.degreeEdges)
+
+ if (shouldBuildInnerResults) {
+ val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
+ edgeWithScore
+ }
+
+ /* process step group by */
+ val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
+ StepResult(edgeWithScores = results, grouped = Nil, degreeEdges = degrees, cursors = cursors, failCount = failCount)
+
+ } else {
+ val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
+ val edge = edgeWithScore.edge
+ val score = edgeWithScore.score
+ val label = edgeWithScore.label
+
+ /* Select */
+ val mergedPropsWithTs = edge.propertyValuesInner(propsSelectColumns)
+
+ // val newEdge = edge.copy(propsWithTs = mergedPropsWithTs)
+ val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
+
+ val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
+ /* OrderBy */
+ val orderByValues =
+ if (queryOption.orderByKeys.isEmpty) (score, edge.tsInnerVal, None, None)
+ else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
+
+ /* StepGroupBy */
+ val stepGroupByValues = newEdgeWithScore.toValues(step.groupBy.keys)
+
+ /* GroupBy */
+ val groupByValues = newEdgeWithScore.toValues(queryOption.groupBy.keys)
+
+ /* FilterOut */
+ val filterOutValues = newEdgeWithScore.toValues(queryOption.filterOutFields)
+
+ newEdgeWithScore.copy(orderByValues = orderByValues,
+ stepGroupByValues = stepGroupByValues,
+ groupByValues = groupByValues,
+ filterOutValues = filterOutValues)
+ }
+
+ /* process step group by */
+ val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
+
+ /* process ordered list */
+ val ordered = if (queryOption.groupBy.keys.isEmpty) StepResult.orderBy(queryOption, results) else Nil
+
+ /* process grouped list */
+ val grouped =
+ if (queryOption.groupBy.keys.isEmpty) Nil
+ else {
+ val agg = new scala.collection.mutable.HashMap[StepResult.GroupByKey, (Double, StepResult.Values)]()
+ results.groupBy { edgeWithScore =>
+ // edgeWithScore.groupByValues.map(_.map(_.toString))
+ edgeWithScore.groupByValues
+ }.foreach { case (k, ls) =>
+ val (scoreSum, merged) = StepResult.mergeOrdered(ls, Nil, queryOption)
+
+ val newScoreSum = scoreSum
+
+ /*
+ * watch out here. by calling toString on Any, we lose type information which will be used
+ * later for toJson.
+ */
+ if (merged.nonEmpty) {
+ val newKey = merged.head.groupByValues
+ agg += ((newKey, (newScoreSum, merged)))
+ }
+ }
+ agg.toSeq.sortBy(_._2._1 * -1)
+ }
+
+ StepResult(edgeWithScores = ordered, grouped = grouped, degreeEdges = degrees, cursors = cursors, failCount = failCount)
+ }
+ }
+ }
+ }
+
+ def toEdgeWithScores(queryRequest: QueryRequest,
+ stepResult: StepResult,
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]]): Seq[EdgeWithScore] = {
+ val queryOption = queryRequest.query.queryOption
+ val queryParam = queryRequest.queryParam
+ val prevScore = queryRequest.prevStepScore
+ val labelWeight = queryRequest.labelWeight
+ val edgeWithScores = stepResult.edgeWithScores
+
+ val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
+ val parents = if (shouldBuildParents) {
+ parentEdges.getOrElse(queryRequest.vertex.id, Nil).map { edgeWithScore =>
+ val edge = edgeWithScore.edge
+ val score = edgeWithScore.score
+ val label = edgeWithScore.label
+
+ /* Select */
+ val mergedPropsWithTs =
+ if (queryOption.selectColumns.isEmpty) {
+ edge.propertyValuesInner()
+ } else {
+ val initial = Map(LabelMeta.timestamp -> edge.propertyValueInner(LabelMeta.timestamp))
+ edge.propertyValues(queryOption.selectColumns) ++ initial
+ }
+
+ val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
+ edgeWithScore.copy(edge = newEdge)
+ }
+ } else Nil
+
+ // skip
+ if (queryOption.ignorePrevStepCache) stepResult.edgeWithScores
+ else {
+ val degreeScore = 0.0
+
+ val sampled =
+ if (queryRequest.queryParam.sample >= 0) sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
+ else edgeWithScores
+
+ val withScores = for {
+ edgeWithScore <- sampled
+ } yield {
+ val edge = edgeWithScore.edge
+ val edgeScore = edgeWithScore.score
+ val score = queryParam.scorePropagateOp match {
+ case "plus" => edgeScore + prevScore
+ case "divide" =>
+ if ((prevScore + queryParam.scorePropagateShrinkage) == 0) 0
+ else edgeScore / (prevScore + queryParam.scorePropagateShrinkage)
+ case _ => edgeScore * prevScore
+ }
+
+ val tsVal = processTimeDecay(queryParam, edge)
+ val newScore = degreeScore + score
+ // val newEdge = if (queryOption.returnTree) edge.copy(parentEdges = parents) else edge
+ val newEdge = edge.copyParentEdges(parents)
+ edgeWithScore.copy(edge = newEdge, score = newScore * labelWeight * tsVal)
+ }
+
+ val normalized =
+ if (queryParam.shouldNormalize) normalize(withScores)
+ else withScores
+
+ normalized
+ }
+ }
+
+ def buildResult[R](query: Query,
+ stepIdx: Int,
+ stepResultLs: Seq[(QueryRequest, StepResult)],
+ parentEdges: Map[VertexId, Seq[EdgeWithScore]])
+ (createFunc: (EdgeWithScore, Seq[LabelMeta]) => R)
+ (implicit ev: WithScore[R]): ListBuffer[R] = {
+ import scala.collection._
+
+ val results = ListBuffer.empty[R]
+ val sequentialLs: ListBuffer[(HashKey, FilterHashKey, R, QueryParam)] = ListBuffer.empty
+ val duplicates: mutable.HashMap[HashKey, ListBuffer[(FilterHashKey, R)]] = mutable.HashMap.empty
+ val edgesToExclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
+ val edgesToInclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
+
+ var numOfDuplicates = 0
+ val queryOption = query.queryOption
+ val step = query.steps(stepIdx)
+ val excludeLabelWithDirSet = step.queryParams.filter(_.exclude).map(l => l.labelWithDir).toSet
+ val includeLabelWithDirSet = step.queryParams.filter(_.include).map(l => l.labelWithDir).toSet
+
+ stepResultLs.foreach { case (queryRequest, stepInnerResult) =>
+ val queryParam = queryRequest.queryParam
+ val label = queryParam.label
+ val shouldBeExcluded = excludeLabelWithDirSet.contains(queryParam.labelWithDir)
+ val shouldBeIncluded = includeLabelWithDirSet.contains(queryParam.labelWithDir)
+
+ val propsSelectColumns = (for {
+ column <- queryOption.propsSelectColumns
+ labelMeta <- label.metaPropsInvMap.get(column)
+ } yield labelMeta)
+
+ for {
+ edgeWithScore <- toEdgeWithScores(queryRequest, stepInnerResult, parentEdges)
+ } {
+ val edge = edgeWithScore.edge
+ val (hashKey, filterHashKey) = toHashKey(queryParam, edge, isDegree = false)
+ // params += (hashKey -> queryParam) //
+
+ /* check if this edge should be exlcuded. */
+ if (shouldBeExcluded) {
+ edgesToExclude.add(filterHashKey)
+ } else {
+ if (shouldBeIncluded) {
+ edgesToInclude.add(filterHashKey)
+ }
+ val newEdgeWithScore = createFunc(edgeWithScore, propsSelectColumns)
+
+ sequentialLs += ((hashKey, filterHashKey, newEdgeWithScore, queryParam))
+ duplicates.get(hashKey) match {
+ case None =>
+ val newLs = ListBuffer.empty[(FilterHashKey, R)]
+ newLs += (filterHashKey -> newEdgeWithScore)
+ duplicates += (hashKey -> newLs) //
+ case Some(old) =>
+ numOfDuplicates += 1
+ old += (filterHashKey -> newEdgeWithScore) //
+ }
+ }
+ }
+ }
+
+
+ if (numOfDuplicates == 0) {
+ // no duplicates at all.
+ for {
+ (hashKey, filterHashKey, edgeWithScore, _) <- sequentialLs
+ if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
+ } {
+ results += edgeWithScore
+ }
+ } else {
+ // need to resolve duplicates.
+ val seen = new mutable.HashSet[HashKey]()
+ for {
+ (hashKey, filterHashKey, edgeWithScore, queryParam) <- sequentialLs
+ if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
+ if !seen.contains(hashKey)
+ } {
+ // val queryParam = params(hashKey)
+ processDuplicates(queryParam, duplicates(hashKey)).foreach { case (_, duplicate) =>
+ if (ev.score(duplicate) >= queryParam.threshold) {
+ seen += hashKey
+ results += duplicate
+ }
+ }
+ }
+ }
+ results
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/42b7702e/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 8bb95fc..82d0c6a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -22,6 +22,7 @@ package org.apache.s2graph.core
import java.util
import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
import java.util.concurrent.{Executors, TimeUnit}
+
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.configuration.{BaseConfiguration, Configuration}
import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
@@ -31,8 +32,9 @@ import org.apache.s2graph.core.index.IndexProvider
import org.apache.s2graph.core.io.tinkerpop.optimize.S2GraphStepStrategy
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
-import org.apache.s2graph.core.storage.{ MutateResponse, SKeyValue, Storage}
+import org.apache.s2graph.core.storage.{MutateResponse, SKeyValue, Storage}
import org.apache.s2graph.core.types._
+import org.apache.s2graph.core.PostProcess._
import org.apache.s2graph.core.utils.{DeferCache, Extensions, logger}
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies
@@ -42,13 +44,13 @@ import org.apache.tinkerpop.gremlin.structure.Graph.{Features, Variables}
import org.apache.tinkerpop.gremlin.structure.io.{GraphReader, GraphWriter, Io, Mapper}
import org.apache.tinkerpop.gremlin.structure.{Edge, Element, Graph, T, Transaction, Vertex}
import play.api.libs.json.{JsObject, Json}
-import scala.annotation.tailrec
+
import scala.collection.JavaConversions._
import scala.collection.mutable
-import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent._
import scala.concurrent.duration.Duration
-import scala.util.{Random, Try}
+import scala.util.Try
object S2Graph {
@@ -156,386 +158,6 @@ object S2Graph {
}
ConfigFactory.parseMap(kvs)
}
-
- /** Global helper functions */
- @tailrec
- final def randomInt(sampleNumber: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = {
- if (range < sampleNumber || set.size == sampleNumber) set
- else randomInt(sampleNumber, range, set + Random.nextInt(range))
- }
-
- def sample(queryRequest: QueryRequest, edges: Seq[EdgeWithScore], n: Int): Seq[EdgeWithScore] = {
- if (edges.size <= n) {
- edges
- } else {
- val plainEdges = if (queryRequest.queryParam.offset == 0) {
- edges.tail
- } else edges
-
- val randoms = randomInt(n, plainEdges.size)
- var samples = List.empty[EdgeWithScore]
- var idx = 0
- plainEdges.foreach { e =>
- if (randoms.contains(idx)) samples = e :: samples
- idx += 1
- }
- samples
- }
- }
-
- def normalize(edgeWithScores: Seq[EdgeWithScore]): Seq[EdgeWithScore] = {
- val sum = edgeWithScores.foldLeft(0.0) { case (acc, cur) => acc + cur.score }
- edgeWithScores.map { edgeWithScore =>
- edgeWithScore.copy(score = edgeWithScore.score / sum)
- }
- }
-
- def alreadyVisitedVertices(edgeWithScoreLs: Seq[EdgeWithScore]): Map[(LabelWithDirection, S2VertexLike), Boolean] = {
- val vertices = for {
- edgeWithScore <- edgeWithScoreLs
- edge = edgeWithScore.edge
- vertex = if (edge.labelWithDir.dir == GraphUtil.directions("out")) edge.tgtVertex else edge.srcVertex
- } yield (edge.labelWithDir, vertex) -> true
-
- vertices.toMap
- }
-
- /** common methods for filter out, transform, aggregate queryResult */
- def convertEdges(queryParam: QueryParam, edge: S2EdgeLike, nextStepOpt: Option[Step]): Seq[S2EdgeLike] = {
- for {
- convertedEdge <- queryParam.edgeTransformer.transform(queryParam, edge, nextStepOpt) if !edge.isDegree
- } yield convertedEdge
- }
-
- def processTimeDecay(queryParam: QueryParam, edge: S2EdgeLike) = {
- /* process time decay */
- val tsVal = queryParam.timeDecay match {
- case None => 1.0
- case Some(timeDecay) =>
- val tsVal = try {
- val innerValWithTsOpt = edge.propertyValue(timeDecay.labelMeta.name)
- innerValWithTsOpt.map { innerValWithTs =>
- val innerVal = innerValWithTs.innerVal
- timeDecay.labelMeta.dataType match {
- case InnerVal.LONG => innerVal.value match {
- case n: BigDecimal => n.bigDecimal.longValue()
- case _ => innerVal.toString().toLong
- }
- case _ => innerVal.toString().toLong
- }
- } getOrElse (edge.ts)
- } catch {
- case e: Exception =>
- logger.error(s"processTimeDecay error. ${edge.toLogString}", e)
- edge.ts
- }
- val timeDiff = queryParam.timestamp - tsVal
- timeDecay.decay(timeDiff)
- }
-
- tsVal
- }
-
- def processDuplicates[R](queryParam: QueryParam,
- duplicates: Seq[(FilterHashKey, R)])(implicit ev: WithScore[R]): Seq[(FilterHashKey, R)] = {
-
- if (queryParam.label.consistencyLevel != "strong") {
- //TODO:
- queryParam.duplicatePolicy match {
- case DuplicatePolicy.First => Seq(duplicates.head)
- case DuplicatePolicy.Raw => duplicates
- case DuplicatePolicy.CountSum =>
- val countSum = duplicates.size
- val (headFilterHashKey, headEdgeWithScore) = duplicates.head
- Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, countSum))
- case _ =>
- val scoreSum = duplicates.foldLeft(0.0) { case (prev, current) => prev + ev.score(current._2) }
- val (headFilterHashKey, headEdgeWithScore) = duplicates.head
- Seq(headFilterHashKey -> ev.withNewScore(headEdgeWithScore, scoreSum))
- }
- } else {
- duplicates
- }
- }
-
- def toHashKey(queryParam: QueryParam, edge: S2EdgeLike, isDegree: Boolean): (HashKey, FilterHashKey) = {
- val src = edge.srcVertex.innerId.hashCode()
- val tgt = edge.tgtVertex.innerId.hashCode()
- val hashKey = (src, edge.labelWithDir.labelId, edge.labelWithDir.dir, tgt, isDegree)
- val filterHashKey = (src, tgt)
-
- (hashKey, filterHashKey)
- }
-
- def filterEdges(q: Query,
- stepIdx: Int,
- queryRequests: Seq[QueryRequest],
- queryResultLsFuture: Future[Seq[StepResult]],
- queryParams: Seq[QueryParam],
- alreadyVisited: Map[(LabelWithDirection, S2VertexLike), Boolean] = Map.empty,
- buildLastStepInnerResult: Boolean = true,
- parentEdges: Map[VertexId, Seq[EdgeWithScore]])
- (implicit ec: scala.concurrent.ExecutionContext): Future[StepResult] = {
-
- queryResultLsFuture.map { queryRequestWithResultLs =>
- val (cursors, failCount) = {
- val _cursors = ArrayBuffer.empty[Array[Byte]]
- var _failCount = 0
-
- queryRequestWithResultLs.foreach { stepResult =>
- _cursors.append(stepResult.cursors: _*)
- _failCount += stepResult.failCount
- }
-
- _cursors -> _failCount
- }
-
-
- if (queryRequestWithResultLs.isEmpty) StepResult.Empty.copy(failCount = failCount)
- else {
- val isLastStep = stepIdx == q.steps.size - 1
- val queryOption = q.queryOption
- val step = q.steps(stepIdx)
-
- val currentStepResults = queryRequests.view.zip(queryRequestWithResultLs)
- val shouldBuildInnerResults = !isLastStep || buildLastStepInnerResult
- val degrees = queryRequestWithResultLs.flatMap(_.degreeEdges)
-
- if (shouldBuildInnerResults) {
- val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
- edgeWithScore
- }
-
- /* process step group by */
- val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
- StepResult(edgeWithScores = results, grouped = Nil, degreeEdges = degrees, cursors = cursors, failCount = failCount)
-
- } else {
- val _results = buildResult(q, stepIdx, currentStepResults, parentEdges) { (edgeWithScore, propsSelectColumns) =>
- val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val label = edgeWithScore.label
-
- /* Select */
- val mergedPropsWithTs = edge.propertyValuesInner(propsSelectColumns)
-
-// val newEdge = edge.copy(propsWithTs = mergedPropsWithTs)
- val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
-
- val newEdgeWithScore = edgeWithScore.copy(edge = newEdge)
- /* OrderBy */
- val orderByValues =
- if (queryOption.orderByKeys.isEmpty) (score, edge.tsInnerVal, None, None)
- else StepResult.toTuple4(newEdgeWithScore.toValues(queryOption.orderByKeys))
-
- /* StepGroupBy */
- val stepGroupByValues = newEdgeWithScore.toValues(step.groupBy.keys)
-
- /* GroupBy */
- val groupByValues = newEdgeWithScore.toValues(queryOption.groupBy.keys)
-
- /* FilterOut */
- val filterOutValues = newEdgeWithScore.toValues(queryOption.filterOutFields)
-
- newEdgeWithScore.copy(orderByValues = orderByValues,
- stepGroupByValues = stepGroupByValues,
- groupByValues = groupByValues,
- filterOutValues = filterOutValues)
- }
-
- /* process step group by */
- val results = StepResult.filterOutStepGroupBy(_results, step.groupBy)
-
- /* process ordered list */
- val ordered = if (queryOption.groupBy.keys.isEmpty) StepResult.orderBy(queryOption, results) else Nil
-
- /* process grouped list */
- val grouped =
- if (queryOption.groupBy.keys.isEmpty) Nil
- else {
- val agg = new scala.collection.mutable.HashMap[StepResult.GroupByKey, (Double, StepResult.Values)]()
- results.groupBy { edgeWithScore =>
- // edgeWithScore.groupByValues.map(_.map(_.toString))
- edgeWithScore.groupByValues
- }.foreach { case (k, ls) =>
- val (scoreSum, merged) = StepResult.mergeOrdered(ls, Nil, queryOption)
-
- val newScoreSum = scoreSum
-
- /*
- * watch out here. by calling toString on Any, we lose type information which will be used
- * later for toJson.
- */
- if (merged.nonEmpty) {
- val newKey = merged.head.groupByValues
- agg += ((newKey, (newScoreSum, merged)))
- }
- }
- agg.toSeq.sortBy(_._2._1 * -1)
- }
-
- StepResult(edgeWithScores = ordered, grouped = grouped, degreeEdges = degrees, cursors = cursors, failCount = failCount)
- }
- }
- }
- }
-
- private def toEdgeWithScores(queryRequest: QueryRequest,
- stepResult: StepResult,
- parentEdges: Map[VertexId, Seq[EdgeWithScore]]): Seq[EdgeWithScore] = {
- val queryOption = queryRequest.query.queryOption
- val queryParam = queryRequest.queryParam
- val prevScore = queryRequest.prevStepScore
- val labelWeight = queryRequest.labelWeight
- val edgeWithScores = stepResult.edgeWithScores
-
- val shouldBuildParents = queryOption.returnTree || queryParam.whereHasParent
- val parents = if (shouldBuildParents) {
- parentEdges.getOrElse(queryRequest.vertex.id, Nil).map { edgeWithScore =>
- val edge = edgeWithScore.edge
- val score = edgeWithScore.score
- val label = edgeWithScore.label
-
- /* Select */
- val mergedPropsWithTs =
- if (queryOption.selectColumns.isEmpty) {
- edge.propertyValuesInner()
- } else {
- val initial = Map(LabelMeta.timestamp -> edge.propertyValueInner(LabelMeta.timestamp))
- edge.propertyValues(queryOption.selectColumns) ++ initial
- }
-
- val newEdge = edge.copyEdgeWithState(mergedPropsWithTs)
- edgeWithScore.copy(edge = newEdge)
- }
- } else Nil
-
- // skip
- if (queryOption.ignorePrevStepCache) stepResult.edgeWithScores
- else {
- val degreeScore = 0.0
-
- val sampled =
- if (queryRequest.queryParam.sample >= 0) sample(queryRequest, edgeWithScores, queryRequest.queryParam.sample)
- else edgeWithScores
-
- val withScores = for {
- edgeWithScore <- sampled
- } yield {
- val edge = edgeWithScore.edge
- val edgeScore = edgeWithScore.score
- val score = queryParam.scorePropagateOp match {
- case "plus" => edgeScore + prevScore
- case "divide" =>
- if ((prevScore + queryParam.scorePropagateShrinkage) == 0) 0
- else edgeScore / (prevScore + queryParam.scorePropagateShrinkage)
- case _ => edgeScore * prevScore
- }
-
- val tsVal = processTimeDecay(queryParam, edge)
- val newScore = degreeScore + score
- // val newEdge = if (queryOption.returnTree) edge.copy(parentEdges = parents) else edge
- val newEdge = edge.copyParentEdges(parents)
- edgeWithScore.copy(edge = newEdge, score = newScore * labelWeight * tsVal)
- }
-
- val normalized =
- if (queryParam.shouldNormalize) normalize(withScores)
- else withScores
-
- normalized
- }
- }
-
- private def buildResult[R](query: Query,
- stepIdx: Int,
- stepResultLs: Seq[(QueryRequest, StepResult)],
- parentEdges: Map[VertexId, Seq[EdgeWithScore]])
- (createFunc: (EdgeWithScore, Seq[LabelMeta]) => R)
- (implicit ev: WithScore[R]): ListBuffer[R] = {
- import scala.collection._
-
- val results = ListBuffer.empty[R]
- val sequentialLs: ListBuffer[(HashKey, FilterHashKey, R, QueryParam)] = ListBuffer.empty
- val duplicates: mutable.HashMap[HashKey, ListBuffer[(FilterHashKey, R)]] = mutable.HashMap.empty
- val edgesToExclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
- val edgesToInclude: mutable.HashSet[FilterHashKey] = mutable.HashSet.empty
-
- var numOfDuplicates = 0
- val queryOption = query.queryOption
- val step = query.steps(stepIdx)
- val excludeLabelWithDirSet = step.queryParams.filter(_.exclude).map(l => l.labelWithDir).toSet
- val includeLabelWithDirSet = step.queryParams.filter(_.include).map(l => l.labelWithDir).toSet
-
- stepResultLs.foreach { case (queryRequest, stepInnerResult) =>
- val queryParam = queryRequest.queryParam
- val label = queryParam.label
- val shouldBeExcluded = excludeLabelWithDirSet.contains(queryParam.labelWithDir)
- val shouldBeIncluded = includeLabelWithDirSet.contains(queryParam.labelWithDir)
-
- val propsSelectColumns = (for {
- column <- queryOption.propsSelectColumns
- labelMeta <- label.metaPropsInvMap.get(column)
- } yield labelMeta)
-
- for {
- edgeWithScore <- toEdgeWithScores(queryRequest, stepInnerResult, parentEdges)
- } {
- val edge = edgeWithScore.edge
- val (hashKey, filterHashKey) = toHashKey(queryParam, edge, isDegree = false)
- // params += (hashKey -> queryParam) //
-
- /* check if this edge should be exlcuded. */
- if (shouldBeExcluded) {
- edgesToExclude.add(filterHashKey)
- } else {
- if (shouldBeIncluded) {
- edgesToInclude.add(filterHashKey)
- }
- val newEdgeWithScore = createFunc(edgeWithScore, propsSelectColumns)
-
- sequentialLs += ((hashKey, filterHashKey, newEdgeWithScore, queryParam))
- duplicates.get(hashKey) match {
- case None =>
- val newLs = ListBuffer.empty[(FilterHashKey, R)]
- newLs += (filterHashKey -> newEdgeWithScore)
- duplicates += (hashKey -> newLs) //
- case Some(old) =>
- numOfDuplicates += 1
- old += (filterHashKey -> newEdgeWithScore) //
- }
- }
- }
- }
-
-
- if (numOfDuplicates == 0) {
- // no duplicates at all.
- for {
- (hashKey, filterHashKey, edgeWithScore, _) <- sequentialLs
- if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
- } {
- results += edgeWithScore
- }
- } else {
- // need to resolve duplicates.
- val seen = new mutable.HashSet[HashKey]()
- for {
- (hashKey, filterHashKey, edgeWithScore, queryParam) <- sequentialLs
- if !edgesToExclude.contains(filterHashKey) || edgesToInclude.contains(filterHashKey)
- if !seen.contains(hashKey)
- } {
- // val queryParam = params(hashKey)
- processDuplicates(queryParam, duplicates(hashKey)).foreach { case (_, duplicate) =>
- if (ev.score(duplicate) >= queryParam.threshold) {
- seen += hashKey
- results += duplicate
- }
- }
- }
- }
- results
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/42b7702e/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index d0a59b2..3074b4e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.storage
import org.apache.hadoop.hbase.util.Bytes
-import org.apache.s2graph.core.S2Graph.{convertEdges, normalize, processTimeDecay, sample}
+import org.apache.s2graph.core.PostProcess._
import org.apache.s2graph.core._
import org.apache.s2graph.core.mysqls.LabelMeta
import org.apache.s2graph.core.parsers.WhereParser
[09/23] incubator-s2graph git commit: apply MutationHelper and
GraphElementBuilder.
Posted by st...@apache.org.
apply MutationHelper and GraphElementBuilder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/aa66822b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/aa66822b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/aa66822b
Branch: refs/heads/master
Commit: aa66822b08d0045e3870af2a9b82523947f553ce
Parents: 937b55a
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 3 21:41:57 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 4 07:01:07 2017 +0900
----------------------------------------------------------------------
.../loader/subscriber/GraphSubscriber.scala | 2 +-
.../loader/subscriber/TransferToHFile.scala | 10 +-
.../s2graph/loader/subscriber/WalLogStat.scala | 3 +-
.../loader/subscriber/WalLogToHDFS.scala | 3 +-
.../scala/org/apache/s2graph/core/S2Graph.scala | 718 ++-----------------
.../org/apache/s2graph/core/S2GraphLike.scala | 238 ++++++
.../s2graph/core/features/S2Features.scala | 19 +
.../apache/s2graph/core/storage/Storage.scala | 112 +--
.../hbase/AsynchbaseStorageReadable.scala | 2 +-
.../tall/SnapshotEdgeDeserializable.scala | 2 +-
.../wide/SnapshotEdgeDeserializable.scala | 2 +-
.../core/storage/hbase/IndexEdgeTest.scala | 4 +-
12 files changed, 386 insertions(+), 729 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
index 2352cdf..a371b6b 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
@@ -106,7 +106,7 @@ object GraphSubscriberHelper extends WithKafka {
(statFunc: (String, Int) => Unit): Iterable[GraphElement] = {
(for (msg <- msgs) yield {
statFunc("total", 1)
- g.toGraphElement(msg, labelMapping) match {
+ g.elementBuilder.toGraphElement(msg, labelMapping) match {
case Some(e) if e.isInstanceOf[S2Edge] =>
statFunc("EdgeParseOk", 1)
e.asInstanceOf[S2Edge]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
index 618c1bd..a7b4e00 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
@@ -84,11 +84,11 @@ object TransferToHFile extends SparkApp {
} yield output
}
def buildPutRequests(snapshotEdge: SnapshotEdge): List[PutRequest] = {
- val kvs = GraphSubscriberHelper.g.getStorage(snapshotEdge.label).snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
+ val kvs = GraphSubscriberHelper.g.getStorage(snapshotEdge.label).serDe.snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
}
def buildPutRequests(indexEdge: IndexEdge): List[PutRequest] = {
- val kvs = GraphSubscriberHelper.g.getStorage(indexEdge.label).indexEdgeSerializer(indexEdge).toKeyValues.toList
+ val kvs = GraphSubscriberHelper.g.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.toList
kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
}
def buildDegreePutRequests(vertexId: String, labelName: String, direction: String, degreeVal: Long): List[PutRequest] = {
@@ -104,7 +104,7 @@ object TransferToHFile extends SparkApp {
val edge = GraphSubscriberHelper.g.newEdge(vertex, vertex, label, dir, propsWithTs=propsWithTs)
edge.edgesWithIndex.flatMap { indexEdge =>
- GraphSubscriberHelper.g.getStorage(indexEdge.label).indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
+ GraphSubscriberHelper.g.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
new PutRequest(kv.table, kv.row, kv.cf, Array.empty[Byte], Bytes.toBytes(degreeVal), kv.timestamp)
}
}
@@ -125,7 +125,7 @@ object TransferToHFile extends SparkApp {
def toKeyValues(strs: Seq[String], labelMapping: Map[String, String], autoEdgeCreate: Boolean): Iterator[KeyValue] = {
val kvList = new java.util.ArrayList[KeyValue]
for (s <- strs) {
- val elementList = GraphSubscriberHelper.g.toGraphElement(s, labelMapping).toSeq
+ val elementList = GraphSubscriberHelper.g.elementBuilder.toGraphElement(s, labelMapping).toSeq
for (element <- elementList) {
if (element.isInstanceOf[S2Edge]) {
val edge = element.asInstanceOf[S2Edge]
@@ -136,7 +136,7 @@ object TransferToHFile extends SparkApp {
}
} else if (element.isInstanceOf[S2Vertex]) {
val vertex = element.asInstanceOf[S2Vertex]
- val putRequestList = GraphSubscriberHelper.g.getStorage(vertex.service).vertexSerializer(vertex).toKeyValues.map { kv =>
+ val putRequestList = GraphSubscriberHelper.g.getStorage(vertex.service).serDe.vertexSerializer(vertex).toKeyValues.map { kv =>
new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp)
}
for (p <- putRequestList) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
index 40f936d..eca77f9 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogStat.scala
@@ -21,7 +21,6 @@ package org.apache.s2graph.loader.subscriber
import kafka.producer.KeyedMessage
import kafka.serializer.StringDecoder
-import org.apache.s2graph.core.S2Graph$
import org.apache.s2graph.spark.spark.{WithKafka, SparkApp}
import org.apache.spark.streaming.Durations._
import org.apache.spark.streaming.kafka.HasOffsetRanges
@@ -69,7 +68,7 @@ object WalLogStat extends SparkApp with WithKafka {
val phase = System.getProperty("phase")
GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
partition.map { case (key, msg) =>
- GraphSubscriberHelper.g.toGraphElement(msg) match {
+ GraphSubscriberHelper.g.elementBuilder.toGraphElement(msg) match {
case Some(elem) =>
val serviceName = elem.serviceName
msg.split("\t", 7) match {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
index 23c3cda..605a994 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/WalLogToHDFS.scala
@@ -22,7 +22,6 @@ package org.apache.s2graph.loader.subscriber
import java.text.SimpleDateFormat
import java.util.Date
import kafka.serializer.StringDecoder
-import org.apache.s2graph.core.S2Graph$
import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.streaming.Durations._
@@ -92,7 +91,7 @@ object WalLogToHDFS extends SparkApp with WithKafka {
GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
partition.flatMap { case (key, msg) =>
- val optMsg = GraphSubscriberHelper.g.toGraphElement(msg).flatMap { element =>
+ val optMsg = GraphSubscriberHelper.g.elementBuilder.toGraphElement(msg).flatMap { element =>
val arr = msg.split("\t", 7)
val service = element.serviceName
val label = arr(5)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 82d0c6a..5e23f9b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -483,11 +483,11 @@ object S2Graph {
new Graph.OptOut(test="org.apache.tinkerpop.gremlin.structure.io.IoTest", method="*", reason="no")
// all failed.
))
-class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph {
+class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2GraphLike {
import S2Graph._
- private var apacheConfiguration: Configuration = _
+ var apacheConfiguration: Configuration = _
def dbSession() = scalikejdbc.AutoSession
@@ -575,6 +575,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
val indexProvider = IndexProvider.apply(config)
+ val elementBuilder = new GraphElementBuilder(this)
+
def getStorage(service: Service): Storage = {
storagePool.getOrElse(s"service:${service.serviceName}", defaultStorage)
}
@@ -639,8 +641,6 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
Try {
if (q.steps.isEmpty) fallback
else {
-
- val queryOption = q.queryOption
def fetch: Future[StepResult] = {
val startStepInnerResult = QueryResult.fromVertices(this, q)
q.steps.zipWithIndex.foldLeft(Future.successful(startStepInnerResult)) { case (prevStepInnerResultFuture, (step, stepIdx)) =>
@@ -795,7 +795,6 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
Query(vertices, Vector(step))
}
- // Extensions.retryOnSuccessWithBackoff(MaxRetryNum, Random.nextInt(MaxBackOff) + 1) {
val retryFuture = Extensions.retryOnSuccess(DeleteAllFetchCount) {
fetchAndDeleteAll(queries, requestTs)
} { case (allDeleted, deleteSuccess) =>
@@ -832,16 +831,19 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
def deleteAllFetchedEdgesLs(stepInnerResultLs: Seq[StepResult],
requestTs: Long): Future[(Boolean, Boolean)] = {
stepInnerResultLs.foreach { stepInnerResult =>
- logger.error(s"[!!!!!!]: ${stepInnerResult.edgeWithScores.size}")
if (stepInnerResult.isFailure) throw new RuntimeException("fetched result is fallback.")
}
val futures = for {
stepInnerResult <- stepInnerResultLs
- deleteStepInnerResult = buildEdgesToDelete(stepInnerResult, requestTs)
- if deleteStepInnerResult.edgeWithScores.nonEmpty
+ filtered = stepInnerResult.edgeWithScores.filter { edgeWithScore =>
+ (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
+ }
+ edgesToDelete = elementBuilder.buildEdgesToDelete(filtered, requestTs)
+ if edgesToDelete.nonEmpty
} yield {
- val head = deleteStepInnerResult.edgeWithScores.head
+ val head = edgesToDelete.head
val label = head.edge.innerLabel
+ val stepResult = StepResult(edgesToDelete, Nil, Nil, false)
val ret = label.schemaVersion match {
case HBaseType.VERSION3 | HBaseType.VERSION4 =>
if (label.consistencyLevel == "strong") {
@@ -849,9 +851,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
* read: snapshotEdge on queryResult = O(N)
* write: N x (relatedEdges x indices(indexedEdge) + 1(snapshotEdge))
*/
- mutateEdges(deleteStepInnerResult.edgeWithScores.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
+ mutateEdges(edgesToDelete.map(_.edge), withWait = true).map(_.forall(_.isSuccess))
} else {
- deleteAllFetchedEdgesAsyncOld(getStorage(label))(deleteStepInnerResult, requestTs, MaxRetryNum)
+ getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
case _ =>
@@ -859,7 +861,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
* read: x
* write: N x ((1(snapshotEdge) + 2(1 for incr, 1 for delete) x indices)
*/
- deleteAllFetchedEdgesAsyncOld(getStorage(label))(deleteStepInnerResult, requestTs, MaxRetryNum)
+ getStorage(label).deleteAllFetchedEdgesAsyncOld(stepResult, requestTs, MaxRetryNum)
}
ret
}
@@ -872,71 +874,6 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- private def deleteAllFetchedEdgesAsyncOld(storage: Storage)(stepInnerResult: StepResult,
- requestTs: Long,
- retryNum: Int): Future[Boolean] = {
- if (stepInnerResult.isEmpty) Future.successful(true)
- else {
- val head = stepInnerResult.edgeWithScores.head
- val zkQuorum = head.edge.innerLabel.hbaseZkAddr
- val futures = for {
- edgeWithScore <- stepInnerResult.edgeWithScores
- } yield {
- val edge = edgeWithScore.edge
- val score = edgeWithScore.score
-
- val edgeSnapshot = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedSnapshotEdgeMutations = storage.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
-
- val edgeForward = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
- val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
- storage.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- storage.buildIncrementsAsync(indexEdge, -1L)
- }
-
- /* reverted direction */
- val edgeRevert = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
- val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
- storage.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
- storage.buildIncrementsAsync(indexEdge, -1L)
- }
-
- val mutations = reversedIndexedEdgesMutations ++ reversedSnapshotEdgeMutations ++ forwardIndexedEdgeMutations
-
- storage.writeToStorage(zkQuorum, mutations, withWait = true)
- }
-
- Future.sequence(futures).map { rets => rets.forall(_.isSuccess) }
- }
- }
-
- def buildEdgesToDelete(stepInnerResult: StepResult, requestTs: Long): StepResult = {
- val filtered = stepInnerResult.edgeWithScores.filter { edgeWithScore =>
- (edgeWithScore.edge.ts < requestTs) && !edgeWithScore.edge.isDegree
- }
- if (filtered.isEmpty) StepResult.Empty
- else {
- val head = filtered.head
- val label = head.edge.innerLabel
- val edgeWithScoreLs = filtered.map { edgeWithScore =>
- val edge = edgeWithScore.edge
- val copiedEdge = label.consistencyLevel match {
- case "strong" =>
- edge.copyEdge(op = GraphUtil.operations("delete"),
- version = requestTs, propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
- case _ =>
- edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
- }
-
- val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
- // logger.debug(s"delete edge from deleteAll: ${edgeToDelete.edge.toLogString}")
- edgeToDelete
- }
- //Degree edge?
- StepResult(edgeWithScores = edgeWithScoreLs, grouped = Nil, degreeEdges = Nil, false)
- }
- }
-
def mutateElements(elements: Seq[GraphElement],
withWait: Boolean = false): Future[Seq[MutateResponse]] = {
@@ -967,8 +904,6 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
- // def mutateEdges(edges: Seq[Edge], withWait: Boolean = false): Future[Seq[Boolean]] = storage.mutateEdges(edges, withWait)
-
def mutateEdges(edges: Seq[S2EdgeLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
val edgeWithIdxs = edges.zipWithIndex
@@ -985,20 +920,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
val idxs = edgeGroup.map(_._2)
/* multiple edges with weak consistency level will be processed as batch */
- val mutations = edges.flatMap { edge =>
- val (_, edgeUpdate) =
- if (edge.getOp() == GraphUtil.operations("delete")) S2Edge.buildDeleteBulk(None, edge)
- else S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = storage.increments(edgeUpdate.deepCopy)
-
- if (bufferIncr.nonEmpty) storage.writeToStorage(zkQuorum, bufferIncr, withWait = false)
- storage.buildVertexPutsAsync(edge) ++ storage.indexedEdgeMutations(edgeUpdate.deepCopy) ++ storage.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
- }
-
- storage.writeToStorage(zkQuorum, mutations, withWait).map { ret =>
- idxs.map(idx => idx -> ret.isSuccess)
- }
+ storage.mutateWeakEdges(zkQuorum, edges, withWait)
}
Future.sequence(futures)
}
@@ -1012,7 +934,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
val edges = edgeGroup.map(_._1)
val idxs = edgeGroup.map(_._2)
val storage = getStorage(label)
- mutateStrongEdges(storage)(edges, withWait = true).map { rets =>
+ val zkQuorum = label.hbaseZkAddr
+ storage.mutateStrongEdges(zkQuorum, edges, withWait = true).map { rets =>
idxs.zip(rets)
}
}
@@ -1026,119 +949,24 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
}
- private def mutateStrongEdges(storage: Storage)(_edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[Boolean]] = {
-
- val edgeWithIdxs = _edges.zipWithIndex
- val grouped = edgeWithIdxs.groupBy { case (edge, idx) =>
- (edge.innerLabel, edge.srcVertex.innerId, edge.tgtVertex.innerId)
- } toSeq
-
- val mutateEdges = grouped.map { case ((_, _, _), edgeGroup) =>
- val edges = edgeGroup.map(_._1)
- val idxs = edgeGroup.map(_._2)
- // After deleteAll, process others
- val mutateEdgeFutures = edges.toList match {
- case head :: tail =>
- val edgeFuture = mutateEdgesInner(storage)(edges, checkConsistency = true , withWait)
-
- //TODO: decide what we will do on failure on vertex put
- val puts = storage.buildVertexPutsAsync(head)
- val vertexFuture = storage.writeToStorage(head.innerLabel.hbaseZkAddr, puts, withWait)
- Seq(edgeFuture, vertexFuture)
- case Nil => Nil
- }
-
- val composed = for {
- // deleteRet <- Future.sequence(deleteAllFutures)
- mutateRet <- Future.sequence(mutateEdgeFutures)
- } yield mutateRet
-
- composed.map(_.forall(_.isSuccess)).map { ret => idxs.map( idx => idx -> ret) }
- }
-
- Future.sequence(mutateEdges).map { squashedRets =>
- squashedRets.flatten.sortBy { case (idx, ret) => idx }.map(_._2)
- }
- }
-
-
- private def mutateEdgesInner(storage: Storage)(edges: Seq[S2EdgeLike],
- checkConsistency: Boolean,
- withWait: Boolean): Future[MutateResponse] = {
- assert(edges.nonEmpty)
- // TODO:: remove after code review: unreachable code
- if (!checkConsistency) {
-
- val zkQuorum = edges.head.innerLabel.hbaseZkAddr
- val futures = edges.map { edge =>
- val (_, edgeUpdate) = S2Edge.buildOperation(None, Seq(edge))
-
- val (bufferIncr, nonBufferIncr) = storage.increments(edgeUpdate.deepCopy)
- val mutations =
- storage.indexedEdgeMutations(edgeUpdate.deepCopy) ++ storage.snapshotEdgeMutations(edgeUpdate.deepCopy) ++ nonBufferIncr
-
- if (bufferIncr.nonEmpty) storage.writeToStorage(zkQuorum, bufferIncr, withWait = false)
-
- storage.writeToStorage(zkQuorum, mutations, withWait)
- }
- Future.sequence(futures).map { rets => new MutateResponse(rets.forall(_.isSuccess)) }
- } else {
- storage.fetchSnapshotEdgeInner(edges.head).flatMap { case (snapshotEdgeOpt, kvOpt) =>
- storage.retry(1)(edges, 0, snapshotEdgeOpt).map(new MutateResponse(_))
- }
- }
- }
-
def mutateVertices(vertices: Seq[S2VertexLike], withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- def mutateVertex(storage: Storage)(vertex: S2VertexLike, withWait: Boolean): Future[MutateResponse] = {
- if (vertex.op == GraphUtil.operations("delete")) {
- storage.writeToStorage(vertex.hbaseZkAddr,
- storage.vertexSerializer(vertex).toKeyValues.map(_.copy(operation = SKeyValue.Delete)), withWait)
- } else if (vertex.op == GraphUtil.operations("deleteAll")) {
- logger.info(s"deleteAll for vertex is truncated. $vertex")
- Future.successful(MutateResponse.Success) // Ignore withWait parameter, because deleteAll operation may takes long time
- } else {
- storage.writeToStorage(vertex.hbaseZkAddr, storage.buildPutsAll(vertex), withWait)
- }
- }
-
- def mutateVertices(storage: Storage)(vertices: Seq[S2VertexLike],
- withWait: Boolean = false): Future[Seq[MutateResponse]] = {
- val futures = vertices.map { vertex => mutateVertex(storage)(vertex, withWait) }
+ def mutateVertices(storage: Storage)(zkQuorum: String, vertices: Seq[S2VertexLike],
+ withWait: Boolean = false): Future[Seq[MutateResponse]] = {
+ val futures = vertices.map { vertex => storage.mutateVertex(zkQuorum, vertex, withWait) }
Future.sequence(futures)
}
val verticesWithIdx = vertices.zipWithIndex
val futures = verticesWithIdx.groupBy { case (v, idx) => v.service }.map { case (service, vertexGroup) =>
- mutateVertices(getStorage(service))(vertexGroup.map(_._1), withWait).map(_.zip(vertexGroup.map(_._2)))
+ mutateVertices(getStorage(service))(service.cluster, vertexGroup.map(_._1), withWait).map(_.zip(vertexGroup.map(_._2)))
}
Future.sequence(futures).map { ls => ls.flatten.toSeq.sortBy(_._2).map(_._1) }
}
-
-
def incrementCounts(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
- def incrementCounts(storage: Storage)(edges: Seq[S2EdgeLike], withWait: Boolean): Future[Seq[MutateResponse]] = {
- val futures = for {
- edge <- edges
- } yield {
- val kvs = for {
- relEdge <- edge.relatedEdges
- edgeWithIndex <- EdgeMutate.filterIndexOption(relEdge.edgesWithIndexValid)
- } yield {
- val countWithTs = edge.propertyValueInner(LabelMeta.count)
- val countVal = countWithTs.innerVal.toString().toLong
- storage.buildIncrementsCountAsync(edgeWithIndex, countVal).head
- }
- storage.writeToStorage(edge.innerLabel.hbaseZkAddr, kvs, withWait = withWait)
- }
-
- Future.sequence(futures)
- }
-
val edgesWithIdx = edges.zipWithIndex
val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
- incrementCounts(getStorage(label))(edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
+ getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
}
Future.sequence(futures).map { ls =>
ls.flatten.toSeq.sortBy(_._2).map(_._1)
@@ -1147,11 +975,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
def updateDegree(edge: S2EdgeLike, degreeVal: Long = 0): Future[MutateResponse] = {
val label = edge.innerLabel
-
val storage = getStorage(label)
- val kvs = storage.buildDegreePuts(edge, degreeVal)
- storage.writeToStorage(edge.innerLabel.service.cluster, kvs, withWait = true)
+ storage.updateDegree(label.hbaseZkAddr, edge, degreeVal)
}
def isRunning(): Boolean = running.get()
@@ -1164,166 +990,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
localLongId.set(0l)
}
- def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] = Try {
- val parts = GraphUtil.split(s)
- val logType = parts(2)
- val element = if (logType == "edge" | logType == "e") {
- /* current only edge is considered to be bulk loaded */
- labelMapping.get(parts(5)) match {
- case None =>
- case Some(toReplace) =>
- parts(5) = toReplace
- }
- toEdge(parts)
- } else if (logType == "vertex" | logType == "v") {
- toVertex(parts)
- } else {
- throw new GraphExceptions.JsonParseException("log type is not exist in log.")
- }
-
- element
- } recover {
- case e: Exception =>
- logger.error(s"[toElement]: $e", e)
- None
- } get
-
-
- def toVertex(s: String): Option[S2VertexLike] = {
- toVertex(GraphUtil.split(s))
- }
-
- def toEdge(s: String): Option[S2EdgeLike] = {
- toEdge(GraphUtil.split(s))
- }
-
- def toEdge(parts: Array[String]): Option[S2EdgeLike] = Try {
- val (ts, operation, logType, srcId, tgtId, label) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
- val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
- val tempDirection = if (parts.length >= 8) parts(7) else "out"
- val direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
- val edge = toEdge(srcId, tgtId, label, direction, props, ts.toLong, operation)
- Option(edge)
- } recover {
- case e: Exception =>
- logger.error(s"[toEdge]: $e", e)
- throw e
- } get
-
- def toVertex(parts: Array[String]): Option[S2VertexLike] = Try {
- val (ts, operation, logType, srcId, serviceName, colName) = (parts(0), parts(1), parts(2), parts(3), parts(4), parts(5))
- val props = if (parts.length >= 7) fromJsonToProperties(Json.parse(parts(6)).asOpt[JsObject].getOrElse(Json.obj())) else Map.empty[String, Any]
- val vertex = toVertex(serviceName, colName, srcId, props, ts.toLong, operation)
- Option(vertex)
- } recover {
- case e: Throwable =>
- logger.error(s"[toVertex]: $e", e)
- throw e
- } get
-
- def toEdge(srcId: Any,
- tgtId: Any,
- labelName: String,
- direction: String,
- props: Map[String, Any] = Map.empty,
- ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2EdgeLike = {
- val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
-
- val srcColumn = if (direction == "out") label.srcColumn else label.tgtColumn
- val tgtColumn = if (direction == "out") label.tgtColumn else label.srcColumn
-
- val srcVertexIdInnerVal = toInnerVal(srcId, srcColumn.columnType, label.schemaVersion)
- val tgtVertexIdInnerVal = toInnerVal(tgtId, tgtColumn.columnType, label.schemaVersion)
-
- val srcVertex = newVertex(SourceVertexId(srcColumn, srcVertexIdInnerVal), System.currentTimeMillis())
- val tgtVertex = newVertex(TargetVertexId(tgtColumn, tgtVertexIdInnerVal), System.currentTimeMillis())
- val dir = GraphUtil.toDir(direction).getOrElse(throw new RuntimeException(s"$direction is not supported."))
-
- val propsPlusTs = props ++ Map(LabelMeta.timestamp.name -> ts)
- val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts)
- val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
-
- new S2Edge(this, srcVertex, tgtVertex, label, dir, op = op, version = ts).copyEdgeWithState(propsWithTs)
- }
-
- def toVertex(serviceName: String,
- columnName: String,
- id: Any,
- props: Map[String, Any] = Map.empty,
- ts: Long = System.currentTimeMillis(),
- operation: String = "insert"): S2VertexLike = {
-
- val service = Service.findByName(serviceName).getOrElse(throw new java.lang.IllegalArgumentException(s"$serviceName is not found."))
- val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new java.lang.IllegalArgumentException(s"$columnName is not found."))
- val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
-
- val srcVertexId = id match {
- case vid: VertexId => id.asInstanceOf[VertexId]
- case _ => VertexId(column, toInnerVal(id, column.columnType, column.schemaVersion))
- }
-
- val propsInner = column.propsToInnerVals(props) ++
- Map(ColumnMeta.timestamp -> InnerVal.withLong(ts, column.schemaVersion))
-
- val vertex = new S2Vertex(this, srcVertexId, ts, S2Vertex.EmptyProps, op)
- S2Vertex.fillPropsWithTs(vertex, propsInner)
- vertex
- }
-
- def toRequestEdge(queryRequest: QueryRequest, parentEdges: Seq[EdgeWithScore]): S2EdgeLike = {
- val srcVertex = queryRequest.vertex
- val queryParam = queryRequest.queryParam
- val tgtVertexIdOpt = queryParam.tgtVertexInnerIdOpt
- val label = queryParam.label
- val labelWithDir = queryParam.labelWithDir
- val (srcColumn, tgtColumn) = label.srcTgtColumn(labelWithDir.dir)
- val propsWithTs = label.EmptyPropsWithTs
-
- tgtVertexIdOpt match {
- case Some(tgtVertexId) => // _to is given.
- /* we use toSnapshotEdge so dont need to swap src, tgt */
- val src = srcVertex.innerId
- val tgt = tgtVertexId
- val (srcVId, tgtVId) = (SourceVertexId(srcColumn, src), TargetVertexId(tgtColumn, tgt))
- val (srcV, tgtV) = (newVertex(srcVId), newVertex(tgtVId))
-
- newEdge(srcV, tgtV, label, labelWithDir.dir, propsWithTs = propsWithTs)
- case None =>
- val src = srcVertex.innerId
- val srcVId = SourceVertexId(srcColumn, src)
- val srcV = newVertex(srcVId)
-
- newEdge(srcV, srcV, label, labelWithDir.dir, propsWithTs = propsWithTs, parentEdges = parentEdges)
- }
- }
- /**
- * helper to create new Edge instance from given parameters on memory(not actually stored in storage).
- *
- * Since we are using mutable map for property value(propsWithTs),
- * we should make sure that reference for mutable map never be shared between multiple Edge instances.
- * To guarantee this, we never create Edge directly, but rather use this helper which is available on S2Graph.
- *
- * Note that we are using following convention
- * 1. `add*` for method that actually store instance into storage,
- * 2. `new*` for method that only create instance on memory, but not store it into storage.
- *
- * @param srcVertex
- * @param tgtVertex
- * @param innerLabel
- * @param dir
- * @param op
- * @param version
- * @param propsWithTs
- * @param parentEdges
- * @param originalEdgeOpt
- * @param pendingEdgeOpt
- * @param statusCode
- * @param lockTs
- * @param tsInnerValOpt
- * @return
- */
def newEdge(srcVertex: S2VertexLike,
tgtVertex: S2VertexLike,
innerLabel: Label,
@@ -1336,89 +1003,22 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
pendingEdgeOpt: Option[S2EdgeLike] = None,
statusCode: Byte = 0,
lockTs: Option[Long] = None,
- tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike = {
- val edge = S2Edge(
- this,
- srcVertex,
- tgtVertex,
- innerLabel,
- dir,
- op,
- version,
- S2Edge.EmptyProps,
- parentEdges,
- originalEdgeOpt,
- pendingEdgeOpt,
- statusCode,
- lockTs,
- tsInnerValOpt)
- S2Edge.fillPropsWithTs(edge, propsWithTs)
- edge
- }
-
- /**
- * helper to create new SnapshotEdge instance from given parameters on memory(not actually stored in storage).
- *
- * Note that this is only available to S2Graph, not structure.Graph so only internal code should use this method.
- * @param srcVertex
- * @param tgtVertex
- * @param label
- * @param dir
- * @param op
- * @param version
- * @param propsWithTs
- * @param pendingEdgeOpt
- * @param statusCode
- * @param lockTs
- * @param tsInnerValOpt
- * @return
- */
- private[core] def newSnapshotEdge(srcVertex: S2VertexLike,
- tgtVertex: S2VertexLike,
- label: Label,
- dir: Int,
- op: Byte,
- version: Long,
- propsWithTs: S2Edge.State,
- pendingEdgeOpt: Option[S2EdgeLike],
- statusCode: Byte = 0,
- lockTs: Option[Long],
- tsInnerValOpt: Option[InnerValLike] = None): SnapshotEdge = {
- val snapshotEdge = new SnapshotEdge(this, srcVertex, tgtVertex, label, dir, op, version, S2Edge.EmptyProps,
- pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
- S2Edge.fillPropsWithTs(snapshotEdge, propsWithTs)
- snapshotEdge
- }
-
- def newVertexId(serviceName: String)(columnName: String)(id: Any): VertexId = {
- val service = Service.findByName(serviceName).getOrElse(throw new RuntimeException(s"$serviceName is not found."))
- val column = ServiceColumn.find(service.id.get, columnName).getOrElse(throw new RuntimeException(s"$columnName is not found."))
- newVertexId(service, column, id)
- }
+ tsInnerValOpt: Option[InnerValLike] = None): S2EdgeLike =
+ elementBuilder.newEdge(srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs,
+ parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
- /**
- * helper to create S2Graph's internal VertexId instance with given parameters.
- * @param service
- * @param column
- * @param id
- * @return
- */
def newVertexId(service: Service,
column: ServiceColumn,
- id: Any): VertexId = {
- val innerVal = CanInnerValLike.anyToInnerValLike.toInnerVal(id)(column.schemaVersion)
- new VertexId(column, innerVal)
- }
+ id: Any): VertexId =
+ elementBuilder.newVertexId(service, column, id)
def newVertex(id: VertexId,
ts: Long = System.currentTimeMillis(),
props: S2Vertex.Props = S2Vertex.EmptyProps,
op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
- val vertex = new S2Vertex(this, id, ts, S2Vertex.EmptyProps, op, belongLabelIds)
- S2Vertex.fillPropsWithTs(vertex, props)
- vertex
- }
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike =
+ elementBuilder.newVertex(id, ts, props, op, belongLabelIds)
+
def getVertex(vertexId: VertexId): Option[S2VertexLike] = {
val v = newVertex(vertexId)
@@ -1441,248 +1041,26 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
stepResultLs.foreach(_.edgeWithScores.foreach(es => ls.add(es.edge)))
ls.iterator()
}
-// getEdges(query).map { stepResult =>
-// val ls = new util.ArrayList[Edge]()
-// stepResult.edgeWithScores.foreach(es => ls.add(es.edge))
-// ls.iterator()
-// }
- }
-
- /**
- * used by graph.traversal().V()
- * @param ids: array of VertexId values. note that last parameter can be used to control if actually fetch vertices from storage or not.
- * since S2Graph use user-provided id as part of edge, it is possible to
- * fetch edge without fetch start vertex. default is false which means we are not fetching vertices from storage.
- * @return
- */
- override def vertices(ids: AnyRef*): util.Iterator[structure.Vertex] = {
- val fetchVertices = ids.lastOption.map { lastParam =>
- if (lastParam.isInstanceOf[Boolean]) lastParam.asInstanceOf[Boolean]
- else true
- }.getOrElse(true)
-
- if (ids.isEmpty) {
- //TODO: default storage need to be fixed.
- Await.result(defaultStorage.fetchVerticesAll(), WaitTimeout).iterator
- } else {
- val vertices = ids.collect {
- case s2Vertex: S2VertexLike => s2Vertex
- case vId: VertexId => newVertex(vId)
- case vertex: Vertex => newVertex(vertex.id().asInstanceOf[VertexId])
- case other @ _ => newVertex(VertexId.fromString(other.toString))
- }
-
- if (fetchVertices) {
- val future = getVertices(vertices).map { vs =>
- val ls = new util.ArrayList[structure.Vertex]()
- ls.addAll(vs)
- ls.iterator()
- }
- Await.result(future, WaitTimeout)
- } else {
- vertices.iterator
- }
- }
- }
-
- override def edges(edgeIds: AnyRef*): util.Iterator[structure.Edge] = {
- if (edgeIds.isEmpty) {
- // FIXME
- Await.result(defaultStorage.fetchEdgesAll(), WaitTimeout).iterator
- } else {
- Await.result(edgesAsync(edgeIds: _*), WaitTimeout)
- }
- }
-
- def edgesAsync(edgeIds: AnyRef*): Future[util.Iterator[structure.Edge]] = {
- val s2EdgeIds = edgeIds.collect {
- case s2Edge: S2EdgeLike => s2Edge.id().asInstanceOf[EdgeId]
- case id: EdgeId => id
- case s: String => EdgeId.fromString(s)
- }
- val edgesToFetch = for {
- id <- s2EdgeIds
- } yield {
- toEdge(id.srcVertexId, id.tgtVertexId, id.labelName, id.direction)
- }
-
- checkEdges(edgesToFetch).map { stepResult =>
- val ls = new util.ArrayList[structure.Edge]
- stepResult.edgeWithScores.foreach { es => ls.add(es.edge) }
- ls.iterator()
- }
- }
- override def tx(): Transaction = {
- if (!features.graph.supportsTransactions) throw Graph.Exceptions.transactionsNotSupported
- ???
- }
-
- override def variables(): Variables = new S2GraphVariables
-
- override def configuration(): Configuration = apacheConfiguration
-
- override def addVertex(label: String): Vertex = {
- if (label == null) throw Element.Exceptions.labelCanNotBeNull
- if (label.isEmpty) throw Element.Exceptions.labelCanNotBeEmpty
-
- addVertex(Seq(T.label, label): _*)
- }
-
- def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2VertexLike = {
- idValue match {
- case vId: VertexId =>
- toVertex(vId.column.service.serviceName, vId.column.columnName, vId, kvsMap)
- case _ =>
- val serviceColumnNames = kvsMap.getOrElse(T.label.toString, DefaultColumnName).toString
-
- val names = serviceColumnNames.split(S2Vertex.VertexLabelDelimiter)
- val (serviceName, columnName) =
- if (names.length == 1) (DefaultServiceName, names(0))
- else throw new RuntimeException("malformed data on vertex label.")
-
- toVertex(serviceName, columnName, idValue, kvsMap)
- }
- }
-
- override def addVertex(kvs: AnyRef*): structure.Vertex = {
- if (!features().vertex().supportsUserSuppliedIds() && kvs.contains(T.id)) {
- throw Vertex.Exceptions.userSuppliedIdsNotSupported
- }
-
- val kvsMap = S2Property.kvsToProps(kvs)
- kvsMap.get(T.id.name()) match {
- case Some(idValue) if !S2Property.validType(idValue) =>
- throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported()
- case _ =>
- }
-
- kvsMap.foreach { case (k, v) => S2Property.assertValidProp(k, v) }
-
- if (kvsMap.contains(T.label.name()) && kvsMap(T.label.name).toString.isEmpty)
- throw Element.Exceptions.labelCanNotBeEmpty
-
- val vertex = kvsMap.get(T.id.name()) match {
- case None => // do nothing
- val id = nextLocalLongId
- makeVertex(Long.box(id), kvsMap)
- case Some(idValue) if S2Property.validType(idValue) =>
- makeVertex(idValue, kvsMap)
- case _ =>
- throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported
- }
-
- addVertexInner(vertex)
-
- vertex
- }
-
- def addVertex(id: VertexId,
- ts: Long = System.currentTimeMillis(),
- props: S2Vertex.Props = S2Vertex.EmptyProps,
- op: Byte = 0,
- belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
- val vertex = newVertex(id, ts, props, op, belongLabelIds)
-
- val future = mutateVertices(Seq(vertex), withWait = true).map { rets =>
- if (rets.forall(_.isSuccess)) vertex
- else throw new RuntimeException("addVertex failed.")
- }
- Await.ready(future, WaitTimeout)
-
- vertex
- }
-
- def addVertexInner(vertex: S2VertexLike): S2VertexLike = {
- val future = mutateVertices(Seq(vertex), withWait = true).flatMap { rets =>
- if (rets.forall(_.isSuccess)) {
- indexProvider.mutateVerticesAsync(Seq(vertex))
- } else throw new RuntimeException("addVertex failed.")
- }
- Await.ready(future, WaitTimeout)
-
- vertex
- }
-
- /* tp3 only */
- def addEdge(srcVertex: S2VertexLike, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
- val containsId = kvs.contains(T.id)
-
- tgtVertex match {
- case otherV: S2VertexLike =>
- if (!features().edge().supportsUserSuppliedIds() && containsId) {
- throw Exceptions.userSuppliedIdsNotSupported()
- }
-
- val props = S2Property.kvsToProps(kvs)
-
- props.foreach { case (k, v) => S2Property.assertValidProp(k, v) }
-
- //TODO: direction, operation, _timestamp need to be reserved property key.
-
- try {
- val direction = props.get("direction").getOrElse("out").toString
- val ts = props.get(LabelMeta.timestamp.name).map(_.toString.toLong).getOrElse(System.currentTimeMillis())
- val operation = props.get("operation").map(_.toString).getOrElse("insert")
- val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
- val dir = GraphUtil.toDir(direction).getOrElse(throw new RuntimeException(s"$direction is not supported."))
- val propsPlusTs = props ++ Map(LabelMeta.timestamp.name -> ts)
- val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts)
- val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
-
- val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs)
-
- val future = mutateEdges(Seq(edge), withWait = true).flatMap { rets =>
- indexProvider.mutateEdgesAsync(Seq(edge))
- }
- Await.ready(future, WaitTimeout)
-
- edge
- } catch {
- case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e)
- }
- case null => throw new java.lang.IllegalArgumentException
- case _ => throw new RuntimeException("only S2Graph vertex can be used.")
- }
}
- override def close(): Unit = {
- shutdown()
- }
-
- override def compute[C <: GraphComputer](aClass: Class[C]): C = ???
-
- override def compute(): GraphComputer = {
- if (!features.graph.supportsComputer) {
- throw Graph.Exceptions.graphComputerNotSupported
- }
- ???
- }
-
- class S2GraphFeatures extends Features {
- import org.apache.s2graph.core.{features => FS}
- override def edge(): Features.EdgeFeatures = new FS.S2EdgeFeatures
-
- override def graph(): Features.GraphFeatures = new FS.S2GraphFeatures
-
- override def supports(featureClass: Class[_ <: Features.FeatureSet], feature: String): Boolean =
- super.supports(featureClass, feature)
-
- override def vertex(): Features.VertexFeatures = new FS.S2VertexFeatures
-
- override def toString: String = {
- s"FEATURES:\nEdgeFeatures:${edge}\nGraphFeatures:${graph}\nVertexFeatures:${vertex}"
- }
- }
-
- private val s2Features = new S2GraphFeatures
-
- override def features() = s2Features
-
- override def toString(): String = "[s2graph]"
+ def toVertex(serviceName: String,
+ columnName: String,
+ id: Any,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2VertexLike =
+ elementBuilder.toVertex(serviceName, columnName, id, props, ts, operation)
- override def io[I <: Io[_ <: GraphReader.ReaderBuilder[_ <: GraphReader], _ <: GraphWriter.WriterBuilder[_ <: GraphWriter], _ <: Mapper.Builder[_]]](builder: Io.Builder[I]): I = {
- builder.graph(this).registry(S2GraphIoRegistry.instance).create().asInstanceOf[I]
- }
+ def toEdge(srcId: Any,
+ tgtId: Any,
+ labelName: String,
+ direction: String,
+ props: Map[String, Any] = Map.empty,
+ ts: Long = System.currentTimeMillis(),
+ operation: String = "insert"): S2EdgeLike =
+ elementBuilder.toEdge(srcId, tgtId, labelName, direction, props, ts, operation)
+ def toGraphElement(s: String, labelMapping: Map[String, String] = Map.empty): Option[GraphElement] =
+ elementBuilder.toGraphElement(s, labelMapping)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
new file mode 100644
index 0000000..03a92c6
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphLike.scala
@@ -0,0 +1,238 @@
+package org.apache.s2graph.core
+import java.util
+
+import org.apache.commons.configuration.Configuration
+import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
+import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
+import org.apache.s2graph.core.features.{S2Features, S2GraphVariables}
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.types.VertexId
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer
+import org.apache.tinkerpop.gremlin.structure
+import org.apache.tinkerpop.gremlin.structure.Edge.Exceptions
+import org.apache.tinkerpop.gremlin.structure.Graph.{Features, Variables}
+import org.apache.tinkerpop.gremlin.structure.io.{GraphReader, GraphWriter, Io, Mapper}
+import org.apache.tinkerpop.gremlin.structure.{Edge, Element, Graph, T, Transaction, Vertex}
+
+import scala.concurrent.{Await, Future}
+import scala.collection.JavaConversions._
+
+trait S2GraphLike extends Graph {
+ this: S2Graph =>
+
+ var apacheConfiguration: Configuration
+
+ private val s2Features = new S2Features
+
+ override def features() = s2Features
+
+ def vertices(ids: AnyRef*): util.Iterator[structure.Vertex] = {
+ val fetchVertices = ids.lastOption.map { lastParam =>
+ if (lastParam.isInstanceOf[Boolean]) lastParam.asInstanceOf[Boolean]
+ else true
+ }.getOrElse(true)
+
+ if (ids.isEmpty) {
+ //TODO: default storage need to be fixed.
+ Await.result(defaultStorage.fetchVerticesAll(), WaitTimeout).iterator
+ } else {
+ val vertices = ids.collect {
+ case s2Vertex: S2VertexLike => s2Vertex
+ case vId: VertexId => newVertex(vId)
+ case vertex: Vertex => newVertex(vertex.id().asInstanceOf[VertexId])
+ case other@_ => newVertex(VertexId.fromString(other.toString))
+ }
+
+ if (fetchVertices) {
+ val future = getVertices(vertices).map { vs =>
+ val ls = new util.ArrayList[structure.Vertex]()
+ ls.addAll(vs)
+ ls.iterator()
+ }
+ Await.result(future, WaitTimeout)
+ } else {
+ vertices.iterator
+ }
+ }
+ }
+
+ def edges(edgeIds: AnyRef*): util.Iterator[structure.Edge] = {
+ if (edgeIds.isEmpty) {
+ // FIXME
+ Await.result(defaultStorage.fetchEdgesAll(), WaitTimeout).iterator
+ } else {
+ Await.result(edgesAsync(edgeIds: _*), WaitTimeout)
+ }
+ }
+
+ def edgesAsync(edgeIds: AnyRef*): Future[util.Iterator[structure.Edge]] = {
+ val s2EdgeIds = edgeIds.collect {
+ case s2Edge: S2EdgeLike => s2Edge.id().asInstanceOf[EdgeId]
+ case id: EdgeId => id
+ case s: String => EdgeId.fromString(s)
+ }
+ val edgesToFetch = for {
+ id <- s2EdgeIds
+ } yield {
+ elementBuilder.toEdge(id.srcVertexId, id.tgtVertexId, id.labelName, id.direction)
+ }
+
+ checkEdges(edgesToFetch).map { stepResult =>
+ val ls = new util.ArrayList[structure.Edge]
+ stepResult.edgeWithScores.foreach { es => ls.add(es.edge) }
+ ls.iterator()
+ }
+ }
+
+ def tx(): Transaction = {
+ if (!features.graph.supportsTransactions) throw Graph.Exceptions.transactionsNotSupported
+ ???
+ }
+
+ def variables(): Variables = new S2GraphVariables
+
+ def configuration(): Configuration = apacheConfiguration
+
+ def addVertex(label: String): Vertex = {
+ if (label == null) throw Element.Exceptions.labelCanNotBeNull
+ if (label.isEmpty) throw Element.Exceptions.labelCanNotBeEmpty
+
+ addVertex(Seq(T.label, label): _*)
+ }
+
+ def makeVertex(idValue: AnyRef, kvsMap: Map[String, AnyRef]): S2VertexLike = {
+ idValue match {
+ case vId: VertexId =>
+ elementBuilder.toVertex(vId.column.service.serviceName, vId.column.columnName, vId, kvsMap)
+ case _ =>
+ val serviceColumnNames = kvsMap.getOrElse(T.label.toString, DefaultColumnName).toString
+
+ val names = serviceColumnNames.split(S2Vertex.VertexLabelDelimiter)
+ val (serviceName, columnName) =
+ if (names.length == 1) (DefaultServiceName, names(0))
+ else throw new RuntimeException("malformed data on vertex label.")
+
+ elementBuilder.toVertex(serviceName, columnName, idValue, kvsMap)
+ }
+ }
+
+ def addVertex(kvs: AnyRef*): structure.Vertex = {
+ if (!features().vertex().supportsUserSuppliedIds() && kvs.contains(T.id)) {
+ throw Vertex.Exceptions.userSuppliedIdsNotSupported
+ }
+
+ val kvsMap = S2Property.kvsToProps(kvs)
+ kvsMap.get(T.id.name()) match {
+ case Some(idValue) if !S2Property.validType(idValue) =>
+ throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported()
+ case _ =>
+ }
+
+ kvsMap.foreach { case (k, v) => S2Property.assertValidProp(k, v) }
+
+ if (kvsMap.contains(T.label.name()) && kvsMap(T.label.name).toString.isEmpty)
+ throw Element.Exceptions.labelCanNotBeEmpty
+
+ val vertex = kvsMap.get(T.id.name()) match {
+ case None => // do nothing
+ val id = nextLocalLongId
+ makeVertex(Long.box(id), kvsMap)
+ case Some(idValue) if S2Property.validType(idValue) =>
+ makeVertex(idValue, kvsMap)
+ case _ =>
+ throw Vertex.Exceptions.userSuppliedIdsOfThisTypeNotSupported
+ }
+
+ addVertexInner(vertex)
+
+ vertex
+ }
+
+ def addVertex(id: VertexId,
+ ts: Long = System.currentTimeMillis(),
+ props: S2Vertex.Props = S2Vertex.EmptyProps,
+ op: Byte = 0,
+ belongLabelIds: Seq[Int] = Seq.empty): S2VertexLike = {
+ val vertex = newVertex(id, ts, props, op, belongLabelIds)
+
+ val future = mutateVertices(Seq(vertex), withWait = true).map { rets =>
+ if (rets.forall(_.isSuccess)) vertex
+ else throw new RuntimeException("addVertex failed.")
+ }
+ Await.ready(future, WaitTimeout)
+
+ vertex
+ }
+
+ def addVertexInner(vertex: S2VertexLike): S2VertexLike = {
+ val future = mutateVertices(Seq(vertex), withWait = true).flatMap { rets =>
+ if (rets.forall(_.isSuccess)) {
+ indexProvider.mutateVerticesAsync(Seq(vertex))
+ } else throw new RuntimeException("addVertex failed.")
+ }
+ Await.ready(future, WaitTimeout)
+
+ vertex
+ }
+
+ /* tp3 only */
+ def addEdge(srcVertex: S2VertexLike, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
+ val containsId = kvs.contains(T.id)
+
+ tgtVertex match {
+ case otherV: S2VertexLike =>
+ if (!features().edge().supportsUserSuppliedIds() && containsId) {
+ throw Exceptions.userSuppliedIdsNotSupported()
+ }
+
+ val props = S2Property.kvsToProps(kvs)
+
+ props.foreach { case (k, v) => S2Property.assertValidProp(k, v) }
+
+ //TODO: direction, operation, _timestamp need to be reserved property key.
+
+ try {
+ val direction = props.get("direction").getOrElse("out").toString
+ val ts = props.get(LabelMeta.timestamp.name).map(_.toString.toLong).getOrElse(System.currentTimeMillis())
+ val operation = props.get("operation").map(_.toString).getOrElse("insert")
+ val label = Label.findByName(labelName).getOrElse(throw new LabelNotExistException(labelName))
+ val dir = GraphUtil.toDir(direction).getOrElse(throw new RuntimeException(s"$direction is not supported."))
+ val propsPlusTs = props ++ Map(LabelMeta.timestamp.name -> ts)
+ val propsWithTs = label.propsToInnerValsWithTs(propsPlusTs, ts)
+ val op = GraphUtil.toOp(operation).getOrElse(throw new RuntimeException(s"$operation is not supported."))
+
+ val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs)
+
+ val future = mutateEdges(Seq(edge), withWait = true).flatMap { rets =>
+ indexProvider.mutateEdgesAsync(Seq(edge))
+ }
+ Await.ready(future, WaitTimeout)
+
+ edge
+ } catch {
+ case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e)
+ }
+ case null => throw new java.lang.IllegalArgumentException
+ case _ => throw new RuntimeException("only S2Graph vertex can be used.")
+ }
+ }
+
+ def close(): Unit = {
+ shutdown()
+ }
+
+ def compute[C <: GraphComputer](aClass: Class[C]): C = ???
+
+ def compute(): GraphComputer = {
+ if (!features.graph.supportsComputer) {
+ throw Graph.Exceptions.graphComputerNotSupported
+ }
+ ???
+ }
+
+ def io[I <: Io[_ <: GraphReader.ReaderBuilder[_ <: GraphReader], _ <: GraphWriter.WriterBuilder[_ <: GraphWriter], _ <: Mapper.Builder[_]]](builder: Io.Builder[I]): I = {
+ builder.graph(this).registry(S2GraphIoRegistry.instance).create().asInstanceOf[I]
+ }
+
+ override def toString(): String = "[s2graph]"
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala b/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
new file mode 100644
index 0000000..36c9ecc
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/features/S2Features.scala
@@ -0,0 +1,19 @@
+package org.apache.s2graph.core.features
+
+import org.apache.tinkerpop.gremlin.structure.Graph.Features
+
+class S2Features extends Features {
+ import org.apache.s2graph.core.{features => FS}
+ override def edge(): Features.EdgeFeatures = new FS.S2EdgeFeatures
+
+ override def graph(): Features.GraphFeatures = new FS.S2GraphFeatures
+
+ override def supports(featureClass: Class[_ <: Features.FeatureSet], feature: String): Boolean =
+ super.supports(featureClass, feature)
+
+ override def vertex(): Features.VertexFeatures = new FS.S2VertexFeatures
+
+ override def toString: String = {
+ s"FEATURES:\nEdgeFeatures:${edge}\nGraphFeatures:${graph}\nVertexFeatures:${vertex}"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index e6075ec..2a8f1e2 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -22,9 +22,10 @@ package org.apache.s2graph.core.storage
import com.typesafe.config.Config
import org.apache.s2graph.core._
-import org.apache.s2graph.core.storage.serde.Deserializable
+import org.apache.s2graph.core.storage.serde.{Deserializable, MutationHelper}
import org.apache.s2graph.core.storage.serde.indexedge.tall.IndexEdgeDeserializable
import org.apache.s2graph.core.types._
+
import scala.concurrent.{ExecutionContext, Future}
abstract class Storage(val graph: S2Graph,
@@ -61,49 +62,51 @@ abstract class Storage(val graph: S2Graph,
*/
lazy val conflictResolver: WriteWriteConflictResolver = new WriteWriteConflictResolver(graph, serDe, io, mutator, fetcher)
- /** IO **/
- def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): serde.Serializable[SnapshotEdge] =
- serDe.snapshotEdgeSerializer(snapshotEdge)
-
- def indexEdgeSerializer(indexEdge: IndexEdge): serde.Serializable[IndexEdge] =
- serDe.indexEdgeSerializer(indexEdge)
-
- def vertexSerializer(vertex: S2VertexLike): serde.Serializable[S2VertexLike] =
- serDe.vertexSerializer(vertex)
-
- def snapshotEdgeDeserializer(schemaVer: String): Deserializable[SnapshotEdge] =
- serDe.snapshotEdgeDeserializer(schemaVer)
-
- def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable =
- serDe.indexEdgeDeserializer(schemaVer)
-
- def vertexDeserializer(schemaVer: String): Deserializable[S2VertexLike] =
- serDe.vertexDeserializer(schemaVer)
+ lazy val mutationHelper: MutationHelper = new MutationHelper(this)
+
+// /** IO **/
+// def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): serde.Serializable[SnapshotEdge] =
+// serDe.snapshotEdgeSerializer(snapshotEdge)
+//
+// def indexEdgeSerializer(indexEdge: IndexEdge): serde.Serializable[IndexEdge] =
+// serDe.indexEdgeSerializer(indexEdge)
+//
+// def vertexSerializer(vertex: S2Vertex): serde.Serializable[S2Vertex] =
+// serDe.vertexSerializer(vertex)
+//
+// def snapshotEdgeDeserializer(schemaVer: String): Deserializable[SnapshotEdge] =
+// serDe.snapshotEdgeDeserializer(schemaVer)
+//
+// def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable =
+// serDe.indexEdgeDeserializer(schemaVer)
+//
+// def vertexDeserializer(schemaVer: String): Deserializable[S2Vertex] =
+// serDe.vertexDeserializer(schemaVer)
/** Mutation Builder */
- def increments(edgeMutate: EdgeMutate): (Seq[SKeyValue], Seq[SKeyValue]) =
- io.increments(edgeMutate)
-
- def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
- io.indexedEdgeMutations(edgeMutate)
-
- def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
- io.buildIncrementsAsync(indexedEdge, amount)
-
- def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
- io.buildIncrementsCountAsync(indexedEdge, amount)
-
- def buildVertexPutsAsync(edge: S2EdgeLike): Seq[SKeyValue] =
- io.buildVertexPutsAsync(edge)
-
- def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
- io.snapshotEdgeMutations(edgeMutate)
-
- def buildDegreePuts(edge: S2EdgeLike, degreeVal: Long): Seq[SKeyValue] =
- io.buildDegreePuts(edge, degreeVal)
-
- def buildPutsAll(vertex: S2VertexLike): Seq[SKeyValue] =
- io.buildPutsAll(vertex)
+// def increments(edgeMutate: EdgeMutate): (Seq[SKeyValue], Seq[SKeyValue]) =
+// io.increments(edgeMutate)
+//
+// def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+// io.indexedEdgeMutations(edgeMutate)
+//
+// def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
+// io.buildIncrementsAsync(indexedEdge, amount)
+//
+// def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
+// io.buildIncrementsCountAsync(indexedEdge, amount)
+//
+// def buildVertexPutsAsync(edge: S2Edge): Seq[SKeyValue] =
+// io.buildVertexPutsAsync(edge)
+//
+// def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
+// io.snapshotEdgeMutations(edgeMutate)
+//
+// def buildDegreePuts(edge: S2Edge, degreeVal: Long): Seq[SKeyValue] =
+// io.buildDegreePuts(edge, degreeVal)
+//
+// def buildPutsAll(vertex: S2Vertex): Seq[SKeyValue] =
+// io.buildPutsAll(vertex)
/** Mutation **/
@@ -129,8 +132,8 @@ abstract class Storage(val graph: S2Graph,
fetcher.fetchSnapshotEdgeInner(edge)
/** Conflict Resolver **/
- def retry(tryNum: Int)(edges: Seq[S2EdgeLike], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2EdgeLike])(implicit ec: ExecutionContext): Future[Boolean] =
- conflictResolver.retry(tryNum)(edges, statusCode, fetchedSnapshotEdgeOpt)
+// def retry(tryNum: Int)(edges: Seq[S2Edge], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2Edge])(implicit ec: ExecutionContext): Future[Boolean] =
+// conflictResolver.retry(tryNum)(edges, statusCode, fetchedSnapshotEdgeOpt)
/** Management **/
@@ -145,4 +148,25 @@ abstract class Storage(val graph: S2Graph,
def shutdown(): Unit = management.shutdown()
def info: Map[String, String] = Map("className" -> this.getClass.getSimpleName)
+
+ def deleteAllFetchedEdgesAsyncOld(stepInnerResult: StepResult,
+ requestTs: Long,
+ retryNum: Int)(implicit ec: ExecutionContext): Future[Boolean] =
+ mutationHelper.deleteAllFetchedEdgesAsyncOld(stepInnerResult, requestTs, retryNum)
+
+ def mutateVertex(zkQuorum: String, vertex: S2VertexLike, withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
+ mutationHelper.mutateVertex(zkQuorum: String, vertex, withWait)
+
+ def mutateStrongEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] =
+ mutationHelper.mutateStrongEdges(zkQuorum, _edges, withWait)
+
+
+ def mutateWeakEdges(zkQuorum: String, _edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[(Int, Boolean)]] =
+ mutationHelper.mutateWeakEdges(zkQuorum, _edges, withWait)
+
+ def incrementCounts(zkQuorum: String, edges: Seq[S2EdgeLike], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[MutateResponse]] =
+ mutationHelper.incrementCounts(zkQuorum, edges, withWait)
+
+ def updateDegree(zkQuorum: String, edge: S2EdgeLike, degreeVal: Long = 0)(implicit ec: ExecutionContext): Future[MutateResponse] =
+ mutationHelper.updateDegree(zkQuorum, edge, degreeVal)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
index 6be5f60..bdb6e99 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageReadable.scala
@@ -303,7 +303,7 @@ class AsynchbaseStorageReadable(val graph: S2Graph,
val cacheTTL = queryParam.cacheTTLInMillis
/* with version 4, request's type is (Scanner, (Int, Int)). otherwise GetRequest. */
- val edge = graph.toRequestEdge(queryRequest, parentEdges)
+ val edge = graph.elementBuilder.toRequestEdge(queryRequest, parentEdges)
val request = buildRequest(queryRequest, edge)
val (intervalMaxBytes, intervalMinBytes) = queryParam.buildInterval(Option(edge))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
index b618962..408822a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/tall/SnapshotEdgeDeserializable.scala
@@ -97,7 +97,7 @@ class SnapshotEdgeDeserializable(graph: S2Graph) extends Deserializable[Snapshot
Option(pendingEdge)
}
- val snapshotEdge = graph.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
+ val snapshotEdge = graph.elementBuilder.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
label, labelWithDir.dir, op, version, props.toMap, statusCode = statusCode,
pendingEdgeOpt = _pendingEdgeOpt, lockTs = None, tsInnerValOpt = Option(tsInnerVal))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
index 8c961ce..ff8eb80 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/snapshotedge/wide/SnapshotEdgeDeserializable.scala
@@ -88,7 +88,7 @@ class SnapshotEdgeDeserializable(graph: S2Graph) extends Deserializable[Snapshot
Option(pendingEdge)
}
- val snapshotEdge = graph.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
+ val snapshotEdge = graph.elementBuilder.newSnapshotEdge(graph.newVertex(srcVertexId, ts), graph.newVertex(tgtVertexId, ts),
label, labelWithDir.dir, op, version, props.toMap, statusCode = statusCode,
pendingEdgeOpt = _pendingEdgeOpt, lockTs = None, tsInnerValOpt = Option(tsInnerVal))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa66822b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
index 0cbaa81..24e98a1 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/storage/hbase/IndexEdgeTest.scala
@@ -46,8 +46,8 @@ class IndexEdgeTest extends FunSuite with Matchers with TestCommonWithModels {
val labelOpt = Option(l)
val edge = graph.newEdge(vertex, tgtVertex, l, labelWithDir.dir, 0, ts, props, tsInnerValOpt = Option(InnerVal.withLong(ts, l.schemaVersion)))
val indexEdge = edge.edgesWithIndex.find(_.labelIndexSeq == LabelIndex.DefaultSeq).head
- val kvs = graph.getStorage(l).indexEdgeSerializer(indexEdge).toKeyValues
- val _indexEdgeOpt = graph.getStorage(l).indexEdgeDeserializer(l.schemaVersion).fromKeyValues(kvs, None)
+ val kvs = graph.getStorage(l).serDe.indexEdgeSerializer(indexEdge).toKeyValues
+ val _indexEdgeOpt = graph.getStorage(l).serDe.indexEdgeDeserializer(l.schemaVersion).fromKeyValues(kvs, None)
_indexEdgeOpt should not be empty
edge == _indexEdgeOpt.get should be(true)
[20/23] incubator-s2graph git commit: remove unused comments.
Posted by st...@apache.org.
remove unused comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6c610874
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6c610874
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6c610874
Branch: refs/heads/master
Commit: 6c610874a3fd2a9a52b5b3ec788b4620adfd819e
Parents: eabe757
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Nov 14 14:25:51 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 14 14:25:51 2017 +0900
----------------------------------------------------------------------
.../apache/s2graph/core/storage/Storage.scala | 50 --------------------
1 file changed, 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6c610874/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
index 18e6fa1..18f6b1e 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/Storage.scala
@@ -64,52 +64,7 @@ abstract class Storage(val graph: S2GraphLike,
lazy val mutationHelper: MutationHelper = new MutationHelper(this)
-// /** IO **/
-// def snapshotEdgeSerializer(snapshotEdge: SnapshotEdge): serde.Serializable[SnapshotEdge] =
-// serDe.snapshotEdgeSerializer(snapshotEdge)
-//
-// def indexEdgeSerializer(indexEdge: IndexEdge): serde.Serializable[IndexEdge] =
-// serDe.indexEdgeSerializer(indexEdge)
-//
-// def vertexSerializer(vertex: S2Vertex): serde.Serializable[S2Vertex] =
-// serDe.vertexSerializer(vertex)
-//
-// def snapshotEdgeDeserializer(schemaVer: String): Deserializable[SnapshotEdge] =
-// serDe.snapshotEdgeDeserializer(schemaVer)
-//
-// def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable =
-// serDe.indexEdgeDeserializer(schemaVer)
-//
-// def vertexDeserializer(schemaVer: String): Deserializable[S2Vertex] =
-// serDe.vertexDeserializer(schemaVer)
-
- /** Mutation Builder */
-// def increments(edgeMutate: EdgeMutate): (Seq[SKeyValue], Seq[SKeyValue]) =
-// io.increments(edgeMutate)
-//
-// def indexedEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
-// io.indexedEdgeMutations(edgeMutate)
-//
-// def buildIncrementsAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
-// io.buildIncrementsAsync(indexedEdge, amount)
-//
-// def buildIncrementsCountAsync(indexedEdge: IndexEdge, amount: Long = 1L): Seq[SKeyValue] =
-// io.buildIncrementsCountAsync(indexedEdge, amount)
-//
-// def buildVertexPutsAsync(edge: S2Edge): Seq[SKeyValue] =
-// io.buildVertexPutsAsync(edge)
-//
-// def snapshotEdgeMutations(edgeMutate: EdgeMutate): Seq[SKeyValue] =
-// io.snapshotEdgeMutations(edgeMutate)
-//
-// def buildDegreePuts(edge: S2Edge, degreeVal: Long): Seq[SKeyValue] =
-// io.buildDegreePuts(edge, degreeVal)
-//
-// def buildPutsAll(vertex: S2Vertex): Seq[SKeyValue] =
-// io.buildPutsAll(vertex)
-
/** Mutation **/
-
def writeToStorage(cluster: String, kvs: Seq[SKeyValue], withWait: Boolean)(implicit ec: ExecutionContext): Future[MutateResponse] =
mutator.writeToStorage(cluster, kvs, withWait)
@@ -131,12 +86,7 @@ abstract class Storage(val graph: S2GraphLike,
def fetchSnapshotEdgeInner(edge: S2EdgeLike)(implicit ec: ExecutionContext): Future[(Option[S2EdgeLike], Option[SKeyValue])] =
fetcher.fetchSnapshotEdgeInner(edge)
- /** Conflict Resolver **/
-// def retry(tryNum: Int)(edges: Seq[S2Edge], statusCode: Byte, fetchedSnapshotEdgeOpt: Option[S2Edge])(implicit ec: ExecutionContext): Future[Boolean] =
-// conflictResolver.retry(tryNum)(edges, statusCode, fetchedSnapshotEdgeOpt)
-
/** Management **/
-
def flush(): Unit = management.flush()
def createTable(config: Config, tableNameStr: String): Unit = management.createTable(config, tableNameStr)
[11/23] incubator-s2graph git commit: add S2EdgeBuilder.
Posted by st...@apache.org.
add S2EdgeBuilder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/2b5df1dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/2b5df1dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/2b5df1dd
Branch: refs/heads/master
Commit: 2b5df1dd0ddc913dca92353b191e8982368f08c0
Parents: 7d08225
Author: DO YUNG YOON <st...@apache.org>
Authored: Mon Nov 6 23:08:25 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Nov 7 00:50:54 2017 +0900
----------------------------------------------------------------------
.../s2graph/core/GraphElementBuilder.scala | 4 +-
.../org/apache/s2graph/core/QueryParam.scala | 21 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 63 ++++-
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 103 ++++++++
.../org/apache/s2graph/core/S2EdgeLike.scala | 258 ++++---------------
.../apache/s2graph/core/storage/StorageIO.scala | 4 +-
.../core/storage/serde/MutationHelper.scala | 6 +-
7 files changed, 234 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index 21179aa..c9133b1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -279,10 +279,10 @@ class GraphElementBuilder(graph: S2Graph) {
val edge = edgeWithScore.edge
val copiedEdge = label.consistencyLevel match {
case "strong" =>
- edge.copyEdge(op = GraphUtil.operations("delete"),
+ edge.builder.copyEdge(op = GraphUtil.operations("delete"),
version = requestTs, propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
case _ =>
- edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
+ edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()), ts = requestTs)
}
val edgeToDelete = edgeWithScore.copy(edge = copiedEdge)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
index e98ef37..3d0d076 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/QueryParam.scala
@@ -185,7 +185,7 @@ case class EdgeTransformer(jsValue: JsValue) {
replace(queryParam, fmt, fieldNames.flatMap(fieldName => toInnerValOpt(queryParam, edge, fieldName)), nextStepOpt)
}
}
- } yield edge.updateTgtVertex(innerVal).copy(originalEdgeOpt = Option(edge))
+ } yield edge.builder.updateTgtVertex(innerVal).copyOriginalEdgeOpt(Option(edge))
edges
@@ -251,7 +251,26 @@ case class RankParam(keySeqAndWeights: Seq[(LabelMeta, Double)] = Seq((LabelMeta
}
bytes
}
+
+ def score(edge: S2EdgeLike): Double = {
+ if (keySeqAndWeights.size <= 0) 1.0f
+ else {
+ var sum: Double = 0
+
+ for ((labelMeta, w) <- keySeqAndWeights) {
+ if (edge.getPropsWithTs().containsKey(labelMeta.name)) {
+ val innerValWithTs = edge.getPropsWithTs().get(labelMeta.name)
+ val cost = try innerValWithTs.innerVal.toString.toDouble catch {
+ case e: Exception => 1.0
+ }
+ sum += w * cost
+ }
+ }
+ sum
+ }
+ }
}
+
object QueryParam {
lazy val Empty = QueryParam(labelName = "")
lazy val DefaultThreshold = Double.MinValue
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
index 3529991..03678c8 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Edge.scala
@@ -40,6 +40,17 @@ import scala.concurrent.Await
import scala.util.hashing.MurmurHash3
object SnapshotEdge {
+ def apply(e: S2EdgeLike): SnapshotEdge = {
+ val (smaller, larger) = (e.srcForVertex, e.tgtForVertex)
+
+ val snapshotEdge = SnapshotEdge(e.innerGraph, smaller, larger, e.innerLabel,
+ GraphUtil.directions("out"), e.getOp(), e.getVersion(), e.getPropsWithTs(),
+ pendingEdgeOpt = e.getPendingEdgeOpt(), statusCode = e.getStatusCode(), lockTs = e.getLockTs(), tsInnerValOpt = e.getTsInnerValOpt())
+
+ snapshotEdge.property(LabelMeta.timestamp.name, e.ts, e.ts)
+
+ snapshotEdge
+ }
def copyFrom(e: SnapshotEdge): SnapshotEdge = {
val copy =
@@ -418,8 +429,8 @@ object EdgeMutate {
}
}
-case class EdgeMutate(edgesToDelete: List[IndexEdge] = List.empty[IndexEdge],
- edgesToInsert: List[IndexEdge] = List.empty[IndexEdge],
+case class EdgeMutate(edgesToDelete: Seq[IndexEdge] = Nil,
+ edgesToInsert: Seq[IndexEdge] = Nil,
newSnapshotEdge: Option[SnapshotEdge] = None) {
def deepCopy: EdgeMutate = copy(
@@ -799,5 +810,53 @@ object S2Edge {
// def fromString(s: String): Option[Edge] = Graph.toEdge(s)
+ def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
+ if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
+
+ def srcForVertex(e: S2EdgeLike): S2VertexLike = {
+ val belongLabelIds = Seq(e.labelWithDir.labelId)
+ if (e.labelWithDir.dir == GraphUtil.directions("in")) {
+ val tgtColumn = getServiceColumn(e.tgtVertex, e.innerLabel.tgtColumn)
+ e.innerGraph.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ val srcColumn = getServiceColumn(e.srcVertex, e.innerLabel.srcColumn)
+ e.innerGraph.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+ def tgtForVertex(e: S2EdgeLike): S2VertexLike = {
+ val belongLabelIds = Seq(e.labelWithDir.labelId)
+ if (e.labelWithDir.dir == GraphUtil.directions("in")) {
+ val srcColumn = getServiceColumn(e.srcVertex, e.innerLabel.srcColumn)
+ e.innerGraph.newVertex(VertexId(srcColumn, e.srcVertex.innerId), e.srcVertex.ts, e.srcVertex.props, belongLabelIds = belongLabelIds)
+ } else {
+ val tgtColumn = getServiceColumn(e.tgtVertex, e.innerLabel.tgtColumn)
+ e.innerGraph.newVertex(VertexId(tgtColumn, e.tgtVertex.innerId), e.tgtVertex.ts, e.tgtVertex.props, belongLabelIds = belongLabelIds)
+ }
+ }
+
+ def updatePropsWithTs(e: S2EdgeLike, others: Props = S2Edge.EmptyProps): Props = {
+ val emptyProp = S2Edge.EmptyProps
+
+ e.getPropsWithTs().forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ others.forEach(new BiConsumer[String, S2Property[_]] {
+ override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
+ })
+
+ emptyProp
+ }
+
+ def propertyValue(e: S2EdgeLike, key: String): Option[InnerValLikeWithTs] = {
+ key match {
+ case "from" | "_from" => Option(InnerValLikeWithTs(e.srcVertex.innerId, e.ts))
+ case "to" | "_to" => Option(InnerValLikeWithTs(e.tgtVertex.innerId, e.ts))
+ case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(e.innerLabel.label, e.innerLabel.schemaVersion), e.ts))
+ case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(e.direction, e.innerLabel.schemaVersion), e.ts))
+ case _ =>
+ e.innerLabel.metaPropsInvMap.get(key).map(labelMeta => e.propertyValueInner(labelMeta))
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
new file mode 100644
index 0000000..ea9598e
--- /dev/null
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeBuilder.scala
@@ -0,0 +1,103 @@
+package org.apache.s2graph.core
+
+import org.apache.s2graph.core.S2Edge.State
+import org.apache.s2graph.core.mysqls.{Label, LabelIndex, LabelMeta}
+import org.apache.s2graph.core.types.{InnerValLike, TargetVertexId, VertexId}
+import org.apache.tinkerpop.gremlin.structure.Property
+
+import scala.collection.JavaConverters._
+
+class S2EdgeBuilder(edge: S2EdgeLike) {
+ def srcForVertex = S2Edge.srcForVertex(edge)
+
+ def tgtForVertex = S2Edge.tgtForVertex(edge)
+
+ def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
+
+ def reverseDirEdge = copyEdge(dir = GraphUtil.toggleDir(edge.getDir))
+
+ def reverseSrcTgtEdge = copyEdge(srcVertex = edge.tgtVertex, tgtVertex = edge.srcVertex)
+
+ def isDegree = edge.getPropsWithTs().containsKey(LabelMeta.degree.name)
+
+ def propsPlusTsValid = edge.getPropsWithTs().asScala.filter(kv => LabelMeta.isValidSeq(kv._2.labelMeta.seq)).asJava
+
+ def labelOrders = LabelIndex.findByLabelIdAll(edge.labelWithDir.labelId)
+
+ def edgesWithIndex = for (labelOrder <- labelOrders) yield {
+ IndexEdge(edge.innerGraph, edge.srcVertex, edge.tgtVertex, edge.innerLabel, edge.getDir(), edge.getOp(),
+ edge.getVersion(), labelOrder.seq, edge.getPropsWithTs(), tsInnerValOpt = edge.getTsInnerValOpt())
+ }
+
+ def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
+ IndexEdge(edge.innerGraph, edge.srcVertex, edge.tgtVertex, edge.innerLabel, edge.getDir(), edge.getOp(),
+ edge.getVersion(), labelOrder.seq, propsPlusTsValid, tsInnerValOpt = edge.getTsInnerValOpt())
+ }
+
+ def relatedEdges: Seq[S2EdgeLike] = {
+ if (edge.labelWithDir.isDirected) {
+ val skipReverse = edge.innerLabel.extraOptions.get("skipReverse").map(_.as[Boolean]).getOrElse(false)
+ if (skipReverse) Seq(edge) else Seq(edge, duplicateEdge)
+ } else {
+ // val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
+ // val base = copy(labelWithDir = outDir)
+ val base = copyEdge(dir = GraphUtil.directions("out"))
+ Seq(base, base.reverseSrcTgtEdge)
+ }
+ }
+
+ def copyEdge(innerGraph: S2Graph = edge.innerGraph,
+ srcVertex: S2VertexLike = edge.srcVertex,
+ tgtVertex: S2VertexLike = edge.tgtVertex,
+ innerLabel: Label = edge.innerLabel,
+ dir: Int = edge.getDir(),
+ op: Byte = edge.getOp(),
+ version: Long = edge.getVersion(),
+ propsWithTs: State = S2Edge.propsToState(edge.getPropsWithTs()),
+ parentEdges: Seq[EdgeWithScore] = edge.getParentEdges(),
+ originalEdgeOpt: Option[S2EdgeLike] = edge.getOriginalEdgeOpt(),
+ pendingEdgeOpt: Option[S2EdgeLike] = edge.getPendingEdgeOpt(),
+ statusCode: Byte = edge.getStatusCode(),
+ lockTs: Option[Long] = edge.getLockTs(),
+ tsInnerValOpt: Option[InnerValLike] = edge.getTsInnerValOpt(),
+ ts: Long = edge.getTs()): S2EdgeLike = {
+ val edge = new S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, S2Edge.EmptyProps,
+ parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
+ S2Edge.fillPropsWithTs(edge, propsWithTs)
+ edge.propertyInner(LabelMeta.timestamp.name, ts, ts)
+ edge
+ }
+
+ def copyEdgeWithState(state: State): S2EdgeLike = {
+ val newEdge = new S2Edge(edge.innerGraph, edge.srcVertex, edge.tgtVertex, edge.innerLabel,
+ edge.getDir(), edge.getOp(), edge.getVersion(), S2Edge.EmptyProps,
+ edge.getParentEdges(), edge.getOriginalEdgeOpt(), edge.getPendingEdgeOpt(),
+ edge.getStatusCode(), edge.getLockTs(), edge.getTsInnerValOpt())
+
+ S2Edge.fillPropsWithTs(newEdge, state)
+ newEdge
+ }
+
+ def updateTgtVertex(id: InnerValLike): S2EdgeLike = {
+ val newId = TargetVertexId(edge.tgtVertex.id.column, id)
+ val newTgtVertex = edge.innerGraph.newVertex(newId, edge.tgtVertex.ts, edge.tgtVertex.props)
+ copyEdge(tgtVertex = newTgtVertex)
+ }
+
+ def edgeId: EdgeId = {
+ val timestamp = if (edge.innerLabel.consistencyLevel == "strong") 0l else edge.ts
+ // EdgeId(srcVertex.innerId, tgtVertex.innerId, label(), "out", timestamp)
+ val (srcColumn, tgtColumn) = edge.innerLabel.srcTgtColumn(edge.getDir())
+ if (edge.getDir() == GraphUtil.directions("out"))
+ EdgeId(VertexId(srcColumn, edge.srcVertex.id.innerId), VertexId(tgtColumn, edge.tgtVertex.id.innerId), edge.label(), "out", timestamp)
+ else
+ EdgeId(VertexId(tgtColumn, edge.tgtVertex.id.innerId), VertexId(srcColumn, edge.srcVertex.id.innerId), edge.label(), "out", timestamp)
+ }
+
+ def propertyInner[V](key: String, value: V, ts: Long): Property[V] = {
+ val labelMeta = edge.innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
+ val newProp = new S2Property[V](edge, labelMeta, key, value, ts)
+ edge.getPropsWithTs().put(key, newProp)
+ newProp
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
index f823d60..9963be7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2EdgeLike.scala
@@ -17,6 +17,8 @@ import scala.collection.JavaConverters._
trait S2EdgeLike extends Edge with GraphElement {
this: S2Edge =>
+ val builder: S2EdgeBuilder = new S2EdgeBuilder(this)
+
val innerGraph: S2Graph
val srcVertex: S2VertexLike
var tgtVertex: S2VertexLike
@@ -48,6 +50,14 @@ trait S2EdgeLike extends Edge with GraphElement {
lazy val labelName = innerLabel.label
lazy val direction = GraphUtil.fromDirection(dir)
+ def getTs(): Long = ts
+ def getOriginalEdgeOpt(): Option[S2EdgeLike] = originalEdgeOpt
+ def getParentEdges(): Seq[EdgeWithScore] = parentEdges
+ def getPendingEdgeOpt(): Option[S2EdgeLike] = pendingEdgeOpt
+ def getPropsWithTs(): Props = propsWithTs
+ def getLockTs(): Option[Long] = lockTs
+ def getStatusCode(): Byte = statusCode
+ def getDir(): Int = dir
def setTgtVertex(v: S2VertexLike): Unit = tgtVertex = v
def getOp(): Byte = op
def setOp(newOp: Byte): Unit = op = newOp
@@ -60,30 +70,10 @@ trait S2EdgeLike extends Edge with GraphElement {
def serializePropsWithTs(): Array[Byte] = HBaseSerializable.propsToKeyValuesWithTs(propsWithTs.asScala.map(kv => kv._2.labelMeta.seq -> kv._2.innerValWithTs).toSeq)
- def updatePropsWithTs(others: Props = S2Edge.EmptyProps): Props = {
- val emptyProp = S2Edge.EmptyProps
-
- propsWithTs.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
+ def updatePropsWithTs(others: Props = S2Edge.EmptyProps): Props =
+ S2Edge.updatePropsWithTs(this, others)
- others.forEach(new BiConsumer[String, S2Property[_]] {
- override def accept(key: String, value: S2Property[_]): Unit = emptyProp.put(key, value)
- })
-
- emptyProp
- }
-
- def propertyValue(key: String): Option[InnerValLikeWithTs] = {
- key match {
- case "from" | "_from" => Option(InnerValLikeWithTs(srcVertex.innerId, ts))
- case "to" | "_to" => Option(InnerValLikeWithTs(tgtVertex.innerId, ts))
- case "label" => Option(InnerValLikeWithTs(InnerVal.withStr(innerLabel.label, schemaVer), ts))
- case "direction" => Option(InnerValLikeWithTs(InnerVal.withStr(direction, schemaVer), ts))
- case _ =>
- innerLabel.metaPropsInvMap.get(key).map(labelMeta => propertyValueInner(labelMeta))
- }
- }
+ def propertyValue(key: String): Option[InnerValLikeWithTs] = S2Edge.propertyValue(this, key)
def propertyValueInner(labelMeta: LabelMeta): InnerValLikeWithTs = {
// propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse()
@@ -116,172 +106,55 @@ trait S2EdgeLike extends Edge with GraphElement {
}
}
- lazy val properties = toProps()
-
- def props = propsWithTs.asScala.mapValues(_.innerVal)
-
- def relatedEdges = {
- if (labelWithDir.isDirected) {
- val skipReverse = innerLabel.extraOptions.get("skipReverse").map(_.as[Boolean]).getOrElse(false)
- if (skipReverse) List(this) else List(this, duplicateEdge)
- } else {
- // val outDir = labelWithDir.copy(dir = GraphUtil.directions("out"))
- // val base = copy(labelWithDir = outDir)
- val base = copy(dir = GraphUtil.directions("out"))
- List(base, base.reverseSrcTgtEdge)
- }
- }
+ def relatedEdges = builder.relatedEdges
- def srcForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
- innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- } else {
- val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
- innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- }
- }
+ def srcForVertex = builder.srcForVertex
- def tgtForVertex = {
- val belongLabelIds = Seq(labelWithDir.labelId)
- if (labelWithDir.dir == GraphUtil.directions("in")) {
- val srcColumn = getServiceColumn(srcVertex, innerLabel.srcColumn)
- innerGraph.newVertex(VertexId(srcColumn, srcVertex.innerId), srcVertex.ts, srcVertex.props, belongLabelIds = belongLabelIds)
- } else {
- val tgtColumn = getServiceColumn(tgtVertex, innerLabel.tgtColumn)
- innerGraph.newVertex(VertexId(tgtColumn, tgtVertex.innerId), tgtVertex.ts, tgtVertex.props, belongLabelIds = belongLabelIds)
- }
- }
+ def tgtForVertex = builder.tgtForVertex
- def duplicateEdge = reverseSrcTgtEdge.reverseDirEdge
+ def duplicateEdge = builder.duplicateEdge
// def reverseDirEdge = copy(labelWithDir = labelWithDir.dirToggled)
- def reverseDirEdge = copy(dir = GraphUtil.toggleDir(dir))
+ def reverseDirEdge = builder.reverseDirEdge
- def reverseSrcTgtEdge = copy(srcVertex = tgtVertex, tgtVertex = srcVertex)
+ def reverseSrcTgtEdge = builder.reverseSrcTgtEdge
- def labelOrders = LabelIndex.findByLabelIdAll(labelWithDir.labelId)
+ def isDegree = builder.isDegree
- def isDegree = propsWithTs.containsKey(LabelMeta.degree.name)
+ def edgesWithIndex = builder.edgesWithIndex
- def propsPlusTsValid = propsWithTs.asScala.filter(kv => LabelMeta.isValidSeq(kv._2.labelMeta.seq)).asJava
-
- def edgesWithIndex = for (labelOrder <- labelOrders) yield {
- IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsWithTs, tsInnerValOpt = tsInnerValOpt)
- }
-
- def edgesWithIndexValid = for (labelOrder <- labelOrders) yield {
- IndexEdge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, labelOrder.seq, propsPlusTsValid, tsInnerValOpt = tsInnerValOpt)
- }
+ def edgesWithIndexValid = builder.edgesWithIndexValid
/** force direction as out on invertedEdge */
- def toSnapshotEdge: SnapshotEdge = {
- val (smaller, larger) = (srcForVertex, tgtForVertex)
-
- // val newLabelWithDir = LabelWithDirection(labelWithDir.labelId, GraphUtil.directions("out"))
-
- propertyInner(LabelMeta.timestamp.name, ts, ts)
- val ret = SnapshotEdge(innerGraph, smaller, larger, innerLabel,
- GraphUtil.directions("out"), op, version, propsWithTs,
- pendingEdgeOpt = pendingEdgeOpt, statusCode = statusCode, lockTs = lockTs, tsInnerValOpt = tsInnerValOpt)
- ret
- }
-
- def defaultPropsWithName = Json.obj("from" -> srcVertex.innerId.toString(), "to" -> tgtVertex.innerId.toString(),
- "label" -> innerLabel.label, "service" -> innerLabel.serviceName)
-
- def propsWithName =
- for {
- (_, v) <- propsWithTs.asScala
- meta = v.labelMeta
- jsValue <- innerValToJsValue(v.innerVal, meta.dataType)
- } yield meta.name -> jsValue
-
- def updateTgtVertex(id: InnerValLike) = {
- val newId = TargetVertexId(tgtVertex.id.column, id)
- val newTgtVertex = innerGraph.newVertex(newId, tgtVertex.ts, tgtVertex.props)
- S2Edge(innerGraph, srcVertex, newTgtVertex, innerLabel, dir, op, version, propsWithTs, tsInnerValOpt = tsInnerValOpt)
- }
-
- def updateOriginalEdgeOpt(newOriginalEdgeOpt: S2EdgeLike): S2EdgeLike = {
- S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, propsWithTs, parentEdges,
- Option(newOriginalEdgeOpt), pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
- }
-
- def rank(r: RankParam): Double =
- if (r.keySeqAndWeights.size <= 0) 1.0f
- else {
- var sum: Double = 0
-
- for ((labelMeta, w) <- r.keySeqAndWeights) {
- if (propsWithTs.containsKey(labelMeta.name)) {
- val innerValWithTs = propsWithTs.get(labelMeta.name)
- val cost = try innerValWithTs.innerVal.toString.toDouble catch {
- case e: Exception =>
- logger.error("toInnerval failed in rank", e)
- 1.0
- }
- sum += w * cost
- }
- }
- sum
- }
+ def toSnapshotEdge: SnapshotEdge = SnapshotEdge.apply(this)
def checkProperty(key: String): Boolean = propsWithTs.containsKey(key)
- def copyEdge(srcVertex: S2VertexLike = srcVertex,
- tgtVertex: S2VertexLike = tgtVertex,
- innerLabel: Label = innerLabel,
- dir: Int = dir,
- op: Byte = op,
- version: Long = version,
- propsWithTs: State = S2Edge.propsToState(this.propsWithTs),
- parentEdges: Seq[EdgeWithScore] = parentEdges,
- originalEdgeOpt: Option[S2EdgeLike] = originalEdgeOpt,
- pendingEdgeOpt: Option[S2EdgeLike] = pendingEdgeOpt,
- statusCode: Byte = statusCode,
- lockTs: Option[Long] = lockTs,
- tsInnerValOpt: Option[InnerValLike] = tsInnerValOpt,
- ts: Long = ts): S2EdgeLike = {
- val edge = new S2Edge(innerGraph, srcVertex, tgtVertex, innerLabel, dir, op, version, S2Edge.EmptyProps,
- parentEdges, originalEdgeOpt, pendingEdgeOpt, statusCode, lockTs, tsInnerValOpt)
- S2Edge.fillPropsWithTs(edge, propsWithTs)
- edge.propertyInner(LabelMeta.timestamp.name, ts, ts)
- edge
- }
-
- def copyEdgeWithState(state: State, ts: Long): S2EdgeLike = {
- val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
- S2Edge.fillPropsWithTs(newEdge, state)
- newEdge.propertyInner(LabelMeta.timestamp.name, ts, ts)
- newEdge
- }
-
def copyEdgeWithState(state: State): S2EdgeLike = {
- val newEdge = copy(propsWithTs = S2Edge.EmptyProps)
- S2Edge.fillPropsWithTs(newEdge, state)
- newEdge
+ builder.copyEdgeWithState(state)
}
def copyOp(newOp: Byte): S2EdgeLike = {
- copy(op = newOp)
+ builder.copyEdge(op = newOp)
}
def copyVersion(newVersion: Long): S2EdgeLike = {
- copy(version = newVersion)
+ builder.copyEdge(version = newVersion)
}
def copyParentEdges(parents: Seq[EdgeWithScore]): S2EdgeLike = {
- copy(parentEdges = parents)
+ builder.copyEdge(parentEdges = parents)
}
+ def copyOriginalEdgeOpt(newOriginalEdgeOpt: Option[S2EdgeLike]): S2EdgeLike =
+ builder.copyEdge(originalEdgeOpt = newOriginalEdgeOpt)
+
def copyStatusCode(newStatusCode: Byte): S2EdgeLike = {
- copy(statusCode = newStatusCode)
+ builder.copyEdge(statusCode = newStatusCode)
}
def copyLockTs(newLockTs: Option[Long]): S2EdgeLike = {
- copy(lockTs = newLockTs)
+ builder.copyEdge(lockTs = newLockTs)
}
def vertices(direction: Direction): util.Iterator[structure.Vertex] = {
@@ -289,19 +162,9 @@ trait S2EdgeLike extends Edge with GraphElement {
direction match {
case Direction.OUT =>
- // val newVertexId = this.direction match {
- // case "out" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
- // case "in" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
- // case _ => throw new IllegalArgumentException("direction can only be out/in.")
- // }
val newVertexId = edgeId.srcVertexId
innerGraph.getVertex(newVertexId).foreach(arr.add)
case Direction.IN =>
- // val newVertexId = this.direction match {
- // case "in" => VertexId(innerLabel.srcColumn, srcVertex.innerId)
- // case "out" => VertexId(innerLabel.tgtColumn, tgtVertex.innerId)
- // case _ => throw new IllegalArgumentException("direction can only be out/in.")
- // }
val newVertexId = edgeId.tgtVertexId
innerGraph.getVertex(newVertexId).foreach(arr.add)
case _ =>
@@ -331,39 +194,33 @@ trait S2EdgeLike extends Edge with GraphElement {
}
def property[V](key: String): Property[V] = {
- val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new java.lang.IllegalStateException(s"$key is not configured on Edge."))
if (propsWithTs.containsKey(key)) propsWithTs.get(key).asInstanceOf[Property[V]]
- else {
- Property.empty()
- // val default = innerLabel.metaPropsDefaultMapInner(labelMeta)
- // propertyInner(key, default.innerVal.value, default.ts).asInstanceOf[Property[V]]
- }
+ else Property.empty()
}
- // just for tinkerpop: save to storage, do not use for internal
def property[V](key: String, value: V): Property[V] = {
S2Property.assertValidProp(key, value)
val v = propertyInner(key, value, System.currentTimeMillis())
- val newTs = props.get(LabelMeta.timestamp.name).map(_.toString.toLong + 1).getOrElse(System.currentTimeMillis())
- val newEdge = this.copyEdge(ts = newTs)
+ val newTs =
+ if (propsWithTs.containsKey(LabelMeta.timestamp.name))
+ propsWithTs.get(LabelMeta.timestamp.name).innerVal.toString().toLong + 1
+ else
+ System.currentTimeMillis()
+
+ val newEdge = builder.copyEdge(ts = newTs)
Await.result(innerGraph.mutateEdges(Seq(newEdge), withWait = true), innerGraph.WaitTimeout)
v
}
- def propertyInner[V](key: String, value: V, ts: Long): Property[V] = {
- val labelMeta = innerLabel.metaPropsInvMap.getOrElse(key, throw new RuntimeException(s"$key is not configured on Edge."))
- val newProp = new S2Property[V](this, labelMeta, key, value, ts)
- propsWithTs.put(key, newProp)
- newProp
- }
+ def propertyInner[V](key: String, value: V, ts: Long): Property[V] = builder.propertyInner(key, value, ts)
def remove(): Unit = {
if (graph.features().edge().supportsRemoveEdges()) {
val requestTs = System.currentTimeMillis()
- val edgeToDelete = this.copyEdge(op = GraphUtil.operations("delete"),
+ val edgeToDelete = builder.copyEdge(op = GraphUtil.operations("delete"),
version = version + S2Edge.incrementVersion, propsWithTs = S2Edge.propsToState(updatePropsWithTs()), ts = requestTs)
// should we delete related edges also?
val future = innerGraph.mutateEdges(Seq(edgeToDelete), withWait = true)
@@ -376,43 +233,14 @@ trait S2EdgeLike extends Edge with GraphElement {
def graph(): Graph = innerGraph
- lazy val edgeId: EdgeId = {
- // NOTE: xxxForVertex makes direction to be "out"
- val timestamp = if (this.innerLabel.consistencyLevel == "strong") 0l else ts
- // EdgeId(srcVertex.innerId, tgtVertex.innerId, label(), "out", timestamp)
- val (srcColumn, tgtColumn) = innerLabel.srcTgtColumn(dir)
- if (direction == "out")
- EdgeId(VertexId(srcColumn, srcVertex.id.innerId), VertexId(tgtColumn, tgtVertex.id.innerId), label(), "out", timestamp)
- else
- EdgeId(VertexId(tgtColumn, tgtVertex.id.innerId), VertexId(srcColumn, srcVertex.id.innerId), label(), "out", timestamp)
- }
+ lazy val edgeId: EdgeId = builder.edgeId
def id(): AnyRef = edgeId
def label(): String = innerLabel.label
- private def toProps(): Map[String, Any] = {
- for {
- (labelMeta, defaultVal) <- innerLabel.metaPropsDefaultMapInner
- } yield {
- // labelMeta.name -> propsWithTs.get(labelMeta.name).map(_.innerValWithTs).getOrElse(defaultVal).innerVal.value
- val value =
- if (propsWithTs.containsKey(labelMeta.name)) {
- propsWithTs.get(labelMeta.name).value
- } else {
- defaultVal.innerVal.value
- }
- labelMeta.name -> value
- }
- }
-
def toLogString: String = {
// val allPropsWithName = defaultPropsWithName ++ Json.toJson(propsWithName).asOpt[JsObject].getOrElse(Json.obj())
List(ts, GraphUtil.fromOp(op), "e", srcVertex.innerId, tgtVertex.innerId, innerLabel.label, propsWithTs).mkString("\t")
}
-
- private def getServiceColumn(vertex: S2VertexLike, defaultServiceColumn: ServiceColumn) =
- if (vertex.id.column == ServiceColumn.Default) defaultServiceColumn else vertex.id.column
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
index 28b15d0..1b9c94b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageIO.scala
@@ -121,7 +121,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
if where == WhereParser.success || where.filter(edge)
convertedEdge <- if (isDefaultTransformer) Seq(edge) else convertEdges(queryParam, edge, nextStepOpt)
} yield {
- val score = edge.rank(queryParam.rank)
+ val score = queryParam.rank.score(edge)
EdgeWithScore(convertedEdge, score, label)
}
StepResult(edgeWithScores = edgeWithScores, grouped = Nil, degreeEdges = degreeEdges, cursors = lastCursor)
@@ -134,7 +134,7 @@ class StorageIO(val graph: S2Graph, val serDe: StorageSerDe) {
if where == WhereParser.success || where.filter(edge)
convertedEdge <- if (isDefaultTransformer) Seq(edge) else convertEdges(queryParam, edge, nextStepOpt)
} yield {
- val edgeScore = edge.rank(queryParam.rank)
+ val edgeScore = queryParam.rank.score(edge)
val score = queryParam.scorePropagateOp match {
case "plus" => edgeScore + prevScore
case "divide" =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/2b5df1dd/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
index e2621af..79c9dc3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/serde/MutationHelper.scala
@@ -30,17 +30,17 @@ class MutationHelper(storage: Storage) {
val edge = edgeWithScore.edge
val score = edgeWithScore.score
- val edgeSnapshot = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeSnapshot = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
val reversedSnapshotEdgeMutations = serDe.snapshotEdgeSerializer(edgeSnapshot.toSnapshotEdge).toKeyValues.map(_.copy(operation = SKeyValue.Put))
- val edgeForward = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeForward = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
val forwardIndexedEdgeMutations = edgeForward.edgesWithIndex.flatMap { indexEdge =>
serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
io.buildIncrementsAsync(indexEdge, -1L)
}
/* reverted direction */
- val edgeRevert = edge.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
+ val edgeRevert = edge.builder.copyEdge(propsWithTs = S2Edge.propsToState(edge.updatePropsWithTs()))
val reversedIndexedEdgesMutations = edgeRevert.duplicateEdge.edgesWithIndex.flatMap { indexEdge =>
serDe.indexEdgeSerializer(indexEdge).toKeyValues.map(_.copy(operation = SKeyValue.Delete)) ++
io.buildIncrementsAsync(indexEdge, -1L)
[23/23] incubator-s2graph git commit: [S2GRAPH-170]: Create Interface
for S2Edge/S2Vertex/S2Graph
Posted by st...@apache.org.
[S2GRAPH-170]: Create Interface for S2Edge/S2Vertex/S2Graph
JIRA:
[S2GRAPH-170] https://issues.apache.org/jira/browse/S2GRAPH-170
Pull Request:
Closes #127
Author
DO YUNG YOON <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/16f18ffe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/16f18ffe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/16f18ffe
Branch: refs/heads/master
Commit: 16f18ffe05a786dedfcba3e10b024345896c1ba2
Parents: 79e1ffa 41885a8
Author: DO YUNG YOON <st...@apache.org>
Authored: Sun Nov 19 11:26:39 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sun Nov 19 11:31:05 2017 +0900
----------------------------------------------------------------------
CHANGES | 1 +
.../loader/subscriber/GraphSubscriber.scala | 67 +-
.../loader/subscriber/TransferToHFile.scala | 14 +-
.../s2graph/loader/subscriber/WalLogStat.scala | 3 +-
.../loader/subscriber/WalLogToHDFS.scala | 3 +-
s2core/build.sbt | 3 +-
.../s2graph/core/GraphElementBuilder.scala | 336 +++
.../org/apache/s2graph/core/GraphUtil.scala | 11 +-
.../org/apache/s2graph/core/Management.scala | 2 +-
.../org/apache/s2graph/core/PostProcess.scala | 56 +-
.../org/apache/s2graph/core/QueryParam.scala | 39 +-
.../org/apache/s2graph/core/QueryResult.scala | 16 +-
.../scala/org/apache/s2graph/core/S2Edge.scala | 519 +----
.../org/apache/s2graph/core/S2EdgeBuilder.scala | 115 ++
.../org/apache/s2graph/core/S2EdgeLike.scala | 255 +++
.../s2graph/core/S2EdgePropertyHelper.scala | 89 +
.../scala/org/apache/s2graph/core/S2Graph.scala | 1942 ++----------------
.../org/apache/s2graph/core/S2GraphLike.scala | 371 ++++
.../org/apache/s2graph/core/S2GraphTp.scala | 117 ++
.../org/apache/s2graph/core/S2Property.scala | 2 +-
.../org/apache/s2graph/core/S2Vertex.scala | 239 +--
.../apache/s2graph/core/S2VertexBuilder.scala | 54 +
.../org/apache/s2graph/core/S2VertexLike.scala | 202 ++
.../apache/s2graph/core/S2VertexProperty.scala | 2 +-
.../apache/s2graph/core/TraversalHelper.scala | 604 ++++++
.../s2graph/core/features/S2Features.scala | 38 +
.../s2graph/core/index/IndexProvider.scala | 24 +-
.../s2graph/core/parsers/WhereParser.scala | 40 +-
.../s2graph/core/rest/RequestParser.scala | 12 +-
.../apache/s2graph/core/rest/RestHandler.scala | 4 +-
.../apache/s2graph/core/storage/Storage.scala | 84 +-
.../apache/s2graph/core/storage/StorageIO.scala | 32 +-
.../s2graph/core/storage/StorageReadable.scala | 16 +-
.../s2graph/core/storage/StorageSerDe.scala | 6 +-
.../storage/WriteWriteConflictResolver.scala | 43 +-
.../core/storage/hbase/AsynchbaseStorage.scala | 2 +-
.../hbase/AsynchbaseStorageReadable.scala | 16 +-
.../storage/hbase/AsynchbaseStorageSerDe.scala | 8 +-
.../core/storage/serde/MutationHelper.scala | 190 ++
.../tall/IndexEdgeDeserializable.scala | 53 +-
.../wide/IndexEdgeDeserializable.scala | 45 +-
.../tall/SnapshotEdgeDeserializable.scala | 14 +-
.../tall/SnapshotEdgeSerializable.scala | 8 +-
.../wide/SnapshotEdgeDeserializable.scala | 15 +-
.../wide/SnapshotEdgeSerializable.scala | 8 +-
.../serde/vertex/VertexDeserializable.scala | 73 -
.../serde/vertex/VertexSerializable.scala | 62 -
.../vertex/tall/VertexDeserializable.scala | 11 +-
.../serde/vertex/tall/VertexSerializable.scala | 4 +-
.../vertex/wide/VertexDeserializable.scala | 11 +-
.../serde/vertex/wide/VertexSerializable.scala | 4 +-
.../org/apache/s2graph/core/S2EdgeTest.scala | 24 +-
.../s2graph/core/TestCommonWithModels.scala | 3 +-
.../s2graph/core/parsers/WhereParserTest.scala | 24 +-
.../s2graph/core/storage/StorageIOTest.scala | 12 +-
.../core/storage/hbase/IndexEdgeTest.scala | 10 +-
.../core/tinkerpop/S2GraphProvider.scala | 4 +-
.../process/S2GraphProcessStandardTest.scala | 4 +-
.../S2GraphStructureIntegrateTest.scala | 4 +-
.../S2GraphStructureStandardTest.scala | 4 +-
.../rest/play/controllers/EdgeController.scala | 14 +-
61 files changed, 3051 insertions(+), 2937 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16f18ffe/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 5f61edf,6e685ae..0c4da4c
--- a/CHANGES
+++ b/CHANGES
@@@ -48,6 -47,6 +48,7 @@@ Release Notes - S2Graph - Version 0.2.
* [S2GRAPH-165] - Add TinkerPop support documentation on README
* [S2GRAPH-167] - Provide Vertex CSV File BulkLoad
* [S2GRAPH-169] - Separate multiple functionalities on Storage class into multiple Interface.
++ * [S2GRAPH-170] - Create Interface for S2Edge/S2Vertex/S2Graph.
** New Feature
* [S2GRAPH-123] - Support different index on out/in direction.