You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2017/07/31 01:05:22 UTC
[08/25] incubator-s2graph git commit: test more on IndexProviderTest.
test more on IndexProviderTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a76dfdc2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a76dfdc2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a76dfdc2
Branch: refs/heads/master
Commit: a76dfdc207fb26b401f601d45f7aa9e793405a13
Parents: 1cd00df
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu Jul 13 10:13:15 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu Jul 13 14:20:32 2017 +0900
----------------------------------------------------------------------
.../core/io/tinkerpop/optimize/S2GraphStep.java | 24 +++---
.../tinkerpop/optimize/S2GraphStepStrategy.java | 1 -
.../optimize/S2GraphTraversalUtil.java | 82 ++++++++++----------
.../scala/org/apache/s2graph/core/S2Graph.scala | 16 +++-
.../s2graph/core/index/IndexProvider.scala | 6 +-
.../s2graph/core/index/IndexProviderTest.scala | 66 +++++++++++-----
.../core/tinkerpop/structure/S2GraphTest.scala | 4 +-
7 files changed, 119 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
----------------------------------------------------------------------
diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
index e769718..f2bc6c8 100644
--- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
+++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
@@ -6,6 +6,7 @@ import org.apache.s2graph.core.S2Graph;
import org.apache.s2graph.core.index.IndexProvider;
import org.apache.s2graph.core.index.IndexProvider$;
import org.apache.s2graph.core.mysqls.Label;
+import org.apache.s2graph.core.types.VertexId;
import org.apache.s2graph.core.utils.logger;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
@@ -16,16 +17,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
-import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
import java.util.*;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> {
private final List<HasContainer> hasContainers = new ArrayList<>();
@@ -67,12 +62,19 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> {
String queryString = IndexProvider$.MODULE$.buildQueryString(hasContainers);
Boolean isVertex = Vertex.class.isAssignableFrom(this.returnClass);
- List<String> ids = new ArrayList<>();
-
- if (isVertex) graph.indexProvider().fetchVertexIds(queryString);
- else graph.indexProvider().fetchEdgeIds(queryString);
+ List<VertexId> vids = new ArrayList<>();
+ List<EdgeId> eids = new ArrayList<>();
- return (Iterator) (Vertex.class.isAssignableFrom(this.returnClass) ? graph.vertices(ids) : graph.edges(ids));
+ if (isVertex) {
+ List<VertexId> ids = graph.indexProvider().fetchVertexIds(queryString);
+ if (ids.isEmpty()) return (Iterator) graph.vertices();
+ else return (Iterator) graph.vertices(ids);
+ }
+ else {
+ List<EdgeId> ids = graph.indexProvider().fetchEdgeIds(queryString);
+ if (ids.isEmpty()) return (Iterator) graph.edges();
+ else return (Iterator) graph.edges(ids);
+ }
});
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java
----------------------------------------------------------------------
diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java
index e30a502..8f40d15 100644
--- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java
+++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStepStrategy.java
@@ -30,7 +30,6 @@ public class S2GraphStepStrategy extends AbstractTraversalStrategy<TraversalStra
final S2GraphStep<?, ?> s2GraphStep = new S2GraphStep<>(originalGraphStep);
TraversalHelper.replaceStep(originalGraphStep, (Step) s2GraphStep, traversal);
-
} else {
Object[] ids = originalGraphStep.getIds();
if (ids[0] instanceof Element) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java
----------------------------------------------------------------------
diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java
index 72f6999..81a1261 100644
--- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java
+++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphTraversalUtil.java
@@ -1,41 +1,41 @@
-package org.apache.s2graph.core.io.tinkerpop.optimize;
-
-import org.apache.s2graph.core.S2Vertex;
-import org.apache.tinkerpop.gremlin.process.traversal.Step;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.wrapped.WrappedVertex;
-
-import java.util.Optional;
-
-public class S2GraphTraversalUtil {
-
- public static S2Vertex getS2Vertex(Element v) {
- while (v instanceof WrappedVertex) {
- v = ((WrappedVertex<Vertex>) v).getBaseVertex();
- }
- if (v instanceof S2Vertex) {
- return (S2Vertex) v;
- } else throw new IllegalArgumentException("Expected traverser of JanusGraph vertex but found: " + v);
- }
-
- public static S2Vertex getS2Vertex(Traverser<? extends Element> traverser) {
- return getS2Vertex(traverser.get());
- }
-
-
- public static Step getNextNonIdentityStep(final Step start) {
- Step currentStep = start.getNextStep();
- //Skip over identity steps
- while (currentStep instanceof IdentityStep) currentStep = currentStep.getNextStep();
- return currentStep;
- }
-
-
-}
+//package org.apache.s2graph.core.io.tinkerpop.optimize;
+//
+//import org.apache.s2graph.core.S2Vertex;
+//import org.apache.tinkerpop.gremlin.process.traversal.Step;
+//import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+//import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+//import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep;
+//import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+//import org.apache.tinkerpop.gremlin.structure.Edge;
+//import org.apache.tinkerpop.gremlin.structure.Element;
+//import org.apache.tinkerpop.gremlin.structure.Graph;
+//import org.apache.tinkerpop.gremlin.structure.Vertex;
+//import org.apache.tinkerpop.gremlin.structure.util.wrapped.WrappedVertex;
+//
+//import java.util.Optional;
+//
+//public class S2GraphTraversalUtil {
+//
+// public static S2Vertex getS2Vertex(Element v) {
+// while (v instanceof WrappedVertex) {
+// v = ((WrappedVertex<Vertex>) v).getBaseVertex();
+// }
+// if (v instanceof S2Vertex) {
+// return (S2Vertex) v;
+// } else throw new IllegalArgumentException("Expected traverser of JanusGraph vertex but found: " + v);
+// }
+//
+// public static S2Vertex getS2Vertex(Traverser<? extends Element> traverser) {
+// return getS2Vertex(traverser.get());
+// }
+//
+//
+// public static Step getNextNonIdentityStep(final Step start) {
+// Step currentStep = start.getNextStep();
+// //Skip over identity steps
+// while (currentStep instanceof IdentityStep) currentStep = currentStep.getNextStep();
+// return currentStep;
+// }
+//
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 8a53335..8ee675d 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -1826,6 +1826,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
}
addVertexInner(vertex)
+ indexProvider.mutateVertices(Seq(vertex))
+
+ vertex
}
def addVertex(id: VertexId,
@@ -1834,11 +1837,14 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
op: Byte = 0,
belongLabelIds: Seq[Int] = Seq.empty): S2Vertex = {
val vertex = newVertex(id, ts, props, op, belongLabelIds)
+
val future = mutateVertices(Seq(vertex), withWait = true).map { rets =>
if (rets.forall(identity)) vertex
else throw new RuntimeException("addVertex failed.")
}
- Await.result(future, WaitTimeout)
+ Await.ready(future, WaitTimeout)
+
+ vertex
}
def addVertexInner(vertex: S2Vertex): S2Vertex = {
@@ -1846,7 +1852,9 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
if (rets.forall(identity)) vertex
else throw new RuntimeException("addVertex failed.")
}
- Await.result(future, WaitTimeout)
+ Await.ready(future, WaitTimeout)
+
+ vertex
}
def addEdge(srcVertex: S2Vertex, labelName: String, tgtVertex: Vertex, kvs: AnyRef*): Edge = {
@@ -1876,10 +1884,10 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends Graph
val edge = newEdge(srcVertex, otherV, label, dir, op = op, version = ts, propsWithTs = propsWithTs)
- indexProvider.mutateEdges(Seq(edge))
-
val future = mutateEdges(Seq(edge), withWait = true)
Await.ready(future, WaitTimeout)
+
+ indexProvider.mutateEdges(Seq(edge))
edge
} catch {
case e: LabelNotExistException => throw new java.lang.IllegalArgumentException(e)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index 3d5b164..b0d2a4b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -99,7 +99,8 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
docs.scoreDocs.foreach { scoreDoc =>
val document = searcher.doc(scoreDoc.doc)
- ids.add(Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get)
+ val id = Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get
+// ids.add(id);
}
reader.close()
@@ -118,7 +119,8 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
docs.scoreDocs.foreach { scoreDoc =>
val document = searcher.doc(scoreDoc.doc)
- ids.add(Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get)
+ val id = Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get
+// ids.add(id)
}
reader.close()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
index f56ac71..b4e5ed2 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
@@ -4,10 +4,11 @@ import org.apache.s2graph.core.Integrate.IntegrateCommon
import org.apache.s2graph.core.{Management, S2Vertex}
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs}
-
+import scala.collection.JavaConversions._
class IndexProviderTest extends IntegrateCommon {
val indexProvider = IndexProvider(config)
+ val numOfTry = 1
test("test vertex write/query") {
import TestUtil._
@@ -28,22 +29,23 @@ class IndexProviderTest extends IntegrateCommon {
val vertex = graph.newVertex(vertexId)
S2Vertex.fillPropsWithTs(vertex, propsWithTs)
- val vertices = Seq(vertex) ++ (0 until 10).map{ ith =>
- val v = graph.newVertex(vertexId)
- S2Vertex.fillPropsWithTs(v, otherPropsWithTs)
- v
- }
+ val otherVertex = graph.newVertex(vertexId)
+ S2Vertex.fillPropsWithTs(otherVertex, otherPropsWithTs)
+
+ val numOfOthers = 10
+ val vertices = Seq(vertex) ++ (0 until numOfOthers).map(_ => otherVertex)
println(s"[# of vertices]: ${vertices.size}")
vertices.foreach(v => println(s"[Vertex]: $v"))
indexProvider.mutateVertices(vertices)
- import scala.collection.JavaConversions._
- val ids = indexProvider.fetchVertexIds("_timestamp: 1")
- ids.head shouldBe vertex.id
+ (0 until numOfTry).foreach { ith =>
+ var ids = indexProvider.fetchVertexIds("_timestamp: 1")
+ ids.head shouldBe vertex.id
- ids.foreach { id =>
- println(s"[Id]: $id")
+ ids.foreach { id =>
+ println(s"[Id]: $id")
+ }
}
}
test("test edge write/query ") {
@@ -64,21 +66,47 @@ class IndexProviderTest extends IntegrateCommon {
testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(20L, 1L, "v4")
)
val edge = graph.newEdge(vertex, vertex, testLabel, 0, propsWithTs = propsWithTs)
- val edges = Seq(edge) ++ (0 until 10).map{ ith =>
- graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs)
- }
+ val otherEdge = graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs)
+ val numOfOthers = 10
+ val edges = Seq(edge) ++ (0 until numOfOthers).map(_ => otherEdge)
println(s"[# of edges]: ${edges.size}")
edges.foreach(e => println(s"[Edge]: $e"))
indexProvider.mutateEdges(edges)
- import scala.collection.JavaConversions._
- val edgeIds = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1")
- edgeIds.head shouldBe edge.edgeId
+ // match
+ (0 until numOfTry).foreach { _ =>
+
+ val ids = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1")
+ ids.head shouldBe edge.edgeId
- edgeIds.foreach { edgeId =>
- println(s"[EdgeId]: $edgeId")
+ ids.foreach { id =>
+ println(s"[Id]: $id")
+ }
}
+ // match and not
+ (0 until numOfTry).foreach { _ =>
+ val ids = indexProvider.fetchEdgeIds("time: 20 AND NOT _timestamp: 1")
+ // ids.size shouldBe 0
+ ids.size shouldBe numOfOthers
+
+ ids.foreach { id =>
+ id shouldBe otherEdge.edgeId
+ println(s"[Id]: $id")
+ }
+ }
+
+ // range
+ (0 until numOfTry).foreach { _ =>
+ val ids = indexProvider.fetchEdgeIds("time: [0 TO 10]")
+ // ids.size shouldBe 0
+ ids.size shouldBe 1
+
+ ids.foreach { id =>
+ id shouldBe edge.edgeId
+ println(s"[Id]: $id")
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a76dfdc2/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
index 1ef7890..63badc0 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
@@ -466,8 +466,8 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels {
val ls = graph.traversal().V().has("name", "josh")
val l = ls.toList
- logger.error(s"[Size]: ${l.size}")
- logger.error(l.toArray.toSeq.mkString("\n"))
+ println(s"[Size]: ${l.size}")
+ println(l.toArray.toSeq.mkString("\n"))
println(ls.toList)
ls
}