You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2017/07/31 01:05:20 UTC
[06/25] incubator-s2graph git commit: add IndexProviderTest.
add IndexProviderTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/1cd00df0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/1cd00df0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/1cd00df0
Branch: refs/heads/master
Commit: 1cd00df0a7a58f454182d997c0a8c8337a50486a
Parents: 0594fd0
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu Jul 13 07:09:40 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu Jul 13 07:09:40 2017 +0900
----------------------------------------------------------------------
.../core/io/tinkerpop/optimize/S2GraphStep.java | 7 ++-
.../s2graph/core/index/IndexProvider.scala | 34 ++++++++---
.../core/Integrate/IntegrateCommon.scala | 2 +-
.../s2graph/core/index/IndexProviderTest.scala | 63 +++++++++++++++++---
4 files changed, 90 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
----------------------------------------------------------------------
diff --git a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
index d494995..e769718 100644
--- a/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
+++ b/s2core/src/main/java/org/apache/s2graph/core/io/tinkerpop/optimize/S2GraphStep.java
@@ -66,7 +66,12 @@ public class S2GraphStep<S, E extends Element> extends GraphStep<S, E> {
String queryString = IndexProvider$.MODULE$.buildQueryString(hasContainers);
Boolean isVertex = Vertex.class.isAssignableFrom(this.returnClass);
- List<String> ids = graph.indexProvider().fetchIds(queryString, isVertex);
+
+ List<String> ids = new ArrayList<>();
+
+ if (isVertex) graph.indexProvider().fetchVertexIds(queryString);
+ else graph.indexProvider().fetchEdgeIds(queryString);
+
return (Iterator) (Vertex.class.isAssignableFrom(this.returnClass) ? graph.vertices(ids) : graph.edges(ids));
});
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index c8b2ebb..3d5b164 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -10,7 +10,7 @@ import org.apache.lucene.store.RAMDirectory
import org.apache.s2graph.core.io.Conversions
import org.apache.s2graph.core.{EdgeId, S2Edge, S2Vertex}
import org.apache.s2graph.core.mysqls._
-import org.apache.s2graph.core.types.InnerValLike
+import org.apache.s2graph.core.types.{InnerValLike, VertexId}
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer
import play.api.libs.json.Json
@@ -38,12 +38,14 @@ object IndexProvider {
trait IndexProvider {
//TODO: Seq nee do be changed into stream
- def fetchIds(queryString: String, isVertex: Boolean = true): java.util.List[String]
+ def fetchEdgeIds(queryString: String): java.util.List[EdgeId]
+
+ def fetchVertexIds(queryString: String): java.util.List[VertexId]
def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean]
def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean]
-`
+
def shutdown(): Unit
}
@@ -85,9 +87,9 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
edges.map(_ => true)
}
- override def fetchIds(queryString: String, isVertex: Boolean = true): java.util.List[String] = {
- val field = if (isVertex) vidField else eidField
- val ids = new java.util.ArrayList[String]
+ override def fetchEdgeIds(queryString: String): java.util.List[EdgeId] = {
+ val field = eidField
+ val ids = new java.util.ArrayList[EdgeId]
val q = new QueryParser(field, analyzer).parse(queryString)
val hitsPerPage = 10
val reader = DirectoryReader.open(directory)
@@ -97,13 +99,31 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
docs.scoreDocs.foreach { scoreDoc =>
val document = searcher.doc(scoreDoc.doc)
- ids.add(document.get(field))
+ ids.add(Conversions.s2EdgeIdReads.reads(Json.parse(document.get(field))).get)
}
reader.close()
ids
}
+ override def fetchVertexIds(queryString: String): java.util.List[VertexId] = {
+ val field = vidField
+ val ids = new java.util.ArrayList[VertexId]
+ val q = new QueryParser(field, analyzer).parse(queryString)
+ val hitsPerPage = 10
+ val reader = DirectoryReader.open(directory)
+ val searcher = new IndexSearcher(reader)
+
+ val docs = searcher.search(q, hitsPerPage)
+
+ docs.scoreDocs.foreach { scoreDoc =>
+ val document = searcher.doc(scoreDoc.doc)
+ ids.add(Conversions.s2VertexIdReads.reads(Json.parse(document.get(field))).get)
+ }
+
+ reader.close()
+ ids
+ }
override def shutdown(): Unit = {
writer.close()
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
index 3d72432..fd496ee 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/IntegrateCommon.scala
@@ -20,7 +20,7 @@
package org.apache.s2graph.core.Integrate
import com.typesafe.config._
-import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.core.mysqls.{Label, Model}
import org.apache.s2graph.core.rest.{RequestParser, RestHandler}
import org.apache.s2graph.core.utils.logger
import org.apache.s2graph.core._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/1cd00df0/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
index e9150e9..f56ac71 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
@@ -1,35 +1,84 @@
package org.apache.s2graph.core.index
import org.apache.s2graph.core.Integrate.IntegrateCommon
+import org.apache.s2graph.core.{Management, S2Vertex}
import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types.{InnerVal, InnerValLikeWithTs}
-import org.apache.s2graph.core.utils.logger
+
class IndexProviderTest extends IntegrateCommon {
val indexProvider = IndexProvider(config)
- test("test write/query ") {
+ test("test vertex write/query") {
+ import TestUtil._
+// Management.addVertexProp(testServiceName, testColumnName, "time", "long")
+
+ val testService = Service.findByName(TestUtil.testServiceName).get
+ val testColumn = ServiceColumn.find(testService.id.get, TestUtil.testColumnName).get
+ val vertexId = graph.newVertexId(testServiceName)(testColumnName)(1L)
+
+ val propsWithTs = Map(
+// testColumn.metasInvMap("time") -> InnerVal.withLong(1L, "v4")
+ ColumnMeta.timestamp -> InnerVal.withLong(1L, "v4")
+ )
+ val otherPropsWithTs = Map(
+// testColumn.metasInvMap("time") -> InnerVal.withLong(2L, "v4")
+ ColumnMeta.timestamp -> InnerVal.withLong(2L, "v4")
+ )
+ val vertex = graph.newVertex(vertexId)
+ S2Vertex.fillPropsWithTs(vertex, propsWithTs)
+
+ val vertices = Seq(vertex) ++ (0 until 10).map{ ith =>
+ val v = graph.newVertex(vertexId)
+ S2Vertex.fillPropsWithTs(v, otherPropsWithTs)
+ v
+ }
+
+ println(s"[# of vertices]: ${vertices.size}")
+ vertices.foreach(v => println(s"[Vertex]: $v"))
+ indexProvider.mutateVertices(vertices)
+
+ import scala.collection.JavaConversions._
+ val ids = indexProvider.fetchVertexIds("_timestamp: 1")
+ ids.head shouldBe vertex.id
+
+ ids.foreach { id =>
+ println(s"[Id]: $id")
+ }
+ }
+ test("test edge write/query ") {
import TestUtil._
val testLabelName = TestUtil.testLabelName
val testLabel = Label.findByName(testLabelName).getOrElse(throw new IllegalArgumentException)
val vertexId = graph.newVertexId(testServiceName)(testColumnName)(1L)
+ val otherVertexId = graph.newVertexId(testServiceName)(testColumnName)(2L)
val vertex = graph.newVertex(vertexId)
+ val otherVertex = graph.newVertex(otherVertexId)
+
val propsWithTs = Map(
LabelMeta.timestamp -> InnerValLikeWithTs.withLong(1L, 1L, "v4"),
testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(10L, 1L, "v4")
)
+ val otherPropsWithTs = Map(
+ LabelMeta.timestamp -> InnerValLikeWithTs.withLong(2L, 2L, "v4"),
+ testLabel.metaPropsInvMap("time") -> InnerValLikeWithTs.withLong(20L, 1L, "v4")
+ )
val edge = graph.newEdge(vertex, vertex, testLabel, 0, propsWithTs = propsWithTs)
- val edges = Seq(edge)
+ val edges = Seq(edge) ++ (0 until 10).map{ ith =>
+ graph.newEdge(otherVertex, otherVertex, testLabel, 0, propsWithTs = otherPropsWithTs)
+ }
- logger.error(s"[# of edges]: ${edges.size}")
- edges.foreach(e => logger.debug(s"[Edge]: $e"))
+ println(s"[# of edges]: ${edges.size}")
+ edges.foreach(e => println(s"[Edge]: $e"))
indexProvider.mutateEdges(edges)
import scala.collection.JavaConversions._
- val edgeIds = indexProvider.fetchIds("time: 10", isVertex = false)
+ val edgeIds = indexProvider.fetchEdgeIds("time: 10 AND _timestamp: 1")
+ edgeIds.head shouldBe edge.edgeId
edgeIds.foreach { edgeId =>
- logger.debug(s"[EdgeId]: $edgeId")
+ println(s"[EdgeId]: $edgeId")
}
+
}
}