You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2018/04/23 04:29:37 UTC
[5/8] incubator-s2graph git commit: add test case for search vertex
query
add test case for search vertex query
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/8ed6d20f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/8ed6d20f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/8ed6d20f
Branch: refs/heads/master
Commit: 8ed6d20faa907e08f43a22360bcbeaab43d273e8
Parents: f187040
Author: daewon <da...@apache.org>
Authored: Fri Apr 20 14:50:17 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Fri Apr 20 14:50:17 2018 +0900
----------------------------------------------------------------------
.../scala/org/apache/s2graph/core/S2Graph.scala | 1 +
.../s2graph/core/index/IndexProvider.scala | 2 -
.../core/index/LuceneIndexProvider.scala | 25 ++--
.../graphql/repository/GraphRepository.scala | 7 +-
.../s2graph/graphql/types/FieldResolver.scala | 23 +++-
.../apache/s2graph/graphql/types/S2Type.scala | 16 +--
.../apache/s2graph/graphql/ScenarioTest.scala | 134 +++++++++++++------
7 files changed, 130 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index 4a28f0a..7f19cb4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -476,6 +476,7 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
val futures = edgesWithIdx.groupBy { case (e, idx) => e.innerLabel }.map { case (label, edgeGroup) =>
getStorage(label).incrementCounts(label.hbaseZkAddr, edgeGroup.map(_._1), withWait).map(_.zip(edgeGroup.map(_._2)))
}
+
Future.sequence(futures).map { ls =>
ls.flatten.toSeq.sortBy(_._2).map(_._1)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index ffbebf4..f573ff6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -35,8 +35,6 @@ import scala.util.Try
object IndexProvider {
import GlobalIndex._
- //TODO: Fix Me
- val hitsPerPage = 100000
val IdField = "id"
def apply(config: Config)(implicit ec: ExecutionContext): IndexProvider = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala
index 8d5f997..16c094a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/LuceneIndexProvider.scala
@@ -24,12 +24,12 @@ import java.util
import com.typesafe.config.Config
import org.apache.lucene.analysis.core.KeywordAnalyzer
-import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.document.{Document, Field, StringField}
import org.apache.lucene.index.{DirectoryReader, IndexWriter, IndexWriterConfig, Term}
import org.apache.lucene.queryparser.classic.{ParseException, QueryParser}
import org.apache.lucene.search.{IndexSearcher, Query}
import org.apache.lucene.store.{BaseDirectory, RAMDirectory, SimpleFSDirectory}
+import org.apache.lucene.search.TopScoreDocCollector
import org.apache.s2graph.core.io.Conversions
import org.apache.s2graph.core.mysqls.GlobalIndex
import org.apache.s2graph.core.types.VertexId
@@ -53,6 +53,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
val writers = mutable.Map.empty[String, IndexWriter]
val directories = mutable.Map.empty[String, BaseDirectory]
val baseDirectory = scala.util.Try(config.getString("index.provider.base.dir")).getOrElse(".")
+ val MAX_RESULTS = 100000
private def getOrElseDirectory(indexName: String): BaseDirectory = {
val pathname = s"${baseDirectory}/${indexName}"
@@ -136,7 +137,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
vertices.foreach { vertex =>
toDocument(vertex, forceToIndex).foreach { doc =>
val vId = vertex.id.toString()
-// logger.error(s"DOC: ${doc}")
+ // logger.error(s"DOC: ${doc}")
writer.updateDocument(new Term(vidField, vId), doc)
}
@@ -166,21 +167,21 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
edges.map(_ => true)
}
- private def fetchInner[T](q: Query, indexKey: String, field: String, reads: Reads[T]): util.List[T] = {
+ private def fetchInner[T](q: Query, offset: Int, limit: Int, indexKey: String, field: String, reads: Reads[T]): util.List[T] = {
val ids = new java.util.HashSet[T]
- var reader: DirectoryReader = null
+ var reader: DirectoryReader = null
try {
val reader = DirectoryReader.open(getOrElseDirectory(indexKey))
val searcher = new IndexSearcher(reader)
+ val collector = TopScoreDocCollector.create(MAX_RESULTS)
+ val startIndex = offset * limit
- val docs = searcher.search(q, hitsPerPage)
-// logger.error(s"total hit: ${docs.scoreDocs.length}")
+ searcher.search(q, collector)
- docs.scoreDocs.foreach { scoreDoc =>
+ val hits = collector.topDocs(startIndex, limit)
+ hits.scoreDocs.foreach { scoreDoc =>
val document = searcher.doc(scoreDoc.doc)
-// logger.error(s"DOC_IN_L: ${document.toString}")
-
val id = reads.reads(Json.parse(document.get(field))).get
ids.add(id)
}
@@ -199,7 +200,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
try {
val q = new QueryParser(field, analyzer).parse(queryString)
- fetchInner[VertexId](q, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads)
+ fetchInner[VertexId](q, 0, 100, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads)
} catch {
case ex: ParseException =>
logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex)
@@ -213,7 +214,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
try {
val q = new QueryParser(field, analyzer).parse(queryString)
- fetchInner[EdgeId](q, GlobalIndex.EdgeIndexName, field, Conversions.s2EdgeIdReads)
+ fetchInner[EdgeId](q, 0, 100, GlobalIndex.EdgeIndexName, field, Conversions.s2EdgeIdReads)
} catch {
case ex: ParseException =>
logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex)
@@ -230,7 +231,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
val field = vidField
try {
val q = new QueryParser(field, analyzer).parse(queryString)
- fetchInner[VertexId](q, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads)
+ fetchInner[VertexId](q, vertexQueryParam.offset, vertexQueryParam.limit, GlobalIndex.VertexIndexName, vidField, Conversions.s2VertexIdReads)
} catch {
case ex: ParseException =>
logger.error(s"[IndexProvider]: ${queryString} parse failed.", ex)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
index a83b7f2..e5a04b1 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
@@ -41,15 +41,14 @@ object GraphRepository {
}
implicit val edgeHasId = new HasId[(S2VertexLike, QueryParam, Seq[S2EdgeLike]), DeferFetchEdges] {
- override def id(value: (S2VertexLike, QueryParam, Seq[S2EdgeLike])): DeferFetchEdges =
- DeferFetchEdges(value._1, value._2)
+ override def id(value: (S2VertexLike, QueryParam, Seq[S2EdgeLike])): DeferFetchEdges = DeferFetchEdges(value._1, value._2)
}
val vertexFetcher =
- Fetcher((ctx: GraphRepository, ids: Seq[VertexQueryParam]) => {
+ Fetcher((ctx: GraphRepository, queryParams: Seq[VertexQueryParam]) => {
implicit val ec = ctx.ec
- Future.traverse(ids)(ctx.getVertices).map(vs => ids.zip(vs))
+ Future.traverse(queryParams)(ctx.getVertices).map(vs => queryParams.zip(vs))
})
val edgeFetcher = Fetcher((ctx: GraphRepository, ids: Seq[DeferFetchEdges]) => {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
index 4f092dd..1625f58 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
@@ -49,19 +49,29 @@ object FieldResolver {
(vertex, qp)
}
- def serviceColumnOnService(column: ServiceColumn, c: Context[GraphRepository, Any]): (Seq[S2VertexLike], Boolean) = {
+ def serviceColumnOnService(column: ServiceColumn, c: Context[GraphRepository, Any]): VertexQueryParam = {
+ val prefix = s"${GlobalIndex.serviceField}:${column.service.serviceName} AND ${GlobalIndex.serviceColumnField}:${column.columnName}"
+
val ids = c.argOpt[Any]("id").toSeq ++ c.argOpt[List[Any]]("ids").toList.flatten
+ val offset = c.arg[Int]("offset")
+ val limit = c.arg[Int]("limit")
+
val vertices = ids.map(vid => c.ctx.toS2VertexLike(vid, column))
+ val searchOpt = c.argOpt[String]("search").map { qs =>
+ if (qs.trim.nonEmpty) s"(${prefix}) AND (${qs})"
+ else prefix
+ }
val columnFields = column.metasInvMap.keySet
val selectedFields = AstHelper.selectedFields(c.astFields)
-
val canSkipFetch = selectedFields.forall(f => f == "id" || !columnFields(f))
- (vertices, canSkipFetch)
+ val vertexQueryParam = VertexQueryParam(offset, limit, searchOpt, vertices.map(_.id), !canSkipFetch)
+
+ vertexQueryParam
}
- def serviceColumnOnLabel(c: Context[GraphRepository, Any]): (S2VertexLike, Boolean) = {
+ def serviceColumnOnLabel(c: Context[GraphRepository, Any]): VertexQueryParam = {
val edge = c.value.asInstanceOf[S2EdgeLike]
val vertex = edge.tgtForVertex
@@ -69,9 +79,10 @@ object FieldResolver {
val selectedFields = AstHelper.selectedFields(c.astFields)
val columnFields = column.metasInvMap.keySet
-
val canSkipFetch = selectedFields.forall(f => f == "id" || !columnFields(f))
- (vertex, canSkipFetch)
+ val vertexQueryParam = VertexQueryParam(0, 1, None, Seq(vertex.id), !canSkipFetch)
+
+ vertexQueryParam
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala
index b532263..c189fe3 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/S2Type.scala
@@ -139,20 +139,15 @@ object S2Type {
arguments = List(
Argument("id", OptionInputType(toScalarType(column.columnType))),
Argument("ids", OptionInputType(ListInputType(toScalarType(column.columnType)))),
- Argument("search", OptionInputType(StringType))
+ Argument("search", OptionInputType(StringType)),
+ Argument("offset", OptionInputType(IntType), defaultValue = 0),
+ Argument("limit", OptionInputType(IntType), defaultValue = 100)
),
description = Option("desc here"),
resolve = c => {
implicit val ec = c.ctx.ec
- val (vertices, canSkipFetchVertex) = FieldResolver.serviceColumnOnService(column, c)
- val searchOpt = c.argOpt[String]("search").map { qs =>
- val prefix = s"(${GlobalIndex.serviceField}:${service.serviceName} AND ${GlobalIndex.serviceColumnField}:${column.columnName})"
- if (qs.trim.nonEmpty) Seq(prefix, qs).mkString(" AND ")
- else prefix
- }
-
- val vertexQueryParam = VertexQueryParam(0, 100, searchOpt, vertices.map(_.id), !canSkipFetchVertex)
+ val vertexQueryParam = FieldResolver.serviceColumnOnService(column, c)
DeferredValue(GraphRepository.vertexFetcher.defer(vertexQueryParam)).map(m => m._2)
}
): Field[GraphRepository, Any]
@@ -178,9 +173,8 @@ object S2Type {
lazy val serviceColumnField: Field[GraphRepository, Any] = Field(column.columnName, labelColumnType, resolve = c => {
implicit val ec = c.ctx.ec
- val (vertex, canSkipFetchVertex) = FieldResolver.serviceColumnOnLabel(c)
+ val vertexQueryParam = FieldResolver.serviceColumnOnLabel(c)
- val vertexQueryParam = VertexQueryParam(0, 100, None, Seq(vertex.id), !canSkipFetchVertex)
DeferredValue(GraphRepository.vertexFetcher.defer(vertexQueryParam)).map(m => m._2.head)
})
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8ed6d20f/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala b/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala
index 81bf884..58e473d 100644
--- a/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala
+++ b/s2graphql/src/test/scala/org/apache/s2graph/graphql/ScenarioTest.scala
@@ -24,6 +24,7 @@ import org.scalatest._
import play.api.libs.json._
import sangria.execution.ValidationError
import sangria.macros._
+import sangria.parser.QueryParser
class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
var testGraph: TestGraph = _
@@ -358,6 +359,11 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
id: "shon"
age: 19
gender: "F"
+ },
+ {
+ id: "rain"
+ age: 21
+ gender: "T"
}]
}) {
isSuccess
@@ -374,8 +380,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
"isSuccess": true
}, {
"isSuccess": true
- }]
- }
+ }, {
+ "isSuccess": true
+ }
+ ]}
}
""")
@@ -428,13 +436,52 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
actual shouldBe expected
}
+
+ it("should fetch vertices using VertexIndex: 'gender in (F, M)') from kakao.user") {
+ def query(search: String) = {
+ QueryParser.parse(
+ s"""
+ query FetchVertices {
+ kakao {
+ user(search: "${search}", ids: ["rain"]) {
+ id
+ age
+ gender
+ }
+ }
+ }""").get
+ }
+
+ val actualGender = testGraph.queryAsJs(query("gender: M OR gender: F"))
+ val expected = Json.parse(
+ """
+ {
+ "data": {
+ "kakao": {
+ "user": [
+ {"id": "rain", "age": 21, "gender": "T"},
+ {"id": "daewon", "age": 20, "gender": "M"},
+ {"id": "shon", "age": 19, "gender": "F"}
+ ]
+ }
+ }
+ }
+ """)
+
+ actualGender shouldBe expected
+
+ val actualAge = testGraph.queryAsJs(query("age: 19 OR age: 20"))
+ actualAge shouldBe expected
+
+ val actualAgeBetween = testGraph.queryAsJs(query("age: [10 TO 20]"))
+ actualAgeBetween shouldBe expected
+ }
}
- }
- describe("Add edge to label 'friends' and fetch ") {
- it("should add edges: daewon -> shon(score: 2) to friends") {
- val query =
- graphql"""
+ describe("Add edge to label 'friends' and fetch ") {
+ it("should add edges: daewon -> shon(score: 2) to friends") {
+ val query =
+ graphql"""
mutation {
addEdge(
@@ -449,10 +496,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
"""
- val actual = testGraph.queryAsJs(query)
+ val actual = testGraph.queryAsJs(query)
- val expected = Json.parse(
- """
+ val expected = Json.parse(
+ """
{
"data": {
"addEdge": [{
@@ -462,12 +509,12 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
""")
- actual shouldBe expected
- }
+ actual shouldBe expected
+ }
- it("should fetch edges: friends of kakao.user(daewon) ") {
- val query =
- graphql"""
+ it("should fetch edges: friends of kakao.user(daewon) ") {
+ val query =
+ graphql"""
query {
kakao {
@@ -495,9 +542,9 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
"""
- val actual = testGraph.queryAsJs(query)
- val expected = Json.parse(
- """
+ val actual = testGraph.queryAsJs(query)
+ val expected = Json.parse(
+ """
{
"data" : {
"kakao" : {
@@ -526,15 +573,15 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
""")
- actual shouldBe expected
+ actual shouldBe expected
+ }
}
- }
- describe("Management: Delete label, service column") {
+ describe("Management: Delete label, service column") {
- it("should delete label: 'friends' and serviceColumn: 'user' on kakao") {
- val query =
- graphql"""
+ it("should delete label: 'friends' and serviceColumn: 'user' on kakao") {
+ val query =
+ graphql"""
mutation {
Management {
@@ -553,9 +600,9 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
"""
- val actual = testGraph.queryAsJs(query)
- val expected = Json.parse(
- """
+ val actual = testGraph.queryAsJs(query)
+ val expected = Json.parse(
+ """
{
"data": {
"Management": {
@@ -570,12 +617,12 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
""")
- actual shouldBe expected
- }
+ actual shouldBe expected
+ }
- it("should fetch failed label: 'friends' and serviceColumn: 'user'") {
- val query =
- graphql"""
+ it("should fetch failed label: 'friends' and serviceColumn: 'user'") {
+ val query =
+ graphql"""
query {
Management {
@@ -590,13 +637,13 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
"""
- // label 'friends' was deleted
- assertThrows[ValidationError] {
- testGraph.queryAsJs(query)
- }
+ // label 'friends' was deleted
+ assertThrows[ValidationError] {
+ testGraph.queryAsJs(query)
+ }
- val queryServiceColumn =
- graphql"""
+ val queryServiceColumn =
+ graphql"""
query {
Management {
@@ -609,10 +656,10 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
"""
- // serviceColumn 'user' was deleted
- val actual = testGraph.queryAsJs(queryServiceColumn)
- val expected = Json.parse(
- """
+ // serviceColumn 'user' was deleted
+ val actual = testGraph.queryAsJs(queryServiceColumn)
+ val expected = Json.parse(
+ """
{
"data": {
"Management": {
@@ -624,7 +671,8 @@ class ScenarioTest extends FunSpec with Matchers with BeforeAndAfterAll {
}
""")
- actual shouldBe expected
+ actual shouldBe expected
+ }
}
}
}