You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2018/11/23 01:35:44 UTC
[1/4] incubator-s2graph git commit: bug fix on offset/limit with
schema v3. - note that HBase version 1.2.8, 1.3.2.1,
1.4.8 have bug on ColumnPaginationFilter. - details on HBASE-20565.
Repository: incubator-s2graph
Updated Branches:
refs/heads/master 42077bf8d -> 66b656f69
bug fix on offset/limit with schema v3.
- note that HBase version 1.2.8, 1.3.2.1, 1.4.8 have bug on ColumnPaginationFilter.
- details on HBASE-20565.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/646860f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/646860f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/646860f0
Branch: refs/heads/master
Commit: 646860f0e12ab0fce0edc7743f2bad379e5d97eb
Parents: f53661e
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Nov 3 06:52:42 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Nov 3 06:52:42 2018 +0900
----------------------------------------------------------------------
.travis.yml | 2 +-
.travis/install-hbase.sh | 21 +++
project/Common.scala | 1 +
s2core/build.sbt | 2 +-
.../core/storage/hbase/AsynchbaseStorage.scala | 2 +-
.../s2graph/core/Integrate/ExampleTest.scala | 130 +++++++++++++++++++
.../graphql/repository/GraphRepository.scala | 4 +-
.../s2graph/graphql/types/FieldResolver.scala | 2 +-
8 files changed, 158 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index bc5e6cb..8664e15 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@
language: scala
env:
- - HBASE_VERSION=1.2.6.1
+ - HBASE_VERSION=1.4.8
cache:
directories:
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/.travis/install-hbase.sh
----------------------------------------------------------------------
diff --git a/.travis/install-hbase.sh b/.travis/install-hbase.sh
new file mode 100755
index 0000000..5744fd8
--- /dev/null
+++ b/.travis/install-hbase.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#set -xe
+#
+#if [ ! -d "$HOME/hbase-$HBASE_VERSION/bin" ]; then
+# cd $HOME && wget -q -O - http://mirror.apache-kr.org/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz | tar xz
+#fi
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index e4a1b61..42628f0 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -26,6 +26,7 @@ object Common {
val specs2Version = "3.8.5"
val hbaseVersion = "1.2.6.1"
+ val asynchbaseVersion = "1.7.2"
val hadoopVersion = "2.7.3"
val tinkerpopVersion = "3.2.5"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/build.sbt
----------------------------------------------------------------------
diff --git a/s2core/build.sbt b/s2core/build.sbt
index 0368715..76ec221 100644
--- a/s2core/build.sbt
+++ b/s2core/build.sbt
@@ -41,7 +41,7 @@ libraryDependencies ++= Seq(
"com.h2database" % "h2" % "1.4.192",
"com.stumbleupon" % "async" % "1.4.1",
"io.netty" % "netty" % "3.9.4.Final" force(),
- "org.hbase" % "asynchbase" % "1.7.2" excludeLogging(),
+ "org.hbase" % "asynchbase" % asynchbaseVersion excludeLogging(),
"net.bytebuddy" % "byte-buddy" % "1.4.26",
"org.apache.tinkerpop" % "gremlin-core" % tinkerpopVersion excludeLogging(),
"org.apache.tinkerpop" % "gremlin-test" % tinkerpopVersion % "test",
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index f670e9c..0c49e71 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -297,7 +297,7 @@ object AsynchbaseStorage {
get.setMaxTimestamp(maxTs)
get.setTimeout(queryParam.rpcTimeout)
- val pagination = new ColumnPaginationFilter(queryParam.limit, queryParam.offset)
+ val pagination = new ColumnPaginationFilter(queryParam.innerLimit, queryParam.innerOffset)
val columnRangeFilterOpt = queryParam.intervalOpt.map { interval =>
new ColumnRangeFilter(intervalMaxBytes, true, intervalMinBytes, true)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala
new file mode 100644
index 0000000..76e30d9
--- /dev/null
+++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala
@@ -0,0 +1,130 @@
+package org.apache.s2graph.core.Integrate
+
+import org.apache.s2graph.core.schema._
+import play.api.libs.json.{JsObject, Json}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+class ExampleTest extends IntegrateCommon {
+
+ import TestUtil._
+
+ private def prepareTestData() = {
+ // create service KakaoFavorites
+ val createServicePayload = Json.parse(
+ """
+ |{"serviceName": "KakaoFavorites", "compressionAlgorithm" : "gz"}
+ |""".stripMargin)
+
+ val createFriendsPayload = Json.parse(
+ s"""{
+ | "label": "friends",
+ | "srcServiceName": "KakaoFavorites",
+ | "srcColumnName": "userName",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "KakaoFavorites",
+ | "tgtColumnName": "userName",
+ | "tgtColumnType": "string",
+ | "isDirected": "false",
+ | "indices": [],
+ | "props": [],
+ | "consistencyLevel": "strong"
+ |}""".stripMargin)
+
+ val createPostPayload = Json.parse(
+ """{
+ | "label": "post",
+ | "srcServiceName": "KakaoFavorites",
+ | "srcColumnName": "userName",
+ | "srcColumnType": "string",
+ | "tgtServiceName": "KakaoFavorites",
+ | "tgtColumnName": "url",
+ | "tgtColumnType": "string",
+ | "isDirected": "true",
+ | "indices": [],
+ | "props": [],
+ | "consistencyLevel": "strong"
+ |}""".stripMargin)
+
+ val insertFriendsPayload = Json.parse(
+ """[
+ | {"from":"Elmo","to":"Big Bird","label":"friends","props":{},"timestamp":1444360152477},
+ | {"from":"Elmo","to":"Ernie","label":"friends","props":{},"timestamp":1444360152478},
+ | {"from":"Elmo","to":"Bert","label":"friends","props":{},"timestamp":1444360152479},
+ |
+ | {"from":"Cookie Monster","to":"Grover","label":"friends","props":{},"timestamp":1444360152480},
+ | {"from":"Cookie Monster","to":"Kermit","label":"friends","props":{},"timestamp":1444360152481},
+ | {"from":"Cookie Monster","to":"Oscar","label":"friends","props":{},"timestamp":1444360152482}
+ |]""".stripMargin)
+
+ val insertPostPayload = Json.parse(
+ """[
+ | {"from":"Big Bird","to":"www.kakaocorp.com/en/main","label":"post","props":{},"timestamp":1444360152477},
+ | {"from":"Big Bird","to":"github.com/kakao/s2graph","label":"post","props":{},"timestamp":1444360152478},
+ | {"from":"Ernie","to":"groups.google.com/forum/#!forum/s2graph","label":"post","props":{},"timestamp":1444360152479},
+ | {"from":"Grover","to":"hbase.apache.org/forum/#!forum/s2graph","label":"post","props":{},"timestamp":1444360152480},
+ | {"from":"Kermit","to":"www.playframework.com","label":"post","props":{},"timestamp":1444360152481},
+ | {"from":"Oscar","to":"www.scala-lang.org","label":"post","props":{},"timestamp":1444360152482}
+ |]""".stripMargin)
+
+
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = parser.toServiceElements(createServicePayload)
+ management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+
+ Service.findByName("KakaoFavorites", useCache = false)
+
+ Label.findByName("friends", useCache = false).foreach { label =>
+ Label.delete(label.id.get)
+ }
+ parser.toLabelElements(createFriendsPayload)
+
+ Label.findByName("post", useCache = false).foreach { label =>
+ Label.delete(label.id.get)
+ }
+
+ parser.toLabelElements(createPostPayload)
+ Await.result(graph.mutateEdges(parser.parseJsonFormat(insertFriendsPayload, operation = "insert").map(_._1), withWait = true), Duration("10 seconds"))
+ Await.result(graph.mutateEdges(parser.parseJsonFormat(insertPostPayload, operation = "insert").map(_._1), withWait = true), Duration("10 seconds"))
+ }
+
+ override def initTestData(): Unit = {
+ prepareTestData()
+ }
+
+ test("[S2GRAPH-243]: Limit bug on 'graph/getEdges' offset 0, limit 3") {
+ val queryJson = Json.parse(
+ """{
+ | "select": ["to"],
+ | "srcVertices": [{"serviceName": "KakaoFavorites", "columnName": "userName", "id":"Elmo"}],
+ | "steps": [
+ | {"step": [{"label": "friends", "direction": "out", "offset": 0, "limit": 3}]}
+ | ]
+ |}""".stripMargin)
+
+ val response = getEdgesSync(queryJson)
+
+ val expectedFriends = Seq("Bert", "Ernie", "Big Bird")
+ val friends = (response \ "results").as[Seq[JsObject]].map(obj => (obj \ "to").as[String])
+
+ friends shouldBe expectedFriends
+ }
+
+ test("[S2GRAPH-243]: Limit bug on 'graph/getEdges' offset 1, limit 2") {
+ val queryJson = Json.parse(
+ """{
+ | "select": ["to"],
+ | "srcVertices": [{"serviceName": "KakaoFavorites", "columnName": "userName", "id":"Elmo"}],
+ | "steps": [
+ | {"step": [{"label": "friends", "direction": "out", "offset": 1, "limit": 2}]}
+ | ]
+ |}""".stripMargin)
+
+ val response = getEdgesSync(queryJson)
+
+ val expectedFriends = Seq("Ernie", "Big Bird")
+ val friends = (response \ "results").as[Seq[JsObject]].map(obj => (obj \ "to").as[String])
+
+ friends shouldBe expectedFriends
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
index d5212c8..460a817 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
@@ -193,14 +193,14 @@ class GraphRepository(val graph: S2GraphLike) {
def getEdges(vertices: Seq[S2VertexLike], queryParam: QueryParam): Future[Seq[S2EdgeLike]] = {
val step = Step(Seq(queryParam))
- val q = Query(vertices, steps = Vector(step))
+ val q = Query(vertices, steps = Vector(step), QueryOption(returnDegree = false))
graph.getEdges(q).map(_.edgeWithScores.map(_.edge))
}
def getEdges(vertex: S2VertexLike, queryParam: QueryParam): Future[Seq[S2EdgeLike]] = {
val step = Step(Seq(queryParam))
- val q = Query(Seq(vertex), steps = Vector(step))
+ val q = Query(Seq(vertex), steps = Vector(step), QueryOption(returnDegree = false))
graph.getEdges(q).map(_.edgeWithScores.map(_.edge))
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
index 1994e5e..4de1b2e 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala
@@ -53,7 +53,7 @@ object FieldResolver {
val vertex = c.value.asInstanceOf[S2VertexLike]
val dir = c.arg[String]("direction")
- val offset = c.arg[Int]("offset") + 1 // +1 for skip degree edge: currently not support
+ val offset = c.arg[Int]("offset") // +1 for skip degree edge: currently not support
val limit = c.arg[Int]("limit")
val whereClauseOpt = c.argOpt[String]("filter")
val where = c.ctx.parser.extractWhere(label, whereClauseOpt)
[3/4] incubator-s2graph git commit: change hbase version on travis.yml
Posted by st...@apache.org.
change hbase version on travis.yml
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d6d9cc59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d6d9cc59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d6d9cc59
Branch: refs/heads/master
Commit: d6d9cc59c9e76c96535f7ccde4e8c7a1030f8f87
Parents: 320189f
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 23 10:18:42 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 23 10:18:42 2018 +0900
----------------------------------------------------------------------
.travis.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d6d9cc59/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 8664e15..bc5e6cb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@
language: scala
env:
- - HBASE_VERSION=1.4.8
+ - HBASE_VERSION=1.2.6.1
cache:
directories:
[4/4] incubator-s2graph git commit: [S2GRAPH-243] Limit bug on
'graph/getEdges'
Posted by st...@apache.org.
[S2GRAPH-243] Limit bug on 'graph/getEdges'
JIRA:
[S2GRAPH-243] https://issues.apache.org/jira/browse/S2GRAPH-243
Pull Request:
Closes #187
Author
DO YUNG YOON <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/66b656f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/66b656f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/66b656f6
Branch: refs/heads/master
Commit: 66b656f69d5b6ff4cf305d72d2457deb9135c4f6
Parents: d6d9cc5
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 23 10:35:32 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 23 10:35:32 2018 +0900
----------------------------------------------------------------------
CHANGES | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/66b656f6/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index bb7f512..df8fbf2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -58,6 +58,7 @@ Release Notes - S2Graph - Version 0.2.0
* [S2GRAPH-196] - Apply Query Parameter to Label Fetch in S2GraphQL
* [S2GRAPH-220] - Filter clause is not working on AnnoyModelFetcher
* [S2GRAPH-228] - GraphQL empty variable fields on post request cause spray.json.DeserializationException
+ * [S2GRAPH-243] - Limit bug on 'graph/getEdges'
** Improvement
* [S2GRAPH-72] - Support Apache TinkerPop and Gremlin
[2/4] incubator-s2graph git commit: Merge remote-tracking branch
'apache/master' into S2GRAPH-243
Posted by st...@apache.org.
Merge remote-tracking branch 'apache/master' into S2GRAPH-243
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/320189fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/320189fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/320189fc
Branch: refs/heads/master
Commit: 320189fc019f4cb42d1b9714b39b2c081ef34e7d
Parents: 646860f 42077bf
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Nov 23 10:18:20 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Nov 23 10:18:20 2018 +0900
----------------------------------------------------------------------
.gitignore | 3 +
CHANGES | 1 +
doc/Makefile | 19 ++
doc/make.bat | 35 ++++
doc/readme.md | 56 ++++++
doc/source/conf.py | 186 +++++++++++++++++++
doc/source/getting_started/index.rst | 43 +++++
doc/source/getting_started/your_first_graph.rst | 178 ++++++++++++++++++
doc/source/index.rst | 30 +++
9 files changed, 551 insertions(+)
----------------------------------------------------------------------