You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by da...@apache.org on 2016/01/04 07:40:34 UTC
[01/46] incubator-s2graph git commit: README update for refactoring.
This resolves S2GRAPH-2.
Repository: incubator-s2graph
Updated Branches:
refs/heads/feature/test_daewon [created] b99389f33
README update for refactoring. This resolves S2GRAPH-2.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/767940b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/767940b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/767940b1
Branch: refs/heads/feature/test_daewon
Commit: 767940b13815f6b43b1e381ea97b6a3c64e75739
Parents: de250c8
Author: jojo <jo...@daumkakao.com>
Authored: Thu Dec 17 15:11:19 2015 +0900
Committer: jojo <jo...@daumkakao.com>
Committed: Thu Dec 17 15:11:19 2015 +0900
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/767940b1/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 5e9088f..07a49f6 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ vagrant ssh
// in the virtual environment..
cd s2graph
-activator run
+activator 'project s2rest_play' run
```
Finally, join the [mailing list](https://groups.google.com/forum/#!forum/s2graph)!
[08/46] incubator-s2graph git commit: performance tunning on where
parser.
Posted by da...@apache.org.
performance tunning on where parser.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a6acec72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a6acec72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a6acec72
Branch: refs/heads/feature/test_daewon
Commit: a6acec72396e23863586e040ca7bca3ea00f57d7
Parents: 6216412
Author: SteamShon <sh...@gmail.com>
Authored: Tue Dec 22 17:02:33 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Tue Dec 22 17:02:33 2015 +0900
----------------------------------------------------------------------
.../com/kakao/s2graph/core/QueryParam.scala | 8 ++++-
.../s2graph/core/parsers/WhereParser.scala | 31 ++++++++++++--------
.../kakao/s2graph/core/rest/RequestParser.scala | 7 +++--
.../storage/hbase/AsynchbaseQueryBuilder.scala | 2 +-
4 files changed, 31 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
index 0cefb3f..a903f4d 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/QueryParam.scala
@@ -258,6 +258,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System
var hasFilters: Map[Byte, InnerValLike] = Map.empty[Byte, InnerValLike]
var where: Try[Where] = Success(WhereParser.success)
+ var whereRawOpt: Option[String] = None
var duplicatePolicy = DuplicatePolicy.First
var rpcTimeoutInMillis = 1000
var maxAttempt = 2
@@ -297,7 +298,7 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System
def toCacheKeyRaw(bytes: Array[Byte]): Array[Byte] = {
val transformBytes = transformer.toHashKeyBytes
//TODO: change this to binrary format.
- val whereBytes = Bytes.toBytes(where.toString())
+ val whereBytes = Bytes.toBytes(whereRawOpt.getOrElse(""))
val durationBytes = duration.map { case (min, max) =>
val minTs = min / cacheTTLInMillis
val maxTs = max / cacheTTLInMillis
@@ -457,6 +458,11 @@ case class QueryParam(labelWithDir: LabelWithDirection, timestamp: Long = System
this
}
+ def whereRawOpt(sqlOpt: Option[String]): QueryParam = {
+ this.whereRawOpt = sqlOpt
+ this
+ }
+
def isSnapshotEdge = tgtVertexInnerIdOpt.isDefined
override def toString = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
index db28eee..15996ea 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
@@ -47,18 +47,22 @@ trait ExtractValue extends JSONParser {
}
}
- private def findParentEdge(edge: Edge, key: String): (String, Edge) = {
- @tailrec def find(edge: Edge, depth: Int): Edge =
- if (depth > 0) find(edge.parentEdges.head.edge, depth - 1)
- else edge
+ @tailrec
+ private def findParent(edge: Edge, depth: Int): Edge =
+ if (depth > 0) findParent(edge.parentEdges.head.edge, depth - 1)
+ else edge
- val split = key.split(parent)
- val depth = split.length - 1
- val propKey = split.last
+ private def findParentEdge(edge: Edge, key: String): (String, Edge) = {
+ if (!key.startsWith(parent)) (key, edge)
+ else {
+ val split = key.split(parent)
+ val depth = split.length - 1
+ val propKey = split.last
- val parentEdge = find(edge, depth)
+ val parentEdge = findParent(edge, depth)
- (propKey, parentEdge)
+ (propKey, parentEdge)
+ }
}
}
@@ -97,9 +101,12 @@ case class Eq(propKey: String, value: String) extends Clause {
case class IN(propKey: String, values: Set[String]) extends Clause {
override def filter(edge: Edge): Boolean = {
val propVal = propToInnerVal(edge, propKey)
- val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) }
-
- valuesToCompare.contains(propVal)
+ values.exists { value =>
+ valueToCompare(edge, propKey, value) == propVal
+ }
+// val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) }
+//
+// valuesToCompare.contains(propVal)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index 5b1fe1b..96d8f60 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -92,8 +92,8 @@ class RequestParser(config: Config) extends JSONParser {
ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
}
- def extractWhere(labelMap: Map[String, Label], jsValue: JsValue) = {
- (jsValue \ "where").asOpt[String] match {
+ def extractWhere(labelMap: Map[String, Label], whereClauseOpt: Option[String]) = {
+ whereClauseOpt match {
case None => Success(WhereParser.success)
case Some(where) =>
WhereParser(labelMap).parse(where) match {
@@ -279,7 +279,8 @@ class RequestParser(config: Config) extends JSONParser {
case None => label.indexSeqsMap.get(scoring.map(kv => kv._1)).map(_.seq).getOrElse(LabelIndex.DefaultSeq)
case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
}
- val where = extractWhere(labelMap, labelGroup)
+ val whereClauseOpt = (labelGroup \ "where").asOpt[String]
+ val where = extractWhere(labelMap, whereClauseOpt)
val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout)
val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a6acec72/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
index 3d4b5c1..d8e4099 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
@@ -27,7 +27,7 @@ class AsynchbaseQueryBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionC
val expreAfterAccess = storage.config.getInt("future.cache.expire.after.access")
val futureCache = CacheBuilder.newBuilder()
- .recordStats()
+// .recordStats()
.initialCapacity(maxSize)
.concurrencyLevel(Runtime.getRuntime.availableProcessors())
.expireAfterWrite(expreAfterWrite, TimeUnit.MILLISECONDS)
[26/46] incubator-s2graph git commit: Fix MySql connection lost error
Posted by da...@apache.org.
Fix MySql connection lost error
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f249c3e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f249c3e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f249c3e0
Branch: refs/heads/feature/test_daewon
Commit: f249c3e0bc1252be76c7e8852f1f44d4dbf4a9f1
Parents: e407707
Author: daewon <bl...@gmail.com>
Authored: Mon Dec 28 12:05:24 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Dec 28 12:05:24 2015 +0900
----------------------------------------------------------------------
.../s2graph/core/TestCommonWithModels.scala | 81 +++++++++++---------
.../kakao/s2graph/core/models/ModelTest.scala | 14 ++--
.../s2graph/core/mysqls/ExperimentSpec.scala | 3 -
.../hbase/AsynchbaseQueryBuilderTest.scala | 12 +--
4 files changed, 60 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f249c3e0/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
index fca7dc0..d350ab4 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
@@ -2,23 +2,43 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
import com.kakao.s2graph.core.mysqls._
+import org.scalatest.BeforeAndAfterAll
+import scalikejdbc.AutoSession
//import com.kakao.s2graph.core.models._
import com.kakao.s2graph.core.types.{InnerVal, LabelWithDirection}
-import com.typesafe.config.ConfigFactory
+import com.typesafe.config.{Config, ConfigFactory}
import scala.concurrent.ExecutionContext
-trait TestCommonWithModels {
+trait TestCommonWithModels {
import InnerVal._
import types.HBaseType._
- val config = ConfigFactory.load()
+ var graph: Graph = _
+ var config: Config = _
+
+ def initTests() = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+
+ implicit val session = AutoSession
+
+ deleteTestLabel()
+ deleteTestService()
+
+ createTestService()
+ createTestLabel()
+ }
+
+ def zkQuorum = config.getString("hbase.zookeeper.quorum")
+ def cluster = config.getString("hbase.zookeeper.quorum")
+
+ implicit val session = AutoSession
- val zkQuorum = config.getString("hbase.zookeeper.quorum")
val serviceName = "_test_service"
val serviceNameV2 = "_test_service_v2"
val columnName = "user_id"
@@ -31,7 +51,6 @@ trait TestCommonWithModels {
val tgtColumnType = "string"
val tgtColumnTypeV2 = "string"
- val cluster = config.getString("hbase.zookeeper.quorum")
val hTableName = "_test_cases"
val preSplitSize = 0
val labelName = "_test_label"
@@ -54,29 +73,21 @@ trait TestCommonWithModels {
val consistencyLevel = "strong"
val hTableTTL = None
- val graph = new Graph(config)(ExecutionContext.Implicits.global)
-
- def initTests() = {
- deleteTestLabel()
- deleteTestService()
-
- Thread.sleep(1000)
-
- createTestService()
- createTestLabel()
- }
def createTestService() = {
+ implicit val session = AutoSession
Management.createService(serviceName, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
Management.createService(serviceNameV2, cluster, hTableName, preSplitSize, hTableTTL = None, "gz")
}
def deleteTestService() = {
+ implicit val session = AutoSession
Management.deleteService(serviceName)
Management.deleteService(serviceNameV2)
}
def deleteTestLabel() = {
+ implicit val session = AutoSession
Management.deleteLabel(labelName)
Management.deleteLabel(labelNameV2)
Management.deleteLabel(undirectedLabelName)
@@ -85,6 +96,7 @@ trait TestCommonWithModels {
def createTestLabel() = {
+ implicit val session = AutoSession
Management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
isDirected = true, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION1, false, "lg4")
@@ -98,31 +110,28 @@ trait TestCommonWithModels {
isDirected = false, serviceName, testIdxProps, testProps, consistencyLevel, Some(hTableName), hTableTTL, VERSION2, false, "lg4")
}
- /** */
- initTests()
-
- lazy val service = Service.findByName(serviceName, useCache = false).get
- lazy val serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
+ def service = Service.findByName(serviceName, useCache = false).get
+ def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
- lazy val column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
- lazy val columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
+ def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
+ def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
- lazy val tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
- lazy val tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
+ def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
+ def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
- lazy val label = Label.findByName(labelName, useCache = false).get
- lazy val labelV2 = Label.findByName(labelNameV2, useCache = false).get
+ def label = Label.findByName(labelName, useCache = false).get
+ def labelV2 = Label.findByName(labelNameV2, useCache = false).get
- lazy val undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
- lazy val undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
+ def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
+ def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
- lazy val dir = GraphUtil.directions("out")
- lazy val op = GraphUtil.operations("insert")
- lazy val labelOrderSeq = LabelIndex.DefaultSeq
+ def dir = GraphUtil.directions("out")
+ def op = GraphUtil.operations("insert")
+ def labelOrderSeq = LabelIndex.DefaultSeq
- lazy val labelWithDir = LabelWithDirection(label.id.get, dir)
- lazy val labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
+ def labelWithDir = LabelWithDirection(label.id.get, dir)
+ def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
- lazy val queryParam = QueryParam(labelWithDir)
- lazy val queryParamV2 = QueryParam(labelWithDirV2)
+ def queryParam = QueryParam(labelWithDir)
+ def queryParamV2 = QueryParam(labelWithDirV2)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f249c3e0/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
index 478da12..7ca62e9 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/models/ModelTest.scala
@@ -5,14 +5,18 @@ import java.util.concurrent.ExecutorService
import com.kakao.s2graph.core.mysqls.{Label, Model}
import com.kakao.s2graph.core.{TestCommonWithModels, TestCommon, Graph}
import com.typesafe.config.ConfigFactory
-import org.scalatest.{FunSuite, Matchers}
+import org.scalatest.{BeforeAndAfterAll, Sequential, FunSuite, Matchers}
import scala.concurrent.ExecutionContext
-/**
- * Created by shon on 5/12/15.
- */
-class ModelTest extends FunSuite with Matchers with TestCommonWithModels {
+class ModelTest extends FunSuite with Matchers with TestCommonWithModels with BeforeAndAfterAll {
+ override def beforeAll(): Unit = {
+ initTests()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
// val serviceName = "testService"
// val newServiceName = "newTestService"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f249c3e0/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
index 2e8cb50..7a0b91c 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
@@ -6,9 +6,6 @@ import com.typesafe.config.ConfigFactory
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
import scalikejdbc._
-/**
- * Created by hsleep on 2015. 11. 30..
- */
class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
val Ttl = 2
override def beforeAll(): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f249c3e0/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
index b404559..9b876c0 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
@@ -1,14 +1,13 @@
package com.kakao.s2graph.core.storage.hbase
import com.kakao.s2graph.core.Graph
+import com.typesafe.config.ConfigFactory
import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.GetRequest
import org.scalatest.{FunSuite, Matchers}
-import scala.concurrent.ExecutionContext.Implicits.global
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 9..
- */
+import scala.concurrent.ExecutionContext
+
class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
val dummyRequests = {
for {
@@ -18,8 +17,9 @@ class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
}
}
- val config = Graph.DefaultConfig
- val graph = new Graph(config)
+ implicit val ec = ExecutionContext.Implicits.global
+ val config = ConfigFactory.load()
+ val graph = new Graph(config)(ec)
val qb = new AsynchbaseQueryBuilder(graph.storage.asInstanceOf[AsynchbaseStorage])
[25/46] incubator-s2graph git commit: Update buckets schema
Posted by da...@apache.org.
Update buckets schema
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e4077071
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e4077071
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e4077071
Branch: refs/heads/feature/test_daewon
Commit: e4077071155e0b78fa16a22dc3b6839c4c8d3816
Parents: 3a3dc2f
Author: daewon <bl...@gmail.com>
Authored: Mon Dec 28 11:41:33 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Dec 28 11:41:33 2015 +0900
----------------------------------------------------------------------
dev_support/graph_mysql/schema.sql | 5 ++-
.../s2graph/core/mysqls/ExperimentSpec.scala | 33 ++------------------
2 files changed, 4 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e4077071/dev_support/graph_mysql/schema.sql
----------------------------------------------------------------------
diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql
index d9f1f8b..9190033 100644
--- a/dev_support/graph_mysql/schema.sql
+++ b/dev_support/graph_mysql/schema.sql
@@ -162,12 +162,10 @@ CREATE TABLE `experiments` (
-- ----------------------------
-- Table structure for `buckets`
-- ----------------------------
-DROP TABLE IF EXISTS `buckets`;
CREATE TABLE `buckets` (
`id` integer NOT NULL AUTO_INCREMENT,
`experiment_id` integer NOT NULL,
- `uuid_mods` varchar(64) NOT NULL,
- `traffic_ratios` varchar(64) NOT NULL,
+ `modular` varchar(64) NOT NULL,
`http_verb` varchar(8) NOT NULL,
`api_path` text NOT NULL,
`uuid_key` varchar(128),
@@ -176,6 +174,7 @@ CREATE TABLE `buckets` (
`timeout` int NOT NULL DEFAULT 1000,
`impression_id` varchar(64) NOT NULL,
`is_graph_query` tinyint NOT NULL DEFAULT 1,
+ `is_empty` tinyint NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
UNIQUE KEY `ux_impression_id` (`impression_id`),
INDEX `idx_experiment_id` (`experiment_id`),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e4077071/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
index 2bc764d..2e8cb50 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/mysqls/ExperimentSpec.scala
@@ -19,37 +19,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
val props = new Properties()
props.setProperty("cache.ttl.seconds", Ttl.toString)
Model.apply(ConfigFactory.load(ConfigFactory.parseProperties(props)))
- /*
-CREATE TABLE `experiments` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `service_id` integer NOT NULL,
- `service_name` varchar(128) NOT NULL,
- `name` varchar(64) NOT NULL,
- `description` varchar(255) NOT NULL,
- `experiment_type` varchar(8) NOT NULL DEFAULT 'u',
- `total_modular` int NOT NULL DEFAULT 100,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_service_id_name` (`service_id`, `name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-CREATE TABLE `buckets` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `experiment_id` integer NOT NULL,
- `uuid_mods` varchar(64) NOT NULL,
- `traffic_ratios` varchar(64) NOT NULL,
- `http_verb` varchar(8) NOT NULL,
- `api_path` text NOT NULL,
- `uuid_key` varchar(128),
- `uuid_placeholder` varchar(64),
- `request_body` text NOT NULL,
- `timeout` int NOT NULL DEFAULT 1000,
- `impression_id` varchar(64) NOT NULL,
- `is_graph_query` tinyint NOT NULL DEFAULT 1,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_impression_id` (`impression_id`),
- INDEX `idx_experiment_id` (`experiment_id`),
- INDEX `idx_impression_id` (`impression_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
- */
+
implicit val session = AutoSession
sql"""DELETE FROM buckets""".update().apply()
sql"""DELETE FROM experiments""".update().apply()
@@ -58,6 +28,7 @@ CREATE TABLE `buckets` (
sql"""INSERT INTO
buckets(experiment_id, modular, http_verb, api_path, request_body, impression_id)
VALUES($expId, "1~100", "POST", "/a/b/c", "None", "imp1")""".update().apply()
+
}
"Experiment" should "find bucket list" in {
[06/46] incubator-s2graph git commit: Merge pull request #214 from
kakao/feature/refactor_postProcess
Posted by da...@apache.org.
Merge pull request #214 from kakao/feature/refactor_postProcess
remove jsNumber.as on sortBy in PostProcess.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e8ae5dcf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e8ae5dcf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e8ae5dcf
Branch: refs/heads/feature/test_daewon
Commit: e8ae5dcfc40b67134b0277d5d7e4690b42c69005
Parents: 5acead8 828a4c7
Author: Doyung Yoon <sh...@gmail.com>
Authored: Mon Dec 21 14:41:23 2015 +0900
Committer: Doyung Yoon <sh...@gmail.com>
Committed: Mon Dec 21 14:41:23 2015 +0900
----------------------------------------------------------------------
.../main/scala/com/kakao/s2graph/core/PostProcess.scala | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[04/46] incubator-s2graph git commit: add config
's2graph.read-only.url' for counter
Posted by da...@apache.org.
add config 's2graph.read-only.url' for counter
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b159a7fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b159a7fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b159a7fb
Branch: refs/heads/feature/test_daewon
Commit: b159a7fbf5f39b335c7709e0eb64f23a6cbfda7f
Parents: 5acead8
Author: Jaesang Kim <ho...@gmail.com>
Authored: Mon Dec 21 14:21:05 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Mon Dec 21 14:21:05 2015 +0900
----------------------------------------------------------------------
.../main/scala/s2/config/S2CounterConfig.scala | 5 ++--
.../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++-----------------
.../counter/core/v2/RankingStorageGraph.scala | 5 ++--
.../main/scala/s2/config/StreamingConfig.scala | 1 +
.../src/test/resources/application.conf | 1 +
5 files changed, 11 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
index d0db73c..fcd0e6a 100644
--- a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
+++ b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
@@ -28,7 +28,7 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
lazy val DB_DEFAULT_USER = getOrElse("db.default.user", "graph")
lazy val DB_DEFAULT_PASSWORD = getOrElse("db.default.password", "graph")
- // REDIS
+ // Redis
lazy val REDIS_INSTANCES = (for {
s <- config.getStringList("redis.instances")
} yield {
@@ -36,8 +36,9 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
(sp(0), if (sp.length > 1) sp(1).toInt else 6379)
}).toList
- // graph
+ // Graph
lazy val GRAPH_URL = getOrElse("s2graph.url", "http://localhost:9000")
+ lazy val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
// Cache
lazy val CACHE_TTL_SECONDS = getOrElse("cache.ttl.seconds", 600)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
index add7c61..7dcff6c 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
@@ -35,6 +35,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
private val labelPostfix = "_counts"
val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
val graphOp = new GraphOperation(config)
import ExactStorageGraph._
@@ -74,26 +75,6 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
seqOfSeq.flatten.toSeq.groupBy(_._1).mapValues { seq => seq.map(_._2).toMap }
}
Await.result(future, 10 second)
-// val (keyWithEq, reqJsLs) = toIncrementCountRequests(policy, counts).unzip(x => ((x._1, x._2), x._3))
-//
-// val future = wsClient.url(s"$s2graphUrl/graphs/edges/incrementCount").post(Json.toJson(reqJsLs)).map { resp =>
-// resp.status match {
-// case HttpStatus.SC_OK =>
-// val respSeq = resp.json.as[Seq[RespGraph]]
-//
-// val keyWithEqResult = {
-// for {
-// ((key, eq), RespGraph(success, result)) <- keyWithEq.zip(respSeq)
-// } yield {
-// (key, (eq, result))
-// }
-// }.groupBy(_._1).mapValues{ seq => seq.map(_._2).toMap }
-// keyWithEqResult
-// case _ =>
-// throw new RuntimeException(s"update failed: $policy $counts")
-// }
-// }
-// Await.result(future, 10 second)
}
def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
@@ -201,7 +182,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
val reqJs = Json.parse(reqJsStr)
// log.warn(s"query: ${reqJs.toString()}")
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(reqJs).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(reqJs).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val respJs = resp.json
@@ -261,7 +242,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
val query = Json.obj("srcVertices" -> Json.arr(src), "steps" -> Json.arr(step))
// println(s"query: ${query.toString()}")
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(query).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(query).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val respJs = resp.json
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
index b0c0a41..9ada8d4 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
@@ -39,6 +39,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
private val labelPostfix = "_topK"
val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
val prepareCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(10000, 600))
val graphOp = new GraphOperation(config)
@@ -246,7 +247,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
log.debug(strJs)
val payload = Json.parse(strJs)
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(payload).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
(resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
@@ -278,7 +279,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
)
)
- val future = wsClient.url(s"$s2graphUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
+ val future = wsClient.url(s"$s2graphReadOnlyUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val checkRespJs = resp.json
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
index 0880bdd..ba5e863 100644
--- a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
+++ b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
@@ -20,4 +20,5 @@ object StreamingConfig extends ConfigFunctions(S2ConfigFactory.config) {
// graph url
val GRAPH_URL = getOrElse("s2graph.url", "")
+ val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/resources/application.conf b/s2counter_loader/src/test/resources/application.conf
index 369a6fa..02d68b2 100644
--- a/s2counter_loader/src/test/resources/application.conf
+++ b/s2counter_loader/src/test/resources/application.conf
@@ -75,3 +75,4 @@ profile.prefetch.size=100
# s2graph
s2graph.url = "http://"${host}":9000"
+s2graph.real-only.url = ${s2graph.url}
[09/46] incubator-s2graph git commit: Add netty dependency
Posted by da...@apache.org.
Add netty dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/57fae9d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/57fae9d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/57fae9d9
Branch: refs/heads/feature/test_daewon
Commit: 57fae9d948c9e70007b857ad2df42ff867fa6562
Parents: a6acec7
Author: daewon <bl...@gmail.com>
Authored: Tue Dec 22 17:04:28 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Tue Dec 22 17:04:28 2015 +0900
----------------------------------------------------------------------
s2core/lib/netty-3.9.4.Final.jar | Bin 0 -> 1310154 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/57fae9d9/s2core/lib/netty-3.9.4.Final.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/netty-3.9.4.Final.jar b/s2core/lib/netty-3.9.4.Final.jar
new file mode 100644
index 0000000..7ee6ba4
Binary files /dev/null and b/s2core/lib/netty-3.9.4.Final.jar differ
[15/46] incubator-s2graph git commit: Merge remote-tracking branch
'origin/develop' into feature/handler_json_parsing_error
Posted by da...@apache.org.
Merge remote-tracking branch 'origin/develop' into feature/handler_json_parsing_error
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e8c35881
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e8c35881
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e8c35881
Branch: refs/heads/feature/test_daewon
Commit: e8c358810ae2658d19228086a0f528c4f9397f09
Parents: e607fc9 e8befb6
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 23 11:59:07 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 23 11:59:07 2015 +0900
----------------------------------------------------------------------
.../main/scala/s2/config/S2CounterConfig.scala | 5 ++--
.../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++-----------------
.../counter/core/v2/RankingStorageGraph.scala | 6 ++---
s2counter_loader/build.sbt | 2 +-
.../main/scala/s2/config/StreamingConfig.scala | 1 +
.../src/test/resources/application.conf | 1 +
6 files changed, 12 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e8c35881/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
[17/46] incubator-s2graph git commit: Resolve Dependency issue on s2ml
Posted by da...@apache.org.
Resolve Dependency issue on s2ml
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/9f528db1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/9f528db1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/9f528db1
Branch: refs/heads/feature/test_daewon
Commit: 9f528db1ee8596f83a446967a81e5f7d6d337405
Parents: d358931
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 23 14:11:49 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 23 14:11:49 2015 +0900
----------------------------------------------------------------------
project/plugins.sbt | 2 ++
s2ml/build.sbt | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/9f528db1/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index a710e0e..27d0f60 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -8,3 +8,5 @@ addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.3")
+
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.0")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/9f528db1/s2ml/build.sbt
----------------------------------------------------------------------
diff --git a/s2ml/build.sbt b/s2ml/build.sbt
index 2c7bf6d..698a32c 100644
--- a/s2ml/build.sbt
+++ b/s2ml/build.sbt
@@ -10,7 +10,7 @@ val sparkVersion = "1.5.1"
resolvers += "Sonatype OSS Releases" at "http://oss.sonatype.org/content/repositories/releases/"
libraryDependencies ++= Seq(
- "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
+ "org.apache.spark" %% "spark-core" % sparkVersion % "provided" exclude("org.jboss.netty", "netty"),
"org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
"org.apache.spark" %% "spark-hive" % sparkVersion % "provided",
"com.typesafe.play" %% "play-ws" % Common.playVersion,
[18/46] incubator-s2graph git commit: Fix the bug: VertexSpec
Posted by da...@apache.org.
Fix the bug: VertexSpec
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c6b607fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c6b607fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c6b607fe
Branch: refs/heads/feature/test_daewon
Commit: c6b607fe44ce8f5d89438b8715050a2d794a4088
Parents: 9f528db
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 23 14:44:27 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 23 14:44:27 2015 +0900
----------------------------------------------------------------------
s2rest_play/test/controllers/VertexSpec.scala | 62 +++++++++++++---------
1 file changed, 38 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c6b607fe/s2rest_play/test/controllers/VertexSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/VertexSpec.scala b/s2rest_play/test/controllers/VertexSpec.scala
index 1fc79c3..46461ea 100644
--- a/s2rest_play/test/controllers/VertexSpec.scala
+++ b/s2rest_play/test/controllers/VertexSpec.scala
@@ -1,38 +1,52 @@
package controllers
+import com.kakao.s2graph.core.Graph
+import com.kakao.s2graph.core.rest.RestCaller
+import com.typesafe.config.ConfigFactory
import play.api.libs.json._
import play.api.test.FakeApplication
import play.api.test.Helpers._
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
class VertexSpec extends SpecCommon {
// init()
"vetex tc" should {
"tc1" in {
-//
-// running(FakeApplication()) {
-// val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
-//
-// val (serviceName, columnName) = (testServiceName, testColumnName)
-//
-// val data = vertexInsertsPayload(serviceName, columnName, ids)
-// val payload = Json.parse(Json.toJson(data).toString)
-// println(payload)
-//
-// val jsResult = contentAsString(VertexController.tryMutates(payload, "insert",
-// Option(serviceName), Option(columnName), withWait = true))
-//
-// val query = vertexQueryJson(serviceName, columnName, ids)
-// val ret = contentAsJson(QueryController.getVerticesInner(query))
-// println(">>>", ret)
-// val fetched = ret.as[Seq[JsValue]]
-// for {
-// (d, f) <- data.zip(fetched)
-// } yield {
-// (d \ "id") must beEqualTo((f \ "id"))
-// ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age"))
-// }
-// }
+
+ running(FakeApplication()) {
+ val config = ConfigFactory.load()
+ val graph = new Graph(config)
+ val rest = new RestCaller(graph)
+
+ val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
+
+ val (serviceName, columnName) = (testServiceName, testColumnName)
+
+ val data = vertexInsertsPayload(serviceName, columnName, ids)
+ val payload = Json.parse(Json.toJson(data).toString)
+ println(payload)
+
+ val jsResult = contentAsString(VertexController.tryMutates(payload, "insert",
+ Option(serviceName), Option(columnName), withWait = true))
+
+ val query = vertexQueryJson(serviceName, columnName, ids)
+ val res = rest.uriMatch("/graphs/getVertices", query)
+
+
+ val ret = Await.result(res, Duration("30 seconds"))
+
+ println(">>>", ret)
+ val fetched = ret.as[Seq[JsValue]]
+ for {
+ (d, f) <- data.zip(fetched)
+ } yield {
+ (d \ "id") must beEqualTo((f \ "id"))
+ ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age"))
+ }
+ }
true
}
}
[02/46] incubator-s2graph git commit: Change error response:
timeoutResult => emptyResult
Posted by da...@apache.org.
Change error response: timeoutResult => emptyResult
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5acead83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5acead83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5acead83
Branch: refs/heads/feature/test_daewon
Commit: 5acead832ad9506f08feca0b47dc1aeaf847dc33
Parents: b5b8fcb
Author: daewon <bl...@gmail.com>
Authored: Mon Dec 21 14:13:31 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Dec 21 14:13:31 2015 +0900
----------------------------------------------------------------------
s2rest_netty/src/main/scala/Server.scala | 3 ++-
s2rest_play/app/controllers/ApplicationController.scala | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5acead83/s2rest_netty/src/main/scala/Server.scala
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala
index 39540d0..3f1e70a 100644
--- a/s2rest_netty/src/main/scala/Server.scala
+++ b/s2rest_netty/src/main/scala/Server.scala
@@ -94,7 +94,8 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S
simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders)
case e: Exception =>
logger.error(s"${jsonQuery}, ${e.getMessage}", e)
- simpleResponse(ctx, InternalServerError, byteBufOpt = None, channelFutureListenerOpt = Option(Close))
+ val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8)
+ simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5acead83/s2rest_play/app/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ApplicationController.scala b/s2rest_play/app/controllers/ApplicationController.scala
index ec21216..484d282 100644
--- a/s2rest_play/app/controllers/ApplicationController.scala
+++ b/s2rest_play/app/controllers/ApplicationController.scala
@@ -21,7 +21,7 @@ object ApplicationController extends Controller {
Future.successful(BadRequest(Json.obj("message" -> ex.getMessage)).as(applicationJsonHeader))
private def errorResults =
- Future.successful(Ok(PostProcess.timeoutResults).as(applicationJsonHeader))
+ Future.successful(Ok(PostProcess.emptyResults).as(applicationJsonHeader))
def requestFallback(body: JsValue): PartialFunction[Throwable, Future[Result]] = {
case e: BadQueryException =>
[33/46] incubator-s2graph git commit: Issue#220. bug fix for
isInnerCall on Query Builder.
Posted by da...@apache.org.
Issue#220. bug fix for isInnerCall on Query Builder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/ca589e64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/ca589e64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/ca589e64
Branch: refs/heads/feature/test_daewon
Commit: ca589e649590cd5bcce2bdfca2361f3a2bd0f2f6
Parents: f75b3b3
Author: SteamShon <sh...@gmail.com>
Authored: Tue Dec 29 17:32:50 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Tue Dec 29 17:32:50 2015 +0900
----------------------------------------------------------------------
.../kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ca589e64/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
index d8e4099..0343cde 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala
@@ -227,7 +227,7 @@ class AsynchbaseQueryBuilder(storage: AsynchbaseStorage)(implicit ec: ExecutionC
val defers: Seq[Deferred[QueryRequestWithResult]] = for {
(queryRequest, prevStepScore) <- queryRequestWithScoreLs
parentEdges <- prevStepEdges.get(queryRequest.vertex.id)
- } yield fetch(queryRequest, prevStepScore, isInnerCall = true, parentEdges)
+ } yield fetch(queryRequest, prevStepScore, isInnerCall = false, parentEdges)
val grouped: Deferred[util.ArrayList[QueryRequestWithResult]] = Deferred.group(defers)
grouped withCallback {
[24/46] incubator-s2graph git commit: set root project
Posted by da...@apache.org.
set root project
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/3a3dc2f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/3a3dc2f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/3a3dc2f4
Branch: refs/heads/feature/test_daewon
Commit: 3a3dc2f4410a57f693238375d9442dbf8cbb54f0
Parents: f45b845
Author: daewon <bl...@gmail.com>
Authored: Thu Dec 24 16:55:19 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Thu Dec 24 16:55:19 2015 +0900
----------------------------------------------------------------------
build.sbt | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3a3dc2f4/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index ac86dbf..ad08bdb 100755
--- a/build.sbt
+++ b/build.sbt
@@ -44,11 +44,5 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
lazy val s2ml = project.settings(commonSettings: _*)
-// test in s2core := {}
-// test in s2rest_play := {}
-
-test in spark := {}
-test in loader := {}
-test in s2counter_core := {}
-test in s2counter_loader := {}
-test in s2ml := {}
+lazy val root = (project in file(".")).
+ aggregate(s2core, s2rest_play)
[32/46] incubator-s2graph git commit: # 208 Move test cases from
s2rest_play to s2graph_core
Posted by da...@apache.org.
# 208 Move test cases from s2rest_play to s2graph_core
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/610d519d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/610d519d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/610d519d
Branch: refs/heads/feature/test_daewon
Commit: 610d519d3a8cf2c140499d0b4e4d3c75faaf08c9
Parents: 395b00f
Author: daewon <bl...@gmail.com>
Authored: Tue Dec 29 17:01:17 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Tue Dec 29 17:01:17 2015 +0900
----------------------------------------------------------------------
.../scala/com/kakao/s2graph/core/Graph.scala | 2 +-
.../kakao/s2graph/core/rest/RequestParser.scala | 11 +
.../scala/com/kakao/s2graph/core/EdgeTest.scala | 11 +-
.../kakao/s2graph/core/Integrate/CrudTest.scala | 141 ++--
.../core/Integrate/IntegrateCommon.scala | 107 +--
.../s2graph/core/Integrate/QueryTest.scala | 86 +--
.../core/Integrate/StrongLabelDeleteTest.scala | 282 ++++++++
.../core/Integrate/WeakLabelDeleteTest.scala | 129 ++++
.../app/controllers/EdgeController.scala | 9 +-
.../benchmark/PostProcessBenchmarkSpec.scala | 6 +-
.../test/controllers/PostProcessSpec.scala | 2 +-
s2rest_play/test/controllers/SpecCommon.scala | 35 +-
.../controllers/StrongLabelDeleteSpec.scala | 690 +++++++++----------
.../test/controllers/WeakLabelDeleteSpec.scala | 252 +++----
14 files changed, 1076 insertions(+), 687 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
index 9a540d2..5bba009 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala
@@ -312,7 +312,7 @@ object Graph {
} get
}
-class Graph(_config: Config)(implicit ec: ExecutionContext) {
+class Graph(_config: Config)(implicit val ec: ExecutionContext) {
val config = _config.withFallback(Graph.DefaultConfig)
val cacheSize = config.getInt("cache.max.size")
// val cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Seq[QueryResult]]()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index 04ac113..f22dfcf 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -503,4 +503,15 @@ class RequestParser(config: Config) extends JSONParser {
element <- Graph.toGraphElement(str)
} yield element
}
+
+ def toDeleteParam(json: JsValue) = {
+ val labelName = (json \ "label").as[String]
+ val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil)
+ val direction = (json \ "direction").asOpt[String].getOrElse("out")
+
+ val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
+ val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
+ val vertices = toVertices(labelName, direction, ids)
+ (labels, direction, ids, ts, vertices)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
index d86541d..2aad32f 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
@@ -7,6 +7,7 @@ import org.scalatest.FunSuite
import org.scalatest.matchers.Matcher
class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
+ initTests()
test("toLogString") {
val testLabelName = labelNameV2
@@ -24,11 +25,11 @@ class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
}).mkString("\n")
val expected = Seq(
- Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"),
- Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"),
- Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"),
- Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"),
- Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}")
+ Seq("1445240543366", "update", "e", "1", "2", testLabelName, "{\"is_blocked\":true}"),
+ Seq("1445240543362", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false}"),
+ Seq("1445240543364", "insert", "e", "1", "2", testLabelName, "{\"is_hidden\":false,\"weight\":10}"),
+ Seq("1445240543363", "delete", "e", "1", "2", testLabelName),
+ Seq("1445240543365", "update", "e", "1", "2", testLabelName, "{\"time\":1,\"weight\":-10}")
).map(_.mkString("\t")).mkString("\n")
assert(bulkEdge === expected)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
index d178419..f3bbdd9 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -1,14 +1,13 @@
package com.kakao.s2graph.core.Integrate
-import com.kakao.s2graph.core.{Management, PostProcess}
import com.kakao.s2graph.core.mysqls._
import play.api.libs.json.{JsObject, Json}
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-
class CrudTest extends IntegrateCommon {
+ import CrudHelper._
+ import TestUtil._
+
test("test CRUD") {
var tcNum = 0
var tcString = ""
@@ -152,87 +151,77 @@ class CrudTest extends IntegrateCommon {
tcRunner.run(tcNum, tcString, bulkQueries, expected)
}
- class CrudTestRunner {
- var seed = 0
-
- def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
- for {
- labelName <- List(testLabelName, testLabelName2)
- i <- 0 until NumOfEachTest
- } {
- seed += 1
- val srcId = seed.toString
- val tgtId = srcId
-
- val maxTs = opWithProps.map(t => t._1).max
- /** insert edges */
- println(s"---- TC${tcNum}_init ----")
- val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
- Util.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
- })
+ object CrudHelper {
- Util.insertEdges(bulkEdges:_*)
+ class CrudTestRunner {
+ var seed = 0
+ def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
for {
- label <- Label.findByName(labelName)
- direction <- List("out", "in")
- cacheTTL <- List(-1L)
+ labelName <- List(testLabelName, testLabelName2)
+ i <- 0 until NumOfEachTest
} {
- val (serviceName, columnName, id, otherId) = direction match {
- case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
- case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
- }
-
- val qId = if (labelName == testLabelName) id else "\"" + id + "\""
- val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
-
- val jsResult = Util.getEdges(query)
-
- val results = jsResult \ "results"
- val deegrees = (jsResult \ "degrees").as[List[JsObject]]
- val propsLs = (results \\ "props").seq
- (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
-
- val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
- val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
-
- from should be(id.toString)
- to should be(otherId.toString)
- (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
-
- for ((key, expectedVal) <- expected) {
- propsLs.last.as[JsObject].keys.contains(key) should be(true)
- (propsLs.last \ key).toString should be(expectedVal)
+ seed += 1
+ val srcId = seed.toString
+ val tgtId = srcId
+
+ val maxTs = opWithProps.map(t => t._1).max
+
+ /** insert edges */
+ println(s"---- TC${tcNum}_init ----")
+ val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
+ TestUtil.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
+ })
+
+ TestUtil.insertEdgesSync(bulkEdges: _*)
+
+ for {
+ label <- Label.findByName(labelName)
+ direction <- List("out", "in")
+ cacheTTL <- List(-1L)
+ } {
+ val (serviceName, columnName, id, otherId) = direction match {
+ case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
+ case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
+ }
+
+ val qId = if (labelName == testLabelName) id else "\"" + id + "\""
+ val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
+
+ val jsResult = TestUtil.getEdgesSync(query)
+
+ val results = jsResult \ "results"
+ val deegrees = (jsResult \ "degrees").as[List[JsObject]]
+ val propsLs = (results \\ "props").seq
+ (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
+
+ val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
+ val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
+
+ from should be(id.toString)
+ to should be(otherId.toString)
+ (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
+
+ for ((key, expectedVal) <- expected) {
+ propsLs.last.as[JsObject].keys.contains(key) should be(true)
+ (propsLs.last \ key).toString should be(expectedVal)
+ }
}
}
}
- }
- def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = {
- val s =
- s"""{
- "srcVertices": [
- {
- "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }
- ],
- "steps": [
- [
- {
- "label": "$labelName",
- "direction": "$dir",
- "offset": 0,
- "limit": 10,
- "cacheTTL": $cacheTTL
- }
- ]
- ]
- }"""
- Json.parse(s)
+ def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = Json.parse(
+ s""" { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id } ],
+ "steps": [ [ {
+ "label": "$labelName",
+ "direction": "$dir",
+ "offset": 0,
+ "limit": 10,
+ "cacheTTL": $cacheTTL }]]}""")
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
index 6db250d..230fa9c 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -8,9 +8,12 @@ import org.scalatest._
import play.api.libs.json.{JsValue, Json}
import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.{Await, ExecutionContext, Future}
trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
+
+ import TestUtil._
+
var graph: Graph = _
var parser: RequestParser = _
var config: Config = _
@@ -78,19 +81,32 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
/**
* Test Helpers
*/
- object Util {
-// def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
-// val arr = for {
-// (label, dir, from, to) <- params
-// } yield {
-// Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
-// }
-//
-// val s = Json.toJson(arr)
-// s
-// }
-
- def getEdges(queryJson: JsValue): JsValue = {
+ object TestUtil {
+ implicit def ec = graph.ec
+
+ // def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
+ // val arr = for {
+ // (label, dir, from, to) <- params
+ // } yield {
+ // Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
+ // }
+ //
+ // val s = Json.toJson(arr)
+ // s
+ // }
+
+ def deleteAllSync(jsValue: JsValue) = {
+ val future = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
+ val (labels, direction, ids, ts, vertices) = parser.toDeleteParam(json)
+ val future = graph.deleteAllAdjacentEdges(vertices.toList, labels, GraphUtil.directions(direction), ts)
+
+ future
+ })
+
+ Await.result(future, HttpRequestWaitingTime)
+ }
+
+ def getEdgesSync(queryJson: JsValue): JsValue = {
val ret = graph.getEdges(parser.toQuery(queryJson))
val result = Await.result(ret, HttpRequestWaitingTime)
val jsResult = PostProcess.toSimpleVertexArrJson(result)
@@ -98,37 +114,41 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
jsResult
}
- def insertEdges(bulkEdges: String*) = {
+ def insertEdgesSync(bulkEdges: String*) = {
val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
val jsResult = Await.result(req, HttpRequestWaitingTime)
jsResult
}
+ def insertEdgesAsync(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ req
+ }
+
def toEdge(elems: Any*): String = elems.mkString("\t")
- }
- // common tables
- protected val testServiceName = "s2graph"
- protected val testLabelName = "s2graph_label_test"
- protected val testLabelName2 = "s2graph_label_test_2"
- protected val testLabelNameV1 = "s2graph_label_test_v1"
- protected val testLabelNameWeak = "s2graph_label_test_weak"
- protected val testColumnName = "user_id_test"
- protected val testColumnType = "long"
- protected val testTgtColumnName = "item_id_test"
- protected val testHTableName = "test-htable"
- protected val newHTableName = "new-htable"
- protected val index1 = "idx_1"
- protected val index2 = "idx_2"
-
- val NumOfEachTest = 30
- val HttpRequestWaitingTime = Duration("Inf")
-
- val createService = s"""{"serviceName" : "$testServiceName"}"""
-
- val testLabelNameCreate =
- s"""
+ // common tables
+ val testServiceName = "s2graph"
+ val testLabelName = "s2graph_label_test"
+ val testLabelName2 = "s2graph_label_test_2"
+ val testLabelNameV1 = "s2graph_label_test_v1"
+ val testLabelNameWeak = "s2graph_label_test_weak"
+ val testColumnName = "user_id_test"
+ val testColumnType = "long"
+ val testTgtColumnName = "item_id_test"
+ val testHTableName = "test-htable"
+ val newHTableName = "new-htable"
+ val index1 = "idx_1"
+ val index2 = "idx_2"
+
+ val NumOfEachTest = 30
+ val HttpRequestWaitingTime = Duration("Inf")
+
+ val createService = s"""{"serviceName" : "$testServiceName"}"""
+
+ val testLabelNameCreate =
+ s"""
{
"label": "$testLabelName",
"srcServiceName": "$testServiceName",
@@ -169,8 +189,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
"hTableName": "$testHTableName"
}"""
- val testLabelName2Create =
- s"""
+ val testLabelName2Create =
+ s"""
{
"label": "$testLabelName2",
"srcServiceName": "$testServiceName",
@@ -208,8 +228,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
"compressionAlgorithm": "gz"
}"""
- val testLabelNameV1Create =
- s"""
+ val testLabelNameV1Create =
+ s"""
{
"label": "$testLabelNameV1",
"srcServiceName": "$testServiceName",
@@ -247,8 +267,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
"compressionAlgorithm": "gz"
}"""
- val testLabelNameWeakCreate =
- s"""
+ val testLabelNameWeakCreate =
+ s"""
{
"label": "$testLabelNameWeak",
"srcServiceName": "$testServiceName",
@@ -284,4 +304,5 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
"isDirected": true,
"compressionAlgorithm": "gz"
}"""
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
index 505927b..bd81b7b 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
@@ -8,7 +8,7 @@ import scala.util.{Success, Try}
class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
- import Util._
+ import TestUtil._
val insert = "insert"
val e = "e"
@@ -36,16 +36,16 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
}
""")
- var edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
+ var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
(edges \ "size").toString should be("1")
- edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
+ edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
(edges \ "size").toString should be("2")
- edges = getEdges(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
(edges \ "size").toString should be("1")
- edges = getEdges(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
+ edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
(edges \ "size").toString should be("2")
}
@@ -68,19 +68,19 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
]]
}""")
- var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
+ var result = getEdgesSync(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
(result \ "results").as[List[JsValue]].size should be(1)
- result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)"))
+ result = getEdgesSync(queryWhere(0, "is_hidden=true and _to in (1)"))
(result \ "results").as[List[JsValue]].size should be(1)
- result = getEdges(queryWhere(0, "_from=0"))
+ result = getEdgesSync(queryWhere(0, "_from=0"))
(result \ "results").as[List[JsValue]].size should be(2)
- result = getEdges(queryWhere(2, "_from=2 or weight in (-1)"))
+ result = getEdgesSync(queryWhere(2, "_from=2 or weight in (-1)"))
(result \ "results").as[List[JsValue]].size should be(2)
- result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)"))
+ result = getEdgesSync(queryWhere(2, "_from=2 and weight in (10, 20)"))
(result \ "results").as[List[JsValue]].size should be(2)
}
@@ -109,7 +109,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
]]
}""")
- val result = getEdges(queryExclude(0))
+ val result = getEdgesSync(queryExclude(0))
(result \ "results").as[List[JsValue]].size should be(1)
}
@@ -132,7 +132,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
)
}
- val result = getEdges(queryGroupBy(0, Seq("weight")))
+ val result = getEdgesSync(queryGroupBy(0, Seq("weight")))
(result \ "size").as[Int] should be(2)
val weights = (result \\ "groupBy").map { js =>
(js \ "weight").as[Int]
@@ -161,13 +161,13 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
]]
}""")
- var result = getEdges(queryTransform(0, "[[\"_to\"]]"))
+ var result = getEdgesSync(queryTransform(0, "[[\"_to\"]]"))
(result \ "results").as[List[JsValue]].size should be(2)
- result = getEdges(queryTransform(0, "[[\"weight\"]]"))
+ result = getEdgesSync(queryTransform(0, "[[\"weight\"]]"))
(result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted)
- result = getEdges(queryTransform(0, "[[\"_from\"]]"))
+ result = getEdgesSync(queryTransform(0, "[[\"_from\"]]"))
val results = (result \ "results").as[JsValue]
(result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted)
}
@@ -187,11 +187,11 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
}
// weight order
- var result = getEdges(queryIndex(Seq(0), "idx_1"))
+ var result = getEdgesSync(queryIndex(Seq(0), "idx_1"))
((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40))
// timestamp order
- result = getEdges(queryIndex(Seq(0), "idx_2"))
+ result = getEdgesSync(queryIndex(Seq(0), "idx_2"))
((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30))
}
@@ -237,13 +237,13 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
}
// get all
- var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ var result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
(result \ "results").as[List[JsValue]].size should be(4)
// inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
(result \ "results").as[List[JsValue]].size should be(3)
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
(result \ "results").as[List[JsValue]].size should be(1)
val bulkEdges = Seq(
@@ -252,21 +252,21 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
)
- insertEdges(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
// duration test after udpate
// get all
- result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 0, to = 5000))
(result \ "results").as[List[JsValue]].size should be(4)
// inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 4000))
(result \ "results").as[List[JsValue]].size should be(3)
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ result = getEdgesSync(queryDuration(Seq(0, 2), from = 1000, to = 2000))
(result \ "results").as[List[JsValue]].size should be(1)
- def a: JsValue = getEdges(queryDuration(Seq(0, 2), from = 3000, to = 2000))
+ def a: JsValue = getEdgesSync(queryDuration(Seq(0, 2), from = 3000, to = 2000))
Try(a).recover {
case e: BadQueryException => JsNull
} should be(Success(JsNull))
@@ -301,9 +301,9 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
val src = 100
val tgt = 200
- insertEdges(toEdge(1001, "insert", "e", src, tgt, testLabelName))
+ insertEdgesSync(toEdge(1001, "insert", "e", src, tgt, testLabelName))
- val result = Util.getEdges(queryParents(src))
+ val result = TestUtil.getEdgesSync(queryParents(src))
val parents = (result \ "results").as[Seq[JsValue]]
val ret = parents.forall {
edge => (edge \ "parents").as[Seq[JsValue]].size == 1
@@ -342,23 +342,23 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
)
- insertEdges(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
- var result = getEdges(querySingle(src, offset = 0, limit = 2))
+ var result = getEdgesSync(querySingle(src, offset = 0, limit = 2))
var edges = (result \ "results").as[List[JsValue]]
edges.size should be(2)
(edges(0) \ "to").as[Long] should be(4)
(edges(1) \ "to").as[Long] should be(3)
- result = getEdges(querySingle(src, offset = 1, limit = 2))
+ result = getEdgesSync(querySingle(src, offset = 1, limit = 2))
edges = (result \ "results").as[List[JsValue]]
edges.size should be(2)
(edges(0) \ "to").as[Long] should be(3)
(edges(1) \ "to").as[Long] should be(2)
- result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+ result = getEdgesSync(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
edges = (result \ "results").as[List[JsValue]]
edges.size should be(1)
}
@@ -406,12 +406,12 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
)
- insertEdges(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
// get edges
- val edges = getEdges(queryScore(0, Map("weight" -> 1)))
- val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
- val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
+ val edges = getEdgesSync(queryScore(0, Map("weight" -> 1)))
+ val orderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
+ val ascOrderByScore = getEdgesSync(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
val edgesTo = edges \ "results" \\ "to"
val orderByTo = orderByScore \ "results" \\ "to"
@@ -523,27 +523,27 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
toEdge(ts, insert, e, 322, 3322, testLabelName)
)
- insertEdges(bulkEdges: _*)
+ insertEdgesSync(bulkEdges: _*)
- val result1 = getEdges(queryWithSampling(testId, sampleSize))
+ val result1 = getEdgesSync(queryWithSampling(testId, sampleSize))
(result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
- val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize))
+ val result2 = getEdgesSync(twoStepQueryWithSampling(testId, sampleSize))
(result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
- val result3 = getEdges(twoQueryWithSampling(testId, sampleSize))
+ val result3 = getEdgesSync(twoQueryWithSampling(testId, sampleSize))
(result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
}
test("limit") {
- insertEdges(
+ insertEdgesSync(
toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)))
- val edges = getEdges(querySingle(0, limit = 1))
- val limitEdges = getEdges(queryGlobalLimit(0, 1))
+ val edges = getEdgesSync(querySingle(0, limit = 1))
+ val limitEdges = getEdgesSync(queryGlobalLimit(0, 1))
val edgesTo = edges \ "results" \\ "to"
val limitEdgesTo = limitEdges \ "results" \\ "to"
@@ -615,7 +615,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
override def initTestData(): Unit = {
super.initTestData()
- insertEdges(
+ insertEdgesSync(
toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
new file mode 100644
index 0000000..69a49b4
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/StrongLabelDeleteTest.scala
@@ -0,0 +1,282 @@
+package com.kakao.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, Future}
+import scala.util.Random
+
+class StrongLabelDeleteTest extends IntegrateCommon {
+
+ import StrongDeleteUtil._
+ import TestUtil._
+
+ test("Strong consistency select") {
+ insertEdgesSync(bulkEdges(): _*)
+
+ var result = getEdgesSync(query(0))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ result = getEdgesSync(query(10))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("Strong consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val deleteParam = Json.arr(
+ Json.obj("label" -> testLabelName2,
+ "direction" -> "in",
+ "ids" -> Json.arr("20"),
+ "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteParam)
+
+ result = getEdgesSync(query(11, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, direction = "out"))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, direction = "out"))
+ println(result)
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, direction = "in", columnName = testTgtColumnName))
+ println(result)
+
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ test("update delete") {
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val src = System.currentTimeMillis()
+
+ val (ret, last) = testInner(i, src)
+ ret should be(true)
+ ret
+ }
+
+ ret.forall(identity)
+ }
+
+ test("update delete 2") {
+ val src = System.currentTimeMillis()
+ var ts = 0L
+
+ val ret = for {
+ i <- 0 until testNum
+ } yield {
+ val (ret, lastTs) = testInner(ts, src)
+ val deletedAt = lastTs + 1
+ val deletedAt2 = lastTs + 2
+ ts = deletedAt2 + 1 // nex start ts
+
+ ret should be(true)
+
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+ val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
+
+ val deleteRet = deleteAllSync(deleteAllRequest)
+ val deleteRet2 = deleteAllSync(deleteAllRequest2)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+
+ degreeAfterDeleteAll should be(0)
+ degreeAfterDeleteAll === (0)
+ }
+
+ ret.forall(identity)
+ }
+
+ /** This test stress out test on degree
+ * when contention is low but number of adjacent edges are large
+ * Large set of contention test
+ */
+ test("large degrees") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ // val allRequests = insertRequests ++ deleteRequests
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = insertRequests.size - deleteRequests.size
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ // println(result)
+
+ val ret = resultSize == expectedDegree && resultDegree == resultSize
+ println(s"[MaxSize]: $maxSize")
+ println(s"[DeleteSize]: $deleteSize")
+ println(s"[ResultDegree]: $resultDegree")
+ println(s"[ExpectedDegree]: $expectedDegree")
+ println(s"[ResultSize]: $resultSize")
+ ret should be(true)
+ }
+
+ test("deleteAll") {
+ val labelName = testLabelName2
+ val dir = "out"
+ val maxSize = 100
+ val deleteSize = 10
+ val numOfConcurrentBatch = 100
+ val src = System.currentTimeMillis()
+ val tgts = (0 until maxSize).map { ith => src + ith }
+ val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+ val insertRequests = tgts.map { tgt =>
+ Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+ Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+ }
+ val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+ val futures = allRequests.grouped(numOfConcurrentBatch).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val deletedAt = System.currentTimeMillis()
+ val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+
+ deleteAllSync(deleteAllRequest)
+
+ val result = getEdgesSync(query(id = src))
+ println(result)
+ val resultEdges = (result \ "results").as[Seq[JsValue]]
+ resultEdges.isEmpty should be(true)
+
+ val degreeAfterDeleteAll = getDegree(result)
+ degreeAfterDeleteAll should be(0)
+ }
+
+ object StrongDeleteUtil {
+
+ val labelName = testLabelName2
+ val maxTgtId = 10
+ val batchSize = 10
+ val testNum = 3
+ val numOfBatch = 10
+
+ def testInner(startTs: Long, src: Long) = {
+ val labelName = testLabelName2
+ val lastOps = Array.fill(maxTgtId)("none")
+ var currentTs = startTs
+
+ val allRequests = for {
+ ith <- 0 until numOfBatch
+ jth <- 0 until batchSize
+ } yield {
+ currentTs += 1
+
+ val tgt = Random.nextInt(maxTgtId)
+ val op = if (Random.nextDouble() < 0.5) "delete" else "update"
+
+ lastOps(tgt) = op
+ Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
+ }
+
+ allRequests.foreach(println(_))
+
+ val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequests =>
+ insertEdgesAsync(bulkRequests: _*)
+ }
+
+ Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+
+ val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
+ val queryJson = query(id = src)
+ val result = getEdgesSync(queryJson)
+ val resultSize = (result \ "size").as[Long]
+ val resultDegree = getDegree(result)
+
+ println(lastOps.toList)
+ println(result)
+
+ val ret = resultDegree == expectedDegree && resultSize == resultDegree
+ if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
+
+ (ret, currentTs)
+ }
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""")
+ )
+
+ def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
+ labelName: String = testLabelName2, direction: String = "out") = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$labelName",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": -1,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+
+ def getDegree(jsValue: JsValue): Long = {
+ ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
new file mode 100644
index 0000000..b80d9c7
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/WeakLabelDeleteTest.scala
@@ -0,0 +1,129 @@
+package com.kakao.s2graph.core.Integrate
+
+import java.util.concurrent.TimeUnit
+
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json.{JsObject, JsValue, Json}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+class WeakLabelDeleteTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import TestUtil._
+ import WeakLabelDeleteHelper._
+
+ test("test weak consistency select") {
+ var result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(4)
+ result = getEdgesSync(query(10))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("test weak consistency delete") {
+ var result = getEdgesSync(query(0))
+ println(result)
+
+ /** expect 4 edges */
+ (result \ "results").as[List[JsValue]].size should be(4)
+ val edges = (result \ "results").as[List[JsObject]]
+ val edgesToStore = parser.toEdges(Json.toJson(edges), "delete")
+ val rets = graph.mutateEdges(edgesToStore, withWait = true)
+ Await.result(rets, Duration(20, TimeUnit.MINUTES))
+
+ /** expect noting */
+ result = getEdgesSync(query(0))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ /** insert should be ignored */
+ val edgesToStore2 = parser.toEdges(Json.toJson(edges), "insert")
+ val rets2 = graph.mutateEdges(edgesToStore2, withWait = true)
+ Await.result(rets2, Duration(20, TimeUnit.MINUTES))
+
+ result = getEdgesSync(query(0))
+ (result \ "results").as[List[JsValue]].size should be(0)
+ }
+
+
+ test("test weak consistency deleteAll") {
+ val deletedAt = 100
+ var result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
+ "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
+ println(json)
+ deleteAllSync(json)
+
+ result = getEdgesSync(query(11, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(12, "out"))
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ result = getEdgesSync(query(10, "out"))
+
+ // 10 -> out -> 20 should not be in result.
+ (result \ "results").as[List[JsValue]].size should be(1)
+ (result \\ "to").size should be(1)
+ (result \\ "to").head.as[String] should be("21")
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ println(result)
+ (result \ "results").as[List[JsValue]].size should be(0)
+
+ insertEdgesSync(bulkEdges(startTs = deletedAt + 1): _*)
+
+ result = getEdgesSync(query(20, "in", testTgtColumnName))
+ (result \ "results").as[List[JsValue]].size should be(3)
+ }
+
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+
+ override def initTestData(): Unit = {
+ super.initTestData()
+ insertEdgesSync(bulkEdges(): _*)
+ }
+
+ object WeakLabelDeleteHelper {
+
+ def bulkEdges(startTs: Int = 0) = Seq(
+ toEdge(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}"""),
+ toEdge(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}"""),
+ toEdge(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}"""),
+ toEdge(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""")
+ )
+
+ def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$columnName",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelNameWeak}",
+ "direction": "${direction}",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]]
+ }""")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/app/controllers/EdgeController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/EdgeController.scala b/s2rest_play/app/controllers/EdgeController.scala
index 8df4f9a..e72aff7 100644
--- a/s2rest_play/app/controllers/EdgeController.scala
+++ b/s2rest_play/app/controllers/EdgeController.scala
@@ -157,14 +157,7 @@ object EdgeController extends Controller {
def deleteAllInner(jsValue: JsValue, withWait: Boolean) = {
val deleteResults = Future.sequence(jsValue.as[Seq[JsValue]] map { json =>
-
- val labelName = (json \ "label").as[String]
- val labels = Label.findByName(labelName).map { l => Seq(l) }.getOrElse(Nil)
- val direction = (json \ "direction").asOpt[String].getOrElse("out")
-
- val ids = (json \ "ids").asOpt[List[JsValue]].getOrElse(Nil)
- val ts = (json \ "timestamp").asOpt[Long].getOrElse(System.currentTimeMillis())
- val vertices = requestParser.toVertices(labelName, direction, ids)
+ val (labels, direction, ids, ts, vertices) = requestParser.toDeleteParam(json)
/** logging for delete all request */
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
index 0122f28..3ae08e2 100644
--- a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
@@ -15,10 +15,6 @@ import scala.concurrent.duration._
* Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6..
*/
class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification {
- sequential
-
- import Helper._
-
init()
override def init() = {
@@ -48,7 +44,7 @@ class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with Play
// create edges
val bulkEdges: String = (0 until 500).map { i =>
- edge"${System.currentTimeMillis()} insert e 0 $i $testLabelNameWeak"($(weight=i))
+ Seq(System.currentTimeMillis(), "insert", "e", "0", i, testLabelNameWeak, Json.obj("weight" -> i)).mkString("\t")
}.mkString("\n")
val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/test/controllers/PostProcessSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/PostProcessSpec.scala b/s2rest_play/test/controllers/PostProcessSpec.scala
index 9573791..30ea307 100644
--- a/s2rest_play/test/controllers/PostProcessSpec.scala
+++ b/s2rest_play/test/controllers/PostProcessSpec.scala
@@ -8,7 +8,7 @@ import play.api.test.PlaySpecification
/**
* Created by hsleep on 2015. 11. 4..
*/
-class PostProcessSpec extends SpecCommon with PlaySpecification {
+class PostProcessSpec extends PlaySpecification {
"test order by json" >> {
val jsLs: Seq[Seq[JsValue]] = Seq(
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/test/controllers/SpecCommon.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/SpecCommon.scala b/s2rest_play/test/controllers/SpecCommon.scala
index 7752544..28c4c23 100644
--- a/s2rest_play/test/controllers/SpecCommon.scala
+++ b/s2rest_play/test/controllers/SpecCommon.scala
@@ -12,41 +12,8 @@ import scala.concurrent.duration._
import scala.util.Random
trait SpecCommon extends Specification {
- sequential
- object Helper {
-
- import org.json4s.native.Serialization
-
- type KV = Map[String, Any]
-
- import scala.language.dynamics
-
- def $aa[T](args: T*) = List($a(args: _ *))
-
- def $a[T](args: T*) = args.toList
-
- object $ extends Dynamic {
- def applyDynamicNamed(name: String)(args: (String, Any)*): Map[String, Any] = args.toMap
- }
-
- implicit class anyMapOps(map: Map[String, Any]) {
- def toJson: JsValue = {
- val js = Serialization.write(map)(org.json4s.DefaultFormats)
- Json.parse(js)
- }
- }
-
- implicit class S2Context(val sc: StringContext) {
- def edge(args: Any*)(implicit map: Map[String, Any] = Map.empty): String = {
- val parts = sc.s(args: _*).split("\\s")
- assert(parts.length == 6)
- (parts.toList :+ map.toJson.toString).mkString("\t")
- }
- }
-
- }
-
val curTime = System.currentTimeMillis
+
val t1 = curTime + 0
val t2 = curTime + 1
val t3 = curTime + 2
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala b/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala
index 04ccc79..f1d8754 100644
--- a/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala
+++ b/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala
@@ -1,345 +1,345 @@
-package controllers
-
-import java.util.concurrent.TimeUnit
-
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, Future}
-import scala.util.Random
-
-class StrongLabelDeleteSpec extends SpecCommon {
- init()
-// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
-
- def bulkEdges(startTs: Int = 0) = Seq(
- Seq(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""").mkString("\t")
- ).mkString("\n")
-
- def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
- labelName: String = testLabelName2, direction: String = "out") = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }],
- "steps": [
- [ {
- "label": "$labelName",
- "direction": "${direction}",
- "offset": 0,
- "limit": -1,
- "duplicate": "raw"
- }
- ]]
- }""")
-
- def getEdges(queryJson: JsValue): JsValue = {
-// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
-
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- def getDegree(jsValue: JsValue): Long = {
- ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
- }
-
-
- "strong label delete test" should {
- running(FakeApplication()) {
- // insert bulk and wait ..
- val edges = bulkEdges()
- println(edges)
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
- }
-
- "test strong consistency select" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- result = getEdges(query(10))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- true
- }
- }
-
- "test strong consistency duration. insert -> delete -> insert" in {
- running(FakeApplication()) {
- val ts0 = 1
- val ts1 = 2
- val ts2 = 3
-
- val edges = Seq(
- Seq(5, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t"),
- Seq(10, "delete", "edge", "-10", "-20", testLabelName2).mkString("\t"),
- Seq(20, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t")
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
-
- val result = getEdges(query(-10))
-
- println(result)
-
- true
- }
- }
-
- "test strong consistency deleteAll" in {
- running(FakeApplication()) {
-
- val deletedAt = 100
- var result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- val json = Json.arr(Json.obj("label" -> testLabelName2,
- "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
- println(json)
- contentAsString(EdgeController.deleteAllInner(json, withWait = true))
-
- result = getEdges(query(11, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(12, direction = "out"))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(10, direction = "out"))
- println(result)
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- (result \\ "to").size must equalTo(1)
- (result \\ "to").head.as[String] must equalTo("21")
-
- result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
-
- result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- true
-
- }
- }
- }
-
-
- "labelargeSet of contention" should {
- val labelName = testLabelName2
- val maxTgtId = 10
- val batchSize = 10
- val testNum = 3
- val numOfBatch = 10
-
- def testInner(startTs: Long, src: Long) = {
- val labelName = testLabelName2
- val lastOps = Array.fill(maxTgtId)("none")
- var currentTs = startTs
-
- val allRequests = for {
- ith <- (0 until numOfBatch)
- jth <- (0 until batchSize)
- } yield {
- currentTs += 1
-
- val tgt = Random.nextInt(maxTgtId)
- val op = if (Random.nextDouble() < 0.5) "delete" else "update"
-
- lastOps(tgt) = op
- Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
- }
-
-
- allRequests.foreach(println(_))
-// println(lastOps.count(op => op != "delete" && op != "none"))
-// println(lastOps)
-//
-// Thread.sleep(1000)
-
- val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequest =>
- val bulkEdge = bulkRequest.mkString("\n")
- EdgeController.mutateAndPublish(bulkEdge, withWait = true)
- }
-
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
- val queryJson = query(id = src)
- val result = getEdges(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- println(lastOps.toList)
- println(result)
-
- val ret = resultDegree == expectedDegree && resultSize == resultDegree
- if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
-
- (ret, currentTs)
- }
-
- "update delete" in {
- running(FakeApplication()) {
- val ret = for {
- i <- (0 until testNum)
- } yield {
- val src = System.currentTimeMillis()
-
- val (ret, last) = testInner(i, src)
- ret must beEqualTo(true)
- ret
- }
-
- ret.forall(identity)
- }
- }
-
- "update delete 2" in {
- running(FakeApplication()) {
-
- val src = System.currentTimeMillis()
- var ts = 0L
-
- val ret = for {
- i <- (0 until testNum)
- } yield {
- val (ret, lastTs) = testInner(ts, src)
- val deletedAt = lastTs + 1
- val deletedAt2 = lastTs + 2
- ts = deletedAt2 + 1 // nex start ts
-
- ret must beEqualTo(true)
-
-
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
- val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
-
- val deleteRet = EdgeController.deleteAllInner(deleteAllRequest, withWait = true)
- val deleteRet2 = EdgeController.deleteAllInner(deleteAllRequest2, withWait = true)
-
-
- Await.result(deleteRet, Duration(20, TimeUnit.MINUTES))
- Await.result(deleteRet2, Duration(20, TimeUnit.MINUTES))
-
- val result = getEdges(query(id = src))
- println(result)
-
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty must beEqualTo(true)
-
- val degreeAfterDeleteAll = getDegree(result)
- degreeAfterDeleteAll must beEqualTo(0)
- true
- }
-
- ret.forall(identity)
- }
- }
-
- /** this test stress out test on degree
- * when contention is low but number of adjacent edges are large */
- "large degrees" in {
- running(FakeApplication()) {
-
-
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- // val allRequests = insertRequests ++ deleteRequests
- val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
- EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
- }
- Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
-
- val expectedDegree = insertRequests.size - deleteRequests.size
- val queryJson = query(id = src)
- val result = getEdges(queryJson)
- val resultSize = (result \ "size").as[Long]
- val resultDegree = getDegree(result)
-
- // println(result)
-
- val ret = resultSize == expectedDegree && resultDegree == resultSize
- println(s"[MaxSize]: $maxSize")
- println(s"[DeleteSize]: $deleteSize")
- println(s"[ResultDegree]: $resultDegree")
- println(s"[ExpectedDegree]: $expectedDegree")
- println(s"[ResultSize]: $resultSize")
- ret must beEqualTo(true)
- }
- }
-
- "deleteAll" in {
- running(FakeApplication()) {
-
- val labelName = testLabelName2
- val dir = "out"
- val maxSize = 100
- val deleteSize = 10
- val numOfConcurrentBatch = 100
- val src = System.currentTimeMillis()
- val tgts = (0 until maxSize).map { ith => src + ith }
- val deleteTgts = Random.shuffle(tgts).take(deleteSize)
- val insertRequests = tgts.map { tgt =>
- Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
- Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
- }
- val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
- // val allRequests = insertRequests ++ deleteRequests
- val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
- EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
- }
- Await.result(Future.sequence(futures), Duration(10, TimeUnit.MINUTES))
-
- val deletedAt = System.currentTimeMillis()
- val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
-
- Await.result(EdgeController.deleteAllInner(deleteAllRequest, withWait = true), Duration(10, TimeUnit.MINUTES))
-
- val result = getEdges(query(id = src))
- println(result)
- val resultEdges = (result \ "results").as[Seq[JsValue]]
- resultEdges.isEmpty must beEqualTo(true)
-
- val degreeAfterDeleteAll = getDegree(result)
- degreeAfterDeleteAll must beEqualTo(0)
- }
- }
- }
-}
-
+//package controllers
+//
+//import java.util.concurrent.TimeUnit
+//
+//
+//import com.kakao.s2graph.core.utils.logger
+//import play.api.libs.json._
+//import play.api.test.Helpers._
+//import play.api.test.{FakeApplication, FakeRequest}
+//
+//import scala.concurrent.duration.Duration
+//import scala.concurrent.{Await, Future}
+//import scala.util.Random
+//
+//class StrongLabelDeleteSpec extends SpecCommon {
+// init()
+//// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
+//
+// def bulkEdges(startTs: Int = 0) = Seq(
+// Seq(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}""").mkString("\t"),
+// Seq(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}""").mkString("\t"),
+// Seq(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}""").mkString("\t"),
+// Seq(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}""").mkString("\t"),
+// Seq(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""").mkString("\t")
+// ).mkString("\n")
+//
+// def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName,
+// labelName: String = testLabelName2, direction: String = "out") = Json.parse(
+// s"""
+// { "srcVertices": [
+// { "serviceName": "$serviceName",
+// "columnName": "$columnName",
+// "id": $id
+// }],
+// "steps": [
+// [ {
+// "label": "$labelName",
+// "direction": "${direction}",
+// "offset": 0,
+// "limit": -1,
+// "duplicate": "raw"
+// }
+// ]]
+// }""")
+//
+// def getEdges(queryJson: JsValue): JsValue = {
+//// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES))
+//
+// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
+// contentAsJson(ret)
+// }
+//
+// def getDegree(jsValue: JsValue): Long = {
+// ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L)
+// }
+//
+//
+// "strong label delete test" should {
+// running(FakeApplication()) {
+// // insert bulk and wait ..
+// val edges = bulkEdges()
+// println(edges)
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
+// }
+//
+// "test strong consistency select" in {
+// running(FakeApplication()) {
+// var result = getEdges(query(0))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+// result = getEdges(query(10))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+// true
+// }
+// }
+//
+// "test strong consistency duration. insert -> delete -> insert" in {
+// running(FakeApplication()) {
+// val ts0 = 1
+// val ts1 = 2
+// val ts2 = 3
+//
+// val edges = Seq(
+// Seq(5, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t"),
+// Seq(10, "delete", "edge", "-10", "-20", testLabelName2).mkString("\t"),
+// Seq(20, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t")
+// ).mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true))
+//
+// val result = getEdges(query(-10))
+//
+// println(result)
+//
+// true
+// }
+// }
+//
+// "test strong consistency deleteAll" in {
+// running(FakeApplication()) {
+//
+// val deletedAt = 100
+// var result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+//
+// val json = Json.arr(Json.obj("label" -> testLabelName2,
+// "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
+// println(json)
+// contentAsString(EdgeController.deleteAllInner(json, withWait = true))
+//
+// result = getEdges(query(11, direction = "out"))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// result = getEdges(query(12, direction = "out"))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// result = getEdges(query(10, direction = "out"))
+// println(result)
+// // 10 -> out -> 20 should not be in result.
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+// (result \\ "to").size must equalTo(1)
+// (result \\ "to").head.as[String] must equalTo("21")
+//
+// result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
+//
+// result = getEdges(query(20, direction = "in", columnName = testTgtColumnName))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+//
+// true
+//
+// }
+// }
+// }
+//
+//
+// "labelargeSet of contention" should {
+// val labelName = testLabelName2
+// val maxTgtId = 10
+// val batchSize = 10
+// val testNum = 3
+// val numOfBatch = 10
+//
+// def testInner(startTs: Long, src: Long) = {
+// val labelName = testLabelName2
+// val lastOps = Array.fill(maxTgtId)("none")
+// var currentTs = startTs
+//
+// val allRequests = for {
+// ith <- (0 until numOfBatch)
+// jth <- (0 until batchSize)
+// } yield {
+// currentTs += 1
+//
+// val tgt = Random.nextInt(maxTgtId)
+// val op = if (Random.nextDouble() < 0.5) "delete" else "update"
+//
+// lastOps(tgt) = op
+// Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t")
+// }
+//
+//
+// allRequests.foreach(println(_))
+//// println(lastOps.count(op => op != "delete" && op != "none"))
+//// println(lastOps)
+////
+//// Thread.sleep(1000)
+//
+// val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequest =>
+// val bulkEdge = bulkRequest.mkString("\n")
+// EdgeController.mutateAndPublish(bulkEdge, withWait = true)
+// }
+//
+// Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+//
+// val expectedDegree = lastOps.count(op => op != "delete" && op != "none")
+// val queryJson = query(id = src)
+// val result = getEdges(queryJson)
+// val resultSize = (result \ "size").as[Long]
+// val resultDegree = getDegree(result)
+//
+// println(lastOps.toList)
+// println(result)
+//
+// val ret = resultDegree == expectedDegree && resultSize == resultDegree
+// if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree")
+//
+// (ret, currentTs)
+// }
+//
+// "update delete" in {
+// running(FakeApplication()) {
+// val ret = for {
+// i <- (0 until testNum)
+// } yield {
+// val src = System.currentTimeMillis()
+//
+// val (ret, last) = testInner(i, src)
+// ret must beEqualTo(true)
+// ret
+// }
+//
+// ret.forall(identity)
+// }
+// }
+//
+// "update delete 2" in {
+// running(FakeApplication()) {
+//
+// val src = System.currentTimeMillis()
+// var ts = 0L
+//
+// val ret = for {
+// i <- (0 until testNum)
+// } yield {
+// val (ret, lastTs) = testInner(ts, src)
+// val deletedAt = lastTs + 1
+// val deletedAt2 = lastTs + 2
+// ts = deletedAt2 + 1 // nex start ts
+//
+// ret must beEqualTo(true)
+//
+//
+// val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+// val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2))
+//
+// val deleteRet = EdgeController.deleteAllInner(deleteAllRequest, withWait = true)
+// val deleteRet2 = EdgeController.deleteAllInner(deleteAllRequest2, withWait = true)
+//
+//
+// Await.result(deleteRet, Duration(20, TimeUnit.MINUTES))
+// Await.result(deleteRet2, Duration(20, TimeUnit.MINUTES))
+//
+// val result = getEdges(query(id = src))
+// println(result)
+//
+// val resultEdges = (result \ "results").as[Seq[JsValue]]
+// resultEdges.isEmpty must beEqualTo(true)
+//
+// val degreeAfterDeleteAll = getDegree(result)
+// degreeAfterDeleteAll must beEqualTo(0)
+// true
+// }
+//
+// ret.forall(identity)
+// }
+// }
+//
+// /** this test stress out test on degree
+// * when contention is low but number of adjacent edges are large */
+// "large degrees" in {
+// running(FakeApplication()) {
+//
+//
+// val labelName = testLabelName2
+// val dir = "out"
+// val maxSize = 100
+// val deleteSize = 10
+// val numOfConcurrentBatch = 100
+// val src = System.currentTimeMillis()
+// val tgts = (0 until maxSize).map { ith => src + ith }
+// val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+// val insertRequests = tgts.map { tgt =>
+// Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+// }
+// val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+// Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+// }
+// val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+// // val allRequests = insertRequests ++ deleteRequests
+// val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
+// EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
+// }
+// Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES))
+//
+// val expectedDegree = insertRequests.size - deleteRequests.size
+// val queryJson = query(id = src)
+// val result = getEdges(queryJson)
+// val resultSize = (result \ "size").as[Long]
+// val resultDegree = getDegree(result)
+//
+// // println(result)
+//
+// val ret = resultSize == expectedDegree && resultDegree == resultSize
+// println(s"[MaxSize]: $maxSize")
+// println(s"[DeleteSize]: $deleteSize")
+// println(s"[ResultDegree]: $resultDegree")
+// println(s"[ExpectedDegree]: $expectedDegree")
+// println(s"[ResultSize]: $resultSize")
+// ret must beEqualTo(true)
+// }
+// }
+//
+// "deleteAll" in {
+// running(FakeApplication()) {
+//
+// val labelName = testLabelName2
+// val dir = "out"
+// val maxSize = 100
+// val deleteSize = 10
+// val numOfConcurrentBatch = 100
+// val src = System.currentTimeMillis()
+// val tgts = (0 until maxSize).map { ith => src + ith }
+// val deleteTgts = Random.shuffle(tgts).take(deleteSize)
+// val insertRequests = tgts.map { tgt =>
+// Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+// }
+// val deleteRequests = deleteTgts.take(deleteSize).map { tgt =>
+// Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t")
+// }
+// val allRequests = Random.shuffle(insertRequests ++ deleteRequests)
+// // val allRequests = insertRequests ++ deleteRequests
+// val futures = allRequests.grouped(numOfConcurrentBatch).map { requests =>
+// EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true)
+// }
+// Await.result(Future.sequence(futures), Duration(10, TimeUnit.MINUTES))
+//
+// val deletedAt = System.currentTimeMillis()
+// val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt))
+//
+// Await.result(EdgeController.deleteAllInner(deleteAllRequest, withWait = true), Duration(10, TimeUnit.MINUTES))
+//
+// val result = getEdges(query(id = src))
+// println(result)
+// val resultEdges = (result \ "results").as[Seq[JsValue]]
+// resultEdges.isEmpty must beEqualTo(true)
+//
+// val degreeAfterDeleteAll = getDegree(result)
+// degreeAfterDeleteAll must beEqualTo(0)
+// }
+// }
+// }
+//}
+//
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/610d519d/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala b/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala
index 164c52b..1497b85 100644
--- a/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala
+++ b/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala
@@ -1,126 +1,126 @@
-package controllers
-
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-class WeakLabelDeleteSpec extends SpecCommon {
- init()
-
- def bulkEdges(startTs: Int = 0) = Seq(
- Seq(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
- Seq(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
- Seq(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
- Seq(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""").mkString("\t")
- ).mkString("\n")
-
- "weak label delete test" should {
- running(FakeApplication()) {
- // insert bulk and wait ..
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(), withWait = true))
- }
-
-
- def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
- s"""
- { "srcVertices": [
- { "serviceName": "$testServiceName",
- "columnName": "$columnName",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelNameWeak}",
- "direction": "${direction}",
- "offset": 0,
- "limit": 10,
- "duplicate": "raw"
- }
- ]]
- }""")
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- "test weak consistency select" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(4)
- result = getEdges(query(10))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- true
- }
- }
-
- "test weak consistency delete" in {
- running(FakeApplication()) {
- var result = getEdges(query(0))
- println(result)
-
- /** expect 4 edges */
- (result \ "results").as[List[JsValue]].size must equalTo(4)
- val edges = (result \ "results").as[List[JsObject]]
-
- contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "delete", withWait = true))
-
- /** expect noting */
- result = getEdges(query(0))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- /** insert should be ignored */
- contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "insert", withWait = true))
-
- result = getEdges(query(0))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
- }
- }
-
- "test weak consistency deleteAll" in {
- running(FakeApplication()) {
- val deletedAt = 100
- var result = getEdges(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
- "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
- println(json)
- contentAsString(EdgeController.deleteAllInner(json, withWait = true))
-
-
- result = getEdges(query(11, "out"))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(12, "out"))
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- result = getEdges(query(10, "out"))
-
- // 10 -> out -> 20 should not be in result.
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- (result \\ "to").size must equalTo(1)
- (result \\ "to").head.as[String] must equalTo("21")
-
- result = getEdges(query(20, "in", testTgtColumnName))
- println(result)
- (result \ "results").as[List[JsValue]].size must equalTo(0)
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
-
- result = getEdges(query(20, "in", testTgtColumnName))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
- }
- }
- }
-
-}
-
+//package controllers
+//
+//import play.api.libs.json._
+//import play.api.test.Helpers._
+//import play.api.test.{FakeApplication, FakeRequest}
+//
+//class WeakLabelDeleteSpec extends SpecCommon {
+// init()
+//
+// def bulkEdges(startTs: Int = 0) = Seq(
+// Seq(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
+// Seq(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
+// Seq(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"),
+// Seq(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"),
+// Seq(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"),
+// Seq(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""").mkString("\t")
+// ).mkString("\n")
+//
+// "weak label delete test" should {
+// running(FakeApplication()) {
+// // insert bulk and wait ..
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(), withWait = true))
+// }
+//
+//
+// def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse(
+// s"""
+// { "srcVertices": [
+// { "serviceName": "$testServiceName",
+// "columnName": "$columnName",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelNameWeak}",
+// "direction": "${direction}",
+// "offset": 0,
+// "limit": 10,
+// "duplicate": "raw"
+// }
+// ]]
+// }""")
+//
+// def getEdges(queryJson: JsValue): JsValue = {
+// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
+// contentAsJson(ret)
+// }
+//
+// "test weak consistency select" in {
+// running(FakeApplication()) {
+// var result = getEdges(query(0))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(4)
+// result = getEdges(query(10))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+// true
+// }
+// }
+//
+// "test weak consistency delete" in {
+// running(FakeApplication()) {
+// var result = getEdges(query(0))
+// println(result)
+//
+// /** expect 4 edges */
+// (result \ "results").as[List[JsValue]].size must equalTo(4)
+// val edges = (result \ "results").as[List[JsObject]]
+//
+// contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "delete", withWait = true))
+//
+// /** expect noting */
+// result = getEdges(query(0))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// /** insert should be ignored */
+// contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "insert", withWait = true))
+//
+// result = getEdges(query(0))
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+// }
+// }
+//
+// "test weak consistency deleteAll" in {
+// running(FakeApplication()) {
+// val deletedAt = 100
+// var result = getEdges(query(20, "in", testTgtColumnName))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+//
+// val json = Json.arr(Json.obj("label" -> testLabelNameWeak,
+// "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt))
+// println(json)
+// contentAsString(EdgeController.deleteAllInner(json, withWait = true))
+//
+//
+// result = getEdges(query(11, "out"))
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// result = getEdges(query(12, "out"))
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// result = getEdges(query(10, "out"))
+//
+// // 10 -> out -> 20 should not be in result.
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+// (result \\ "to").size must equalTo(1)
+// (result \\ "to").head.as[String] must equalTo("21")
+//
+// result = getEdges(query(20, "in", testTgtColumnName))
+// println(result)
+// (result \ "results").as[List[JsValue]].size must equalTo(0)
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true))
+//
+// result = getEdges(query(20, "in", testTgtColumnName))
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+// }
+// }
+// }
+//
+//}
+//
[12/46] incubator-s2graph git commit: add short circuit for IN clause
Posted by da...@apache.org.
add short circuit for IN clause
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d358931b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d358931b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d358931b
Branch: refs/heads/feature/test_daewon
Commit: d358931b9bef535dc46cc65e12123e4a7061a45c
Parents: abf356c
Author: SteamShon <sh...@gmail.com>
Authored: Tue Dec 22 17:32:42 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Tue Dec 22 17:32:42 2015 +0900
----------------------------------------------------------------------
.../com/kakao/s2graph/core/rest/RequestParser.scala | 2 +-
.../kakao/s2graph/core/parsers/WhereParserTest.scala | 12 ++++++------
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d358931b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index f9a6d01..2081a18 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -202,7 +202,7 @@ class RequestParser(config: Config) extends JSONParser {
val queryParams =
for {
labelGroup <- queryParamJsVals
- queryParam <- parseQueryParam(labelMap, labelGroup)
+ queryParam <- parseQueryParam(labelGroup)
} yield {
val (_, columnName) =
if (queryParam.labelWithDir.dir == GraphUtil.directions("out")) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d358931b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
index 73d0d76..7663496 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala
@@ -31,8 +31,8 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val labelMap = Map(label.label -> label)
- def validate(labelMap: Map[String, Label])(edge: Edge)(sql: String)(expected: Boolean) = {
- val whereOpt = WhereParser(labelMap).parse(sql)
+ def validate(label: Label)(edge: Edge)(sql: String)(expected: Boolean) = {
+ val whereOpt = WhereParser(label).parse(sql)
whereOpt.isSuccess shouldBe true
println("=================================================================")
@@ -57,7 +57,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val js = Json.obj("is_hidden" -> true, "is_blocked" -> false, "weight" -> 10, "time" -> 3, "name" -> "abc")
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
- val f = validate(labelMap)(edge) _
+ val f = validate(label)(edge) _
/** labelName label is long-long relation */
f(s"_to=${tgtVertex.innerId.toString}")(true)
@@ -77,7 +77,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val propsInner = Management.toProps(label, js.fields).map { case (k, v) => k -> InnerValLikeWithTs(v, ts) }.toMap + dummyTs
val edge = Edge(srcVertex, tgtVertex, labelWithDir, 0.toByte, ts, propsInner)
- val f = validate(labelMap)(edge) _
+ val f = validate(label)(edge) _
// time == 3
f("time >= 3")(true)
@@ -107,7 +107,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
val labelWithDirection = if (schemaVer == VERSION2) labelWithDirV2 else labelWithDir
val edge = Edge(srcVertex, tgtVertex, labelWithDirection, 0.toByte, ts, propsInner)
val lname = if (schemaVer == VERSION2) labelNameV2 else labelName
- val f = validate(labelMap)(edge) _
+ val f = validate(label)(edge) _
f(s"_from = -1 or _to = ${tgtVertex.innerId.value}")(true)
f(s"_from = ${srcVertex.innerId.value} and _to = ${tgtVertex.innerId.value}")(true)
@@ -139,7 +139,7 @@ class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels {
println(parentEdge.toString)
println(grandParentEdge.toString)
- val f = validate(labelMap)(edge) _
+ val f = validate(label)(edge) _
// Compare edge's prop(`_from`) with edge's prop(`name`)
f("_from = 1")(true)
[45/46] incubator-s2graph git commit: Merge branch 'develop' of
github.com:kakao/s2graph into develop
Posted by da...@apache.org.
Merge branch 'develop' of github.com:kakao/s2graph into develop
* 'develop' of github.com:kakao/s2graph:
configurable docker's hostname
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/8e13d4ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/8e13d4ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/8e13d4ea
Branch: refs/heads/feature/test_daewon
Commit: 8e13d4eacfb4b101b66f4243309a168481f03626
Parents: 16fa7d7 dfbda63
Author: daewon <bl...@gmail.com>
Authored: Thu Dec 31 14:12:23 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Thu Dec 31 14:12:23 2015 +0900
----------------------------------------------------------------------
dev_support/docker-compose.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[43/46] incubator-s2graph git commit: Add color
Posted by da...@apache.org.
Add color
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e2677a5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e2677a5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e2677a5d
Branch: refs/heads/feature/test_daewon
Commit: e2677a5dfe137beff9d30936fa05b86c20ac65a9
Parents: d9f6ee8
Author: daewon <bl...@gmail.com>
Authored: Thu Dec 31 10:25:40 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Thu Dec 31 10:25:40 2015 +0900
----------------------------------------------------------------------
s2core/src/main/resources/logback.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e2677a5d/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 9174452..5eb6737 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -3,7 +3,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- %d{ISO8601} [%-5level] [%cyan(%logger{0})] [%thread] - %msg%n
+ %d{ISO8601} [%highlight(%-5level)] [%magenta(%logger{0})] [%thread] - %msg%n
</pattern>
</encoder>
</appender>
[19/46] incubator-s2graph git commit: Merge branch 'develop' of
github.com:kakao/s2graph into develop
Posted by da...@apache.org.
Merge branch 'develop' of github.com:kakao/s2graph into develop
* 'develop' of github.com:kakao/s2graph:
fix scala version to 2.10.6 for s2counter_loader
add config 's2graph.read-only.url' for counter
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d7b48a69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d7b48a69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d7b48a69
Branch: refs/heads/feature/test_daewon
Commit: d7b48a69a8140b8085756d2441642e3ee6ed2a22
Parents: c6b607f e8befb6
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 23 14:46:45 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 23 14:46:45 2015 +0900
----------------------------------------------------------------------
.../main/scala/s2/config/S2CounterConfig.scala | 5 ++--
.../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++-----------------
.../counter/core/v2/RankingStorageGraph.scala | 5 ++--
s2counter_loader/build.sbt | 2 +-
.../main/scala/s2/config/StreamingConfig.scala | 1 +
.../src/test/resources/application.conf | 1 +
6 files changed, 12 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
[23/46] incubator-s2graph git commit: exclude tests(s2ml, s2counter*,
loader, spark) on root project
Posted by da...@apache.org.
exclude tests(s2ml, s2counter*, loader, spark) on root project
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f45b845f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f45b845f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f45b845f
Branch: refs/heads/feature/test_daewon
Commit: f45b845f11608fd2b0fa03afe869866c617b2815
Parents: 6a42db8
Author: daewon <bl...@gmail.com>
Authored: Thu Dec 24 16:19:56 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Thu Dec 24 16:20:23 2015 +0900
----------------------------------------------------------------------
build.sbt | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f45b845f/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 3e0dd41..ac86dbf 100755
--- a/build.sbt
+++ b/build.sbt
@@ -43,3 +43,12 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
.settings(commonSettings: _*)
lazy val s2ml = project.settings(commonSettings: _*)
+
+// test in s2core := {}
+// test in s2rest_play := {}
+
+test in spark := {}
+test in loader := {}
+test in s2counter_core := {}
+test in s2counter_loader := {}
+test in s2ml := {}
[16/46] incubator-s2graph git commit: handle json parse exception
Posted by da...@apache.org.
handle json parse exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f92596f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f92596f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f92596f4
Branch: refs/heads/feature/test_daewon
Commit: f92596f4c1a311fe2f8c45645a96ae067360bfba
Parents: e8c3588
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 23 12:15:08 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 23 12:15:08 2015 +0900
----------------------------------------------------------------------
.../counter/core/v2/RankingStorageGraph.scala | 23 +++++++++++++-------
1 file changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f92596f4/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
index 4517697..48ef1fb 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
@@ -286,14 +286,21 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
""".stripMargin
log.debug(strJs)
- val payload = Json.parse(strJs)
- wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
- resp.status match {
- case HttpStatus.SC_OK =>
- (resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
- case _ =>
- throw new RuntimeException(s"failed getEdges. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
- }
+ Try {
+ Json.parse(strJs)
+ } match {
+ case Success(payload) =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
+ resp.status match {
+ case HttpStatus.SC_OK =>
+ (resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
+ case _ =>
+ throw new RuntimeException(s"failed getEdges. errCode: ${resp.status}, body: ${resp.body}, query: $payload")
+ }
+ }
+ case Failure(ex) =>
+ log.error(s"$ex")
+ Future.successful(Nil)
}
}
[30/46] incubator-s2graph git commit: Make QuerySpec, CrudSpec works
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2rest_play/test/controllers/QuerySpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/QuerySpec.scala b/s2rest_play/test/controllers/QuerySpec.scala
index b55a327..04b6b85 100644
--- a/s2rest_play/test/controllers/QuerySpec.scala
+++ b/s2rest_play/test/controllers/QuerySpec.scala
@@ -1,693 +1,693 @@
-package controllers
-
-import com.kakao.s2graph.core.utils.logger
-import play.api.libs.json._
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
-import play.api.{Application => PlayApplication}
-
-import scala.concurrent.Await
-
-class QuerySpec extends SpecCommon with PlaySpecification {
-
- import Helper._
-
- implicit val app = FakeApplication()
-
- init()
-
- "query test" should {
- running(FakeApplication()) {
-
- // insert bulk and wait ..
- val bulkEdges: String = Seq(
- edge"1000 insert e 0 1 $testLabelName"($(weight = 40, is_hidden = true)),
- edge"2000 insert e 0 2 $testLabelName"($(weight = 30, is_hidden = false)),
- edge"3000 insert e 2 0 $testLabelName"($(weight = 20)),
- edge"4000 insert e 2 1 $testLabelName"($(weight = 10)),
- edge"3000 insert e 10 20 $testLabelName"($(weight = 20)),
- edge"4000 insert e 20 20 $testLabelName"($(weight = 10)),
- edge"1 insert e -1 1000 $testLabelName",
- edge"1 insert e -1 2000 $testLabelName",
- edge"1 insert e -1 3000 $testLabelName",
- edge"1 insert e 1000 10000 $testLabelName",
- edge"1 insert e 1000 11000 $testLabelName",
- edge"1 insert e 2000 11000 $testLabelName",
- edge"1 insert e 2000 12000 $testLabelName",
- edge"1 insert e 3000 12000 $testLabelName",
- edge"1 insert e 3000 13000 $testLabelName",
- edge"1 insert e 10000 100000 $testLabelName",
- edge"2 insert e 11000 200000 $testLabelName",
- edge"3 insert e 12000 300000 $testLabelName").mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
- }
-
- def queryParents(id: Long) = Json.parse( s"""
- {
- "returnTree": true,
- "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- }
- ],[{
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": -1
- }
- ]]
- }""".stripMargin)
-
- def queryExclude(id: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- },
- {
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": 2,
- "exclude": true
- }
- ]]
- }""")
-
- def queryTransform(id: Int, transforms: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "transform": $transforms
- }
- ]]
- }""")
-
- def queryWhere(id: Int, where: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "where": "${where}"
- }
- ]]
- }""")
-
- def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit,
- "_to": $to
- }
- ]]
- }
- """)
-
- def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit
- }
- ]]
- }
- """)
-
- def queryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- }
- ]
- }""")
-
-
- def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- },
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "sample": ${sample}
- }]
- }
- ]
- }""")
-
- def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- {
- "step": [{
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 50,
- "sample": ${sample}
- },
- {
- "label": "${testLabelName2}",
- "direction": "out",
- "offset": 0,
- "limit": 50
- }]
- }
- ]
- }""")
-
- def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id)))
-
- def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
- Json.obj(
- "groupBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- def queryScore(id: Int, scoring: Map[String, Int]): JsValue = {
- val q = Json.obj(
- "srcVertices" -> Json.arr(
- Json.obj(
- "serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "id" -> id
- )
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- println(q)
- q
- }
-
- def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = {
- Json.obj(
- "orderBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- }
-
- def queryGlobalLimit(id: Int, limit: Int): JsValue = {
- Json.obj(
- "limit" -> limit,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "index": "${index}",
- "interval": {
- "from": [
- {
- "${prop}": ${fromVal}
- }
- ],
- "to": [
- {
- "${prop}": ${toVal}
- }
- ]
- }
- }
- ]]
- }
- """)
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- def queryIndex(ids: Seq[Int], indexName: String) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(label = testLabelName, index = indexName))
- val $steps = $a($(step = $step))
-
- val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson
- js
- }
-
- def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(
- label = testLabelName, direction = "out", offset = 0, limit = 100,
- duration = $(from = from, to = to)))
-
- val $steps = $a($(step = $step))
-
- $(srcVertices = $from, steps = $steps).toJson
- }
-
- "interval" >> {
- running(FakeApplication()) {
- // insert test set
- var edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
- println(edges)
- (edges \ "size").toString must_== "1"
-
- edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
- println(edges)
- (edges \ "size").toString must_== "2"
-
- edges = getEdges(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
- println(edges)
- (edges \ "size").toString must_== "1"
-
- edges = getEdges(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
- println(edges)
- (edges \ "size").toString must_== "2"
- }
- }
-
- "union query" in {
- running(FakeApplication()) {
- var result = getEdges(queryUnion(0, 2))
- result.as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryUnion(0, 3))
- result.as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryUnion(0, 4))
- result.as[List[JsValue]].size must equalTo(4)
-
- result = getEdges(queryUnion(0, 5))
- result.as[List[JsValue]].size must equalTo(5)
-
- val union = result.as[List[JsValue]].head
- val single = getEdges(querySingle(0))
-
- (union \\ "from").map(_.toString).sorted must equalTo((single \\ "from").map(_.toString).sorted)
- (union \\ "to").map(_.toString).sorted must equalTo((single \\ "to").map(_.toString).sorted)
- (union \\ "weight").map(_.toString).sorted must equalTo((single \\ "weight").map(_.toString).sorted)
- }
- }
-
- "get edge with where condition" in {
- running(FakeApplication()) {
- var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)"))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- result = getEdges(queryWhere(0, "_from=0"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryWhere(2, "_from=2 or weight in (-1)"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
- }
- }
-
- "get edge exclude" in {
- running(FakeApplication()) {
- val result = getEdges(queryExclude(0))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
- }
- }
-
- "get edge groupBy property" in {
- running(FakeApplication()) {
- val result = getEdges(queryGroupBy(0, Seq("weight")))
- (result \ "size").as[Int] must_== 2
- val weights = (result \\ "groupBy").map { js =>
- (js \ "weight").as[Int]
- }
- weights must contain(exactly(30, 40))
- weights must not contain (10)
- }
- }
-
- "edge transform " in {
- running(FakeApplication()) {
- var result = getEdges(queryTransform(0, "[[\"_to\"]]"))
- (result \ "results").as[List[JsValue]].size must equalTo(2)
-
- result = getEdges(queryTransform(0, "[[\"weight\"]]"))
- (result \\ "to").map(_.toString).sorted must equalTo((result \\ "weight").map(_.toString).sorted)
-
- result = getEdges(queryTransform(0, "[[\"_from\"]]"))
- val results = (result \ "results").as[JsValue]
- (result \\ "to").map(_.toString).sorted must equalTo((results \\ "from").map(_.toString).sorted)
- }
- }
-
- "index" in {
- running(FakeApplication()) {
- // weight order
- var result = getEdges(queryIndex(Seq(0), "idx_1"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(40))
-
- // timestamp order
- result = getEdges(queryIndex(Seq(0), "idx_2"))
- ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(30))
- }
- }
-
- "checkEdges" in {
- running(FakeApplication()) {
- val json = Json.parse( s"""
- [{"from": 0, "to": 1, "label": "$testLabelName"},
- {"from": 0, "to": 2, "label": "$testLabelName"}]
- """)
-
- def checkEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- val res = checkEdges(json)
- val typeRes = res.isInstanceOf[JsArray]
- typeRes must equalTo(true)
-
- val fst = res.as[Seq[JsValue]].head \ "to"
- fst.as[Int] must equalTo(1)
-
- val snd = res.as[Seq[JsValue]].last \ "to"
- snd.as[Int] must equalTo(2)
- }
- }
-
- "duration" in {
- running(FakeApplication()) {
- // get all
- var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size must equalTo(4)
-
- // inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
- // duration test after udpate
- // get all
- result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
- (result \ "results").as[List[JsValue]].size must equalTo(4)
-
- // inclusive, exclusive
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
- (result \ "results").as[List[JsValue]].size must equalTo(3)
-
- result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
- (result \ "results").as[List[JsValue]].size must equalTo(1)
-
- result = getEdges(queryDuration(Seq(0, 2), from = 3000, to = 2000))
- (result \ "message").as[String] must contain("Duration error. Timestamp of From cannot be larger than To")
- }
- }
-
- "returnTree" in {
- running(FakeApplication()) {
- val src = 100
- val tgt = 200
- val labelName = testLabelName
-
- val bulkEdges: String = Seq(
- edge"1001 insert e $src $tgt $labelName"
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- val result = getEdges(queryParents(src))
-
- val parents = (result \ "results").as[Seq[JsValue]]
- val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 }
- ret must equalTo(true)
- }
- }
-
- "pagination and _to" in {
- running(FakeApplication()) {
- val src = System.currentTimeMillis().toInt
- val labelName = testLabelName
- val bulkEdges: String = Seq(
- edge"1001 insert e $src 1 $labelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e $src 2 $labelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e $src 3 $labelName"($(weight = 30)),
- edge"4004 insert e $src 4 $labelName"($(weight = 40))
- ).mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- var result = getEdges(querySingle(src, offset = 0, limit = 2))
- println(result)
- var edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(2)
- (edges(0) \ "to").as[Long] must beEqualTo(4)
- (edges(1) \ "to").as[Long] must beEqualTo(3)
-
- result = getEdges(querySingle(src, offset = 1, limit = 2))
- println(result)
- edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(2)
- (edges(0) \ "to").as[Long] must beEqualTo(3)
- (edges(1) \ "to").as[Long] must beEqualTo(2)
-
- result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
- println(result)
- edges = (result \ "results").as[List[JsValue]]
- edges.size must equalTo(1)
- }
- }
-
- "orderBy" >> {
- running(FakeApplication()) {
- // insert test set
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
- contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- // get edges
- val edges = getEdges(queryScore(0, Map("weight" -> 1)))
- val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
- val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
-
- println(edges)
- println(orderByScore)
- println(ascOrderByScore)
-
- val edgesTo = edges \ "results" \\ "to"
- val orderByTo = orderByScore \ "results" \\ "to"
- val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
-
- edgesTo must_== Seq(JsNumber(2), JsNumber(1))
- edgesTo must_== orderByTo
- ascOrderByTo must_== Seq(JsNumber(1), JsNumber(2))
- edgesTo.reverse must_== ascOrderByTo
- }
- }
-
- "query with sampling" >> {
- running(FakeApplication()) {
- val sampleSize = 2
- val testId = 22
- val bulkEdges = Seq(
- edge"1442985659166 insert e $testId 122 $testLabelName",
- edge"1442985659166 insert e $testId 222 $testLabelName",
- edge"1442985659166 insert e $testId 322 $testLabelName",
-
- edge"1442985659166 insert e $testId 922 $testLabelName2",
- edge"1442985659166 insert e $testId 222 $testLabelName2",
- edge"1442985659166 insert e $testId 322 $testLabelName2",
-
- edge"1442985659166 insert e 122 1122 $testLabelName",
- edge"1442985659166 insert e 122 1222 $testLabelName",
- edge"1442985659166 insert e 122 1322 $testLabelName",
- edge"1442985659166 insert e 222 2122 $testLabelName",
- edge"1442985659166 insert e 222 2222 $testLabelName",
- edge"1442985659166 insert e 222 2322 $testLabelName",
- edge"1442985659166 insert e 322 3122 $testLabelName",
- edge"1442985659166 insert e 322 3222 $testLabelName",
- edge"1442985659166 insert e 322 3322 $testLabelName"
- )
-
- contentAsJson(EdgeController.mutateAndPublish(bulkEdges.mkString("\n"), withWait = true))
-
- val result1 = getEdges(queryWithSampling(testId, sampleSize))
- println(Json.toJson(result1))
- (result1 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize, bulkEdges.size))
-
- val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result2))
- (result2 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
-
- val result3 = getEdges(twoQueryWithSampling(testId, sampleSize))
- println(Json.toJson(result3))
- (result3 \ "results").as[List[JsValue]].size must equalTo(sampleSize + 3) // edges in testLabelName2 = 3
- }
- }
-
- "limit" >> {
- running(FakeApplication()) {
- // insert test set
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
- contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- val edges = getEdges(querySingle(0, limit=1))
- val limitEdges = getEdges(queryGlobalLimit(0, 1))
-
- println(edges)
- println(limitEdges)
-
- val edgesTo = edges \ "results" \\ "to"
- val limitEdgesTo = limitEdges \ "results" \\ "to"
-
- edgesTo must_== limitEdgesTo
- }
- }
- }
-}
+//package controllers
+//
+//import com.kakao.s2graph.core.utils.logger
+//import play.api.libs.json._
+//import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
+//import play.api.{Application => PlayApplication}
+//
+//import scala.concurrent.Await
+//
+//class QuerySpec extends SpecCommon with PlaySpecification {
+//
+// import Helper._
+//
+// implicit val app = FakeApplication()
+//
+// init()
+//
+// "query test" should {
+// running(FakeApplication()) {
+//
+// // insert bulk and wait ..
+// val bulkEdges: String = Seq(
+// edge"1000 insert e 0 1 $testLabelName"($(weight = 40, is_hidden = true)),
+// edge"2000 insert e 0 2 $testLabelName"($(weight = 30, is_hidden = false)),
+// edge"3000 insert e 2 0 $testLabelName"($(weight = 20)),
+// edge"4000 insert e 2 1 $testLabelName"($(weight = 10)),
+// edge"3000 insert e 10 20 $testLabelName"($(weight = 20)),
+// edge"4000 insert e 20 20 $testLabelName"($(weight = 10)),
+// edge"1 insert e -1 1000 $testLabelName",
+// edge"1 insert e -1 2000 $testLabelName",
+// edge"1 insert e -1 3000 $testLabelName",
+// edge"1 insert e 1000 10000 $testLabelName",
+// edge"1 insert e 1000 11000 $testLabelName",
+// edge"1 insert e 2000 11000 $testLabelName",
+// edge"1 insert e 2000 12000 $testLabelName",
+// edge"1 insert e 3000 12000 $testLabelName",
+// edge"1 insert e 3000 13000 $testLabelName",
+// edge"1 insert e 10000 100000 $testLabelName",
+// edge"2 insert e 11000 200000 $testLabelName",
+// edge"3 insert e 12000 300000 $testLabelName").mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+// }
+//
+// def queryParents(id: Long) = Json.parse( s"""
+// {
+// "returnTree": true,
+// "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 2
+// }
+// ],[{
+// "label": "${testLabelName}",
+// "direction": "in",
+// "offset": 0,
+// "limit": -1
+// }
+// ]]
+// }""".stripMargin)
+//
+// def queryExclude(id: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 2
+// },
+// {
+// "label": "${testLabelName}",
+// "direction": "in",
+// "offset": 0,
+// "limit": 2,
+// "exclude": true
+// }
+// ]]
+// }""")
+//
+// def queryTransform(id: Int, transforms: String) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "transform": $transforms
+// }
+// ]]
+// }""")
+//
+// def queryWhere(id: Int, where: String) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 100,
+// "where": "${where}"
+// }
+// ]]
+// }""")
+//
+// def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": $offset,
+// "limit": $limit,
+// "_to": $to
+// }
+// ]]
+// }
+// """)
+//
+// def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": $offset,
+// "limit": $limit
+// }
+// ]]
+// }
+// """)
+//
+// def queryWithSampling(id: Int, sample: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// {
+// "step": [{
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 100,
+// "sample": ${sample}
+// }]
+// }
+// ]
+// }""")
+//
+//
+// def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// {
+// "step": [{
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 100,
+// "sample": ${sample}
+// }]
+// },
+// {
+// "step": [{
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 100,
+// "sample": ${sample}
+// }]
+// }
+// ]
+// }""")
+//
+// def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// {
+// "step": [{
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 50,
+// "sample": ${sample}
+// },
+// {
+// "label": "${testLabelName2}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 50
+// }]
+// }
+// ]
+// }""")
+//
+// def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id)))
+//
+// def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
+// Json.obj(
+// "groupBy" -> props,
+// "srcVertices" -> Json.arr(
+// Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+// ),
+// "steps" -> Json.arr(
+// Json.obj(
+// "step" -> Json.arr(
+// Json.obj(
+// "label" -> testLabelName
+// )
+// )
+// )
+// )
+// )
+// }
+//
+// def queryScore(id: Int, scoring: Map[String, Int]): JsValue = {
+// val q = Json.obj(
+// "srcVertices" -> Json.arr(
+// Json.obj(
+// "serviceName" -> testServiceName,
+// "columnName" -> testColumnName,
+// "id" -> id
+// )
+// ),
+// "steps" -> Json.arr(
+// Json.obj(
+// "step" -> Json.arr(
+// Json.obj(
+// "label" -> testLabelName,
+// "scoring" -> scoring
+// )
+// )
+// )
+// )
+// )
+// println(q)
+// q
+// }
+//
+// def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = {
+// Json.obj(
+// "orderBy" -> props,
+// "srcVertices" -> Json.arr(
+// Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+// ),
+// "steps" -> Json.arr(
+// Json.obj(
+// "step" -> Json.arr(
+// Json.obj(
+// "label" -> testLabelName,
+// "scoring" -> scoring
+// )
+// )
+// )
+// )
+// )
+// }
+//
+// def queryGlobalLimit(id: Int, limit: Int): JsValue = {
+// Json.obj(
+// "limit" -> limit,
+// "srcVertices" -> Json.arr(
+// Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+// ),
+// "steps" -> Json.arr(
+// Json.obj(
+// "step" -> Json.arr(
+// Json.obj(
+// "label" -> testLabelName
+// )
+// )
+// )
+// )
+// )
+// }
+//
+// def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse( s"""
+// { "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "index": "${index}",
+// "interval": {
+// "from": [
+// {
+// "${prop}": ${fromVal}
+// }
+// ],
+// "to": [
+// {
+// "${prop}": ${toVal}
+// }
+// ]
+// }
+// }
+// ]]
+// }
+// """)
+//
+// def getEdges(queryJson: JsValue): JsValue = {
+// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
+// contentAsJson(ret)
+// }
+//
+// def queryIndex(ids: Seq[Int], indexName: String) = {
+// val $from = $a(
+// $(serviceName = testServiceName,
+// columnName = testColumnName,
+// ids = ids))
+//
+// val $step = $a($(label = testLabelName, index = indexName))
+// val $steps = $a($(step = $step))
+//
+// val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson
+// js
+// }
+//
+// def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
+// val $from = $a(
+// $(serviceName = testServiceName,
+// columnName = testColumnName,
+// ids = ids))
+//
+// val $step = $a($(
+// label = testLabelName, direction = "out", offset = 0, limit = 100,
+// duration = $(from = from, to = to)))
+//
+// val $steps = $a($(step = $step))
+//
+// $(srcVertices = $from, steps = $steps).toJson
+// }
+//
+// "interval" >> {
+// running(FakeApplication()) {
+// // insert test set
+// var edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
+// println(edges)
+// (edges \ "size").toString must_== "1"
+//
+// edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
+// println(edges)
+// (edges \ "size").toString must_== "2"
+//
+// edges = getEdges(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
+// println(edges)
+// (edges \ "size").toString must_== "1"
+//
+// edges = getEdges(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
+// println(edges)
+// (edges \ "size").toString must_== "2"
+// }
+// }
+//
+// "union query" in {
+// running(FakeApplication()) {
+// var result = getEdges(queryUnion(0, 2))
+// result.as[List[JsValue]].size must equalTo(2)
+//
+// result = getEdges(queryUnion(0, 3))
+// result.as[List[JsValue]].size must equalTo(3)
+//
+// result = getEdges(queryUnion(0, 4))
+// result.as[List[JsValue]].size must equalTo(4)
+//
+// result = getEdges(queryUnion(0, 5))
+// result.as[List[JsValue]].size must equalTo(5)
+//
+// val union = result.as[List[JsValue]].head
+// val single = getEdges(querySingle(0))
+//
+// (union \\ "from").map(_.toString).sorted must equalTo((single \\ "from").map(_.toString).sorted)
+// (union \\ "to").map(_.toString).sorted must equalTo((single \\ "to").map(_.toString).sorted)
+// (union \\ "weight").map(_.toString).sorted must equalTo((single \\ "weight").map(_.toString).sorted)
+// }
+// }
+//
+// "get edge with where condition" in {
+// running(FakeApplication()) {
+// var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+//
+// result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)"))
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+//
+// result = getEdges(queryWhere(0, "_from=0"))
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+//
+// result = getEdges(queryWhere(2, "_from=2 or weight in (-1)"))
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+//
+// result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)"))
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+// }
+// }
+//
+// "get edge exclude" in {
+// running(FakeApplication()) {
+// val result = getEdges(queryExclude(0))
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+// }
+// }
+//
+// "get edge groupBy property" in {
+// running(FakeApplication()) {
+// val result = getEdges(queryGroupBy(0, Seq("weight")))
+// (result \ "size").as[Int] must_== 2
+// val weights = (result \\ "groupBy").map { js =>
+// (js \ "weight").as[Int]
+// }
+// weights must contain(exactly(30, 40))
+// weights must not contain (10)
+// }
+// }
+//
+// "edge transform " in {
+// running(FakeApplication()) {
+// var result = getEdges(queryTransform(0, "[[\"_to\"]]"))
+// (result \ "results").as[List[JsValue]].size must equalTo(2)
+//
+// result = getEdges(queryTransform(0, "[[\"weight\"]]"))
+// (result \\ "to").map(_.toString).sorted must equalTo((result \\ "weight").map(_.toString).sorted)
+//
+// result = getEdges(queryTransform(0, "[[\"_from\"]]"))
+// val results = (result \ "results").as[JsValue]
+// (result \\ "to").map(_.toString).sorted must equalTo((results \\ "from").map(_.toString).sorted)
+// }
+// }
+//
+// "index" in {
+// running(FakeApplication()) {
+// // weight order
+// var result = getEdges(queryIndex(Seq(0), "idx_1"))
+// ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(40))
+//
+// // timestamp order
+// result = getEdges(queryIndex(Seq(0), "idx_2"))
+// ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(30))
+// }
+// }
+//
+// "checkEdges" in {
+// running(FakeApplication()) {
+// val json = Json.parse( s"""
+// [{"from": 0, "to": 1, "label": "$testLabelName"},
+// {"from": 0, "to": 2, "label": "$testLabelName"}]
+// """)
+//
+// def checkEdges(queryJson: JsValue): JsValue = {
+// val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
+// contentAsJson(ret)
+// }
+//
+// val res = checkEdges(json)
+// val typeRes = res.isInstanceOf[JsArray]
+// typeRes must equalTo(true)
+//
+// val fst = res.as[Seq[JsValue]].head \ "to"
+// fst.as[Int] must equalTo(1)
+//
+// val snd = res.as[Seq[JsValue]].last \ "to"
+// snd.as[Int] must equalTo(2)
+// }
+// }
+//
+// "duration" in {
+// running(FakeApplication()) {
+// // get all
+// var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+// (result \ "results").as[List[JsValue]].size must equalTo(4)
+//
+// // inclusive, exclusive
+// result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+//
+// result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+//
+// val bulkEdges: String = Seq(
+// edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
+// edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
+// edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
+// edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
+// ).mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+// // duration test after udpate
+// // get all
+// result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+// (result \ "results").as[List[JsValue]].size must equalTo(4)
+//
+// // inclusive, exclusive
+// result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+// (result \ "results").as[List[JsValue]].size must equalTo(3)
+//
+// result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+// (result \ "results").as[List[JsValue]].size must equalTo(1)
+//
+// result = getEdges(queryDuration(Seq(0, 2), from = 3000, to = 2000))
+// (result \ "message").as[String] must contain("Duration error. Timestamp of From cannot be larger than To")
+// }
+// }
+//
+// "returnTree" in {
+// running(FakeApplication()) {
+// val src = 100
+// val tgt = 200
+// val labelName = testLabelName
+//
+// val bulkEdges: String = Seq(
+// edge"1001 insert e $src $tgt $labelName"
+// ).mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+//
+// val result = getEdges(queryParents(src))
+//
+// val parents = (result \ "results").as[Seq[JsValue]]
+// val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 }
+// ret must equalTo(true)
+// }
+// }
+//
+// "pagination and _to" in {
+// running(FakeApplication()) {
+// val src = System.currentTimeMillis().toInt
+// val labelName = testLabelName
+// val bulkEdges: String = Seq(
+// edge"1001 insert e $src 1 $labelName"($(weight = 10, is_hidden = true)),
+// edge"2002 insert e $src 2 $labelName"($(weight = 20, is_hidden = false)),
+// edge"3003 insert e $src 3 $labelName"($(weight = 30)),
+// edge"4004 insert e $src 4 $labelName"($(weight = 40))
+// ).mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+//
+// var result = getEdges(querySingle(src, offset = 0, limit = 2))
+// println(result)
+// var edges = (result \ "results").as[List[JsValue]]
+// edges.size must equalTo(2)
+// (edges(0) \ "to").as[Long] must beEqualTo(4)
+// (edges(1) \ "to").as[Long] must beEqualTo(3)
+//
+// result = getEdges(querySingle(src, offset = 1, limit = 2))
+// println(result)
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size must equalTo(2)
+// (edges(0) \ "to").as[Long] must beEqualTo(3)
+// (edges(1) \ "to").as[Long] must beEqualTo(2)
+//
+// result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+// println(result)
+// edges = (result \ "results").as[List[JsValue]]
+// edges.size must equalTo(1)
+// }
+// }
+//
+//// "orderBy" >> {
+//// running(FakeApplication()) {
+//// // insert test set
+//// val bulkEdges: String = Seq(
+//// edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
+//// edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
+//// edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
+//// edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
+//// ).mkString("\n")
+//// contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+////
+//// // get edges
+//// val edges = getEdges(queryScore(0, Map("weight" -> 1)))
+//// val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
+//// val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
+////
+//// println(edges)
+//// println(orderByScore)
+//// println(ascOrderByScore)
+////
+//// val edgesTo = edges \ "results" \\ "to"
+//// val orderByTo = orderByScore \ "results" \\ "to"
+//// val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
+////
+//// edgesTo must_== Seq(JsNumber(2), JsNumber(1))
+//// edgesTo must_== orderByTo
+//// ascOrderByTo must_== Seq(JsNumber(1), JsNumber(2))
+//// edgesTo.reverse must_== ascOrderByTo
+//// }
+//// }
+//
+//// "query with sampling" >> {
+//// running(FakeApplication()) {
+//// val sampleSize = 2
+//// val testId = 22
+//// val bulkEdges = Seq(
+//// edge"1442985659166 insert e $testId 122 $testLabelName",
+//// edge"1442985659166 insert e $testId 222 $testLabelName",
+//// edge"1442985659166 insert e $testId 322 $testLabelName",
+////
+//// edge"1442985659166 insert e $testId 922 $testLabelName2",
+//// edge"1442985659166 insert e $testId 222 $testLabelName2",
+//// edge"1442985659166 insert e $testId 322 $testLabelName2",
+////
+//// edge"1442985659166 insert e 122 1122 $testLabelName",
+//// edge"1442985659166 insert e 122 1222 $testLabelName",
+//// edge"1442985659166 insert e 122 1322 $testLabelName",
+//// edge"1442985659166 insert e 222 2122 $testLabelName",
+//// edge"1442985659166 insert e 222 2222 $testLabelName",
+//// edge"1442985659166 insert e 222 2322 $testLabelName",
+//// edge"1442985659166 insert e 322 3122 $testLabelName",
+//// edge"1442985659166 insert e 322 3222 $testLabelName",
+//// edge"1442985659166 insert e 322 3322 $testLabelName"
+//// )
+////
+//// contentAsJson(EdgeController.mutateAndPublish(bulkEdges.mkString("\n"), withWait = true))
+////
+//// val result1 = getEdges(queryWithSampling(testId, sampleSize))
+//// println(Json.toJson(result1))
+//// (result1 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize, bulkEdges.size))
+////
+//// val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize))
+//// println(Json.toJson(result2))
+//// (result2 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
+////
+//// val result3 = getEdges(twoQueryWithSampling(testId, sampleSize))
+//// println(Json.toJson(result3))
+//// (result3 \ "results").as[List[JsValue]].size must equalTo(sampleSize + 3) // edges in testLabelName2 = 3
+//// }
+//// }
+//
+//// "limit" >> {
+//// running(FakeApplication()) {
+//// // insert test set
+//// val bulkEdges: String = Seq(
+//// edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
+//// edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
+//// edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
+//// edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
+//// ).mkString("\n")
+//// contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+////
+//// val edges = getEdges(querySingle(0, limit=1))
+//// val limitEdges = getEdges(queryGlobalLimit(0, 1))
+////
+//// println(edges)
+//// println(limitEdges)
+////
+//// val edgesTo = edges \ "results" \\ "to"
+//// val limitEdgesTo = limitEdges \ "results" \\ "to"
+////
+//// edgesTo must_== limitEdgesTo
+//// }
+//// }
+// }
+//}
[07/46] incubator-s2graph git commit: extract method
Posted by da...@apache.org.
extract method
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6216412e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6216412e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6216412e
Branch: refs/heads/feature/test_daewon
Commit: 6216412e9121f894ea12162c93bb26accc375a1d
Parents: e8ae5dc
Author: daewon <bl...@gmail.com>
Authored: Mon Dec 21 15:00:07 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Dec 21 15:00:07 2015 +0900
----------------------------------------------------------------------
.../main/scala/com/kakao/s2graph/core/PostProcess.scala | 4 +++-
s2rest_netty/src/main/scala/Server.scala | 10 +++++-----
s2rest_play/app/controllers/ApplicationController.scala | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6216412e/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
index d39c87f..c3f3359 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
@@ -11,9 +11,11 @@ object PostProcess extends JSONParser {
/**
* Result Entity score field name
*/
- val SCORE_FIELD_NAME = "scoreSum"
val timeoutResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isTimeout" -> true)
val emptyResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isEmpty" -> true)
+ def badRequestResults(ex: => Exception) = Json.obj("message" -> ex.getMessage)
+
+ val SCORE_FIELD_NAME = "scoreSum"
val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
def groupEdgeResult(queryRequestWithResultLs: Seq[QueryRequestWithResult], exclude: Seq[QueryRequestWithResult]) = {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6216412e/s2rest_netty/src/main/scala/Server.scala
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala
index 3f1e70a..10d1275 100644
--- a/s2rest_netty/src/main/scala/Server.scala
+++ b/s2rest_netty/src/main/scala/Server.scala
@@ -59,7 +59,7 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S
}
}
- def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, jsonQuery: JsValue, future: Future[(JsValue, String)], startedAt: Long) = {
+ def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, body: JsValue, future: Future[(JsValue, String)], startedAt: Long) = {
val defaultHeaders = List(Names.CONTENT_TYPE -> ApplicationJson)
// NOTE: logging size of result should move to s2core.
@@ -81,7 +81,7 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S
if (impId.isEmpty) headerWithKeepAlive
else (Experiment.impressionKey, impId) :: headerWithKeepAlive
- val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(resJson)} ${jsonQuery}"
+ val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(resJson)} ${body}"
logger.info(log)
val responseHeaders = (Names.CONTENT_LENGTH -> buf.readableBytes().toString) :: headerWithImpKey
@@ -89,11 +89,11 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S
case Failure(ex) => ex match {
case e: BadQueryException =>
- logger.error(s"{$jsonQuery}, ${e.getMessage}", e)
- val buf: ByteBuf = Unpooled.copiedBuffer(Json.obj("message" -> e.getMessage).toString, CharsetUtil.UTF_8)
+ logger.error(s"{$body}, ${e.getMessage}", e)
+ val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8)
simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders)
case e: Exception =>
- logger.error(s"${jsonQuery}, ${e.getMessage}", e)
+ logger.error(s"${body}, ${e.getMessage}", e)
val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8)
simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6216412e/s2rest_play/app/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/controllers/ApplicationController.scala b/s2rest_play/app/controllers/ApplicationController.scala
index 484d282..7c30a5f 100644
--- a/s2rest_play/app/controllers/ApplicationController.scala
+++ b/s2rest_play/app/controllers/ApplicationController.scala
@@ -18,7 +18,7 @@ object ApplicationController extends Controller {
val jsonParser: BodyParser[JsValue] = controllers.s2parse.json
private def badQueryExceptionResults(ex: Exception) =
- Future.successful(BadRequest(Json.obj("message" -> ex.getMessage)).as(applicationJsonHeader))
+ Future.successful(BadRequest(PostProcess.badRequestResults(ex)).as(applicationJsonHeader))
private def errorResults =
Future.successful(Ok(PostProcess.emptyResults).as(applicationJsonHeader))
[20/46] incubator-s2graph git commit: fix compile error
Posted by da...@apache.org.
fix compile error
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b1dc979e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b1dc979e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b1dc979e
Branch: refs/heads/feature/test_daewon
Commit: b1dc979e4623e6fefe7d095eb0e53ada9e130c1f
Parents: f92596f
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 23 14:55:15 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 23 14:55:15 2015 +0900
----------------------------------------------------------------------
s2counter_loader/build.sbt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b1dc979e/s2counter_loader/build.sbt
----------------------------------------------------------------------
diff --git a/s2counter_loader/build.sbt b/s2counter_loader/build.sbt
index d80c36c..a79bc0b 100644
--- a/s2counter_loader/build.sbt
+++ b/s2counter_loader/build.sbt
@@ -20,7 +20,7 @@ libraryDependencies ++= Seq(
"com.google.guava" % "guava" % "16.0.1"
)
-scalaVersion := "2.10.6"
+crossScalaVersions := Seq("2.10.6")
fork := true
[10/46] incubator-s2graph git commit: performance tunning for IN
clause on where parser
Posted by da...@apache.org.
performance tunning for IN clause on where parser
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e9748524
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e9748524
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e9748524
Branch: refs/heads/feature/test_daewon
Commit: e974852433ebedb2dcd9e0e6a6da1317f359ae6d
Parents: a6acec7
Author: SteamShon <sh...@gmail.com>
Authored: Tue Dec 22 17:28:47 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Tue Dec 22 17:28:47 2015 +0900
----------------------------------------------------------------------
.../s2graph/core/parsers/WhereParser.scala | 30 ++++++++++++++++----
.../kakao/s2graph/core/rest/RequestParser.scala | 8 +++---
2 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e9748524/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
index 15996ea..2a62b3b 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/parsers/WhereParser.scala
@@ -98,15 +98,28 @@ case class Eq(propKey: String, value: String) extends Clause {
override def filter(edge: Edge): Boolean = binaryOp(_ == _)(propKey, value)(edge)
}
+case class InWithoutParent(label: Label, propKey: String, values: Set[String]) extends Clause {
+ val innerValLikeLs = values.map { value =>
+ val labelMeta = label.metaPropsInvMap.getOrElse(propKey, throw WhereParserException(s"Where clause contains not existing property name: $propKey"))
+ val dataType = propKey match {
+ case "_to" | "to" => label.tgtColumn.columnType
+ case "_from" | "from" => label.srcColumn.columnType
+ case _ => labelMeta.dataType
+ }
+ toInnerVal(value, dataType, label.schemaVersion)
+ }
+ override def filter(edge: Edge): Boolean = {
+ val propVal = propToInnerVal(edge, propKey)
+ innerValLikeLs.contains(propVal)
+ }
+}
+
case class IN(propKey: String, values: Set[String]) extends Clause {
override def filter(edge: Edge): Boolean = {
val propVal = propToInnerVal(edge, propKey)
values.exists { value =>
valueToCompare(edge, propKey, value) == propVal
}
-// val valuesToCompare = values.map { value => valueToCompare(edge, propKey, value) }
-//
-// valuesToCompare.contains(propVal)
}
}
@@ -136,7 +149,7 @@ object WhereParser {
val success = Where()
}
-case class WhereParser(labelMap: Map[String, Label]) extends JavaTokenParsers with JSONParser {
+case class WhereParser(label: Label) extends JavaTokenParsers with JSONParser {
val anyStr = "[^\\s(),]+".r
@@ -174,8 +187,13 @@ case class WhereParser(labelMap: Map[String, Label]) extends JavaTokenParsers wi
case f ~ minV ~ maxV => Between(f, minV, maxV)
} | identWithDot ~ (notIn | in) ~ ("(" ~> repsep(anyStr, ",") <~ ")") ^^ {
case f ~ op ~ values =>
- if (op.toLowerCase == "in") IN(f, values.toSet)
- else Not(IN(f, values.toSet))
+ val inClause =
+ if (f.startsWith("_parent")) IN(f, values.toSet)
+ else InWithoutParent(label, f, values.toSet)
+ if (op.toLowerCase == "in") inClause
+ else Not(inClause)
+
+
case _ => throw WhereParserException(s"Failed to parse where clause. ")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e9748524/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index 96d8f60..f9a6d01 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -92,11 +92,11 @@ class RequestParser(config: Config) extends JSONParser {
ret.map(_.toMap).getOrElse(Map.empty[Byte, InnerValLike])
}
- def extractWhere(labelMap: Map[String, Label], whereClauseOpt: Option[String]) = {
+ def extractWhere(label: Label, whereClauseOpt: Option[String]) = {
whereClauseOpt match {
case None => Success(WhereParser.success)
case Some(where) =>
- WhereParser(labelMap).parse(where) match {
+ WhereParser(label).parse(where) match {
case s@Success(_) => s
case Failure(ex) => throw BadQueryException(ex.getMessage, ex)
}
@@ -251,7 +251,7 @@ class RequestParser(config: Config) extends JSONParser {
}
}
- private def parseQueryParam(labelMap: Map[String, Label], labelGroup: JsValue): Option[QueryParam] = {
+ private def parseQueryParam(labelGroup: JsValue): Option[QueryParam] = {
for {
labelName <- parse[Option[String]](labelGroup, "label")
} yield {
@@ -280,7 +280,7 @@ class RequestParser(config: Config) extends JSONParser {
case Some(indexName) => label.indexNameMap.get(indexName).map(_.seq).getOrElse(throw new RuntimeException("cannot find index"))
}
val whereClauseOpt = (labelGroup \ "where").asOpt[String]
- val where = extractWhere(labelMap, whereClauseOpt)
+ val where = extractWhere(label, whereClauseOpt)
val includeDegree = (labelGroup \ "includeDegree").asOpt[Boolean].getOrElse(true)
val rpcTimeout = (labelGroup \ "rpcTimeout").asOpt[Int].getOrElse(DefaultRpcTimeout)
val maxAttempt = (labelGroup \ "maxAttempt").asOpt[Int].getOrElse(DefaultMaxAttempt)
[35/46] incubator-s2graph git commit: Merge pull request #219 from
kakao/feature/move_test_from_rest_to_core
Posted by da...@apache.org.
Merge pull request #219 from kakao/feature/move_test_from_rest_to_core
Feature/move test from rest to core
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/67aafe35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/67aafe35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/67aafe35
Branch: refs/heads/feature/test_daewon
Commit: 67aafe3592fc97d27e4366c97a5f71d63efcb2ae
Parents: bd65d26 610d519
Author: Doyung Yoon <st...@apache.org>
Authored: Tue Dec 29 17:36:04 2015 +0900
Committer: Doyung Yoon <st...@apache.org>
Committed: Tue Dec 29 17:36:04 2015 +0900
----------------------------------------------------------------------
.../scala/com/kakao/s2graph/core/Graph.scala | 2 +-
.../kakao/s2graph/core/rest/RequestParser.scala | 21 +
.../scala/com/kakao/s2graph/core/EdgeTest.scala | 29 +
.../com/kakao/s2graph/core/GraphTest.scala | 8 -
.../kakao/s2graph/core/Integrate/CrudTest.scala | 227 +++
.../core/Integrate/IntegrateCommon.scala | 308 ++++
.../s2graph/core/Integrate/QueryTest.scala | 639 ++++++++
.../core/Integrate/StrongLabelDeleteTest.scala | 282 ++++
.../core/Integrate/WeakLabelDeleteTest.scala | 129 ++
.../com/kakao/s2graph/core/JsonParserTest.scala | 3 -
.../com/kakao/s2graph/core/QueryParamTest.scala | 3 -
.../com/kakao/s2graph/core/TestCommon.scala | 6 -
.../s2graph/core/TestCommonWithModels.scala | 15 +-
.../app/controllers/EdgeController.scala | 9 +-
.../benchmark/PostProcessBenchmarkSpec.scala | 6 +-
.../test/controllers/BasicCrudSpec.scala | 500 ++++---
.../test/controllers/PostProcessSpec.scala | 2 +-
s2rest_play/test/controllers/QuerySpec.scala | 1386 +++++++++---------
s2rest_play/test/controllers/SpecCommon.scala | 35 +-
.../controllers/StrongLabelDeleteSpec.scala | 690 ++++-----
.../test/controllers/WeakLabelDeleteSpec.scala | 252 ++--
21 files changed, 3064 insertions(+), 1488 deletions(-)
----------------------------------------------------------------------
[27/46] incubator-s2graph git commit: Merge pull request #218 from
kakao/feature/fix_s2core_test_error
Posted by da...@apache.org.
Merge pull request #218 from kakao/feature/fix_s2core_test_error
Feature/fix s2core test error
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b6fe2969
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b6fe2969
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b6fe2969
Branch: refs/heads/feature/test_daewon
Commit: b6fe29695104b4f07ad4bf59035dbc9ec7fe5bed
Parents: 3a3dc2f f249c3e
Author: Doyung Yoon <st...@apache.org>
Authored: Mon Dec 28 12:09:34 2015 +0900
Committer: Doyung Yoon <st...@apache.org>
Committed: Mon Dec 28 12:09:34 2015 +0900
----------------------------------------------------------------------
dev_support/graph_mysql/schema.sql | 5 +-
.../s2graph/core/TestCommonWithModels.scala | 81 +++++++++++---------
.../kakao/s2graph/core/models/ModelTest.scala | 14 ++--
.../s2graph/core/mysqls/ExperimentSpec.scala | 36 +--------
.../hbase/AsynchbaseQueryBuilderTest.scala | 12 +--
5 files changed, 64 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
[05/46] incubator-s2graph git commit: fix scala version to 2.10.6 for
s2counter_loader
Posted by da...@apache.org.
fix scala version to 2.10.6 for s2counter_loader
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/dea96e68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/dea96e68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/dea96e68
Branch: refs/heads/feature/test_daewon
Commit: dea96e68f91b9862e69b60ce51e0e21ceef89eb7
Parents: b159a7f
Author: Jaesang Kim <ho...@gmail.com>
Authored: Mon Dec 21 14:31:08 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Mon Dec 21 14:31:08 2015 +0900
----------------------------------------------------------------------
s2counter_loader/build.sbt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dea96e68/s2counter_loader/build.sbt
----------------------------------------------------------------------
diff --git a/s2counter_loader/build.sbt b/s2counter_loader/build.sbt
index a79bc0b..d80c36c 100644
--- a/s2counter_loader/build.sbt
+++ b/s2counter_loader/build.sbt
@@ -20,7 +20,7 @@ libraryDependencies ++= Seq(
"com.google.guava" % "guava" % "16.0.1"
)
-crossScalaVersions := Seq("2.10.6")
+scalaVersion := "2.10.6"
fork := true
[21/46] incubator-s2graph git commit: Change response format on
BadRequest
Posted by da...@apache.org.
Change response format on BadRequest
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/77b5afd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/77b5afd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/77b5afd7
Branch: refs/heads/feature/test_daewon
Commit: 77b5afd755771a7c09f969fb014b21f1211fe415
Parents: d7b48a6
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 23 15:31:37 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 23 15:31:37 2015 +0900
----------------------------------------------------------------------
s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala | 6 +++++-
.../main/scala/com/kakao/s2graph/core/rest/RestCaller.scala | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/77b5afd7/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
index c3f3359..83e3a71 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
@@ -1,5 +1,6 @@
package com.kakao.s2graph.core
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
import com.kakao.s2graph.core.mysqls.{ColumnMeta, Label, ServiceColumn, LabelMeta}
import com.kakao.s2graph.core.types.{InnerValLike, InnerVal}
import com.kakao.s2graph.core.utils.logger
@@ -13,7 +14,10 @@ object PostProcess extends JSONParser {
*/
val timeoutResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isTimeout" -> true)
val emptyResults = Json.obj("size" -> 0, "results" -> Json.arr(), "isEmpty" -> true)
- def badRequestResults(ex: => Exception) = Json.obj("message" -> ex.getMessage)
+ def badRequestResults(ex: => Exception) = ex match {
+ case ex: BadQueryException => Json.obj("message" -> ex.msg)
+ case _ => Json.obj("message" -> ex.getMessage)
+ }
val SCORE_FIELD_NAME = "scoreSum"
val reservedColumns = Set("cacheRemain", "from", "to", "label", "direction", "_timestamp", "timestamp", "score", "props")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/77b5afd7/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
index 3278ece..bef1dec 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala
@@ -155,7 +155,7 @@ class RestCaller(graph: Graph)(implicit ec: ExecutionContext) {
Try(Json.parse(body)).recover {
case e: Exception =>
- throw new RuntimeException(s"wrong or missing template parameter: ${e.getMessage}")
+ throw new BadQueryException(s"wrong or missing template parameter: ${e.getMessage.takeWhile(_ != '\n')}")
} get
}
[14/46] incubator-s2graph git commit: Merge pull request #215 from
hsleep/feature/counter_config_for_readonly_graph
Posted by da...@apache.org.
Merge pull request #215 from hsleep/feature/counter_config_for_readonly_graph
add counter config for readonly graph
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e8befb6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e8befb6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e8befb6d
Branch: refs/heads/feature/test_daewon
Commit: e8befb6d3addf13cf9e1ae9061cc35d5bd920caa
Parents: d358931 dea96e6
Author: elric-kang <mi...@gmail.com>
Authored: Wed Dec 23 11:30:34 2015 +0900
Committer: elric-kang <mi...@gmail.com>
Committed: Wed Dec 23 11:30:34 2015 +0900
----------------------------------------------------------------------
.../main/scala/s2/config/S2CounterConfig.scala | 5 ++--
.../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++-----------------
.../counter/core/v2/RankingStorageGraph.scala | 5 ++--
s2counter_loader/build.sbt | 2 +-
.../main/scala/s2/config/StreamingConfig.scala | 1 +
.../src/test/resources/application.conf | 1 +
6 files changed, 12 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
[38/46] incubator-s2graph git commit: Merge pull request #224 from
hsleep/feature/fix_docker_configuration
Posted by da...@apache.org.
Merge pull request #224 from hsleep/feature/fix_docker_configuration
Fix docker-compose configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/8f86bc3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/8f86bc3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/8f86bc3c
Branch: refs/heads/feature/test_daewon
Commit: 8f86bc3c0b1e4966ad0ff2ed920715aceca7ba53
Parents: 67aafe3 a685bd0
Author: Junki Kim <wi...@gmail.com>
Authored: Wed Dec 30 11:52:25 2015 +0900
Committer: Junki Kim <wi...@gmail.com>
Committed: Wed Dec 30 11:52:25 2015 +0900
----------------------------------------------------------------------
dev_support/docker-compose.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[22/46] incubator-s2graph git commit: Merge pull request #216 from
hsleep/feature/handle_json_parsing_error
Posted by da...@apache.org.
Merge pull request #216 from hsleep/feature/handle_json_parsing_error
handle json parsing error
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6a42db83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6a42db83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6a42db83
Branch: refs/heads/feature/test_daewon
Commit: 6a42db833b5e472ee2337da850b88fff4bba65a7
Parents: 77b5afd b1dc979
Author: Doyung Yoon <sh...@gmail.com>
Authored: Thu Dec 24 15:10:18 2015 +0900
Committer: Doyung Yoon <sh...@gmail.com>
Committed: Thu Dec 24 15:10:18 2015 +0900
----------------------------------------------------------------------
.../counter/core/v2/RankingStorageGraph.scala | 63 +++++++++++++++++---
s2counter_loader/build.sbt | 2 +-
.../s2/counter/core/CounterFunctions.scala | 12 ++--
3 files changed, 64 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[28/46] incubator-s2graph git commit: Merge pull request #211 from
HyunsungJo/issue/readme_update_for_refactoring
Posted by da...@apache.org.
Merge pull request #211 from HyunsungJo/issue/readme_update_for_refactoring
README update for refactoring. This resolves S2GRAPH-2.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f75b3b3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f75b3b3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f75b3b3d
Branch: refs/heads/feature/test_daewon
Commit: f75b3b3dc665c2a1ef0e1daf7acb5f658a25e1de
Parents: b6fe296 767940b
Author: Doyung Yoon <st...@apache.org>
Authored: Mon Dec 28 12:09:56 2015 +0900
Committer: Doyung Yoon <st...@apache.org>
Committed: Mon Dec 28 12:09:56 2015 +0900
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[34/46] incubator-s2graph git commit: Merge pull request #221 from
kakao/feature/Issue#220
Posted by da...@apache.org.
Merge pull request #221 from kakao/feature/Issue#220
Issue#220. bug fix for isInnerCall on Query Builder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/bd65d26b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/bd65d26b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/bd65d26b
Branch: refs/heads/feature/test_daewon
Commit: bd65d26b122189916221f2f1a19e59685478b462
Parents: f75b3b3 ca589e6
Author: Doyung Yoon <st...@apache.org>
Authored: Tue Dec 29 17:33:39 2015 +0900
Committer: Doyung Yoon <st...@apache.org>
Committed: Tue Dec 29 17:33:39 2015 +0900
----------------------------------------------------------------------
.../kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilder.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[11/46] incubator-s2graph git commit: Merge branch 'hotfix/20151222'
of https://github.com/kakao/s2graph into hotfix/20151222
Posted by da...@apache.org.
Merge branch 'hotfix/20151222' of https://github.com/kakao/s2graph into hotfix/20151222
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/abf356cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/abf356cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/abf356cf
Branch: refs/heads/feature/test_daewon
Commit: abf356cfffa38d31efdbc9750622ec5c8baeac9a
Parents: e974852 57fae9d
Author: SteamShon <sh...@gmail.com>
Authored: Tue Dec 22 17:29:11 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Tue Dec 22 17:29:11 2015 +0900
----------------------------------------------------------------------
s2core/lib/netty-3.9.4.Final.jar | Bin 0 -> 1310154 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
[13/46] incubator-s2graph git commit: handle json parse exception
Posted by da...@apache.org.
handle json parse exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/e607fc9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/e607fc9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/e607fc9b
Branch: refs/heads/feature/test_daewon
Commit: e607fc9b704ae30d0040dabff62c28ae68150941
Parents: d358931
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 23 11:24:17 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 23 11:25:12 2015 +0900
----------------------------------------------------------------------
.../counter/core/v2/RankingStorageGraph.scala | 41 ++++++++++++++++++++
.../s2/counter/core/CounterFunctions.scala | 12 ++++--
2 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e607fc9b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
index b0c0a41..b12efbd 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
@@ -207,6 +207,46 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
private def getEdges(key: RankingKey, duplicate: String="first"): Future[List[JsValue]] = {
val labelName = counterModel.findById(key.policyId).get.action + labelPostfix
+// val ids = (0 until BUCKET_SHARD_COUNT).map { idx =>
+// s"${makeBucketShardKey(idx, key)}"
+// }
+//
+// val payload = Json.obj(
+// "srcVertices" -> Json.arr(
+// Json.obj(
+// "serviceName" -> SERVICE_NAME,
+// "columnName" -> BUCKET_COLUMN_NAME,
+// "ids" -> ids
+// )
+// ),
+// "steps" -> Json.arr(
+// Json.obj(
+// "step" -> Json.arr(
+// Json.obj(
+// "label" -> labelName,
+// "duplicate" -> duplicate,
+// "direction" -> "out",
+// "offset" -> 0,
+// "limit" -> -1,
+// "interval" -> Json.obj(
+// "from" -> Json.obj(
+// "time_unit" -> key.eq.tq.q.toString,
+// "time_value" -> key.eq.tq.ts
+// ),
+// "to" -> Json.obj(
+// "time_unit" -> key.eq.tq.q.toString,
+// "time_value" -> key.eq.tq.ts
+// ),
+// "scoring" -> Json.obj(
+// "score" -> 1
+// )
+// )
+// )
+// )
+// )
+// )
+// )
+
val ids = {
(0 until BUCKET_SHARD_COUNT).map { shardIdx =>
s""""${makeBucketShardKey(shardIdx, key)}""""
@@ -246,6 +286,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
log.debug(strJs)
val payload = Json.parse(strJs)
+
wsClient.url(s"$s2graphUrl/graphs/getEdges").post(payload).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/e607fc9b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
index 32e3d0c..cc2e54b 100644
--- a/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
+++ b/s2counter_loader/src/main/scala/s2/counter/core/CounterFunctions.scala
@@ -128,6 +128,12 @@ object CounterFunctions extends Logging with WithKafka {
itemRankingRdd.unpersist(false)
}
}
+
+ private def parseLine(line: String): Option[TrxLog] = Try {
+ val js = Json.parse(line)
+ js.toString()
+ js.as[TrxLog]
+ }.toOption
def makeTrxLogRdd(rdd: RDD[(String, String)], numPartitions: Int): RDD[TrxLog] = {
rdd.mapPartitions { part =>
@@ -135,10 +141,8 @@ object CounterFunctions extends Logging with WithKafka {
for {
(k, v) <- part
line <- GraphUtil.parseString(v)
- trxLog = Json.parse(line).as[TrxLog] if trxLog.success
- } yield {
- trxLog
- }
+ trxLog <- parseLine(line) if trxLog.success
+ } yield trxLog
}
}
[40/46] incubator-s2graph git commit: Merge branch
'feature/move_test_from_rest_to_core' into develop
Posted by da...@apache.org.
Merge branch 'feature/move_test_from_rest_to_core' into develop
* feature/move_test_from_rest_to_core:
Make logger works
Move VertexSpec to s2core, remove unused dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d9f6ee85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d9f6ee85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d9f6ee85
Branch: refs/heads/feature/test_daewon
Commit: d9f6ee8549abf417799a0df43188cb95ebf7442b
Parents: 8f86bc3 5a77433
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 30 11:55:36 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 30 11:55:36 2015 +0900
----------------------------------------------------------------------
build.sbt | 8 +-
s2core/src/main/resources/logback.xml | 33 +-
s2core/src/main/resources/reference.conf | 19 +-
.../core/storage/hbase/AsynchbaseStorage.scala | 10 +-
.../com/kakao/s2graph/core/utils/Logger.scala | 1 -
.../kakao/s2graph/core/Integrate/CrudTest.scala | 1 -
.../core/Integrate/IntegrateCommon.scala | 2 +-
.../core/Integrate/VertexTestHelper.scala | 71 ++
.../hbase/AsynchbaseQueryBuilderTest.scala | 3 +-
s2rest_play/build.sbt | 3 +-
.../test/benchmark/JsonBenchmarkSpec.scala | 14 +-
.../benchmark/OrderingUtilBenchmarkSpec.scala | 16 +-
.../benchmark/PostProcessBenchmarkSpec.scala | 367 +++++-----
.../test/benchmark/SamplingBenchmarkSpec.scala | 2 +-
.../test/controllers/AdminControllerSpec.scala | 30 +-
s2rest_play/test/controllers/SpecCommon.scala | 668 +++++++++----------
s2rest_play/test/controllers/VertexSpec.scala | 106 +--
17 files changed, 718 insertions(+), 636 deletions(-)
----------------------------------------------------------------------
[41/46] incubator-s2graph git commit: configurable docker's hostname
Posted by da...@apache.org.
configurable docker's hostname
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/616cc816
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/616cc816
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/616cc816
Branch: refs/heads/feature/test_daewon
Commit: 616cc8160a26067475483a061769403a82244a32
Parents: d9f6ee8
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 30 13:13:03 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 30 13:13:03 2015 +0900
----------------------------------------------------------------------
dev_support/docker-compose.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/616cc816/dev_support/docker-compose.yml
----------------------------------------------------------------------
diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml
index 3fa16b7..9818251 100644
--- a/dev_support/docker-compose.yml
+++ b/dev_support/docker-compose.yml
@@ -16,7 +16,7 @@ graph_mysql:
graph_hbase:
image: nerdammer/hbase:0.98.10.1
container_name: graph_hbase
- hostname: default
+ hostname: "${DOCKER_MACHINE_NAME}"
ports:
- "3306:3306"
- "2181:2181"
[31/46] incubator-s2graph git commit: Make QuerySpec, CrudSpec works
Posted by da...@apache.org.
Make QuerySpec, CrudSpec works
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/395b00f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/395b00f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/395b00f6
Branch: refs/heads/feature/test_daewon
Commit: 395b00f66b851678102f58bf218fcdb36924cc7a
Parents: 0e1c0e0
Author: daewon <bl...@gmail.com>
Authored: Tue Dec 29 14:54:22 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Tue Dec 29 14:54:22 2015 +0900
----------------------------------------------------------------------
.../scala/com/kakao/s2graph/core/EdgeTest.scala | 28 +
.../kakao/s2graph/core/Integrate/CrudTest.scala | 6 +-
.../core/Integrate/IntegrateCommon.scala | 17 +-
.../s2graph/core/Integrate/QueryTest.scala | 694 ++++++++-
.../test/controllers/BasicCrudSpec.scala | 498 +++----
s2rest_play/test/controllers/QuerySpec.scala | 1386 +++++++++---------
6 files changed, 1623 insertions(+), 1006 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
index 6374d43..d86541d 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/EdgeTest.scala
@@ -4,8 +4,36 @@ import com.kakao.s2graph.core.mysqls.LabelMeta
import com.kakao.s2graph.core.types.{InnerVal, InnerValLikeWithTs, VertexId}
import com.kakao.s2graph.core.utils.logger
import org.scalatest.FunSuite
+import org.scalatest.matchers.Matcher
class EdgeTest extends FunSuite with TestCommon with TestCommonWithModels {
+
+ test("toLogString") {
+ val testLabelName = labelNameV2
+ val bulkQueries = List(
+ ("1445240543366", "update", "{\"is_blocked\":true}"),
+ ("1445240543362", "insert", "{\"is_hidden\":false}"),
+ ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
+ ("1445240543363", "delete", "{}"),
+ ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
+
+ val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
+
+ val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
+ Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
+ }).mkString("\n")
+
+ val expected = Seq(
+ Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"),
+ Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"),
+ Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"),
+ Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"),
+ Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}")
+ ).map(_.mkString("\t")).mkString("\n")
+
+ assert(bulkEdge === expected)
+ }
+
test("buildOperation") {
val schemaVersion = "v2"
val vertexId = VertexId(0, InnerVal.withStr("dummy", schemaVersion))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
index a093dcc..d178419 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -169,10 +169,10 @@ class CrudTest extends IntegrateCommon {
/** insert edges */
println(s"---- TC${tcNum}_init ----")
val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
- TestHelper.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
+ Util.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
})
- TestHelper.insertEdges(bulkEdges:_*)
+ Util.insertEdges(bulkEdges:_*)
for {
label <- Label.findByName(labelName)
@@ -187,7 +187,7 @@ class CrudTest extends IntegrateCommon {
val qId = if (labelName == testLabelName) id else "\"" + id + "\""
val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
- val jsResult = TestHelper.getEdges(query)
+ val jsResult = Util.getEdges(query)
val results = jsResult \ "results"
val deegrees = (jsResult \ "degrees").as[List[JsObject]]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
index 2aa8abf..6db250d 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -15,7 +15,7 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
var parser: RequestParser = _
var config: Config = _
- override def beforeAll(): Unit = {
+ override def beforeAll = {
config = ConfigFactory.load()
graph = new Graph(config)(ExecutionContext.Implicits.global)
parser = new RequestParser(graph.config)
@@ -78,7 +78,18 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
/**
* Test Helpers
*/
- object TestHelper {
+ object Util {
+// def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
+// val arr = for {
+// (label, dir, from, to) <- params
+// } yield {
+// Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
+// }
+//
+// val s = Json.toJson(arr)
+// s
+// }
+
def getEdges(queryJson: JsValue): JsValue = {
val ret = graph.getEdges(parser.toQuery(queryJson))
val result = Await.result(ret, HttpRequestWaitingTime)
@@ -90,6 +101,8 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
def insertEdges(bulkEdges: String*) = {
val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
val jsResult = Await.result(req, HttpRequestWaitingTime)
+
+ jsResult
}
def toEdge(elems: Any*): String = elems.mkString("\t")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
index 53696b0..505927b 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
@@ -1,63 +1,639 @@
package com.kakao.s2graph.core.Integrate
-import com.kakao.s2graph.core.Management
-import com.kakao.s2graph.core.mysqls._
-import play.api.libs.json.{JsValue, Json}
-
-class QueryTest extends IntegrateCommon {
-
-// test("test Query") {
-//
-// }
-//
-// test("test returnTree") {
-// def queryParents(id: Long) = Json.parse(
-// s"""
-// {
-// "returnTree": true,
-// "srcVertices": [
-// { "serviceName": "${testServiceName}",
-// "columnName": "${testColumnName}",
-// "id": ${id}
-// }],
-// "steps": [
-// [ {
-// "label": "${testLabelName}",
-// "direction": "out",
-// "offset": 0,
-// "limit": 2
-// }
-// ],[{
-// "label": "${testLabelName}",
-// "direction": "in",
-// "offset": 0,
-// "limit": -1
-// }
-// ]]
-// }""".stripMargin)
-//
-// val src = 100
-// val tgt = 200
-// val labelName = testLabelName
-//
-// insertEdges(
-// toEdge(1001, "insert", "e", src, tgt, labelName)
-// )
-//
-// val result = getEdges(queryParents(src))
-// val parents = (result \ "results").as[Seq[JsValue]]
-// val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 }
-//
-// ret === true
-// }
-
- override def beforeAll(): Unit = {
- super.beforeAll()
-
-
- }
-
- override def afterAll(): Unit = {
- super.afterAll()
+import com.kakao.s2graph.core.GraphExceptions.BadQueryException
+import org.scalatest.BeforeAndAfterEach
+import play.api.libs.json.{JsNull, JsNumber, JsValue, Json}
+
+import scala.util.{Success, Try}
+
+class QueryTest extends IntegrateCommon with BeforeAndAfterEach {
+
+ import Util._
+
+ val insert = "insert"
+ val e = "e"
+ val weight = "weight"
+ val is_hidden = "is_hidden"
+
+ test("interval") {
+ def queryWithInterval(id: Int, index: String, prop: String, fromVal: Int, toVal: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "index": "$index",
+ "interval": {
+ "from": [ { "$prop": $fromVal } ],
+ "to": [ { "$prop": $toVal } ]
+ }
+ }
+ ]]
+ }
+ """)
+
+ var edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdges(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index
+ (edges \ "size").toString should be("2")
+
+ edges = getEdges(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index
+ (edges \ "size").toString should be("1")
+
+ edges = getEdges(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index
+ (edges \ "size").toString should be("2")
+ }
+
+ test("get edge with where condition") {
+ def queryWhere(id: Int, where: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "where": "${where}"
+ }
+ ]]
+ }""")
+
+ var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)"))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ result = getEdges(queryWhere(0, "_from=0"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdges(queryWhere(2, "_from=2 or weight in (-1)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+ }
+
+ test("get edge exclude") {
+ def queryExclude(id: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ },
+ {
+ "label": "${testLabelName}",
+ "direction": "in",
+ "offset": 0,
+ "limit": 2,
+ "exclude": true
+ }
+ ]]
+ }""")
+
+ val result = getEdges(queryExclude(0))
+ (result \ "results").as[List[JsValue]].size should be(1)
+ }
+
+ test("get edge groupBy property") {
+ def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
+ Json.obj(
+ "groupBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+ }
+
+ val result = getEdges(queryGroupBy(0, Seq("weight")))
+ (result \ "size").as[Int] should be(2)
+ val weights = (result \\ "groupBy").map { js =>
+ (js \ "weight").as[Int]
+ }
+ weights should contain(30)
+ weights should contain(40)
+
+ weights should not contain (10)
+ }
+
+ test("edge transform") {
+ def queryTransform(id: Int, transforms: String) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": 0,
+ "transform": $transforms
+ }
+ ]]
+ }""")
+
+ var result = getEdges(queryTransform(0, "[[\"_to\"]]"))
+ (result \ "results").as[List[JsValue]].size should be(2)
+
+ result = getEdges(queryTransform(0, "[[\"weight\"]]"))
+ (result \\ "to").map(_.toString).sorted should be((result \\ "weight").map(_.toString).sorted)
+
+ result = getEdges(queryTransform(0, "[[\"_from\"]]"))
+ val results = (result \ "results").as[JsValue]
+ (result \\ "to").map(_.toString).sorted should be((results \\ "from").map(_.toString).sorted)
+ }
+
+ test("index") {
+ def queryIndex(ids: Seq[Int], indexName: String) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj("label" -> testLabelName, "index" -> indexName))
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ val js = Json.obj("withScore" -> false, "srcVertices" -> $from, "steps" -> $steps)
+ js
+ }
+
+ // weight order
+ var result = getEdges(queryIndex(Seq(0), "idx_1"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(40))
+
+ // timestamp order
+ result = getEdges(queryIndex(Seq(0), "idx_2"))
+ ((result \ "results").as[List[JsValue]].head \\ "weight").head should be(JsNumber(30))
+ }
+
+ // "checkEdges" in {
+ // running(FakeApplication()) {
+ // val json = Json.parse( s"""
+ // [{"from": 0, "to": 1, "label": "$testLabelName"},
+ // {"from": 0, "to": 2, "label": "$testLabelName"}]
+ // """)
+ //
+ // def checkEdges(queryJson: JsValue): JsValue = {
+ // val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get
+ // contentAsJson(ret)
+ // }
+ //
+ // val res = checkEdges(json)
+ // val typeRes = res.isInstanceOf[JsArray]
+ // typeRes must equalTo(true)
+ //
+ // val fst = res.as[Seq[JsValue]].head \ "to"
+ // fst.as[Int] must equalTo(1)
+ //
+ // val snd = res.as[Seq[JsValue]].last \ "to"
+ // snd.as[Int] must equalTo(2)
+ // }
+ // }
+
+
+ test("duration") {
+ def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
+ val $from = Json.arr(
+ Json.obj("serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "ids" -> ids))
+
+ val $step = Json.arr(Json.obj(
+ "label" -> testLabelName, "direction" -> "out", "offset" -> 0, "limit" -> 100,
+ "duration" -> Json.obj("from" -> from, "to" -> to)))
+
+ val $steps = Json.arr(Json.obj("step" -> $step))
+
+ Json.obj("srcVertices" -> $from, "steps" -> $steps)
+ }
+
+ // get all
+ var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+ // inclusive, exclusive
+ result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+ insertEdges(bulkEdges: _*)
+
+ // duration test after udpate
+ // get all
+ result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000))
+ (result \ "results").as[List[JsValue]].size should be(4)
+
+ // inclusive, exclusive
+ result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000))
+ (result \ "results").as[List[JsValue]].size should be(3)
+
+ result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000))
+ (result \ "results").as[List[JsValue]].size should be(1)
+
+ def a: JsValue = getEdges(queryDuration(Seq(0, 2), from = 3000, to = 2000))
+ Try(a).recover {
+ case e: BadQueryException => JsNull
+ } should be(Success(JsNull))
+ }
+
+ test("return tree") {
+ def queryParents(id: Long) = Json.parse(
+ s"""
+ {
+ "returnTree": true,
+ "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 2
+ }
+ ],[{
+ "label": "$testLabelName",
+ "direction": "in",
+ "offset": 0,
+ "limit": -1
+ }
+ ]]
+ }""".stripMargin)
+
+ val src = 100
+ val tgt = 200
+
+ insertEdges(toEdge(1001, "insert", "e", src, tgt, testLabelName))
+
+ val result = Util.getEdges(queryParents(src))
+ val parents = (result \ "results").as[Seq[JsValue]]
+ val ret = parents.forall {
+ edge => (edge \ "parents").as[Seq[JsValue]].size == 1
+ }
+
+ ret should be(true)
+ }
+
+
+
+ test("pagination and _to") {
+ def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "${testServiceName}",
+ "columnName": "${testColumnName}",
+ "id": ${id}
+ }],
+ "steps": [
+ [ {
+ "label": "${testLabelName}",
+ "direction": "out",
+ "offset": $offset,
+ "limit": $limit,
+ "_to": $to
+ }
+ ]]
+ }
+ """)
+
+ val src = System.currentTimeMillis().toInt
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, src, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, src, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, src, 3, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, src, 4, testLabelName, Json.obj(weight -> 40))
+ )
+ insertEdges(bulkEdges: _*)
+
+ var result = getEdges(querySingle(src, offset = 0, limit = 2))
+ var edges = (result \ "results").as[List[JsValue]]
+
+ edges.size should be(2)
+ (edges(0) \ "to").as[Long] should be(4)
+ (edges(1) \ "to").as[Long] should be(3)
+
+ result = getEdges(querySingle(src, offset = 1, limit = 2))
+
+ edges = (result \ "results").as[List[JsValue]]
+ edges.size should be(2)
+ (edges(0) \ "to").as[Long] should be(3)
+ (edges(1) \ "to").as[Long] should be(2)
+
+ result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1))
+ edges = (result \ "results").as[List[JsValue]]
+ edges.size should be(1)
+ }
+ test("order by") {
+ def queryScore(id: Int, scoring: Map[String, Int]): JsValue = Json.obj(
+ "srcVertices" -> Json.arr(
+ Json.obj(
+ "serviceName" -> testServiceName,
+ "columnName" -> testColumnName,
+ "id" -> id
+ )
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+ def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = Json.obj(
+ "orderBy" -> props,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName,
+ "scoring" -> scoring
+ )
+ )
+ )
+ )
+ )
+
+ val bulkEdges = Seq(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40))
+ )
+
+ insertEdges(bulkEdges: _*)
+
+ // get edges
+ val edges = getEdges(queryScore(0, Map("weight" -> 1)))
+ val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC"))))
+ val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC"))))
+
+ val edgesTo = edges \ "results" \\ "to"
+ val orderByTo = orderByScore \ "results" \\ "to"
+ val ascOrderByTo = ascOrderByScore \ "results" \\ "to"
+
+ edgesTo should be(Seq(JsNumber(2), JsNumber(1)))
+ edgesTo should be(orderByTo)
+ ascOrderByTo should be(Seq(JsNumber(1), JsNumber(2)))
+ edgesTo.reverse should be(ascOrderByTo)
+ }
+
+ test("query with sampling") {
+ def queryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ },
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 100,
+ "sample": $sample
+ }]
+ }
+ ]
+ }""")
+
+ def twoQueryWithSampling(id: Int, sample: Int) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ {
+ "step": [{
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50,
+ "sample": $sample
+ },
+ {
+ "label": "$testLabelName2",
+ "direction": "out",
+ "offset": 0,
+ "limit": 50
+ }]
+ }
+ ]
+ }""")
+
+ val sampleSize = 2
+ val ts = "1442985659166"
+ val testId = 22
+
+ val bulkEdges = Seq(
+ toEdge(ts, insert, e, testId, 122, testLabelName),
+ toEdge(ts, insert, e, testId, 222, testLabelName),
+ toEdge(ts, insert, e, testId, 322, testLabelName),
+
+ toEdge(ts, insert, e, testId, 922, testLabelName2),
+ toEdge(ts, insert, e, testId, 222, testLabelName2),
+ toEdge(ts, insert, e, testId, 322, testLabelName2),
+
+ toEdge(ts, insert, e, 122, 1122, testLabelName),
+ toEdge(ts, insert, e, 122, 1222, testLabelName),
+ toEdge(ts, insert, e, 122, 1322, testLabelName),
+ toEdge(ts, insert, e, 222, 2122, testLabelName),
+ toEdge(ts, insert, e, 222, 2222, testLabelName),
+ toEdge(ts, insert, e, 222, 2322, testLabelName),
+ toEdge(ts, insert, e, 322, 3122, testLabelName),
+ toEdge(ts, insert, e, 322, 3222, testLabelName),
+ toEdge(ts, insert, e, 322, 3322, testLabelName)
+ )
+
+ insertEdges(bulkEdges: _*)
+
+ val result1 = getEdges(queryWithSampling(testId, sampleSize))
+ (result1 \ "results").as[List[JsValue]].size should be(math.min(sampleSize, bulkEdges.size))
+
+ val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize))
+ (result2 \ "results").as[List[JsValue]].size should be(math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size))
+
+ val result3 = getEdges(twoQueryWithSampling(testId, sampleSize))
+ (result3 \ "results").as[List[JsValue]].size should be(sampleSize + 3) // edges in testLabelName2 = 3
+ }
+
+ test("limit") {
+ insertEdges(
+ toEdge(1001, insert, e, 0, 1, testLabelName, Json.obj(weight -> 10, is_hidden -> true)),
+ toEdge(2002, insert, e, 0, 2, testLabelName, Json.obj(weight -> 20, is_hidden -> false)),
+ toEdge(3003, insert, e, 2, 0, testLabelName, Json.obj(weight -> 30)),
+ toEdge(4004, insert, e, 2, 1, testLabelName, Json.obj(weight -> 40)))
+
+ val edges = getEdges(querySingle(0, limit = 1))
+ val limitEdges = getEdges(queryGlobalLimit(0, 1))
+
+ val edgesTo = edges \ "results" \\ "to"
+ val limitEdgesTo = limitEdges \ "results" \\ "to"
+
+ edgesTo should be(limitEdgesTo)
+ }
+
+ // test("union query") {
+ // def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id)))
+ //
+ // var result = getEdges(queryUnion(0, 2))
+ // result.as[List[JsValue]].size should be (2)
+ //
+ // result = getEdges(queryUnion(0, 3))
+ // result.as[List[JsValue]].size should be (3)
+ //
+ // result = getEdges(queryUnion(0, 4))
+ // result.as[List[JsValue]].size should be (4)
+ //
+ // result = getEdges(queryUnion(0, 5))
+ // result.as[List[JsValue]].size should be (5)
+ //
+ // val union = result.as[List[JsValue]].head
+ // val single = getEdges(querySingle(0))
+ //
+ // (union \\ "from").map(_.toString).sorted should be ((single \\ "from").map(_.toString).sorted)
+ // (union \\ "to").map(_.toString).sorted should be ((single \\ "to").map(_.toString).sorted)
+ // (union \\ "weight").map(_.toString).sorted should be ((single \\ "weight").map(_.toString).sorted)
+ // }
+
+ def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse(
+ s"""
+ { "srcVertices": [
+ { "serviceName": "$testServiceName",
+ "columnName": "$testColumnName",
+ "id": $id
+ }],
+ "steps": [
+ [ {
+ "label": "$testLabelName",
+ "direction": "out",
+ "offset": $offset,
+ "limit": $limit
+ }
+ ]]
+ }
+ """)
+
+ def queryGlobalLimit(id: Int, limit: Int): JsValue = Json.obj(
+ "limit" -> limit,
+ "srcVertices" -> Json.arr(
+ Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
+ ),
+ "steps" -> Json.arr(
+ Json.obj(
+ "step" -> Json.arr(
+ Json.obj(
+ "label" -> testLabelName
+ )
+ )
+ )
+ )
+ )
+
+ // called by each test, each
+ override def beforeEach = initTestData()
+
+ // called by start test, once
+ override def initTestData(): Unit = {
+ super.initTestData()
+
+ insertEdges(
+ toEdge(1000, insert, e, 0, 1, testLabelName, Json.obj(weight -> 40, is_hidden -> true)),
+ toEdge(2000, insert, e, 0, 2, testLabelName, Json.obj(weight -> 30, is_hidden -> false)),
+ toEdge(3000, insert, e, 2, 0, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 2, 1, testLabelName, Json.obj(weight -> 10)),
+ toEdge(3000, insert, e, 10, 20, testLabelName, Json.obj(weight -> 20)),
+ toEdge(4000, insert, e, 20, 20, testLabelName, Json.obj(weight -> 10)),
+ toEdge(1, insert, e, -1, 1000, testLabelName),
+ toEdge(1, insert, e, -1, 2000, testLabelName),
+ toEdge(1, insert, e, -1, 3000, testLabelName),
+ toEdge(1, insert, e, 1000, 10000, testLabelName),
+ toEdge(1, insert, e, 1000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 11000, testLabelName),
+ toEdge(1, insert, e, 2000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 12000, testLabelName),
+ toEdge(1, insert, e, 3000, 13000, testLabelName),
+ toEdge(1, insert, e, 10000, 100000, testLabelName),
+ toEdge(2, insert, e, 11000, 200000, testLabelName),
+ toEdge(3, insert, e, 12000, 300000, testLabelName)
+ )
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/395b00f6/s2rest_play/test/controllers/BasicCrudSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/BasicCrudSpec.scala b/s2rest_play/test/controllers/BasicCrudSpec.scala
index 07419b3..9da2abd 100644
--- a/s2rest_play/test/controllers/BasicCrudSpec.scala
+++ b/s2rest_play/test/controllers/BasicCrudSpec.scala
@@ -1,249 +1,249 @@
-package controllers
-
-import com.kakao.s2graph.core.Management
-import com.kakao.s2graph.core.mysqls._
-
-//import com.kakao.s2graph.core.models._
-
-import play.api.libs.json._
-import play.api.test.Helpers._
-import play.api.test.{FakeApplication, FakeRequest}
-
-import scala.concurrent.Await
-
-
-class BasicCrudSpec extends SpecCommon {
- var seed = 0
- def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
- for {
- labelName <- List(testLabelName, testLabelName2)
- i <- 0 until NUM_OF_EACH_TEST
- } {
- seed += 1
-// val srcId = ((tcNum * 1000) + i).toString
-// val tgtId = if (labelName == testLabelName) s"${srcId + 1000 + i}" else s"${srcId + 1000 + i}abc"
- val srcId = seed.toString
- val tgtId = srcId
-
- val maxTs = opWithProps.map(t => t._1).max
-
- /** insert edges */
- println(s"---- TC${tcNum}_init ----")
- val bulkEdge = (for ((ts, op, props) <- opWithProps) yield {
- List(ts, op, "e", srcId, tgtId, labelName, props).mkString("\t")
- }).mkString("\n")
-
- val req = EdgeController.mutateAndPublish(bulkEdge, withWait = true)
- val res = Await.result(req, HTTP_REQ_WAITING_TIME)
-
- res.header.status must equalTo(200)
-
- println(s"---- TC${tcNum}_init ----")
-// Thread.sleep(100)
-
- for {
- label <- Label.findByName(labelName)
- direction <- List("out", "in")
- cacheTTL <- List(-1L)
- } {
- val (serviceName, columnName, id, otherId) = direction match {
- case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
- case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
- }
- val qId = if (labelName == testLabelName) id else "\"" + id + "\""
- val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(query)).get
- val jsResult = commonCheck(ret)
-
- val results = jsResult \ "results"
- val deegrees = (jsResult \ "degrees").as[List[JsObject]]
- val propsLs = (results \\ "props").seq
- (deegrees.head \ LabelMeta.degree.name).as[Int] must equalTo(1)
-
- val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
- val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
-
- from must equalTo(id.toString)
- to must equalTo(otherId.toString)
-// (results \\ "_timestamp").seq.last.as[Long] must equalTo(maxTs)
- for ((key, expectedVal) <- expected) {
- propsLs.last.as[JsObject].keys.contains(key) must equalTo(true)
- (propsLs.last \ key).toString must equalTo(expectedVal)
- }
- Await.result(ret, HTTP_REQ_WAITING_TIME)
- }
- }
- }
-
- init()
- "Basic Crud " should {
- "tc1" in {
- running(FakeApplication()) {
-
- var tcNum = 0
- var tcString = ""
- var bulkQueries = List.empty[(Long, String, String)]
- var expected = Map.empty[String, String]
-
- tcNum = 7
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 8
- tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
- bulkQueries = List(
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 9
- tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 10
- tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 11
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "insert", "{\"time\": 10}"),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 12
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
- (t1, "insert", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 13
- tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 14
- tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
- bulkQueries = List(
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 15
- tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
- bulkQueries = List(
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 16
- tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
- bulkQueries = List(
- (t2, "delete", ""),
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 17
- tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t2, "delete", ""),
- (t1, "update", "{\"time\": 10}"))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- tcNum = 18
- tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
- bulkQueries = List(
- (t3, "update", "{\"time\": 10, \"weight\": 20}"),
- (t1, "update", "{\"time\": 10}"),
- (t2, "delete", ""))
- expected = Map("time" -> "10", "weight" -> "20")
-
- runTC(tcNum, tcString, bulkQueries, expected)
-
- tcNum = 19
- tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
- bulkQueries = List(
- (t5, "update", "{\"is_blocked\": true}"),
- (t1, "insert", "{\"is_hidden\": false}"),
- (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
- (t2, "delete", ""),
- (t4, "update", "{\"time\": 1, \"weight\": -10}"))
- expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
-
- runTC(tcNum, tcString, bulkQueries, expected)
- true
- }
- }
- }
-
- "toLogString" in {
- running(FakeApplication()) {
- val bulkQueries = List(
- ("1445240543366", "update", "{\"is_blocked\":true}"),
- ("1445240543362", "insert", "{\"is_hidden\":false}"),
- ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
- ("1445240543363", "delete", "{}"),
- ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
-
- val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
-
- val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
- Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
- }).mkString("\n")
-
- val expected = Seq(
- Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"),
- Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"),
- Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"),
- Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"),
- Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}")
- ).map(_.mkString("\t")).mkString("\n")
-
- bulkEdge must equalTo(expected)
-
- true
- }
- }
-}
-
-
+//package controllers
+//
+//import com.kakao.s2graph.core.Management
+//import com.kakao.s2graph.core.mysqls._
+//
+////import com.kakao.s2graph.core.models._
+//
+//import play.api.libs.json._
+//import play.api.test.Helpers._
+//import play.api.test.{FakeApplication, FakeRequest}
+//
+//import scala.concurrent.Await
+//
+//
+//class BasicCrudSpec extends SpecCommon {
+// var seed = 0
+// def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
+// for {
+// labelName <- List(testLabelName, testLabelName2)
+// i <- 0 until NUM_OF_EACH_TEST
+// } {
+// seed += 1
+//// val srcId = ((tcNum * 1000) + i).toString
+//// val tgtId = if (labelName == testLabelName) s"${srcId + 1000 + i}" else s"${srcId + 1000 + i}abc"
+// val srcId = seed.toString
+// val tgtId = srcId
+//
+// val maxTs = opWithProps.map(t => t._1).max
+//
+// /** insert edges */
+// println(s"---- TC${tcNum}_init ----")
+// val bulkEdge = (for ((ts, op, props) <- opWithProps) yield {
+// List(ts, op, "e", srcId, tgtId, labelName, props).mkString("\t")
+// }).mkString("\n")
+//
+// val req = EdgeController.mutateAndPublish(bulkEdge, withWait = true)
+// val res = Await.result(req, HTTP_REQ_WAITING_TIME)
+//
+// res.header.status must equalTo(200)
+//
+// println(s"---- TC${tcNum}_init ----")
+//// Thread.sleep(100)
+//
+// for {
+// label <- Label.findByName(labelName)
+// direction <- List("out", "in")
+// cacheTTL <- List(-1L)
+// } {
+// val (serviceName, columnName, id, otherId) = direction match {
+// case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
+// case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
+// }
+// val qId = if (labelName == testLabelName) id else "\"" + id + "\""
+// val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
+// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(query)).get
+// val jsResult = commonCheck(ret)
+//
+// val results = jsResult \ "results"
+// val deegrees = (jsResult \ "degrees").as[List[JsObject]]
+// val propsLs = (results \\ "props").seq
+// (deegrees.head \ LabelMeta.degree.name).as[Int] must equalTo(1)
+//
+// val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
+// val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
+//
+// from must equalTo(id.toString)
+// to must equalTo(otherId.toString)
+//// (results \\ "_timestamp").seq.last.as[Long] must equalTo(maxTs)
+// for ((key, expectedVal) <- expected) {
+// propsLs.last.as[JsObject].keys.contains(key) must equalTo(true)
+// (propsLs.last \ key).toString must equalTo(expectedVal)
+// }
+// Await.result(ret, HTTP_REQ_WAITING_TIME)
+// }
+// }
+// }
+//
+// init()
+// "Basic Crud " should {
+// "tc1" in {
+// running(FakeApplication()) {
+//
+// var tcNum = 0
+// var tcString = ""
+// var bulkQueries = List.empty[(Long, String, String)]
+// var expected = Map.empty[String, String]
+//
+// tcNum = 7
+// tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+// bulkQueries = List(
+// (t1, "insert", "{\"time\": 10}"),
+// (t2, "delete", ""),
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 8
+// tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+// bulkQueries = List(
+// (t1, "insert", "{\"time\": 10}"),
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+// (t2, "delete", ""))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 9
+// tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
+// bulkQueries = List(
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+// (t2, "delete", ""),
+// (t1, "insert", "{\"time\": 10}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 10
+// tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
+// bulkQueries = List(
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+// (t1, "insert", "{\"time\": 10}"),
+// (t2, "delete", ""))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 11
+// tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
+// bulkQueries = List(
+// (t2, "delete", ""),
+// (t1, "insert", "{\"time\": 10}"),
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 12
+// tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
+// bulkQueries = List(
+// (t2, "delete", ""),
+// (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+// (t1, "insert", "{\"time\": 10}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 13
+// tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
+// bulkQueries = List(
+// (t1, "update", "{\"time\": 10}"),
+// (t2, "delete", ""),
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// tcNum = 14
+// tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
+// bulkQueries = List(
+// (t1, "update", "{\"time\": 10}"),
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+// (t2, "delete", ""))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// tcNum = 15
+// tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
+// bulkQueries = List(
+// (t2, "delete", ""),
+// (t1, "update", "{\"time\": 10}"),
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// tcNum = 16
+// tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
+// bulkQueries = List(
+// (t2, "delete", ""),
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+// (t1, "update", "{\"time\": 10}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// tcNum = 17
+// tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
+// bulkQueries = List(
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+// (t2, "delete", ""),
+// (t1, "update", "{\"time\": 10}"))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// tcNum = 18
+// tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
+// bulkQueries = List(
+// (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+// (t1, "update", "{\"time\": 10}"),
+// (t2, "delete", ""))
+// expected = Map("time" -> "10", "weight" -> "20")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+//
+// tcNum = 19
+// tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
+// bulkQueries = List(
+// (t5, "update", "{\"is_blocked\": true}"),
+// (t1, "insert", "{\"is_hidden\": false}"),
+// (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
+// (t2, "delete", ""),
+// (t4, "update", "{\"time\": 1, \"weight\": -10}"))
+// expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
+//
+// runTC(tcNum, tcString, bulkQueries, expected)
+// true
+// }
+// }
+// }
+//
+// "toLogString" in {
+// running(FakeApplication()) {
+// val bulkQueries = List(
+// ("1445240543366", "update", "{\"is_blocked\":true}"),
+// ("1445240543362", "insert", "{\"is_hidden\":false}"),
+// ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"),
+// ("1445240543363", "delete", "{}"),
+// ("1445240543365", "update", "{\"time\":1, \"weight\":-10}"))
+//
+// val (srcId, tgtId, labelName) = ("1", "2", testLabelName)
+//
+// val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield {
+// Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString
+// }).mkString("\n")
+//
+// val expected = Seq(
+// Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"),
+// Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"),
+// Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"),
+// Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"),
+// Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}")
+// ).map(_.mkString("\t")).mkString("\n")
+//
+// bulkEdge must equalTo(expected)
+//
+// true
+// }
+// }
+//}
+//
+//
[36/46] incubator-s2graph git commit: Move VertexSpec to s2core,
remove unused dependency
Posted by da...@apache.org.
Move VertexSpec to s2core, remove unused dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5c0af72a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5c0af72a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5c0af72a
Branch: refs/heads/feature/test_daewon
Commit: 5c0af72a36a95161d14f32a3fbaee92e69fc55fa
Parents: 610d519
Author: daewon <bl...@gmail.com>
Authored: Tue Dec 29 19:06:31 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Tue Dec 29 19:06:31 2015 +0900
----------------------------------------------------------------------
build.sbt | 8 +-
s2core/src/main/resources/logback.xml | 26 +-
s2core/src/main/resources/reference.conf | 5 +
.../com/kakao/s2graph/core/utils/Logger.scala | 1 -
.../kakao/s2graph/core/Integrate/CrudTest.scala | 1 -
.../core/Integrate/IntegrateCommon.scala | 2 +-
.../core/Integrate/VertexTestHelper.scala | 71 ++
s2rest_play/build.sbt | 3 +-
.../test/benchmark/JsonBenchmarkSpec.scala | 14 +-
.../benchmark/OrderingUtilBenchmarkSpec.scala | 16 +-
.../benchmark/PostProcessBenchmarkSpec.scala | 367 +++++-----
.../test/benchmark/SamplingBenchmarkSpec.scala | 2 +-
.../test/controllers/AdminControllerSpec.scala | 30 +-
s2rest_play/test/controllers/SpecCommon.scala | 668 +++++++++----------
s2rest_play/test/controllers/VertexSpec.scala | 106 +--
15 files changed, 700 insertions(+), 620 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index ad08bdb..c4035e3 100755
--- a/build.sbt
+++ b/build.sbt
@@ -5,7 +5,7 @@ lazy val commonSettings = Seq(
scalaVersion := "2.11.7",
version := "0.12.1-SNAPSHOT",
scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation", "-feature", "-Xlint"),
- javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map{ case (key, value) => "-D" + key + "=" + value }.toSeq,
+ javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map { case (key, value) => "-D" + key + "=" + value }.toSeq,
testOptions in Test += Tests.Argument("-oDF"),
parallelExecution in Test := false,
resolvers ++= Seq(
@@ -24,6 +24,7 @@ Revolver.settings
lazy val s2rest_play = project.enablePlugins(PlayScala)
.dependsOn(s2core, s2counter_core)
.settings(commonSettings: _*)
+ .settings(testOptions in Test += Tests.Argument("sequential"))
lazy val s2rest_netty = project
.dependsOn(s2core)
@@ -44,5 +45,6 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
lazy val s2ml = project.settings(commonSettings: _*)
-lazy val root = (project in file(".")).
- aggregate(s2core, s2rest_play)
+lazy val root = (project in file("."))
+ .aggregate(s2core, s2rest_play)
+ .settings(commonSettings: _*)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 94f4920..428a70f 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -1,15 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
- <jmxConfigurator/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n
- </pattern>
- </encoder>
- </appender>
+ <jmxConfigurator/>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>
+ %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n
+ </pattern>
+ </encoder>
+ </appender>
- <root level="info">
- <appender-ref ref="STDOUT"/>
- </root>
+ <root name="error" level="error">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+ <root name="application" level="info">
+ <appender-ref ref="STDOUT"/>
+ </root>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf
index b527f7f..5727b28 100644
--- a/s2core/src/main/resources/reference.conf
+++ b/s2core/src/main/resources/reference.conf
@@ -1,7 +1,12 @@
# APP PHASE
phase=dev
+
host=localhost
+logger.root=DEBUG
+
+logger.application=DEBUG
+
hbase.zookeeper.quorum=${host}
# DB
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
index cb4514d..609300a 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Logger.scala
@@ -1,6 +1,5 @@
package com.kakao.s2graph.core.utils
-//import play.api.Logger
import org.slf4j.LoggerFactory
import play.api.libs.json.JsValue
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
index f3bbdd9..1c09778 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -4,7 +4,6 @@ import com.kakao.s2graph.core.mysqls._
import play.api.libs.json.{JsObject, Json}
class CrudTest extends IntegrateCommon {
-
import CrudHelper._
import TestUtil._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
index 230fa9c..6f73f11 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -143,7 +143,7 @@ trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
val index2 = "idx_2"
val NumOfEachTest = 30
- val HttpRequestWaitingTime = Duration("Inf")
+ val HttpRequestWaitingTime = Duration("60 seconds")
val createService = s"""{"serviceName" : "$testServiceName"}"""
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
new file mode 100644
index 0000000..ffbec11
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/VertexTestHelper.scala
@@ -0,0 +1,71 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core.PostProcess
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.Await
+import scala.util.Random
+
+
+class VertexTestHelper extends IntegrateCommon {
+
+ import TestUtil._
+ import VertexTestHelper._
+
+ test("vertex") {
+ val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
+ val (serviceName, columnName) = (testServiceName, testColumnName)
+
+ val data = vertexInsertsPayload(serviceName, columnName, ids)
+ val payload = Json.parse(Json.toJson(data).toString)
+ println(payload)
+
+ val vertices = parser.toVertices(payload, "insert", Option(serviceName), Option(columnName))
+ Await.result(graph.mutateVertices(vertices, withWait = true), HttpRequestWaitingTime)
+
+ val res = graph.getVertices(vertices).map { vertices =>
+ PostProcess.verticesToJson(vertices)
+ }
+
+ val ret = Await.result(res, HttpRequestWaitingTime)
+ val fetched = ret.as[Seq[JsValue]]
+ for {
+ (d, f) <- data.zip(fetched)
+ } yield {
+ (d \ "id") should be(f \ "id")
+ ((d \ "props") \ "age") should be((f \ "props") \ "age")
+ }
+ }
+
+ object VertexTestHelper {
+ def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
+ Json.parse(
+ s"""
+ |[
+ |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
+ ]}
+ |]
+ """.stripMargin)
+ }
+
+ def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
+ ids.map { id =>
+ Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
+ }
+ }
+
+ val vertexPropsKeys = List(
+ ("age", "int")
+ )
+
+ def randomProps() = {
+ (for {
+ (propKey, propType) <- vertexPropsKeys
+ } yield {
+ propKey -> Random.nextInt(100)
+ }).toMap
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
index 09da78c..09216a1 100644
--- a/s2rest_play/build.sbt
+++ b/s2rest_play/build.sbt
@@ -10,8 +10,7 @@ libraryDependencies ++= Seq(
ws,
filters,
"xalan" % "serializer" % "2.7.2", // Download in Intelli J(Download Source/Document)
- "com.github.danielwegener" % "logback-kafka-appender" % "0.0.3",
- "org.json4s" %% "json4s-native" % "3.2.11" % Test
+ "com.github.danielwegener" % "logback-kafka-appender" % "0.0.3"
)
enablePlugins(JavaServerAppPackaging)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
index 7f0ef83..1f7c1ef 100644
--- a/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/JsonBenchmarkSpec.scala
@@ -8,32 +8,32 @@ class JsonBenchmarkSpec extends BenchmarkCommon {
"json benchmark" >> {
duration("map to json") {
- (0 to 100) foreach { n =>
+ (0 to 10) foreach { n =>
val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
Json.toJson(numberMaps)
}
}
duration("directMakeJson") {
- (0 to 100) foreach { n =>
+ (0 to 10) foreach { n =>
var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
+ (0 to 10).foreach { n =>
jsObj += (n.toString -> JsNumber(n * n))
}
}
}
duration("map to json 2") {
- (0 to 500) foreach { n =>
- val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap
+ (0 to 50) foreach { n =>
+ val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap
Json.toJson(numberMaps)
}
}
duration("directMakeJson 2") {
- (0 to 500) foreach { n =>
+ (0 to 50) foreach { n =>
var jsObj = play.api.libs.json.Json.obj()
- (0 to 100).foreach { n =>
+ (0 to 10).foreach { n =>
jsObj += (n.toString -> JsNumber(n * n))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
index 053eb23..d2d3624 100644
--- a/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/OrderingUtilBenchmarkSpec.scala
@@ -21,7 +21,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -32,7 +32,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}.map { x => x._1 }
val sorted2 = duration("MultiOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
}
seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false)))
@@ -42,7 +42,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
"performance MultiOrdering double" >> {
- val tupLs = (0 until 500) map { i =>
+ val tupLs = (0 until 50) map { i =>
Random.nextDouble() -> Random.nextDouble()
}
@@ -51,13 +51,13 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
duration("MultiOrdering double") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false)))
}
}
duration("TupleOrdering double") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -68,7 +68,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
"performance MultiOrdering jsvalue" >> {
- val tupLs = (0 until 500) map { i =>
+ val tupLs = (0 until 50) map { i =>
Random.nextDouble() -> Random.nextLong()
}
@@ -77,7 +77,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
val sorted1 = duration("TupleOrdering double,long") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
tupLs.sortBy { case (x, y) =>
-x -> -y
}
@@ -88,7 +88,7 @@ class OrderingUtilBenchmarkSpec extends BenchmarkCommon {
}
val sorted2 = duration("MultiOrdering jsvalue") {
- (0 until 10000) foreach { _ =>
+ (0 until 1000) foreach { _ =>
seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
}
seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false)))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
index 3ae08e2..6de3dd5 100644
--- a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala
@@ -1,184 +1,130 @@
-package benchmark
-
-import com.kakao.s2graph.core.mysqls.Label
-import com.kakao.s2graph.core.rest.RequestParser
-import com.kakao.s2graph.core.{PostProcess, Graph, Management}
-import com.typesafe.config.ConfigFactory
-import controllers._
-import play.api.libs.json.{JsValue, Json}
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
-
-import scala.concurrent.Await
-import scala.concurrent.duration._
-
-/**
- * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6..
- */
-class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification {
- init()
-
- override def init() = {
- running(FakeApplication()) {
- println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- Management.deleteService(testServiceName)
-
- // 1. createService
- val result = AdminController.createServiceInner(Json.parse(createService))
- println(s">> Service created : $createService, $result")
-
- val labelNames = Map(
- testLabelNameWeak -> testLabelNameWeakCreate
- )
-
- for {
- (labelName, create) <- labelNames
- } {
- Management.deleteLabel(labelName)
- Label.findByName(labelName, useCache = false) match {
- case None =>
- AdminController.createLabelInner(Json.parse(create))
- case Some(label) =>
- println(s">> Label already exist: $create, $label")
- }
- }
-
- // create edges
- val bulkEdges: String = (0 until 500).map { i =>
- Seq(System.currentTimeMillis(), "insert", "e", "0", i, testLabelNameWeak, Json.obj("weight" -> i)).mkString("\t")
- }.mkString("\n")
-
- val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- }
- }
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
-
-// "test performance of getEdges orderBy" >> {
-// running(FakeApplication()) {
-// val strJs =
-// s"""
-// |{
-// | "orderBy": [
-// | {"score": "DESC"},
-// | {"timestamp": "DESC"}
-// | ],
-// | "srcVertices": [
-// | {
-// | "serviceName": "$testServiceName",
-// | "columnName": "$testColumnName",
-// | "ids": [0]
-// | }
-// | ],
-// | "steps": [
-// | {
-// | "step": [
-// | {
-// | "cacheTTL": 60000,
-// | "label": "$testLabelNameWeak",
-// | "offset": 0,
-// | "limit": -1,
-// | "direction": "out",
-// | "scoring": [
-// | {"weight": 1}
-// | ]
-// | }
-// | ]
-// | }
-// | ]
-// |}
-// """.stripMargin
-//
-// object Parser extends RequestParser
+//package benchmark
//
-// val js = Json.parse(strJs)
-//
-// val q = Parser.toQuery(js)
+//import com.kakao.s2graph.core.mysqls.Label
+//import com.kakao.s2graph.core.rest.RequestParser
+//import com.kakao.s2graph.core.{PostProcess, Graph, Management}
+//import com.typesafe.config.ConfigFactory
+//import controllers._
+//import play.api.libs.json.{JsValue, Json}
+//import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
//
-// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
+//import scala.concurrent.Await
+//import scala.concurrent.duration._
//
-// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
+///**
+// * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6..
+// */
+//class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification {
//
-// (resultJs \ "size").as[Int] must_== 500
+// init()
//
-// (0 to 5) foreach { _ =>
-// duration("toSimpleVertexArrJson new orderBy") {
-// (0 to 1000) foreach { _ =>
-// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
-// }
+// override def init() = {
+// running(FakeApplication()) {
+// println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+// Management.deleteService(testServiceName)
+//
+// // 1. createService
+// val result = AdminController.createServiceInner(Json.parse(createService))
+// println(s">> Service created : $createService, $result")
+//
+// val labelNames = Map(
+// testLabelNameWeak -> testLabelNameWeakCreate
+// )
+//
+// for {
+// (labelName, create) <- labelNames
+// } {
+// Management.deleteLabel(labelName)
+// Label.findByName(labelName, useCache = false) match {
+// case None =>
+// AdminController.createLabelInner(Json.parse(create))
+// case Some(label) =>
+// println(s">> Label already exist: $create, $label")
// }
// }
//
-// (resultJs \ "size").as[Int] must_== 500
+// // create edges
+// val bulkEdges: String = (0 until 50).map { i =>
+// Seq(System.currentTimeMillis(), "insert", "e", "0", i, testLabelNameWeak, Json.obj("weight" -> i)).mkString("\t")
+// }.mkString("\n")
+//
+// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
+//
+// println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
// }
// }
-
- "test performance of getEdges" >> {
- running(FakeApplication()) {
- val strJs =
- s"""
- |{
- | "srcVertices": [
- | {
- | "serviceName": "$testServiceName",
- | "columnName": "$testColumnName",
- | "ids": [0]
- | }
- | ],
- | "steps": [
- | {
- | "step": [
- | {
- | "cacheTTL": 60000,
- | "label": "$testLabelNameWeak",
- | "offset": 0,
- | "limit": -1,
- | "direction": "out",
- | "scoring": [
- | {"weight": 1}
- | ]
- | }
- | ]
- | }
- | ]
- |}
- """.stripMargin
-
- val config = ConfigFactory.load()
- val s2graph = new Graph(config)(scala.concurrent.ExecutionContext.Implicits.global)
- val s2parser = new RequestParser(s2graph.config)
- val js = Json.parse(strJs)
-
- val q = s2parser.toQuery(js)
-
- val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
-
- val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
-
- (0 to 5) foreach { _ =>
- duration("toSimpleVertexArrJson new") {
- (0 to 1000) foreach { _ =>
- PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
- }
- }
- }
-
- (resultJs \ "size").as[Int] must_== 500
- }
- }
-
-// "test performance of getEdges withScore=false" >> {
+//
+// def getEdges(queryJson: JsValue): JsValue = {
+// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
+// contentAsJson(ret)
+// }
+//
+// val s2: Graph = com.kakao.s2graph.rest.Global.s2graph
+//
+//// "test performance of getEdges orderBy" >> {
+//// running(FakeApplication()) {
+//// val strJs =
+//// s"""
+//// |{
+//// | "orderBy": [
+//// | {"score": "DESC"},
+//// | {"timestamp": "DESC"}
+//// | ],
+//// | "srcVertices": [
+//// | {
+//// | "serviceName": "$testServiceName",
+//// | "columnName": "$testColumnName",
+//// | "ids": [0]
+//// | }
+//// | ],
+//// | "steps": [
+//// | {
+//// | "step": [
+//// | {
+//// | "cacheTTL": 60000,
+//// | "label": "$testLabelNameWeak",
+//// | "offset": 0,
+//// | "limit": -1,
+//// | "direction": "out",
+//// | "scoring": [
+//// | {"weight": 1}
+//// | ]
+//// | }
+//// | ]
+//// | }
+//// | ]
+//// |}
+//// """.stripMargin
+////
+//// object Parser extends RequestParser
+////
+//// val js = Json.parse(strJs)
+////
+//// val q = Parser.toQuery(js)
+////
+//// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
+////
+//// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
+////
+//// (resultJs \ "size").as[Int] must_== 500
+////
+//// (0 to 5) foreach { _ =>
+//// duration("toSimpleVertexArrJson new orderBy") {
+//// (0 to 1000) foreach { _ =>
+//// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
+//// }
+//// }
+//// }
+////
+//// (resultJs \ "size").as[Int] must_== 500
+//// }
+//// }
+//
+// "test performance of getEdges" >> {
// running(FakeApplication()) {
// val strJs =
// s"""
// |{
-// | "withScore": false,
// | "srcVertices": [
// | {
// | "serviceName": "$testServiceName",
@@ -205,33 +151,88 @@ class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with Play
// |}
// """.stripMargin
//
-// object Parser extends RequestParser
-//
+// val config = ConfigFactory.load()
+// val s2graph = new Graph(config)(scala.concurrent.ExecutionContext.Implicits.global)
+// val s2parser = new RequestParser(s2graph.config)
// val js = Json.parse(strJs)
//
-// val q = Parser.toQuery(js)
+// val q = s2parser.toQuery(js)
//
// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
//
-// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
-//
-// (resultJs \ "size").as[Int] must_== 500
+// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
//
// (0 to 5) foreach { _ =>
-// duration("toSimpleVertexArrJson withScore=false org") {
-// (0 to 1000) foreach { _ =>
-// PostProcess.toSimpleVertexArrJsonOrg(queryResultLs, Nil)
-// }
-// }
-//
-// duration("toSimpleVertexArrJson withScore=false new") {
-// (0 to 1000) foreach { _ =>
+// duration("toSimpleVertexArrJson new") {
+// (0 to 100) foreach { _ =>
// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
// }
// }
// }
//
-// (resultJs \ "size").as[Int] must_== 500
+// (resultJs \ "size").as[Int] must_== 50
// }
// }
-}
+//
+//// "test performance of getEdges withScore=false" >> {
+//// running(FakeApplication()) {
+//// val strJs =
+//// s"""
+//// |{
+//// | "withScore": false,
+//// | "srcVertices": [
+//// | {
+//// | "serviceName": "$testServiceName",
+//// | "columnName": "$testColumnName",
+//// | "ids": [0]
+//// | }
+//// | ],
+//// | "steps": [
+//// | {
+//// | "step": [
+//// | {
+//// | "cacheTTL": 60000,
+//// | "label": "$testLabelNameWeak",
+//// | "offset": 0,
+//// | "limit": -1,
+//// | "direction": "out",
+//// | "scoring": [
+//// | {"weight": 1}
+//// | ]
+//// | }
+//// | ]
+//// | }
+//// | ]
+//// |}
+//// """.stripMargin
+////
+//// object Parser extends RequestParser
+////
+//// val js = Json.parse(strJs)
+////
+//// val q = Parser.toQuery(js)
+////
+//// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds)
+////
+//// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs)
+////
+//// (resultJs \ "size").as[Int] must_== 500
+////
+//// (0 to 5) foreach { _ =>
+//// duration("toSimpleVertexArrJson withScore=false org") {
+//// (0 to 1000) foreach { _ =>
+//// PostProcess.toSimpleVertexArrJsonOrg(queryResultLs, Nil)
+//// }
+//// }
+////
+//// duration("toSimpleVertexArrJson withScore=false new") {
+//// (0 to 1000) foreach { _ =>
+//// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil)
+//// }
+//// }
+//// }
+////
+//// (resultJs \ "size").as[Int] must_== 500
+//// }
+//// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
index 7e442e4..6f7f4fc 100644
--- a/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
+++ b/s2rest_play/test/benchmark/SamplingBenchmarkSpec.scala
@@ -47,7 +47,7 @@ class SamplingBenchmarkSpec extends BenchmarkCommon {
}
// test data
- val testLimit = 100000
+ val testLimit = 1000
val testNum = 10
val testData = (0 to 1000).toList
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/controllers/AdminControllerSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/AdminControllerSpec.scala b/s2rest_play/test/controllers/AdminControllerSpec.scala
index e41fccb..1d92ed2 100644
--- a/s2rest_play/test/controllers/AdminControllerSpec.scala
+++ b/s2rest_play/test/controllers/AdminControllerSpec.scala
@@ -10,18 +10,18 @@ import scala.concurrent.Await
/**
* Created by mojo22jojo(hyunsung.jo@gmail.com) on 15. 10. 13..
*/
-class AdminControllerSpec extends SpecCommon {
- init()
- "EdgeControllerSpec" should {
- "update htable" in {
- running(FakeApplication()) {
- val insertUrl = s"/graphs/updateHTable/$testLabelName/$newHTableName"
-
- val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "text/plain")
-
- Await.result(route(req).get, HTTP_REQ_WAITING_TIME)
- Label.findByName(testLabelName, useCache = true).get.hTableName mustEqual newHTableName
- }
- }
- }
-}
+//class AdminControllerSpec extends SpecCommon {
+// init()
+// "EdgeControllerSpec" should {
+// "update htable" in {
+// running(FakeApplication()) {
+// val insertUrl = s"/graphs/updateHTable/$testLabelName/$newHTableName"
+//
+// val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "text/plain")
+//
+// Await.result(route(req).get, HTTP_REQ_WAITING_TIME)
+// Label.findByName(testLabelName, useCache = true).get.hTableName mustEqual newHTableName
+// }
+// }
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/controllers/SpecCommon.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/SpecCommon.scala b/s2rest_play/test/controllers/SpecCommon.scala
index 28c4c23..5fec7ea 100644
--- a/s2rest_play/test/controllers/SpecCommon.scala
+++ b/s2rest_play/test/controllers/SpecCommon.scala
@@ -1,334 +1,334 @@
-package controllers
-
-import com.kakao.s2graph.core._
-import com.kakao.s2graph.core.mysqls._
-import org.specs2.mutable.Specification
-import play.api.libs.json._
-import play.api.test.FakeApplication
-import play.api.test.Helpers._
-
-import scala.concurrent.Future
-import scala.concurrent.duration._
-import scala.util.Random
-
-trait SpecCommon extends Specification {
- val curTime = System.currentTimeMillis
-
- val t1 = curTime + 0
- val t2 = curTime + 1
- val t3 = curTime + 2
- val t4 = curTime + 3
- val t5 = curTime + 4
-
- protected val testServiceName = "s2graph"
- protected val testLabelName = "s2graph_label_test"
- protected val testLabelName2 = "s2graph_label_test_2"
- protected val testLabelNameV1 = "s2graph_label_test_v1"
- protected val testLabelNameWeak = "s2graph_label_test_weak"
- protected val testColumnName = "user_id_test"
- protected val testColumnType = "long"
- protected val testTgtColumnName = "item_id_test"
- protected val testHTableName = "test-htable"
- protected val newHTableName = "new-htable"
- protected val index1 = "idx_1"
- protected val index2 = "idx_2"
-
- val NUM_OF_EACH_TEST = 100
- val HTTP_REQ_WAITING_TIME = Duration(300, SECONDS)
- val asyncFlushInterval = 100
-
- val createService = s"""{"serviceName" : "$testServiceName"}"""
- val testLabelNameCreate =
- s"""
- {
- "label": "$testLabelName",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testColumnName",
- "tgtColumnType": "long",
- "indices": [
- {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
- {"name": "$index2", "propNames": ["_timestamp"]}
- ],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "schemaVersion": "v2",
- "compressionAlgorithm": "gz",
- "hTableName": "$testHTableName"
- }"""
-
- val testLabelName2Create =
- s"""
- {
- "label": "$testLabelName2",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": false,
- "schemaVersion": "v3",
- "compressionAlgorithm": "gz"
- }"""
-
- val testLabelNameV1Create =
- s"""
- {
- "label": "$testLabelNameV1",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "${testTgtColumnName}_v1",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "strong",
- "isDirected": true,
- "schemaVersion": "v1",
- "compressionAlgorithm": "gz"
- }"""
- val testLabelNameWeakCreate =
- s"""
- {
- "label": "$testLabelNameWeak",
- "srcServiceName": "$testServiceName",
- "srcColumnName": "$testColumnName",
- "srcColumnType": "long",
- "tgtServiceName": "$testServiceName",
- "tgtColumnName": "$testTgtColumnName",
- "tgtColumnType": "string",
- "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
- "props": [
- {
- "name": "time",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "weight",
- "dataType": "long",
- "defaultValue": 0
- },
- {
- "name": "is_hidden",
- "dataType": "boolean",
- "defaultValue": false
- },
- {
- "name": "is_blocked",
- "dataType": "boolean",
- "defaultValue": false
- }
- ],
- "consistencyLevel": "weak",
- "isDirected": true,
- "compressionAlgorithm": "gz"
- }"""
-
- val vertexPropsKeys = List(
- ("age", "int")
- )
-
- val createVertex =
- s"""{
- "serviceName": "$testServiceName",
- "columnName": "$testColumnName",
- "columnType": "$testColumnType",
- "props": [
- {"name": "is_active", "dataType": "boolean", "defaultValue": true},
- {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
- {"name": "nickname", "dataType": "string", "defaultValue": ".."},
- {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
- {"name": "age", "dataType": "integer", "defaultValue": 0}
- ]
- }"""
-
-
- val TS = System.currentTimeMillis()
-
- def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = {
- val s =
- s"""{
- "srcVertices": [
- {
- "serviceName": "$serviceName",
- "columnName": "$columnName",
- "id": $id
- }
- ],
- "steps": [
- [
- {
- "label": "$labelName",
- "direction": "$dir",
- "offset": 0,
- "limit": 10,
- "cacheTTL": $cacheTTL
- }
- ]
- ]
- }"""
- println(s)
- Json.parse(s)
- }
-
- def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
- val arr = for {
- (label, dir, from, to) <- params
- } yield {
- Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
- }
-
- val s = Json.toJson(arr)
- println(s)
- s
- }
-
- def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
- Json.parse(
- s"""
- |[
- |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
- ]}
- |]
- """.stripMargin)
- }
-
- def randomProps() = {
- (for {
- (propKey, propType) <- vertexPropsKeys
- } yield {
- propKey -> Random.nextInt(100)
- }).toMap
- }
-
- def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
- ids.map { id =>
- Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
- }
- }
-
- def commonCheck(rslt: Future[play.api.mvc.Result]): JsValue = {
- status(rslt) must equalTo(OK)
- contentType(rslt) must beSome.which(_ == "application/json")
- val jsRslt = contentAsJson(rslt)
- println("======")
- println(jsRslt)
- println("======")
- jsRslt.as[JsObject].keys.contains("size") must equalTo(true)
- (jsRslt \ "size").as[Int] must greaterThan(0)
- jsRslt.as[JsObject].keys.contains("results") must equalTo(true)
- val jsRsltsObj = jsRslt \ "results"
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("from") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("to") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("_timestamp") must equalTo(true)
- jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("props") must equalTo(true)
- jsRslt
- }
-
- def init() = {
- running(FakeApplication()) {
- println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- Management.deleteService(testServiceName)
-
- // 1. createService
- val result = AdminController.createServiceInner(Json.parse(createService))
- println(s">> Service created : $createService, $result")
-
- val labelNames = Map(testLabelName -> testLabelNameCreate,
- testLabelName2 -> testLabelName2Create,
- testLabelNameV1 -> testLabelNameV1Create,
- testLabelNameWeak -> testLabelNameWeakCreate)
-
- for {
- (labelName, create) <- labelNames
- } {
- Management.deleteLabel(labelName)
- Label.findByName(labelName, useCache = false) match {
- case None =>
- AdminController.createLabelInner(Json.parse(create))
- case Some(label) =>
- println(s">> Label already exist: $create, $label")
- }
- }
-
- // 5. create vertex
- vertexPropsKeys.map { case (key, keyType) =>
- Management.addVertexProp(testServiceName, testColumnName, key, keyType)
- }
-
- println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
- }
- }
-}
-
+//package controllers
+//
+//import com.kakao.s2graph.core._
+//import com.kakao.s2graph.core.mysqls._
+//import org.specs2.mutable.Specification
+//import play.api.libs.json._
+//import play.api.test.FakeApplication
+//import play.api.test.Helpers._
+//
+//import scala.concurrent.Future
+//import scala.concurrent.duration._
+//import scala.util.Random
+//
+//trait SpecCommon extends Specification {
+// val curTime = System.currentTimeMillis
+//
+// val t1 = curTime + 0
+// val t2 = curTime + 1
+// val t3 = curTime + 2
+// val t4 = curTime + 3
+// val t5 = curTime + 4
+//
+// protected val testServiceName = "s2graph"
+// protected val testLabelName = "s2graph_label_test"
+// protected val testLabelName2 = "s2graph_label_test_2"
+// protected val testLabelNameV1 = "s2graph_label_test_v1"
+// protected val testLabelNameWeak = "s2graph_label_test_weak"
+// protected val testColumnName = "user_id_test"
+// protected val testColumnType = "long"
+// protected val testTgtColumnName = "item_id_test"
+// protected val testHTableName = "test-htable"
+// protected val newHTableName = "new-htable"
+// protected val index1 = "idx_1"
+// protected val index2 = "idx_2"
+//
+// val NUM_OF_EACH_TEST = 100
+// val HTTP_REQ_WAITING_TIME = Duration(300, SECONDS)
+// val asyncFlushInterval = 100
+//
+// val createService = s"""{"serviceName" : "$testServiceName"}"""
+// val testLabelNameCreate =
+// s"""
+// {
+// "label": "$testLabelName",
+// "srcServiceName": "$testServiceName",
+// "srcColumnName": "$testColumnName",
+// "srcColumnType": "long",
+// "tgtServiceName": "$testServiceName",
+// "tgtColumnName": "$testColumnName",
+// "tgtColumnType": "long",
+// "indices": [
+// {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
+// {"name": "$index2", "propNames": ["_timestamp"]}
+// ],
+// "props": [
+// {
+// "name": "time",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "weight",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "is_hidden",
+// "dataType": "boolean",
+// "defaultValue": false
+// },
+// {
+// "name": "is_blocked",
+// "dataType": "boolean",
+// "defaultValue": false
+// }
+// ],
+// "consistencyLevel": "strong",
+// "schemaVersion": "v2",
+// "compressionAlgorithm": "gz",
+// "hTableName": "$testHTableName"
+// }"""
+//
+// val testLabelName2Create =
+// s"""
+// {
+// "label": "$testLabelName2",
+// "srcServiceName": "$testServiceName",
+// "srcColumnName": "$testColumnName",
+// "srcColumnType": "long",
+// "tgtServiceName": "$testServiceName",
+// "tgtColumnName": "$testTgtColumnName",
+// "tgtColumnType": "string",
+// "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+// "props": [
+// {
+// "name": "time",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "weight",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "is_hidden",
+// "dataType": "boolean",
+// "defaultValue": false
+// },
+// {
+// "name": "is_blocked",
+// "dataType": "boolean",
+// "defaultValue": false
+// }
+// ],
+// "consistencyLevel": "strong",
+// "isDirected": false,
+// "schemaVersion": "v3",
+// "compressionAlgorithm": "gz"
+// }"""
+//
+// val testLabelNameV1Create =
+// s"""
+// {
+// "label": "$testLabelNameV1",
+// "srcServiceName": "$testServiceName",
+// "srcColumnName": "$testColumnName",
+// "srcColumnType": "long",
+// "tgtServiceName": "$testServiceName",
+// "tgtColumnName": "${testTgtColumnName}_v1",
+// "tgtColumnType": "string",
+// "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+// "props": [
+// {
+// "name": "time",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "weight",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "is_hidden",
+// "dataType": "boolean",
+// "defaultValue": false
+// },
+// {
+// "name": "is_blocked",
+// "dataType": "boolean",
+// "defaultValue": false
+// }
+// ],
+// "consistencyLevel": "strong",
+// "isDirected": true,
+// "schemaVersion": "v1",
+// "compressionAlgorithm": "gz"
+// }"""
+// val testLabelNameWeakCreate =
+// s"""
+// {
+// "label": "$testLabelNameWeak",
+// "srcServiceName": "$testServiceName",
+// "srcColumnName": "$testColumnName",
+// "srcColumnType": "long",
+// "tgtServiceName": "$testServiceName",
+// "tgtColumnName": "$testTgtColumnName",
+// "tgtColumnType": "string",
+// "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+// "props": [
+// {
+// "name": "time",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "weight",
+// "dataType": "long",
+// "defaultValue": 0
+// },
+// {
+// "name": "is_hidden",
+// "dataType": "boolean",
+// "defaultValue": false
+// },
+// {
+// "name": "is_blocked",
+// "dataType": "boolean",
+// "defaultValue": false
+// }
+// ],
+// "consistencyLevel": "weak",
+// "isDirected": true,
+// "compressionAlgorithm": "gz"
+// }"""
+//
+// val vertexPropsKeys = List(
+// ("age", "int")
+// )
+//
+// val createVertex =
+// s"""{
+// "serviceName": "$testServiceName",
+// "columnName": "$testColumnName",
+// "columnType": "$testColumnType",
+// "props": [
+// {"name": "is_active", "dataType": "boolean", "defaultValue": true},
+// {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
+// {"name": "nickname", "dataType": "string", "defaultValue": ".."},
+// {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
+// {"name": "age", "dataType": "integer", "defaultValue": 0}
+// ]
+// }"""
+//
+//
+// val TS = System.currentTimeMillis()
+//
+// def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = {
+// val s =
+// s"""{
+// "srcVertices": [
+// {
+// "serviceName": "$serviceName",
+// "columnName": "$columnName",
+// "id": $id
+// }
+// ],
+// "steps": [
+// [
+// {
+// "label": "$labelName",
+// "direction": "$dir",
+// "offset": 0,
+// "limit": 10,
+// "cacheTTL": $cacheTTL
+// }
+// ]
+// ]
+// }"""
+// println(s)
+// Json.parse(s)
+// }
+//
+// def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = {
+// val arr = for {
+// (label, dir, from, to) <- params
+// } yield {
+// Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to)
+// }
+//
+// val s = Json.toJson(arr)
+// println(s)
+// s
+// }
+//
+// def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = {
+// Json.parse(
+// s"""
+// |[
+// |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")}
+// ]}
+// |]
+// """.stripMargin)
+// }
+//
+// def randomProps() = {
+// (for {
+// (propKey, propType) <- vertexPropsKeys
+// } yield {
+// propKey -> Random.nextInt(100)
+// }).toMap
+// }
+//
+// def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = {
+// ids.map { id =>
+// Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis())
+// }
+// }
+//
+// def commonCheck(rslt: Future[play.api.mvc.Result]): JsValue = {
+// status(rslt) must equalTo(OK)
+// contentType(rslt) must beSome.which(_ == "application/json")
+// val jsRslt = contentAsJson(rslt)
+// println("======")
+// println(jsRslt)
+// println("======")
+// jsRslt.as[JsObject].keys.contains("size") must equalTo(true)
+// (jsRslt \ "size").as[Int] must greaterThan(0)
+// jsRslt.as[JsObject].keys.contains("results") must equalTo(true)
+// val jsRsltsObj = jsRslt \ "results"
+// jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("from") must equalTo(true)
+// jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("to") must equalTo(true)
+// jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("_timestamp") must equalTo(true)
+// jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("props") must equalTo(true)
+// jsRslt
+// }
+//
+// def init() = {
+// running(FakeApplication()) {
+// println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+// Management.deleteService(testServiceName)
+//
+// // 1. createService
+// val result = AdminController.createServiceInner(Json.parse(createService))
+// println(s">> Service created : $createService, $result")
+//
+// val labelNames = Map(testLabelName -> testLabelNameCreate,
+// testLabelName2 -> testLabelName2Create,
+// testLabelNameV1 -> testLabelNameV1Create,
+// testLabelNameWeak -> testLabelNameWeakCreate)
+//
+// for {
+// (labelName, create) <- labelNames
+// } {
+// Management.deleteLabel(labelName)
+// Label.findByName(labelName, useCache = false) match {
+// case None =>
+// AdminController.createLabelInner(Json.parse(create))
+// case Some(label) =>
+// println(s">> Label already exist: $create, $label")
+// }
+// }
+//
+// // 5. create vertex
+// vertexPropsKeys.map { case (key, keyType) =>
+// Management.addVertexProp(testServiceName, testColumnName, key, keyType)
+// }
+//
+// println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+// }
+// }
+//}
+//
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c0af72a/s2rest_play/test/controllers/VertexSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/VertexSpec.scala b/s2rest_play/test/controllers/VertexSpec.scala
index 46461ea..1999213 100644
--- a/s2rest_play/test/controllers/VertexSpec.scala
+++ b/s2rest_play/test/controllers/VertexSpec.scala
@@ -1,53 +1,53 @@
-package controllers
-
-import com.kakao.s2graph.core.Graph
-import com.kakao.s2graph.core.rest.RestCaller
-import com.typesafe.config.ConfigFactory
-import play.api.libs.json._
-import play.api.test.FakeApplication
-import play.api.test.Helpers._
-
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-
-class VertexSpec extends SpecCommon {
- // init()
-
- "vetex tc" should {
- "tc1" in {
-
- running(FakeApplication()) {
- val config = ConfigFactory.load()
- val graph = new Graph(config)
- val rest = new RestCaller(graph)
-
- val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
-
- val (serviceName, columnName) = (testServiceName, testColumnName)
-
- val data = vertexInsertsPayload(serviceName, columnName, ids)
- val payload = Json.parse(Json.toJson(data).toString)
- println(payload)
-
- val jsResult = contentAsString(VertexController.tryMutates(payload, "insert",
- Option(serviceName), Option(columnName), withWait = true))
-
- val query = vertexQueryJson(serviceName, columnName, ids)
- val res = rest.uriMatch("/graphs/getVertices", query)
-
-
- val ret = Await.result(res, Duration("30 seconds"))
-
- println(">>>", ret)
- val fetched = ret.as[Seq[JsValue]]
- for {
- (d, f) <- data.zip(fetched)
- } yield {
- (d \ "id") must beEqualTo((f \ "id"))
- ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age"))
- }
- }
- true
- }
- }
-}
+//package controllers
+//
+//import com.kakao.s2graph.core.Graph
+//import com.kakao.s2graph.core.rest.RestCaller
+//import com.typesafe.config.ConfigFactory
+//import play.api.libs.json._
+//import play.api.test.FakeApplication
+//import play.api.test.Helpers._
+//
+//import scala.concurrent.Await
+//import scala.concurrent.duration.Duration
+//
+//class VertexSpec extends SpecCommon {
+// // init()
+//
+// "vetex tc" should {
+// "tc1" in {
+//
+// running(FakeApplication()) {
+// val config = ConfigFactory.load()
+// val graph = new Graph(config)
+// val rest = new RestCaller(graph)
+//
+// val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0)
+//
+// val (serviceName, columnName) = (testServiceName, testColumnName)
+//
+// val data = vertexInsertsPayload(serviceName, columnName, ids)
+// val payload = Json.parse(Json.toJson(data).toString)
+// println(payload)
+//
+// val jsResult = contentAsString(VertexController.tryMutates(payload, "insert",
+// Option(serviceName), Option(columnName), withWait = true))
+//
+// val query = vertexQueryJson(serviceName, columnName, ids)
+// val res = rest.uriMatch("/graphs/getVertices", query)
+//
+//
+// val ret = Await.result(res, Duration("30 seconds"))
+//
+// println(">>>", ret)
+// val fetched = ret.as[Seq[JsValue]]
+// for {
+// (d, f) <- data.zip(fetched)
+// } yield {
+// (d \ "id") must beEqualTo((f \ "id"))
+// ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age"))
+// }
+// }
+// true
+// }
+// }
+//}
[46/46] incubator-s2graph git commit: Remove unused code
Posted by da...@apache.org.
Remove unused code
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b99389f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b99389f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b99389f3
Branch: refs/heads/feature/test_daewon
Commit: b99389f337db3965e7c9d5378ae17fda21f79303
Parents: 8e13d4e
Author: daewon <bl...@gmail.com>
Authored: Mon Jan 4 11:09:07 2016 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Jan 4 11:09:07 2016 +0900
----------------------------------------------------------------------
s2rest_netty/build.sbt | 3 +-
test/controllers/QueryExtendSpec.scala | 269 ----------------------------
2 files changed, 1 insertion(+), 271 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b99389f3/s2rest_netty/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_netty/build.sbt b/s2rest_netty/build.sbt
index 2e1ad72..666fd80 100644
--- a/s2rest_netty/build.sbt
+++ b/s2rest_netty/build.sbt
@@ -3,6 +3,5 @@ name := "s2rest_netty"
enablePlugins(JavaAppPackaging)
libraryDependencies ++= Seq(
- "io.netty" % "netty-all" % "4.0.33.Final",
- "com.typesafe.play" %% "play-ws" % Common.playVersion
+ "io.netty" % "netty-all" % "4.0.33.Final"
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b99389f3/test/controllers/QueryExtendSpec.scala
----------------------------------------------------------------------
diff --git a/test/controllers/QueryExtendSpec.scala b/test/controllers/QueryExtendSpec.scala
deleted file mode 100644
index fe8899d..0000000
--- a/test/controllers/QueryExtendSpec.scala
+++ /dev/null
@@ -1,269 +0,0 @@
-package controllers
-
-import play.api.libs.json._
-import play.api.test.{FakeApplication, FakeRequest, PlaySpecification}
-import play.api.{Application => PlayApplication}
-
-class QueryExtendSpec extends SpecCommon with PlaySpecification {
-
- import Helper._
-
- implicit val app = FakeApplication()
-
- init()
-
- "query test" should {
- def queryParents(id: Long) = Json.parse(s"""
- {
- "returnTree": true,
- "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- }
- ],[{
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": -1
- }
- ]]
- }""".stripMargin)
-
- def queryExclude(id: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 2
- },
- {
- "label": "${testLabelName}",
- "direction": "in",
- "offset": 0,
- "limit": 2,
- "exclude": true
- }
- ]]
- }""")
-
- def queryTransform(id: Int, transforms: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "transform": $transforms
- }
- ]]
- }""")
-
- def queryWhere(id: Int, where: String) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": 0,
- "limit": 100,
- "where": "${where}"
- }
- ]]
- }""")
-
- def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit,
- "_to": $to
- }
- ]]
- }
- """)
-
- def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s"""
- { "srcVertices": [
- { "serviceName": "${testServiceName}",
- "columnName": "${testColumnName}",
- "id": ${id}
- }],
- "steps": [
- [ {
- "label": "${testLabelName}",
- "direction": "out",
- "offset": $offset,
- "limit": $limit
- }
- ]]
- }
- """)
-
- def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id)))
-
- def queryGroupBy(id: Int, props: Seq[String]): JsValue = {
- Json.obj(
- "groupBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- def queryScore(id: Int, scoring: Map[String, Int]): JsValue = {
- val q = Json.obj(
- "srcVertices" -> Json.arr(
- Json.obj(
- "serviceName" -> testServiceName,
- "columnName" -> testColumnName,
- "id" -> id
- )
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- println(q)
- q
- }
-
- def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = {
- Json.obj(
- "orderBy" -> props,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName,
- "scoring" -> scoring
- )
- )
- )
- )
- )
- }
-
- def queryGlobalLimit(id: Int, limit: Int): JsValue = {
- Json.obj(
- "limit" -> limit,
- "srcVertices" -> Json.arr(
- Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id)
- ),
- "steps" -> Json.arr(
- Json.obj(
- "step" -> Json.arr(
- Json.obj(
- "label" -> testLabelName
- )
- )
- )
- )
- )
- }
-
- def getEdges(queryJson: JsValue): JsValue = {
- val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get
- contentAsJson(ret)
- }
-
- def queryIndex(ids: Seq[Int], indexName: String) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(label = testLabelName, index = indexName))
- val $steps = $a($(step = $step))
-
- val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson
- js
- }
-
- def queryDuration(ids: Seq[Int], from: Int, to: Int) = {
- val $from = $a(
- $(serviceName = testServiceName,
- columnName = testColumnName,
- ids = ids))
-
- val $step = $a($(
- label = testLabelName, direction = "out", offset = 0, limit = 100,
- duration = $(from = from, to = to)))
-
- val $steps = $a($(step = $step))
-
- $(srcVertices = $from, steps = $steps).toJson
- }
-
- "limit" >> {
- running(FakeApplication()) {
- // insert test set
- val bulkEdges: String = Seq(
- edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)),
- edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)),
- edge"3003 insert e 2 0 $testLabelName"($(weight = 30)),
- edge"4004 insert e 2 1 $testLabelName"($(weight = 40))
- ).mkString("\n")
- contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true))
-
- val edges = getEdges(querySingle(0, limit=1))
- val limitEdges = getEdges(queryGlobalLimit(0, 1))
-
- println(edges)
- println(limitEdges)
-
- val edgesTo = edges \ "results" \\ "to"
- val limitEdgesTo = limitEdges \ "results" \\ "to"
-
- edgesTo must_== limitEdgesTo
- }
- }
- }
-}
[42/46] incubator-s2graph git commit: Merge pull request #225 from
hsleep/feature/configurable_docker_hostname
Posted by da...@apache.org.
Merge pull request #225 from hsleep/feature/configurable_docker_hostname
configurable docker's hostname
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/dfbda63c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/dfbda63c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/dfbda63c
Branch: refs/heads/feature/test_daewon
Commit: dfbda63c06f20630b3940863b5594f350de1e52c
Parents: d9f6ee8 616cc81
Author: Junki Kim <wi...@gmail.com>
Authored: Wed Dec 30 14:10:39 2015 +0900
Committer: Junki Kim <wi...@gmail.com>
Committed: Wed Dec 30 14:10:39 2015 +0900
----------------------------------------------------------------------
dev_support/docker-compose.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[29/46] incubator-s2graph git commit: Move BasicCrudSpec to CrudTest
Posted by da...@apache.org.
Move BasicCrudSpec to CrudTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/0e1c0e0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/0e1c0e0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/0e1c0e0a
Branch: refs/heads/feature/test_daewon
Commit: 0e1c0e0a3bb66ee99b05ef0236369a8c17285c3c
Parents: f75b3b3
Author: daewon <bl...@gmail.com>
Authored: Mon Dec 28 18:17:33 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Mon Dec 28 18:17:33 2015 +0900
----------------------------------------------------------------------
.../kakao/s2graph/core/rest/RequestParser.scala | 10 +
.../com/kakao/s2graph/core/GraphTest.scala | 8 -
.../kakao/s2graph/core/Integrate/CrudTest.scala | 238 ++++++++++++++++
.../core/Integrate/IntegrateCommon.scala | 274 +++++++++++++++++++
.../s2graph/core/Integrate/QueryTest.scala | 63 +++++
.../com/kakao/s2graph/core/JsonParserTest.scala | 3 -
.../com/kakao/s2graph/core/QueryParamTest.scala | 3 -
.../com/kakao/s2graph/core/TestCommon.scala | 6 -
.../s2graph/core/TestCommonWithModels.scala | 15 +-
.../test/controllers/BasicCrudSpec.scala | 2 -
10 files changed, 596 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
index 2081a18..04ac113 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala
@@ -493,4 +493,14 @@ class RequestParser(config: Config) extends JSONParser {
(quads, isReverted)
}
+
+ def toGraphElements(str: String): Seq[GraphElement] = {
+ val edgeStrs = str.split("\\n")
+
+ for {
+ edgeStr <- edgeStrs
+ str <- GraphUtil.parseString(edgeStr)
+ element <- Graph.toGraphElement(str)
+ } yield element
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
deleted file mode 100644
index 9a58bd0..0000000
--- a/s2core/src/test/scala/com/kakao/s2graph/core/GraphTest.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.kakao.s2graph.core
-
-/**
- * Created by shon on 5/29/15.
- */
-class GraphTest {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
new file mode 100644
index 0000000..a093dcc
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/CrudTest.scala
@@ -0,0 +1,238 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core.{Management, PostProcess}
+import com.kakao.s2graph.core.mysqls._
+import play.api.libs.json.{JsObject, Json}
+
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+class CrudTest extends IntegrateCommon {
+
+ test("test CRUD") {
+ var tcNum = 0
+ var tcString = ""
+ var bulkQueries = List.empty[(Long, String, String)]
+ var expected = Map.empty[String, String]
+
+ val curTime = System.currentTimeMillis
+ val t1 = curTime + 0
+ val t2 = curTime + 1
+ val t3 = curTime + 2
+ val t4 = curTime + 3
+ val t5 = curTime + 4
+
+ val tcRunner = new CrudTestRunner()
+ tcNum = 1
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 2
+ tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test "
+ bulkQueries = List(
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 3
+ tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 4
+ tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 5
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "insert", "{\"time\": 10}"),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 6
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "insert", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "insert", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 7
+ tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 8
+ tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test "
+ bulkQueries = List(
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 9
+ tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test "
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 10
+ tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test"
+ bulkQueries = List(
+ (t2, "delete", ""),
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 11
+ tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t2, "delete", ""),
+ (t1, "update", "{\"time\": 10}"))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ tcNum = 12
+ tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test "
+ bulkQueries = List(
+ (t3, "update", "{\"time\": 10, \"weight\": 20}"),
+ (t1, "update", "{\"time\": 10}"),
+ (t2, "delete", ""))
+ expected = Map("time" -> "10", "weight" -> "20")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+
+ tcNum = 13
+ tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test "
+ bulkQueries = List(
+ (t5, "update", "{\"is_blocked\": true}"),
+ (t1, "insert", "{\"is_hidden\": false}"),
+ (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"),
+ (t2, "delete", ""),
+ (t4, "update", "{\"time\": 1, \"weight\": -10}"))
+ expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true")
+
+ tcRunner.run(tcNum, tcString, bulkQueries, expected)
+ }
+
+ class CrudTestRunner {
+ var seed = 0
+
+ def run(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
+ for {
+ labelName <- List(testLabelName, testLabelName2)
+ i <- 0 until NumOfEachTest
+ } {
+ seed += 1
+ val srcId = seed.toString
+ val tgtId = srcId
+
+ val maxTs = opWithProps.map(t => t._1).max
+
+ /** insert edges */
+ println(s"---- TC${tcNum}_init ----")
+ val bulkEdges = (for ((ts, op, props) <- opWithProps) yield {
+ TestHelper.toEdge(ts, op, "e", srcId, tgtId, labelName, props)
+ })
+
+ TestHelper.insertEdges(bulkEdges:_*)
+
+ for {
+ label <- Label.findByName(labelName)
+ direction <- List("out", "in")
+ cacheTTL <- List(-1L)
+ } {
+ val (serviceName, columnName, id, otherId) = direction match {
+ case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId)
+ case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId)
+ }
+
+ val qId = if (labelName == testLabelName) id else "\"" + id + "\""
+ val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL)
+
+ val jsResult = TestHelper.getEdges(query)
+
+ val results = jsResult \ "results"
+ val deegrees = (jsResult \ "degrees").as[List[JsObject]]
+ val propsLs = (results \\ "props").seq
+ (deegrees.head \ LabelMeta.degree.name).as[Int] should be(1)
+
+ val from = (results \\ "from").seq.last.toString.replaceAll("\"", "")
+ val to = (results \\ "to").seq.last.toString.replaceAll("\"", "")
+
+ from should be(id.toString)
+ to should be(otherId.toString)
+ (results \\ "_timestamp").seq.last.as[Long] should be(maxTs)
+
+ for ((key, expectedVal) <- expected) {
+ propsLs.last.as[JsObject].keys.contains(key) should be(true)
+ (propsLs.last \ key).toString should be(expectedVal)
+ }
+ }
+ }
+ }
+
+ def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = {
+ val s =
+ s"""{
+ "srcVertices": [
+ {
+ "serviceName": "$serviceName",
+ "columnName": "$columnName",
+ "id": $id
+ }
+ ],
+ "steps": [
+ [
+ {
+ "label": "$labelName",
+ "direction": "$dir",
+ "offset": 0,
+ "limit": 10,
+ "cacheTTL": $cacheTTL
+ }
+ ]
+ ]
+ }"""
+ Json.parse(s)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
new file mode 100644
index 0000000..2aa8abf
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/IntegrateCommon.scala
@@ -0,0 +1,274 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core._
+import com.kakao.s2graph.core.mysqls.Label
+import com.kakao.s2graph.core.rest.RequestParser
+import com.typesafe.config._
+import org.scalatest._
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, ExecutionContext}
+
+trait IntegrateCommon extends FunSuite with Matchers with BeforeAndAfterAll {
+ var graph: Graph = _
+ var parser: RequestParser = _
+ var config: Config = _
+
+ override def beforeAll(): Unit = {
+ config = ConfigFactory.load()
+ graph = new Graph(config)(ExecutionContext.Implicits.global)
+ parser = new RequestParser(graph.config)
+ initTestData()
+ }
+
+ override def afterAll(): Unit = {
+ graph.shutdown()
+ }
+
+ /**
+ * Make Service, Label, Vertex for integrate test
+ */
+ def initTestData() = {
+ println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ Management.deleteService(testServiceName)
+
+ // 1. createService
+ val jsValue = Json.parse(createService)
+ val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) =
+ parser.toServiceElements(jsValue)
+
+ val tryRes =
+ Management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ println(s">> Service created : $createService, $tryRes")
+
+ val labelNames = Map(testLabelName -> testLabelNameCreate,
+ testLabelName2 -> testLabelName2Create,
+ testLabelNameV1 -> testLabelNameV1Create,
+ testLabelNameWeak -> testLabelNameWeakCreate)
+
+ for {
+ (labelName, create) <- labelNames
+ } {
+ Management.deleteLabel(labelName)
+ Label.findByName(labelName, useCache = false) match {
+ case None =>
+ val json = Json.parse(create)
+ val tryRes = for {
+ labelArgs <- parser.toLabelElements(json)
+ label <- (Management.createLabel _).tupled(labelArgs)
+ } yield label
+
+ tryRes.get
+ case Some(label) =>
+ println(s">> Label already exist: $create, $label")
+ }
+ }
+
+ val vertexPropsKeys = List("age" -> "int")
+
+ vertexPropsKeys.map { case (key, keyType) =>
+ Management.addVertexProp(testServiceName, testColumnName, key, keyType)
+ }
+
+ println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+ }
+
+
+ /**
+ * Test Helpers
+ */
+ object TestHelper {
+ def getEdges(queryJson: JsValue): JsValue = {
+ val ret = graph.getEdges(parser.toQuery(queryJson))
+ val result = Await.result(ret, HttpRequestWaitingTime)
+ val jsResult = PostProcess.toSimpleVertexArrJson(result)
+
+ jsResult
+ }
+
+ def insertEdges(bulkEdges: String*) = {
+ val req = graph.mutateElements(parser.toGraphElements(bulkEdges.mkString("\n")), withWait = true)
+ val jsResult = Await.result(req, HttpRequestWaitingTime)
+ }
+
+ def toEdge(elems: Any*): String = elems.mkString("\t")
+ }
+
+ // common tables
+ protected val testServiceName = "s2graph"
+ protected val testLabelName = "s2graph_label_test"
+ protected val testLabelName2 = "s2graph_label_test_2"
+ protected val testLabelNameV1 = "s2graph_label_test_v1"
+ protected val testLabelNameWeak = "s2graph_label_test_weak"
+ protected val testColumnName = "user_id_test"
+ protected val testColumnType = "long"
+ protected val testTgtColumnName = "item_id_test"
+ protected val testHTableName = "test-htable"
+ protected val newHTableName = "new-htable"
+ protected val index1 = "idx_1"
+ protected val index2 = "idx_2"
+
+ val NumOfEachTest = 30
+ val HttpRequestWaitingTime = Duration("Inf")
+
+ val createService = s"""{"serviceName" : "$testServiceName"}"""
+
+ val testLabelNameCreate =
+ s"""
+ {
+ "label": "$testLabelName",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testColumnName",
+ "tgtColumnType": "long",
+ "indices": [
+ {"name": "$index1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]},
+ {"name": "$index2", "propNames": ["_timestamp"]}
+ ],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "schemaVersion": "v2",
+ "compressionAlgorithm": "gz",
+ "hTableName": "$testHTableName"
+ }"""
+
+ val testLabelName2Create =
+ s"""
+ {
+ "label": "$testLabelName2",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": false,
+ "schemaVersion": "v3",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameV1Create =
+ s"""
+ {
+ "label": "$testLabelNameV1",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "${testTgtColumnName}_v1",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "strong",
+ "isDirected": true,
+ "schemaVersion": "v1",
+ "compressionAlgorithm": "gz"
+ }"""
+
+ val testLabelNameWeakCreate =
+ s"""
+ {
+ "label": "$testLabelNameWeak",
+ "srcServiceName": "$testServiceName",
+ "srcColumnName": "$testColumnName",
+ "srcColumnType": "long",
+ "tgtServiceName": "$testServiceName",
+ "tgtColumnName": "$testTgtColumnName",
+ "tgtColumnType": "string",
+ "indices": [{"name": "$index1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}],
+ "props": [
+ {
+ "name": "time",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "weight",
+ "dataType": "long",
+ "defaultValue": 0
+ },
+ {
+ "name": "is_hidden",
+ "dataType": "boolean",
+ "defaultValue": false
+ },
+ {
+ "name": "is_blocked",
+ "dataType": "boolean",
+ "defaultValue": false
+ }
+ ],
+ "consistencyLevel": "weak",
+ "isDirected": true,
+ "compressionAlgorithm": "gz"
+ }"""
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
new file mode 100644
index 0000000..53696b0
--- /dev/null
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/Integrate/QueryTest.scala
@@ -0,0 +1,63 @@
+package com.kakao.s2graph.core.Integrate
+
+import com.kakao.s2graph.core.Management
+import com.kakao.s2graph.core.mysqls._
+import play.api.libs.json.{JsValue, Json}
+
+class QueryTest extends IntegrateCommon {
+
+// test("test Query") {
+//
+// }
+//
+// test("test returnTree") {
+// def queryParents(id: Long) = Json.parse(
+// s"""
+// {
+// "returnTree": true,
+// "srcVertices": [
+// { "serviceName": "${testServiceName}",
+// "columnName": "${testColumnName}",
+// "id": ${id}
+// }],
+// "steps": [
+// [ {
+// "label": "${testLabelName}",
+// "direction": "out",
+// "offset": 0,
+// "limit": 2
+// }
+// ],[{
+// "label": "${testLabelName}",
+// "direction": "in",
+// "offset": 0,
+// "limit": -1
+// }
+// ]]
+// }""".stripMargin)
+//
+// val src = 100
+// val tgt = 200
+// val labelName = testLabelName
+//
+// insertEdges(
+// toEdge(1001, "insert", "e", src, tgt, labelName)
+// )
+//
+// val result = getEdges(queryParents(src))
+// val parents = (result \ "results").as[Seq[JsValue]]
+// val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 }
+//
+// ret === true
+// }
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
index 84349cb..127f2fe 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/JsonParserTest.scala
@@ -3,9 +3,6 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.types.{InnerValLike, InnerVal}
import org.scalatest.{Matchers, FunSuite}
-/**
- * Created by shon on 5/30/15.
- */
class JsonParserTest extends FunSuite with Matchers with TestCommon with JSONParser {
import types.HBaseType._
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
index 374293e..544c17c 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/QueryParamTest.scala
@@ -4,9 +4,6 @@ import com.kakao.s2graph.core.types.LabelWithDirection
import org.apache.hadoop.hbase.util.Bytes
import org.scalatest.{FunSuite, Matchers}
-/**
- * Created by shon on 7/31/15.
- */
class QueryParamTest extends FunSuite with Matchers with TestCommon {
// val version = HBaseType.VERSION2
// val testEdge = Management.toEdge(ts, "insert", "1", "10", labelNameV2, "out", Json.obj("is_blocked" -> true, "phone_number" -> "xxxx", "age" -> 20).toString)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
index fae3d06..b18672f 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommon.scala
@@ -9,13 +9,7 @@ import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.{PutRequest, KeyValue}
-/**
- * Created by shon on 6/1/15.
- */
trait TestCommon {
-
-
-
val ts = System.currentTimeMillis()
val testServiceId = 1
val testColumnId = 1
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
index d350ab4..e2a3363 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/TestCommonWithModels.scala
@@ -2,18 +2,16 @@ package com.kakao.s2graph.core
import com.kakao.s2graph.core.Management.JsonModel.{Index, Prop}
import com.kakao.s2graph.core.mysqls._
-import org.scalatest.BeforeAndAfterAll
import scalikejdbc.AutoSession
//import com.kakao.s2graph.core.models._
-
import com.kakao.s2graph.core.types.{InnerVal, LabelWithDirection}
import com.typesafe.config.{Config, ConfigFactory}
import scala.concurrent.ExecutionContext
-trait TestCommonWithModels {
+trait TestCommonWithModels {
import InnerVal._
import types.HBaseType._
@@ -35,6 +33,7 @@ trait TestCommonWithModels {
}
def zkQuorum = config.getString("hbase.zookeeper.quorum")
+
def cluster = config.getString("hbase.zookeeper.quorum")
implicit val session = AutoSession
@@ -94,7 +93,6 @@ trait TestCommonWithModels {
Management.deleteLabel(undirectedLabelNameV2)
}
-
def createTestLabel() = {
implicit val session = AutoSession
Management.createLabel(labelName, serviceName, columnName, columnType, serviceName, columnName, columnType,
@@ -111,27 +109,36 @@ trait TestCommonWithModels {
}
def service = Service.findByName(serviceName, useCache = false).get
+
def serviceV2 = Service.findByName(serviceNameV2, useCache = false).get
def column = ServiceColumn.find(service.id.get, columnName, useCache = false).get
+
def columnV2 = ServiceColumn.find(serviceV2.id.get, columnNameV2, useCache = false).get
def tgtColumn = ServiceColumn.find(service.id.get, tgtColumnName, useCache = false).get
+
def tgtColumnV2 = ServiceColumn.find(serviceV2.id.get, tgtColumnNameV2, useCache = false).get
def label = Label.findByName(labelName, useCache = false).get
+
def labelV2 = Label.findByName(labelNameV2, useCache = false).get
def undirectedLabel = Label.findByName(undirectedLabelName, useCache = false).get
+
def undirectedLabelV2 = Label.findByName(undirectedLabelNameV2, useCache = false).get
def dir = GraphUtil.directions("out")
+
def op = GraphUtil.operations("insert")
+
def labelOrderSeq = LabelIndex.DefaultSeq
def labelWithDir = LabelWithDirection(label.id.get, dir)
+
def labelWithDirV2 = LabelWithDirection(labelV2.id.get, dir)
def queryParam = QueryParam(labelWithDir)
+
def queryParamV2 = QueryParam(labelWithDirV2)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0e1c0e0a/s2rest_play/test/controllers/BasicCrudSpec.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/test/controllers/BasicCrudSpec.scala b/s2rest_play/test/controllers/BasicCrudSpec.scala
index f4f11b4..07419b3 100644
--- a/s2rest_play/test/controllers/BasicCrudSpec.scala
+++ b/s2rest_play/test/controllers/BasicCrudSpec.scala
@@ -13,8 +13,6 @@ import scala.concurrent.Await
class BasicCrudSpec extends SpecCommon {
- sequential
-
var seed = 0
def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = {
for {
[03/46] incubator-s2graph git commit: remove jsNumber.as on sortBy in
PostProcess.
Posted by da...@apache.org.
remove jsNumber.as on sortBy in PostProcess.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/828a4c7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/828a4c7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/828a4c7b
Branch: refs/heads/feature/test_daewon
Commit: 828a4c7bb92471eb56bd78a9c607966a9c263692
Parents: b5b8fcb
Author: SteamShon <sh...@gmail.com>
Authored: Mon Dec 21 14:16:09 2015 +0900
Committer: SteamShon <sh...@gmail.com>
Committed: Mon Dec 21 14:16:09 2015 +0900
----------------------------------------------------------------------
.../main/scala/com/kakao/s2graph/core/PostProcess.scala | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/828a4c7b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
index 0c72beb..d39c87f 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala
@@ -257,7 +257,7 @@ object PostProcess extends JSONParser {
} yield column -> value
}
- val groupedEdges =
+ val groupedEdgesWithScoreSum =
for {
(groupByKeyVals, groupedRawEdges) <- grouped
} yield {
@@ -266,7 +266,7 @@ object PostProcess extends JSONParser {
val edges = orderBy(query, orderByColumns, groupedRawEdges).map(_._1)
//TODO: refactor this
- if (query.returnAgg)
+ val js = if (query.returnAgg)
Json.obj(
"groupBy" -> Json.toJson(groupByKeyVals.toMap),
"scoreSum" -> scoreSum,
@@ -278,12 +278,14 @@ object PostProcess extends JSONParser {
"scoreSum" -> scoreSum,
"agg" -> Json.arr()
)
+ (js, scoreSum)
}
val groupedSortedJsons = query.limitOpt match {
- case None => groupedEdges.toList.sortBy { jsVal => -1 * (jsVal \ "scoreSum").as[Double] }
+ case None =>
+ groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1)
case Some(limit) =>
- groupedEdges.toList.sortBy { jsVal => -1 * (jsVal \ "scoreSum").as[Double] } take (limit)
+ groupedEdgesWithScoreSum.toList.sortBy { case (jsVal, scoreSum) => scoreSum * -1 }.map(_._1).take(limit)
}
Json.obj(
[37/46] incubator-s2graph git commit: fix `expose` configuration in
`docker-compose.yml`
Posted by da...@apache.org.
fix `expose` configuration in `docker-compose.yml`
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a685bd01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a685bd01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a685bd01
Branch: refs/heads/feature/test_daewon
Commit: a685bd01862cc639628f3068c1d519fd8512a9d6
Parents: 67aafe3
Author: Jaesang Kim <ho...@gmail.com>
Authored: Wed Dec 30 10:19:17 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Wed Dec 30 10:19:17 2015 +0900
----------------------------------------------------------------------
dev_support/docker-compose.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a685bd01/dev_support/docker-compose.yml
----------------------------------------------------------------------
diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml
index c92f173..3fa16b7 100644
--- a/dev_support/docker-compose.yml
+++ b/dev_support/docker-compose.yml
@@ -26,5 +26,5 @@ graph_hbase:
- "60030:60030"
- "9000:9000"
expose:
- - "3306:3306"
+ - "3306"
- "9000"
[44/46] incubator-s2graph git commit: update dev_support README
Posted by da...@apache.org.
update dev_support README
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/16fa7d7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/16fa7d7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/16fa7d7d
Branch: refs/heads/feature/test_daewon
Commit: 16fa7d7d941f94fbf53434d06a022e6a76526fa4
Parents: e2677a5
Author: daewon <bl...@gmail.com>
Authored: Thu Dec 31 14:11:59 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Thu Dec 31 14:11:59 2015 +0900
----------------------------------------------------------------------
dev_support/README.md | 1 +
s2core/src/main/resources/logback.xml | 5 ++---
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16fa7d7d/dev_support/README.md
----------------------------------------------------------------------
diff --git a/dev_support/README.md b/dev_support/README.md
index f720886..556a7d9 100644
--- a/dev_support/README.md
+++ b/dev_support/README.md
@@ -4,6 +4,7 @@
- `sbt "project s2rest_play" docker:publishLocal`
2. Run MySQL and HBase container first.
- change directory to dev-support. `cd dev-support`
+ - `docker-compose build`
- `docker-compose up -d graph_mysql` will run MySQL and HBase at same time.
3. Run graph container
- `docker-compose up -d`
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/16fa7d7d/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 5eb6737..c238209 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -1,9 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <withJansi>true</withJansi>
<encoder>
<pattern>
- %d{ISO8601} [%highlight(%-5level)] [%magenta(%logger{0})] [%thread] - %msg%n
+ %d{ISO8601} [%highlight(%-5level)] [%gray(%logger{0})] [%thread] - %msg%n
</pattern>
</encoder>
</appender>
@@ -20,5 +21,3 @@
<appender-ref ref="STDOUT"/>
</logger>
</configuration>
-
-
[39/46] incubator-s2graph git commit: Make logger works
Posted by da...@apache.org.
Make logger works
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5a77433d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5a77433d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5a77433d
Branch: refs/heads/feature/test_daewon
Commit: 5a77433dd53b84913d74e794564775d0ee90a5a1
Parents: 5c0af72
Author: daewon <bl...@gmail.com>
Authored: Wed Dec 30 11:53:47 2015 +0900
Committer: daewon <bl...@gmail.com>
Committed: Wed Dec 30 11:53:47 2015 +0900
----------------------------------------------------------------------
s2core/src/main/resources/logback.xml | 15 ++++++++-----
s2core/src/main/resources/reference.conf | 22 ++++++++------------
.../core/storage/hbase/AsynchbaseStorage.scala | 10 ++++-----
.../hbase/AsynchbaseQueryBuilderTest.scala | 3 ++-
4 files changed, 26 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5a77433d/s2core/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml
index 428a70f..9174452 100644
--- a/s2core/src/main/resources/logback.xml
+++ b/s2core/src/main/resources/logback.xml
@@ -1,19 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
- <jmxConfigurator/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n
+ %d{ISO8601} [%-5level] [%cyan(%logger{0})] [%thread] - %msg%n
</pattern>
</encoder>
</appender>
- <root name="error" level="error">
+ <root level="INFO">
<appender-ref ref="STDOUT"/>
</root>
- <root name="application" level="info">
+ <logger name="application" level="DEBUG">
<appender-ref ref="STDOUT"/>
- </root>
+ </logger>
+
+ <logger name="error" level="DEBUG">
+ <appender-ref ref="STDOUT"/>
+ </logger>
</configuration>
+
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5a77433d/s2core/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf
index 5727b28..176159a 100644
--- a/s2core/src/main/resources/reference.conf
+++ b/s2core/src/main/resources/reference.conf
@@ -1,22 +1,18 @@
# APP PHASE
-phase=dev
+phase = dev
-host=localhost
+host = localhost
-logger.root=DEBUG
-
-logger.application=DEBUG
-
-hbase.zookeeper.quorum=${host}
+hbase.zookeeper.quorum = ${host}
# DB
-s2graph.models.table.name="models-dev"
-db.default.driver="com.mysql.jdbc.Driver"
-db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
-db.default.user="graph"
-db.default.password="graph"
+s2graph.models.table.name = "models-dev"
+db.default.driver = "com.mysql.jdbc.Driver"
+db.default.url = "jdbc:mysql://"${host}":3306/graph_dev"
+db.default.user = "graph"
+db.default.password = "graph"
-cache.max.size=100000
+cache.max.size = 100000
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5a77433d/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
index acb7573..bad8a13 100644
--- a/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
+++ b/s2core/src/main/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseStorage.scala
@@ -57,7 +57,7 @@ class AsynchbaseStorage(val config: Config, vertexCache: Cache[Integer, Option[V
val queryBuilder = new AsynchbaseQueryBuilder(this)(ec)
val mutationBuilder = new AsynchbaseMutationBuilder(this)(ec)
-// val cacheOpt = Option(cache)
+ // val cacheOpt = Option(cache)
val cacheOpt = None
val vertexCacheOpt = Option(vertexCache)
@@ -312,7 +312,7 @@ class AsynchbaseStorage(val config: Config, vertexCache: Cache[Integer, Option[V
}
}
val newVersion = kvOpt.map(_.timestamp()).getOrElse(edge.ts) + 1
-// snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
+ // snapshotEdgeOpt.map(_.version).getOrElse(edge.ts) + 1
val pendingEdge = edge.copy(version = newVersion, statusCode = 1, lockTs = lockTs)
val base = snapshotEdgeOpt match {
case None =>
@@ -471,9 +471,9 @@ class AsynchbaseStorage(val config: Config, vertexCache: Cache[Integer, Option[V
edgeUpdate: EdgeMutate): Future[Boolean] = {
val label = edge.label
def oldBytes = kvOpt.map(_.value()).getOrElse(Array.empty)
-// def oldBytes = snapshotEdgeOpt.map { e =>
-// snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head.value
-// }.getOrElse(Array.empty)
+ // def oldBytes = snapshotEdgeOpt.map { e =>
+ // snapshotEdgeSerializer(e.toSnapshotEdge).toKeyValues.head.value
+ // }.getOrElse(Array.empty)
def process(lockEdge: SnapshotEdge,
releaseLockEdge: SnapshotEdge,
_edgeMutate: EdgeMutate,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5a77433d/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
index 9b876c0..4c94389 100644
--- a/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
+++ b/s2core/src/test/scala/com/kakao/s2graph/core/storage/hbase/AsynchbaseQueryBuilderTest.scala
@@ -1,6 +1,7 @@
package com.kakao.s2graph.core.storage.hbase
import com.kakao.s2graph.core.Graph
+import com.kakao.s2graph.core.utils.logger
import com.typesafe.config.ConfigFactory
import org.apache.hadoop.hbase.util.Bytes
import org.hbase.async.GetRequest
@@ -46,6 +47,6 @@ class AsynchbaseQueryBuilderTest extends FunSuite with Matchers {
}
val duration = System.nanoTime() - startedAt
- println(s">> bytes: $duration")
+ logger.debug(s">> bytes: $duration")
}
}