You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2019/01/25 22:48:53 UTC
[01/20] incubator-s2graph git commit: s2http initial commit [Forced
Update!]
Repository: incubator-s2graph
Updated Branches:
refs/heads/master ef0b6e51a -> 19254301d (forced update)
s2http initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a16ecb08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a16ecb08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a16ecb08
Branch: refs/heads/master
Commit: a16ecb08819ea01f4498dfdd371717de56387a72
Parents: 66b656f
Author: daewon <da...@apache.org>
Authored: Mon Nov 26 17:11:36 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Mon Nov 26 17:32:33 2018 +0900
----------------------------------------------------------------------
build.sbt | 8 +-
s2core/src/main/resources/reference.conf | 10 +-
s2http/README.md | 47 +++++++
s2http/build.sbt | 41 +++++++
s2http/src/main/resources/application.conf | 27 +++++
s2http/src/main/resources/log4j.properties | 26 ++++
.../apache/s2graph/http/S2GraphAdminRoute.scala | 121 +++++++++++++++++++
.../s2graph/http/S2GraphTraversalRoute.scala | 65 ++++++++++
.../scala/org/apache/s2graph/http/Server.scala | 70 +++++++++++
.../apache/s2graph/http/AdminRouteSpec.scala | 61 ++++++++++
10 files changed, 467 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 03e9bd0..c4d2c35 100755
--- a/build.sbt
+++ b/build.sbt
@@ -46,7 +46,7 @@ lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogba
.settings(dockerSettings: _*)
.enablePlugins(sbtdocker.DockerPlugin, JavaAppPackaging)
-lazy val s2rest_netty = project
+lazy val s2http = project
.dependsOn(s2core)
.settings(commonSettings: _*)
.settings(dockerSettings: _*)
@@ -78,8 +78,8 @@ lazy val s2graph_gremlin = project.dependsOn(s2core)
.settings(commonSettings: _*)
lazy val root = (project in file("."))
- .aggregate(s2core, s2rest_play, s2jobs)
- .dependsOn(s2rest_play, s2rest_netty, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project
+ .aggregate(s2core, s2rest_play, s2jobs, s2http)
+ .dependsOn(s2rest_play, s2http, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project
.settings(commonSettings: _*)
lazy val dockerSettings = Seq(
@@ -124,7 +124,7 @@ Packager.defaultSettings
publishSigned := {
(publishSigned in s2rest_play).value
- (publishSigned in s2rest_netty).value
+ (publishSigned in s2http).value
(publishSigned in s2core).value
(publishSigned in spark).value
(publishSigned in s2jobs).value
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2core/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf
index 73c949a..e0681dc 100644
--- a/s2core/src/main/resources/reference.conf
+++ b/s2core/src/main/resources/reference.conf
@@ -53,9 +53,9 @@ db.default.url = "jdbc:h2:file:./var/metastore;MODE=MYSQL"
db.default.user = "sa"
db.default.password = "sa"
-
-akka {
- loggers = ["akka.event.slf4j.Slf4jLogger"]
- loglevel = "DEBUG"
-}
+//
+//akka {
+// loggers = ["akka.event.slf4j.Slf4jLogger"]
+// loglevel = "DEBUG"
+//}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/README.md
----------------------------------------------------------------------
diff --git a/s2http/README.md b/s2http/README.md
new file mode 100644
index 0000000..18f43a2
--- /dev/null
+++ b/s2http/README.md
@@ -0,0 +1,47 @@
+<!---
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+--->
+# S2Graph HTTP Layer
+
+## Development Setup
+
+### Run Server
+Let's run http server.
+
+```bash
+sbt 'project s2http' '~reStart'
+```
+
+When the server is running, connect to `http://localhost:8000`. If it works normally, you can see the following screen.
+
+```json
+{
+ "port": 8000,
+ "started_at": 1543218853354
+}
+```
+
+### API testing
+```test
+sbt 'project s2http' "test-only *s2graph.http*"
+```
+
+### API List
+ - link to S2GraphDocument
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/build.sbt
----------------------------------------------------------------------
diff --git a/s2http/build.sbt b/s2http/build.sbt
new file mode 100644
index 0000000..944493c
--- /dev/null
+++ b/s2http/build.sbt
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+lazy val akkaHttpVersion = "10.1.5"
+lazy val akkaVersion = "2.5.18"
+
+name := "s2http"
+
+version := "0.1"
+
+description := "s2graph http server"
+
+scalacOptions ++= Seq("-deprecation", "-feature")
+
+libraryDependencies ++= Seq(
+ "com.typesafe.akka" %% "akka-http" % akkaHttpVersion,
+ "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion,
+ "com.typesafe.akka" %% "akka-stream" % akkaVersion,
+
+ "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % Test,
+ "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test,
+
+ "org.scalatest" %% "scalatest" % "3.0.5" % Test
+)
+
+Revolver.settings
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2http/src/main/resources/application.conf b/s2http/src/main/resources/application.conf
new file mode 100644
index 0000000..7019e02
--- /dev/null
+++ b/s2http/src/main/resources/application.conf
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+//db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL",
+//db.default.password = sa
+//db.default.user = sa
+//s2graph.storage.backend = rocks
+//rocks.storage.file.path = rocks_db
+//rocks.storage.mode = production
+//rocks.storage.ttl = -1
+//rocks.storage.read.only = false
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/s2http/src/main/resources/log4j.properties b/s2http/src/main/resources/log4j.properties
new file mode 100644
index 0000000..2070d82
--- /dev/null
+++ b/s2http/src/main/resources/log4j.properties
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=INFO, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
new file mode 100644
index 0000000..c2b832e
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
@@ -0,0 +1,121 @@
+package org.apache.s2graph.http
+
+import akka.http.scaladsl.model._
+import org.apache.s2graph.core.rest.RequestParser
+import org.apache.s2graph.core.{Management, S2Graph}
+
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.server.Directives._
+
+import org.slf4j.LoggerFactory
+import play.api.libs.json._
+
+import scala.util.Try
+
+object S2GraphAdminRoute {
+
+ import scala.util._
+
+ trait AdminMessageFormatter[T] {
+ def toJson(msg: T): JsValue
+ }
+
+ import scala.language.reflectiveCalls
+
+ type ToPlayJson = {def toJson: JsValue}
+
+ object AdminMessageFormatter {
+
+ implicit def toPlayJson[A <: ToPlayJson] = new AdminMessageFormatter[A] {
+ def toJson(js: A) = js.toJson
+ }
+
+ implicit def fromPlayJson[T <: JsValue] = new AdminMessageFormatter[T] {
+ def toJson(js: T) = js
+ }
+ }
+
+ def toHttpEntity[A: AdminMessageFormatter](opt: Option[A], status: StatusCode = StatusCodes.OK, message: String = ""): HttpResponse = {
+ val ev = implicitly[AdminMessageFormatter[A]]
+ val res = opt.map(ev.toJson).getOrElse(Json.obj("message" -> message))
+
+ HttpResponse(
+ status = status,
+ entity = HttpEntity(ContentTypes.`application/json`, res.toString)
+ )
+ }
+
+ def toHttpEntity[A: AdminMessageFormatter](opt: Try[A]): HttpResponse = {
+ val ev = implicitly[AdminMessageFormatter[A]]
+ val (status, res) = opt match {
+ case Success(m) => StatusCodes.Created -> Json.obj("status" -> "ok", "message" -> ev.toJson(m))
+ case Failure(e) => StatusCodes.OK -> Json.obj("status" -> "failure", "message" -> e.toString)
+ }
+
+ toHttpEntity(Option(res), status = status)
+ }
+}
+
+trait S2GraphAdminRoute {
+
+ import S2GraphAdminRoute._
+
+ val s2graph: S2Graph
+ val logger = LoggerFactory.getLogger(this.getClass)
+
+ lazy val management: Management = s2graph.management
+ lazy val requestParser: RequestParser = new RequestParser(s2graph)
+
+ // routes impl
+ lazy val getLabel = path("getLabel" / Segment) { labelName =>
+ val labelOpt = Management.findLabel(labelName)
+
+ complete(toHttpEntity(labelOpt, message = s"Label not found: ${labelName}"))
+ }
+
+ lazy val getService = path("getService" / Segment) { serviceName =>
+ val serviceOpt = Management.findService(serviceName)
+
+ complete(toHttpEntity(serviceOpt, message = s"Service not found: ${serviceName}"))
+ }
+
+ lazy val createService = path("createService") {
+ entity(as[String]) { body =>
+ val params = Json.parse(body)
+
+ val parseTry = Try(requestParser.toServiceElements(params))
+ val serviceTry = for {
+ (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) <- parseTry
+ service <- management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm)
+ } yield service
+
+ complete(toHttpEntity(serviceTry))
+ }
+ }
+
+ lazy val createLabel = path("createLabel") {
+ entity(as[String]) { body =>
+ val params = Json.parse(body)
+
+ val labelTry = for {
+ label <- requestParser.toLabelElements(params)
+ } yield label
+
+ complete(toHttpEntity(labelTry))
+ }
+ }
+
+ // expose routes
+ lazy val adminRoute: Route =
+ get {
+ concat(
+ getService,
+ getLabel
+ )
+ } ~ post {
+ concat(
+ createService,
+ createLabel
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
new file mode 100644
index 0000000..1814436
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
@@ -0,0 +1,65 @@
+package org.apache.s2graph.http
+
+import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.core.rest.RestHandler.CanLookup
+import org.slf4j.LoggerFactory
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.model.headers.RawHeader
+import akka.http.scaladsl.model.{HttpHeader, StatusCodes}
+import org.apache.s2graph.core.GraphExceptions.{BadQueryException, JsonParseException}
+import org.apache.s2graph.core.rest.RestHandler
+
+
+object S2GraphTraversalRoute {
+
+ import scala.collection._
+
+ implicit val akkHttpHeaderLookup = new CanLookup[immutable.Seq[HttpHeader]] {
+ override def lookup(m: immutable.Seq[HttpHeader], key: String): Option[String] = m.find(_.name() == key).map(_.value())
+ }
+}
+
+trait S2GraphTraversalRoute {
+
+ import S2GraphTraversalRoute._
+
+ val s2graph: S2Graph
+ val logger = LoggerFactory.getLogger(this.getClass)
+
+ implicit lazy val ec = s2graph.ec
+ lazy val restHandler = new RestHandler(s2graph)
+
+ // The `/graphs/*` APIs are implemented to be branched from the existing restHandler.doPost.
+ // Implement it first by delegating that function.
+ lazy val delegated: Route = {
+ entity(as[String]) { body =>
+ logger.info(body)
+
+ extractRequest { request =>
+ val result = restHandler.doPost(request.uri.toRelative.toString(), body, request.headers)
+ val responseHeaders = result.headers.toList.map { case (k, v) => RawHeader(k, v) }
+
+ val f = result.body.map(StatusCodes.OK -> _.toString).recover {
+ case BadQueryException(msg, _) => StatusCodes.BadRequest -> msg
+ case JsonParseException(msg) => StatusCodes.BadRequest -> msg
+ case e: Exception => StatusCodes.InternalServerError -> e.toString
+ }
+
+ respondWithHeaders(responseHeaders) {
+ complete(f)
+ }
+ }
+ }
+ }
+
+ // expose routes
+ lazy val traversalRoute: Route =
+ post {
+ concat(
+ delegated // getEdges, experiments, etc.
+ )
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
new file mode 100644
index 0000000..82d2343
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.http
+
+import scala.language.postfixOps
+import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.duration.Duration
+import scala.util.{Failure, Success}
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes}
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.server.Directives._
+import akka.stream.ActorMaterializer
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.S2Graph
+import org.slf4j.LoggerFactory
+
+object Server extends App
+ with S2GraphTraversalRoute
+ with S2GraphAdminRoute {
+
+ implicit val system: ActorSystem = ActorSystem("S2GraphHttpServer")
+ implicit val materializer: ActorMaterializer = ActorMaterializer()
+ implicit val executionContext: ExecutionContext = system.dispatcher
+
+ val config = ConfigFactory.load()
+
+ override val s2graph = new S2Graph(config)
+ override val logger = LoggerFactory.getLogger(this.getClass)
+
+ val port = sys.props.get("http.port").fold(8000)(_.toInt)
+ val serverStatus = s""" { "port": ${port}, "started_at": ${System.currentTimeMillis()} }"""
+
+ val health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverStatus))
+
+ // Allows you to determine routes to expose according to external settings.
+ lazy val routes: Route = concat(
+ pathPrefix("graphs")(traversalRoute),
+ pathPrefix("admin")(adminRoute),
+ get(complete(health))
+ )
+
+ val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, "localhost", port)
+ binding.onComplete {
+ case Success(bound) => logger.info(s"Server online at http://${bound.localAddress.getHostString}:${bound.localAddress.getPort}/")
+ case Failure(e) =>
+ logger.error(s"Server could not start!", e)
+ system.terminate()
+ }
+
+ Await.result(system.whenTerminated, Duration.Inf)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
new file mode 100644
index 0000000..7e5d794
--- /dev/null
+++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
@@ -0,0 +1,61 @@
+package org.apache.s2graph.http
+
+import akka.http.scaladsl.marshalling.Marshal
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.testkit.ScalatestRouteTest
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.S2Graph
+import org.scalatest.concurrent.ScalaFutures
+import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec}
+import org.slf4j.LoggerFactory
+import play.api.libs.json.{JsString, Json}
+
+class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll {
+ val config = ConfigFactory.load()
+ val s2graph = new S2Graph(config)
+ override val logger = LoggerFactory.getLogger(this.getClass)
+
+ override def afterAll(): Unit = {
+ s2graph.shutdown()
+ }
+
+ import akka.http.scaladsl.server.Directives._
+
+ lazy val routes = adminRoute
+
+ "AdminRoute" should {
+ "be able to create service (POST /createService)" in {
+ val serviceParam = Json.obj(
+ "serviceName" -> "kakaoFavorites",
+ "compressionAlgorithm" -> "gz"
+ )
+
+ val serviceEntity = Marshal(serviceParam.toString).to[MessageEntity].futureValue
+ val request = Post("/createService").withEntity(serviceEntity)
+
+ request ~> routes ~> check {
+ status should ===(StatusCodes.Created)
+ contentType should ===(ContentTypes.`application/json`)
+
+ val response = Json.parse(entityAs[String])
+
+ (response \\ "name").head should ===(JsString("kakaoFavorites"))
+ (response \\ "status").head should ===(JsString("ok"))
+ }
+ }
+
+ "return service if present (GET /getService/{serviceName})" in {
+ val request = HttpRequest(uri = "/getService/kakaoFavorites")
+
+ request ~> routes ~> check {
+ status should ===(StatusCodes.OK)
+ contentType should ===(ContentTypes.`application/json`)
+
+ val response = Json.parse(entityAs[String])
+
+ (response \\ "name").head should ===(JsString("kakaoFavorites"))
+ }
+ }
+ }
+}
+
[19/20] incubator-s2graph git commit: Merge branch 'S2GRAPH-248' of
https://github.com/daewon/incubator-s2graph into daewon-S2GRAPH-248
Posted by st...@apache.org.
Merge branch 'S2GRAPH-248' of https://github.com/daewon/incubator-s2graph into daewon-S2GRAPH-248
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/53ff5cd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/53ff5cd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/53ff5cd8
Branch: refs/heads/master
Commit: 53ff5cd89d6df64f44656c662768d8598806e268
Parents: f77a152 8c3f4a2
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu Jan 24 21:14:49 2019 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu Jan 24 21:14:49 2019 +0900
----------------------------------------------------------------------
README.md | 193 ++--
bin/s2graph-daemon.sh | 9 +-
bin/s2graph.sh | 2 +-
bin/start-s2graph.sh | 2 +-
bin/stop-s2graph.sh | 2 +-
build.sbt | 20 +-
dev_support/README.md | 40 +-
doc/source/_static/css/s2graph_style.css | 1 +
doc/source/api/index.rst | 13 +
doc/source/api/management/index.rst | 444 +++++++++
doc/source/api/mutate/index.rst | 10 +
doc/source/api/mutate/mutate_edge.rst | 328 +++++++
doc/source/api/mutate/mutate_vertex.rst | 139 +++
doc/source/api/query/index.rst | 15 +
doc/source/api/query/query_edge.rst | 954 +++++++++++++++++++
doc/source/api/query/query_options.rst | 27 +
doc/source/api/query/query_vertex.rst | 17 +
doc/source/conf.py | 2 +
doc/source/getting_started/your_first_graph.rst | 14 +-
doc/source/index.rst | 7 +-
s2core/src/main/resources/reference.conf | 10 +-
.../org/apache/s2graph/core/Management.scala | 44 +
.../org/apache/s2graph/core/schema/Label.scala | 38 +-
s2graphql/build.sbt | 12 +-
s2graphql/src/main/resources/application.conf | 11 +-
s2graphql/src/main/resources/assets/.gitignore | 1 -
.../src/main/resources/assets/graphiql.html | 151 +++
.../apache/s2graph/graphql/GraphQLServer.scala | 81 +-
.../org/apache/s2graph/graphql/HttpServer.scala | 154 ---
s2graphql/src/test/resources/application.conf | 11 +-
.../org/apache/s2graph/graphql/TestGraph.scala | 2 +-
s2http/README.md | 47 +
s2http/build.sbt | 41 +
s2http/src/main/resources/application.conf | 27 +
s2http/src/main/resources/log4j.properties | 26 +
.../apache/s2graph/http/PlayJsonSupport.scala | 74 ++
.../apache/s2graph/http/S2GraphAdminRoute.scala | 281 ++++++
.../s2graph/http/S2GraphMutateRoute.scala | 97 ++
.../apache/s2graph/http/S2GraphQLRoute.scala | 105 ++
.../s2graph/http/S2GraphTraversalRoute.scala | 62 ++
.../s2graph/http/SangriaGraphQLSupport.scala | 38 +
.../scala/org/apache/s2graph/http/Server.scala | 86 ++
.../apache/s2graph/http/AdminRouteSpec.scala | 94 ++
.../apache/s2graph/http/MutateRouteSpec.scala | 63 ++
.../rest/play/controllers/AdminController.scala | 32 +-
45 files changed, 3393 insertions(+), 434 deletions(-)
----------------------------------------------------------------------
[09/20] incubator-s2graph git commit: update document for new api
route
Posted by st...@apache.org.
update document for new api route
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/455e1551
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/455e1551
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/455e1551
Branch: refs/heads/master
Commit: 455e1551ce95dc6754ffbdf1c9311f09f16c2685
Parents: 955e21d
Author: daewon <da...@apache.org>
Authored: Wed Dec 26 16:18:22 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Dec 26 16:18:22 2018 +0900
----------------------------------------------------------------------
doc/source/getting_started/your_first_graph.rst | 14 ++++----
.../apache/s2graph/http/S2GraphAdminRoute.scala | 34 ++++++++++----------
.../scala/org/apache/s2graph/http/Server.scala | 6 ++--
3 files changed, 28 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/455e1551/doc/source/getting_started/your_first_graph.rst
----------------------------------------------------------------------
diff --git a/doc/source/getting_started/your_first_graph.rst b/doc/source/getting_started/your_first_graph.rst
index 72d6e08..916cf53 100644
--- a/doc/source/getting_started/your_first_graph.rst
+++ b/doc/source/getting_started/your_first_graph.rst
@@ -11,7 +11,7 @@ The following POST query will create a service named ``KakaoFavorites``
.. code:: bash
- curl -XPOST localhost:9000/graphs/createService -H 'Content-Type: Application/json' -d '
+ curl -XPOST localhost:9000/admin/createService -H 'Content-Type: Application/json' -d '
{
"serviceName": "KakaoFavorites",
"compressionAlgorithm" : "gz"
@@ -21,7 +21,7 @@ To make sure the service is created correctly, check out the following
.. code:: bash
- curl -XGET localhost:9000/graphs/getService/KakaoFavorites
+ curl -XGET localhost:9000/admin/getService/KakaoFavorites
Next, we will need some friends.
---------------------------------------------
@@ -29,7 +29,7 @@ In S2Graph, relationships are organized as labels. Create a ``friends`` label wi
.. code:: bash
- curl -XPOST localhost:9000/graphs/createLabel -H 'Content-Type: Application/json' -d '
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
{
"label": "friends",
"srcServiceName": "KakaoFavorites",
@@ -48,13 +48,13 @@ Check if the label has been created correctly:
.. code:: bash
- curl -XGET localhost:9000/graphs/getLabel/friends
+ curl -XGET localhost:9000/admin/getLabel/friends
Now that the label ``friends`` is ready, we can store the friendship data. Entries of a label are called edges, and you can add edges with ``edges/insert`` API:
.. code:: bash
- curl -XPOST localhost:9000/graphs/edges/insert -H 'Content-Type: Application/json' -d '
+ curl -XPOST localhost:9000/mutate/edge/insert -H 'Content-Type: Application/json' -d '
[
{"from":"Elmo","to":"Big Bird","label":"friends","props":{},"timestamp":1444360152477},
{"from":"Elmo","to":"Ernie","label":"friends","props":{},"timestamp":1444360152478},
@@ -95,7 +95,7 @@ We will need a new label ``post`` for this data:
.. code:: bash
- curl -XPOST localhost:9000/graphs/createLabel -H 'Content-Type: Application/json' -d '
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
{
"label": "post",
"srcServiceName": "KakaoFavorites",
@@ -114,7 +114,7 @@ Now, insert some posts of the users:
.. code:: bash
- curl -XPOST localhost:9000/graphs/edges/insert -H 'Content-Type: Application/json' -d '
+ curl -XPOST localhost:9000/mutate/edge/insert -H 'Content-Type: Application/json' -d '
[
{"from":"Big Bird","to":"www.kakaocorp.com/en/main","label":"post","props":{},"timestamp":1444360152477},
{"from":"Big Bird","to":"github.com/kakao/s2graph","label":"post","props":{},"timestamp":1444360152478},
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/455e1551/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
index 47ac86a..1219968 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
@@ -56,27 +56,27 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
// routes impl
/* GET */
- // GET /graphs/getService/:serviceName
+ // GET /admin/getService/:serviceName
lazy val getService = path("getService" / Segment) { serviceName =>
val serviceOpt = Management.findService(serviceName)
complete(toHttpEntity(serviceOpt, message = s"Service not found: ${serviceName}"))
}
- // GET /graphs/getServiceColumn/:serviceName/:columnName
+ // GET /admin/getServiceColumn/:serviceName/:columnName
lazy val getServiceColumn = path("getServiceColumn" / Segment / Segment) { (serviceName, columnName) =>
val ret = Management.findServiceColumn(serviceName, columnName)
complete(toHttpEntity(ret, message = s"ServiceColumn not found: ${serviceName}, ${columnName}"))
}
- // GET /graphs/getLabel/:labelName
+ // GET /admin/getLabel/:labelName
lazy val getLabel = path("getLabel" / Segment) { labelName =>
val labelOpt = Management.findLabel(labelName)
complete(toHttpEntity(labelOpt, message = s"Label not found: ${labelName}"))
}
- // GET /graphs/getLabels/:serviceName
+ // GET /admin/getLabels/:serviceName
lazy val getLabels = path("getLabels" / Segment) { serviceName =>
val ret = Management.findLabels(serviceName)
@@ -84,7 +84,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
/* POST */
- // POST /graphs/createService
+ // POST /admin/createService
lazy val createService = path("createService") {
entity(as[JsValue]) { params =>
@@ -98,7 +98,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/createServiceColumn
+ // POST /admin/createServiceColumn
lazy val createServiceColumn = path("createServiceColumn") {
entity(as[JsValue]) { params =>
@@ -112,7 +112,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/createLabel
+ // POST /admin/createLabel
lazy val createLabel = path("createLabel") {
entity(as[JsValue]) { params =>
val labelTry = requestParser.toLabelElements(params)
@@ -133,7 +133,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/addProp/:labelName
+ // POST /admin/addProp/:labelName
lazy val addProp = path("addProp" / Segment) { labelName =>
entity(as[JsValue]) { params =>
val labelMetaTry = for {
@@ -145,7 +145,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/addServiceColumnProp/:serviceName/:columnName
+ // POST /admin/addServiceColumnProp/:serviceName/:columnName
lazy val addServiceColumnProp = path("addServiceColumnProp" / Segments) { params =>
val (serviceName, columnName, storeInGlobalIndex) = params match {
case s :: c :: Nil => (s, c, false)
@@ -166,7 +166,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/createHTable
+ // POST /admin/createHTable
lazy val createHTable = path("createHTable") {
entity(as[JsValue]) { params =>
params.validate[HTableParams] match {
@@ -182,14 +182,14 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/copyLabel/:oldLabelName/:newLabelName
+ // POST /admin/copyLabel/:oldLabelName/:newLabelName
lazy val copyLabel = path("copyLabel" / Segment / Segment) { (oldLabelName, newLabelName) =>
val copyTry = management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
complete(toHttpEntity(copyTry))
}
- // POST /graphs/renameLabel/:oldLabelName/:newLabelName
+ // POST /admin/renameLabel/:oldLabelName/:newLabelName
lazy val renameLabel = path("renameLabel" / Segment / Segment) { (oldLabelName, newLabelName) =>
Label.findByName(oldLabelName) match {
case None => complete(toHttpEntity(None: Option[JsValue], status = StatusCodes.NotFound, message = s"Label $oldLabelName not found."))
@@ -200,7 +200,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/swapLabels/:leftLabelName/:rightLabelName
+ // POST /admin/swapLabels/:leftLabelName/:rightLabelName
lazy val swapLabel = path("swapLabel" / Segment / Segment) { (leftLabelName, rightLabelName) =>
val left = Label.findByName(leftLabelName, useCache = false)
val right = Label.findByName(rightLabelName, useCache = false)
@@ -215,7 +215,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
- // POST /graphs/updateHTable/:labelName/:newHTableName
+ // POST /admin/updateHTable/:labelName/:newHTableName
lazy val updateHTable = path("updateHTable" / Segment / Segment) { (labelName, newHTableName) =>
val updateTry = Management.updateHTable(labelName, newHTableName).map(Json.toJson(_))
@@ -223,21 +223,21 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
/* PUT */
- // PUT /graphs/deleteLabelReally/:labelName
+ // PUT /admin/deleteLabelReally/:labelName
lazy val deleteLabelReally = path("deleteLabelReally" / Segment) { labelName =>
val ret = Management.deleteLabel(labelName).toOption
complete(toHttpEntity(ret, message = s"Label not found: ${labelName}"))
}
- // PUT /graphs/markDeletedLabel/:labelName
+ // PUT /admin/markDeletedLabel/:labelName
lazy val markDeletedLabel = path("markDeletedLabel" / Segment) { labelName =>
val ret = Management.markDeletedLabel(labelName).toOption.map(Json.toJson(_))
complete(toHttpEntity(ret, message = s"Label not found: ${labelName}"))
}
- // PUT /graphs/deleteServiceColumn/:serviceName/:columnName
+ // PUT /admin/deleteServiceColumn/:serviceName/:columnName
lazy val deleteServiceColumn = path("deleteServiceColumn" / Segment / Segment) { (serviceName, columnName) =>
val ret = Management.deleteColumn(serviceName, columnName).toOption
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/455e1551/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
index 00146f6..03802ed 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
@@ -49,7 +49,9 @@ object Server extends App
override val logger = LoggerFactory.getLogger(this.getClass)
val port = sys.props.get("http.port").fold(8000)(_.toInt)
- val serverStatus = s""" { "port": ${port}, "started_at": ${System.currentTimeMillis()} }"""
+ val interface = sys.props.get("http.interface").fold("0.0.0.0")(identity)
+
+ val serverStatus = s""" { "port": ${port}, "interface": ${interface}, "started_at": ${System.currentTimeMillis()} }"""
val health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverStatus))
@@ -62,7 +64,7 @@ object Server extends App
get(complete(health))
)
- val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, "localhost", port)
+ val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, interface, port)
binding.onComplete {
case Success(bound) => logger.info(s"Server online at http://${bound.localAddress.getHostString}:${bound.localAddress.getPort}/")
case Failure(e) => logger.error(s"Server could not start!", e)
[10/20] incubator-s2graph git commit: add api document
Posted by st...@apache.org.
add api document
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/7e4f6c81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/7e4f6c81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/7e4f6c81
Branch: refs/heads/master
Commit: 7e4f6c81f61f6c718a5e8bdb4b53cb534c21c1b5
Parents: 455e155
Author: daewon <da...@apache.org>
Authored: Wed Dec 26 19:19:35 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Dec 26 19:19:35 2018 +0900
----------------------------------------------------------------------
doc/source/conf.py | 2 ++
doc/source/index.rst | 1 +
.../src/main/scala/org/apache/s2graph/http/Server.scala | 10 ++++++++--
3 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7e4f6c81/doc/source/conf.py
----------------------------------------------------------------------
diff --git a/doc/source/conf.py b/doc/source/conf.py
index afa4c10..5736d04 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -46,6 +46,8 @@ extensions = [
'sphinx.ext.githubpages',
]
+html_style = 'css/s2graph_style.css'
+
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7e4f6c81/doc/source/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 83bf8d5..fd4e755 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -20,6 +20,7 @@ dev@s2graph.incubator.apache.org is for people who want to contribute to S2Graph
getting_started/index
getting_started/your_first_graph
+ api/index
..
Indices and tables
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/7e4f6c81/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
index 03802ed..e70a182 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
@@ -19,6 +19,8 @@
package org.apache.s2graph.http
+import java.time.Instant
+
import scala.language.postfixOps
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.duration.Duration
@@ -51,9 +53,13 @@ object Server extends App
val port = sys.props.get("http.port").fold(8000)(_.toInt)
val interface = sys.props.get("http.interface").fold("0.0.0.0")(identity)
- val serverStatus = s""" { "port": ${port}, "interface": ${interface}, "started_at": ${System.currentTimeMillis()} }"""
+ val startAt = System.currentTimeMillis()
+
+ def uptime = System.currentTimeMillis() - startAt
+
+ def serverHealth = s"""{ "port": ${port}, "interface": "${interface}", "started_at": ${Instant.ofEpochMilli(startAt)}, "uptime": "${uptime} millis" """
- val health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverStatus))
+ def health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverHealth))
// Allows you to determine routes to expose according to external settings.
lazy val routes: Route = concat(
[15/20] incubator-s2graph git commit: add edge query
Posted by st...@apache.org.
add edge query
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/3bb9f029
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/3bb9f029
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/3bb9f029
Branch: refs/heads/master
Commit: 3bb9f0294478568c86b66461d11df2f13e3daa36
Parents: aa309b2
Author: daewon <da...@apache.org>
Authored: Thu Dec 27 18:42:41 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Dec 27 18:42:41 2018 +0900
----------------------------------------------------------------------
doc/source/api/query/index.rst | 17 +
doc/source/api/query/query_edge.rst | 958 ++++++++++++++++++++++++++++
doc/source/api/query/query_options.rst | 27 +
doc/source/api/query/query_vertex.rst | 4 +
4 files changed, 1006 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3bb9f029/doc/source/api/query/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/index.rst b/doc/source/api/query/index.rst
new file mode 100644
index 0000000..26e2b01
--- /dev/null
+++ b/doc/source/api/query/index.rst
@@ -0,0 +1,17 @@
+Query APIs
+==============================================
+
+Once you have your graph data uploaded to S2Graph, you can traverse your graph using our query APIs.
+S2Graph query DSL define ``breadth first search(BFS)`` and various options for ``ranking`` subgraph that BFS return.
+Query DSL has many options for ranking and filtering.
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ query_options
+ query_edge
+
+ ..
+ query_vertex
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3bb9f029/doc/source/api/query/query_edge.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/query_edge.rst b/doc/source/api/query/query_edge.rst
new file mode 100644
index 0000000..2b74daf
--- /dev/null
+++ b/doc/source/api/query/query_edge.rst
@@ -0,0 +1,958 @@
+****************
+Edge Queries
+****************
+
+S2Graph provides a query DSL which has been reported to have a pretty steep learning curve.
+One tip is to try to understand each features by projecting it to that of a RDBMS such MySQL.
+This doesn't work all the time, but there are many similarities between S2Graph and a conventional RDBMS.
+For example, S2Graphs "getEdges" is used to fetch data and traverse multiple steps. This is very similar to the "SELECT" query in MySQL.
+
+Traversing each step is similar to ``join`` operation in RDBMS. One thing to note here is that users must start their traverse from smaller set to terminate BFS early as soon as possible.
+Another tip is to not be shy to ask! Ask any questions on our `mailing list`_. list or open an issue at our `github`_ with the problems that you're having with S2Graph.
+
+.. _mailing list: https://groups.google.com/forum/#!forum/s2graph
+
+.. _github: https://github.com/apache/incubator-s2graph
+
+
+APIs
+-------------------------------------
+
+
+checkEdges - ``POST /graphs/checkEdges``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+return edge for given vertex pair only if edge exist.
+This is more ``general`` way to check edge existence between any given vertex pairs comparing using ``_to`` on query parameter
+
+
+.. code:: bashn
+
+ curl -XPOST localhost:9000/graphs/checkEdges -H 'Content-Type: Application/json' -d '
+ [
+ {"label": "talk_friend", "direction": "out", "from": 1, "to": 100},
+ {"label": "talk_friend", "direction": "out", "from": 1, "to": 101}
+ ]'
+
+
+getEdges - ``POST /graphs/getEdges``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Select edges with query.
+
+**Duplicate Policy**
+
+Here is a very basic query to fetch all edges that start from source vertex "101".
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 101
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]
+ }
+ ]
+ }'
+
+
+``Notice the "duplicate" field``. If a target label's consistency level is ``weak`` and multiple edges exist with the same (from, to, label, direction) id, then the query is expect to have a policy for handling edge duplicates. S2Graph provides four duplicate policies on edges.
+
+
+.. note::
+ - raw: Allow duplicates and return all edges.
+ - first: Return only the first edge if multiple edges exist. This is default.
+ - countSum: Return only one edge, and return how many duplicates exist.
+ - sum: Return only one edge, and return sum of the scores.
+
+
+With duplicate "raw", there are actually three edges with the same (from, to, label, direction) id.
+
+.. code:: bash
+
+ {
+ "size": 3,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -29,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 6,
+ "timestamp": 6,
+ "score": 1,
+ "props": {
+ "_timestamp": 6,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -29,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 5,
+ "timestamp": 5,
+ "score": 1,
+ "props": {
+ "_timestamp": 5,
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -29,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 4,
+ "timestamp": 4,
+ "score": 1,
+ "props": {
+ "_timestamp": 4,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+Duplicate "countSum" returns only one edge with the score sum of 3.
+
+.. code:: bash
+
+ {
+ "size": 1,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -135,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 4,
+ "timestamp": 4,
+ "score": 3,
+ "props": {
+ "_timestamp": 4,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+
+**Select Option Example**
+
+In case you want to control the fields shown in the result edges, use the "select" option.
+
+.. code:: bash
+
+ {
+ "select": ["from", "to", "label"],
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 101
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]
+ }
+ ]
+ }
+
+S2Graph will return only those fields in the result.
+
+.. code:: bash
+
+ {
+ "size": 3,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ }
+ ],
+ "results": [
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak"
+ },
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak"
+ },
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak"
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+Default value of the "select" option is an empty array which means that all edge fields are returned.
+
+
+**groupBy Option Example**
+
+
+Result edges can be grouped by a given field.
+
+.. code:: bash
+
+ {
+ "select": ["from", "to", "label", "direction", "timestamp", "score", "time", "weight", "is_hidden", "is_blocked"],
+ "groupBy": ["from", "to", "label"],
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 101
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]
+ }
+ ]
+ }
+
+
+You can see the result edges are grouped by their "from", "to", and "label" fields.
+
+
+.. code:: bash
+
+ {
+ "size": 1,
+ "results": [
+ {
+ "groupBy": {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak"
+ },
+ "agg": [
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "timestamp": 6,
+ "score": 1,
+ "props": {
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "timestamp": 5,
+ "score": 1,
+ "props": {
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "timestamp": 4,
+ "score": 1,
+ "props": {
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ]
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+
+**filterOut option example**
+
+You can also run two queries concurrently, and filter the result of one query with the result of the other.
+
+.. code:: bash
+
+ {
+ "filterOutFields": ["_to"],
+ "filterOut": {
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 100
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]
+ }
+ ]
+ },
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 101
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw"
+ }
+ ]
+ }
+ ]
+ }
+
+S2Graph will run two concurrent queries, one in the main step, and another in the filter out clause. Here is more practical example.
+
+
+.. coce:: bash
+
+ {
+ "filterOut": {
+ "srcVertices": [
+ {
+ "columnName": "uuid",
+ "id": "Alec",
+ "serviceName": "daumnews"
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "direction": "out",
+ "label": "daumnews_user_view_news",
+ "limit": 100,
+ "offset": 0
+ }
+ ]
+ }
+ ]
+ },
+ "srcVertices": [
+ {
+ "columnName": "uuid",
+ "id": "Alec",
+ "serviceName": "daumnews"
+ }
+ ],
+ "steps": [
+ {
+ "nextStepLimit": 10,
+ "step": [
+ {
+ "direction": "out",
+ "duplicate": "scoreSum",
+ "label": "daumnews_user_view_news",
+ "limit": 100,
+ "offset": 0,
+ "timeDecay": {
+ "decayRate": 0.1,
+ "initial": 1,
+ "timeUnit": 86000000
+ }
+ }
+ ]
+ },
+ {
+ "nextStepLimit": 10,
+ "step": [
+ {
+ "label": "daumnews_news_belongto_category",
+ "limit": 1
+ }
+ ]
+ },
+ {
+ "step": [
+ {
+ "direction": "in",
+ "label": "daumnews_news_belongto_category",
+ "limit": 10
+ }
+ ]
+ }
+ ]
+ }
+
+
+
+The main query from the above will traverse a graph of users and news articles as follows:
+
+1. Fetch the list of news articles that user Alec read.
+2. Get the categories of the result edges of step one.
+3. Fetch other articles that were published in same category.
+
+
+Meanwhile, Alec does not want to get articles that he already read. This can be taken care of with the following query in the filterOut option:
+Articles that Alec has already read.
+
+
+.. code:: bash
+
+ {
+ "size": 5,
+ "degrees": [
+ {
+ "from": "Alec",
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_degree": 6
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -19,
+ "from": "Alec",
+ "to": 20150803143507760,
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_timestamp": 1438591888454,
+ "timestamp": 1438591888454,
+ "score": 0.9342237306639056,
+ "props": {
+ "_timestamp": 1438591888454
+ }
+ },
+ {
+ "cacheRemain": -19,
+ "from": "Alec",
+ "to": 20150803150406010,
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_timestamp": 1438591143640,
+ "timestamp": 1438591143640,
+ "score": 0.9333716513280771,
+ "props": {
+ "_timestamp": 1438591143640
+ }
+ },
+ {
+ "cacheRemain": -19,
+ "from": "Alec",
+ "to": 20150803144908340,
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_timestamp": 1438581933262,
+ "timestamp": 1438581933262,
+ "score": 0.922898833570944,
+ "props": {
+ "_timestamp": 1438581933262
+ }
+ },
+ {
+ "cacheRemain": -19,
+ "from": "Alec",
+ "to": 20150803124627492,
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_timestamp": 1438581485765,
+ "timestamp": 1438581485765,
+ "score": 0.9223930035297659,
+ "props": {
+ "_timestamp": 1438581485765
+ }
+ },
+ {
+ "cacheRemain": -19,
+ "from": "Alec",
+ "to": 20150803113311090,
+ "label": "daumnews_user_view_news",
+ "direction": "out",
+ "_timestamp": 1438580536376,
+ "timestamp": 1438580536376,
+ "score": 0.9213207756669546,
+ "props": {
+ "_timestamp": 1438580536376
+ }
+ }
+ ],
+ "impressionId": 354266627
+ }
+
+
+Without "filterOut"
+
+.. code:: bash
+
+ {
+ "size": 2,
+ "degrees": [
+ {
+ "from": 1028,
+ "label": "daumnews_news_belongto_category",
+ "direction": "in",
+ "_degree": 2
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -33,
+ "from": 1028,
+ "to": 20150803105805092,
+ "label": "daumnews_news_belongto_category",
+ "direction": "in",
+ "_timestamp": 1438590169146,
+ "timestamp": 1438590169146,
+ "score": 0.9342777143725886,
+ "props": {
+ "updateTime": 20150803172249144,
+ "_timestamp": 1438590169146
+ }
+ },
+ {
+ "cacheRemain": -33,
+ "from": 1028,
+ "to": 20150803143507760,
+ "label": "daumnews_news_belongto_category",
+ "direction": "in",
+ "_timestamp": 1438581548486,
+ "timestamp": 1438581548486,
+ "score": 0.9342777143725886,
+ "props": {
+ "updateTime": 20150803145908490,
+ "_timestamp": 1438581548486
+ }
+ }
+ ],
+ "impressionId": -14034523
+ }
+
+
+with "filterOut"
+
+
+.. code:: bash
+
+ {
+ "size": 1,
+ "degrees": [],
+ "results": [
+ {
+ "cacheRemain": 85957406,
+ "from": 1028,
+ "to": 20150803105805092,
+ "label": "daumnews_news_belongto_category",
+ "direction": "in",
+ "_timestamp": 1438590169146,
+ "timestamp": 1438590169146,
+ "score": 0.9343106784173475,
+ "props": {
+ "updateTime": 20150803172249144,
+ "_timestamp": 1438590169146
+ }
+ }
+ ],
+ "impressionId": -14034523
+ }
+
+
+Note that article ``20150803143507760`` has been filtered out.
+
+
+**nextStepLimit Example**
+
+S2Graph provides step-level aggregation so that users can take the top K items from the aggregated results.
+
+**nextStepThreshold Example**
+
+**sample Example**
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ {"sample":2,"step": [{"label": "graph_test", "direction": "out", "offset": 0, "limit": 10, "scoring": {"time": 1, "weight": 1}}]}
+ ]
+ }
+
+
+**transform Example**
+
+With typical two-step query, S2Graph will start the second step from the "_to" (vertex id) values of the first steps' result. With the "transform" option, you can actually use any single field from the result edges' properties of step one.
+
+Add a "transform" option to the query from example 1.
+
+.. code:: bash
+
+ {
+ "select": [],
+ "srcVertices": [
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id_test",
+ "id": 101
+ }
+ ],
+ "steps": [
+ {
+ "step": [
+ {
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "offset": 0,
+ "limit": 10,
+ "duplicate": "raw",
+ "transform": [
+ ["_to"],
+ ["time.$", "time"]
+ ]
+ }
+ ]
+ }
+ ]
+ }
+
+Note that we have six resulting edges. We have two transform rules, the first one simply fetches edges with their target vertex IDs (such as "to": "10"), and the second rule will fetch the same edges but with the "time" values replacing vertex IDs (such as "to": "to": "time.-30").
+
+.. code:: bash
+
+ {
+ "size": 6,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ },
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 6,
+ "timestamp": 6,
+ "score": 1,
+ "props": {
+ "_timestamp": 6,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "time.-30",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 6,
+ "timestamp": 6,
+ "score": 1,
+ "props": {
+ "_timestamp": 6,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 5,
+ "timestamp": 5,
+ "score": 1,
+ "props": {
+ "_timestamp": 5,
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "time.-10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 5,
+ "timestamp": 5,
+ "score": 1,
+ "props": {
+ "_timestamp": 5,
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 4,
+ "timestamp": 4,
+ "score": 1,
+ "props": {
+ "_timestamp": 4,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -8,
+ "from": 101,
+ "to": "time.0",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 4,
+ "timestamp": 4,
+ "score": 1,
+ "props": {
+ "_timestamp": 4,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+
+**Two-Step Traversal Example**
+
+The following query will fetch a user's (id 1) friends of friends by chaining multiple steps:
+
+
+.. code:: bash
+
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ {
+ "step": [
+ {"label": "friends", "direction": "out", "limit": 100}
+ ]
+ },
+ {
+ "step": [
+ {"label": "friends", "direction": "out", "limit": 10}
+ ]
+ }
+ ]
+ }'
+
+**Three-Step Traversal Example**
+
+Add more steps for wider traversals. Be gentle on the limit options since the number of visited edges will increase exponentially and become very heavy on the system.
+
+**More examples**
+
+Example 1. From label "graph_test", select the first 100 edges that start from vertex "account_id = 1", with default sorting.
+
+.. code:: bash
+
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "graph_test", "direction": "out", "offset": 0, "limit": 100
+ }]
+ ]
+ }'
+
+Example 2. Now select between the 50th and 100th edges from the same query.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "graph_test", "direction": "in", "offset": 50, "limit": 50}]
+ ]
+ }'
+
+Example 3. Now add a time range filter so that you will only get the edges that were inserted between 1416214118000 and 1416300000000.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "graph_test", "direction": "in", "offset": 50, "limit": 50, "duration": {"from": 1416214118000, "to": 1416300000000}]
+ ]
+ }'
+
+Example 4. Now add scoring rule to sort the result by indexed properties "time" and "weight", with weights of 1.5 and 10, respectively.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "graph_test", "direction": "in", "offset": 50, "limit": 50, "duration": {"from": 1416214118000, "to": 1416214218000}, "scoring": {"time": 1.5, "weight": 10}]
+ ]
+ }'
+
+
+Example 5. Make a two-step query to fetch friends of friends of a user "account_id = 1". (Limit the first step by 10 friends and the second step by 100.)
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "friends", "direction": "out", "limit": 100}],
+ [{"label": "friends", "direction": "out", "limit": 10}]
+ ]
+ }'
+
+
+Example 6. Make a two-step query to fetch the music playlist of the friends of user "account_id = 1". Limit the first step by 10 friends and the second step by 100 tracks.)
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "talk_friend", "direction": "out", "limit": 100}],
+ [{"label": "play_music", "direction": "out", "limit": 10}]
+ ]
+ }'
+
+
+Example 7. Query the friends of user "account_id = 1" who played the track "track_id = 200".
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getEdges -H 'Content-Type: Application/json' -d '
+ {
+ "srcVertices": [{"serviceName": "s2graph", "columnName": "account_id", "id":1}],
+ "steps": [
+ [{"label": "talk_friend", "direction": "out", "limit": 100}],
+ [{"label": "play_music", "direction": "out", "_to": 200}]
+ ]
+ }'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3bb9f029/doc/source/api/query/query_options.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/query_options.rst b/doc/source/api/query/query_options.rst
new file mode 100644
index 0000000..548a699
--- /dev/null
+++ b/doc/source/api/query/query_options.rst
@@ -0,0 +1,27 @@
+****************
+Query Options
+****************
+
+Query Level Options
+-----------------------
+
+A typical query contains the source vertex as a starting point, a list of labels to traverse, and optional filters or weights for unwanted results and sorting respectively. Query requests are structured as follows
+
+
+QUERY LEVEL TABLE HERE
+
+**Step**
+
+Each step tells S2Graph which labels to traverse in the graph and how to traverse them. Labels in the very first step should be directly connected to the source vertices, labels in the second step should be directly connected to the result vertices of the first step traversal, and so on. A step is specified with a list of ``query params`` which we will cover in detail below.
+
+
+Step Level Option
+-------------------------
+
+STEP LEVEL TABLE HERE
+
+
+Query Param Level Option
+-----------------------------
+
+Query PARAM LEVEL TABLE HERE
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/3bb9f029/doc/source/api/query/query_vertex.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/query_vertex.rst b/doc/source/api/query/query_vertex.rst
new file mode 100644
index 0000000..15f5e3e
--- /dev/null
+++ b/doc/source/api/query/query_vertex.rst
@@ -0,0 +1,4 @@
+****************
+Query Vertices
+****************
+
[04/20] incubator-s2graph git commit: Merge pull request #3 from
SteamShon/S2GRAPH-248
Posted by st...@apache.org.
Merge pull request #3 from SteamShon/S2GRAPH-248
@SteamShon Thanks for your work.!
merge [S2GRAPH-249]
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/1dda954d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/1dda954d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/1dda954d
Branch: refs/heads/master
Commit: 1dda954d3c07ddb0966546501c3078c67e2cbc04
Parents: 4154bbe b8ab86d
Author: daewon <da...@apache.org>
Authored: Thu Nov 29 17:16:24 2018 +0900
Committer: GitHub <no...@github.com>
Committed: Thu Nov 29 17:16:24 2018 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/Management.scala | 44 ++++
.../org/apache/s2graph/core/schema/Label.scala | 38 ++--
.../apache/s2graph/http/S2GraphAdminRoute.scala | 224 ++++++++++++++++++-
.../s2graph/http/S2GraphMutateRoute.scala | 119 ++++++++++
.../scala/org/apache/s2graph/http/Server.scala | 4 +-
.../apache/s2graph/http/AdminRouteSpec.scala | 41 +++-
.../apache/s2graph/http/MutateRouteSpec.scala | 52 +++++
.../rest/play/controllers/AdminController.scala | 32 +--
8 files changed, 492 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
[17/20] incubator-s2graph git commit: add
Posted by st...@apache.org.
add
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/4bcf9d96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/4bcf9d96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/4bcf9d96
Branch: refs/heads/master
Commit: 4bcf9d96625ba700f122b959c063cb34f60fcd7e
Parents: bbdc661
Author: daewon <da...@apache.org>
Authored: Sat Dec 29 11:51:09 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Sat Dec 29 11:51:09 2018 +0900
----------------------------------------------------------------------
doc/source/api/management/index.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4bcf9d96/doc/source/api/management/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/management/index.rst b/doc/source/api/management/index.rst
index 17061b0..be18a36 100644
--- a/doc/source/api/management/index.rst
+++ b/doc/source/api/management/index.rst
@@ -345,11 +345,11 @@ The following is an example of adding index ``play_count`` to a label ``graph_te
.. code:: bash
- // add prop first
+ # add prop first
curl -XPOST localhost:9000/admin/addProp/graph_test -H 'Content-Type: Application/json' -d '
{ "name": "play_count", "defaultValue": 0, "dataType": "integer" }'
- // then add index
+ # then add index
curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
{
"label": "graph_test",
[08/20] incubator-s2graph git commit: update akka http version
Posted by st...@apache.org.
update akka http version
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/955e21d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/955e21d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/955e21d7
Branch: refs/heads/master
Commit: 955e21d7c9ae45f9fd9d79dafefe233ae557d17b
Parents: ac630bf
Author: daewon <da...@apache.org>
Authored: Wed Dec 26 13:48:22 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Dec 26 13:48:22 2018 +0900
----------------------------------------------------------------------
s2http/build.sbt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/955e21d7/s2http/build.sbt
----------------------------------------------------------------------
diff --git a/s2http/build.sbt b/s2http/build.sbt
index 944493c..ef020cd 100644
--- a/s2http/build.sbt
+++ b/s2http/build.sbt
@@ -16,8 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-lazy val akkaHttpVersion = "10.1.5"
-lazy val akkaVersion = "2.5.18"
+lazy val akkaHttpVersion = "10.1.6"
+lazy val akkaVersion = "2.5.19"
name := "s2http"
[07/20] incubator-s2graph git commit: Change startup script project
to s2http
Posted by st...@apache.org.
Change startup script project to s2http
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/ac630bf6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/ac630bf6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/ac630bf6
Branch: refs/heads/master
Commit: ac630bf68c777fe963f1beaba90d7989fe755e4f
Parents: dcaa1f3
Author: daewon <da...@apache.org>
Authored: Wed Dec 12 19:47:31 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Dec 12 19:47:31 2018 +0900
----------------------------------------------------------------------
README.md | 193 ++++++++++++++++++++++-----------------------
bin/s2graph-daemon.sh | 9 +--
bin/s2graph.sh | 2 +-
bin/start-s2graph.sh | 2 +-
bin/stop-s2graph.sh | 2 +-
build.sbt | 14 +---
dev_support/README.md | 40 ++++------
7 files changed, 121 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index f873611..537c397 100644
--- a/README.md
+++ b/README.md
@@ -1,90 +1,87 @@
+<!---
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+--->
-<!---
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
---->
-
-S2Graph [](https://travis-ci.org/apache/incubator-s2graph)
-=======
-
-[**S2Graph**](http://s2graph.apache.org/) is a **graph database** designed to handle transactional graph processing at scale. Its REST API allows you to store, manage and query relational information using **edge** and **vertex** representations in a **fully asynchronous** and **non-blocking** manner.
-
-S2Graph is a implementation of [**Apache TinkerPop**](https://tinkerpop.apache.org/) on [**Apache HBASE**](https://hbase.apache.org/).
-
-This document covers some basic concepts and terms of S2Graph as well as help you get a feel for the S2Graph API.
-
-Building from the source
-========================
-
-To build S2Graph from the source, install the JDK 8 and [SBT](http://www.scala-sbt.org/), and run the following command in the project root:
-
- sbt package
-This will create a distribution of S2Graph that is ready to be deployed.
-
-One can find distribution on `target/apache-s2graph-$version-incubating-bin`.
-
-Quick Start
-===========
-
-Once extracted the downloaded binary release of S2Graph or built from the source as described above, the following files and directories should be found in the directory.
-
-```
-DISCLAIMER
-LICENCE # the Apache License 2.0
-NOTICE
-bin # scripts to manage the lifecycle of S2Graph
-conf # configuration files
-lib # contains the binary
-logs # application logs
-var # application data
-```
-
-This directory layout contains all binary and scripts required to launch S2Graph. The directories `logs` and `var` may not be present initially, and are created once S2Graph is launched.
-
-The following will launch S2Graph, using [HBase](https://hbase.apache.org/) in the standalone mode for data storage and [H2](http://www.h2database.com/html/main.html) as the metadata storage.
-
- sh bin/start-s2graph.sh
-To connect to a remote HBase cluster or use MySQL as the metastore, refer to the instructions in [`conf/application.conf`](conf/application.conf).
-S2Graph is tested on HBase versions 0.98, 1.0, 1.1, and 1.2 (https://hub.docker.com/r/harisekhon/hbase/tags/).
-
-Project Layout
-==============
-
-Here is what you can find in each subproject.
-
-1. `s2core`: The core library, containing the data abstractions for graph entities, storage adapters and utilities.
-2. `s2rest_play`: The REST server built with [Play framework](https://www.playframework.com/), providing the write and query API.
-3. `s2rest_netty`: The REST server built directly using Netty, implementing only the query API.
-4. [`s2graphql`](s2graphql/README.md): The GraphQL server, providing standard web interface for S2Graph.
-~~4. `loader`: A collection of Spark jobs for bulk loading streaming data into S2Graph.~~
-~~5. `spark`: Spark utilities for `loader` and `s2counter_loader`.~~
-5. [`s2jobs`](s2jobs/README.md): A collection of Spark jobs to support OLAP on S2Graph.
-6. `s2counter_core`: The core library providing data structures and logics for `s2counter_loader`.
-7. `s2counter_loader`: Spark streaming jobs that consume Kafka WAL logs and calculate various top-*K* results on-the-fly.
-8. `s2graph_gremlin`: Gremlin plugin for tinkerpop users.
-
-The first four projects are for OLTP-style workloads, currently the main target of S2Graph.
-The remaining projects are for OLAP-style workloads, especially for integrating S2Graph with other projects, such as [Apache Spark](https://spark.apache.org/) and [Kafka](https://kafka.apache.org/).
+S2Graph [](https://travis-ci.org/apache/incubator-s2graph)
+=======
+
+[**S2Graph**](http://s2graph.apache.org/) is a **graph database** designed to handle transactional graph processing at scale. Its REST API allows you to store, manage and query relational information using **edge** and **vertex** representations in a **fully asynchronous** and **non-blocking** manner.
+
+S2Graph is a implementation of [**Apache TinkerPop**](https://tinkerpop.apache.org/) on [**Apache HBASE**](https://hbase.apache.org/).
+
+This document covers some basic concepts and terms of S2Graph as well as help you get a feel for the S2Graph API.
+
+Building from the source
+========================
+
+To build S2Graph from the source, install the JDK 8 and [SBT](http://www.scala-sbt.org/), and run the following command in the project root:
+
+ sbt package
+This will create a distribution of S2Graph that is ready to be deployed.
+
+One can find distribution on `target/apache-s2graph-$version-incubating-bin`.
+
+Quick Start
+===========
+
+Once extracted the downloaded binary release of S2Graph or built from the source as described above, the following files and directories should be found in the directory.
+
+```
+DISCLAIMER
+LICENCE # the Apache License 2.0
+NOTICE
+bin # scripts to manage the lifecycle of S2Graph
+conf # configuration files
+lib # contains the binary
+logs # application logs
+var # application data
+```
+
+This directory layout contains all binary and scripts required to launch S2Graph. The directories `logs` and `var` may not be present initially, and are created once S2Graph is launched.
+
+The following will launch S2Graph, using [HBase](https://hbase.apache.org/) in the standalone mode for data storage and [H2](http://www.h2database.com/html/main.html) as the metadata storage.
+
+ sh bin/start-s2graph.sh
+To connect to a remote HBase cluster or use MySQL as the metastore, refer to the instructions in [`conf/application.conf`](conf/application.conf).
+S2Graph is tested on HBase versions 0.98, 1.0, 1.1, and 1.2 (https://hub.docker.com/r/harisekhon/hbase/tags/).
+
+Project Layout
+==============
+
+Here is what you can find in each subproject.
+
+1. `s2core`: The core library, containing the data abstractions for graph entities, storage adapters and utilities.
+2. `s2http`: The REST server built with [akka-http] the write and query API.
+~~4. `loader`: A collection of Spark jobs for bulk loading streaming data into S2Graph.~~
+~~5. `spark`: Spark utilities for `loader` and `s2counter_loader`.~~
+5. [`s2jobs`](s2jobs/README.md): A collection of Spark jobs to support OLAP on S2Graph.
+6. `s2counter_core`: The core library providing data structures and logics for `s2counter_loader`.
+7. `s2counter_loader`: Spark streaming jobs that consume Kafka WAL logs and calculate various top-*K* results on-the-fly.
+8. `s2graph_gremlin`: Gremlin plugin for tinkerpop users.
+
+The first four projects are for OLTP-style workloads, currently the main target of S2Graph.
+The remaining projects are for OLAP-style workloads, especially for integrating S2Graph with other projects, such as [Apache Spark](https://spark.apache.org/) and [Kafka](https://kafka.apache.org/).
The ~~`loader` and `spark`~~ projects are deprecated by [`s2jobs`](s2jobs/README.md) since version 0.2.0.
-Note that, the OLAP-style workloads are under development and we are planning to provide documentations in the upcoming releases.
-
+Note that, the OLAP-style workloads are under development and we are planning to provide documentations in the upcoming releases.
+
Your First Graph
================
@@ -273,19 +270,19 @@ S2Graph passes `Apache TinkerPop`'s `StructureStandardSuite` and `ProcessStandar
- MultiProperties
- MetaProperties
- UuidIds
-- AnyIds
+- AnyIds
- NumericIds
- StringIds
### Edge Features **not** implemented.
- UuidIds
-- AnyIds
+- AnyIds
- NumericIds
- StringIds
### Vertex property features **not** implemented.
- UuidIds
-- AnyIds
+- AnyIds
- NumericIds
- StringIds
- MapValues
@@ -294,7 +291,7 @@ S2Graph passes `Apache TinkerPop`'s `StructureStandardSuite` and `ProcessStandar
- ByteArrayValues
- DoubleArrayValues
- FloatArrayValues
-- IntegerArrayValues
+- IntegerArrayValues
- StringArrayValues
- LongArrayValues
- SerializableValues
@@ -307,7 +304,7 @@ S2Graph passes `Apache TinkerPop`'s `StructureStandardSuite` and `ProcessStandar
- ByteArrayValues
- DoubleArrayValues
- FloatArrayValues
-- IntegerArrayValues
+- IntegerArrayValues
- StringArrayValues
- LongArrayValues
- SerializableValues
@@ -335,7 +332,7 @@ S2Graph is a singleton that can be shared among multiple threads. You instantiat
Some important properties for configuration.
-#### HBase for data storage.
+#### HBase for data storage.
```
hbase.zookeeper.quorum=localhost:2181
```
@@ -437,26 +434,26 @@ Please checkout advanced example below to understand what data model is availabl
It is possible to separate multiple namespaces into logical spaces.
S2Graph achieve this by following data model. details can be found on https://steamshon.gitbooks.io/s2graph-book/content/the_data_model.html.
-1. Service: the top level abstraction
+1. Service: the top level abstraction
A convenient logical grouping of related entities
-Similar to the database abstraction that most relational databases support.
+Similar to the database abstraction that most relational databases support.
2. Column: belongs to a service.
-A set of homogeneous vertices such as users, news articles or tags.
-Every vertex has a user-provided unique ID that allows the efficient lookup.
+A set of homogeneous vertices such as users, news articles or tags.
+Every vertex has a user-provided unique ID that allows the efficient lookup.
A service typically contains multiple columns.
3. Label: schema for edge
-A set of homogeneous edges such as friendships, views, or clicks.
-Relation between two columns as well as a recursive association within one column.
+A set of homogeneous edges such as friendships, views, or clicks.
+Relation between two columns as well as a recursive association within one column.
The two columns connected with a label may not necessarily be in the same service, allowing us to store and query data that spans over multiple services.
Instead of convert user provided Id into internal unique numeric Id, S2Graph simply composite service and column metadata with user provided Id to guarantee global unique Id.
-Following is simple example to exploit these data model in s2graph.
+Following is simple example to exploit these data model in s2graph.
```
// init graph
graph = S2Graph.open(new BaseConfiguration())
@@ -552,7 +549,7 @@ will be updated.
## Cache
will be updated.
-## Gremlin
+## Gremlin
S2Graph has full support for gremlin. However gremlin’s fine grained graphy nature results in very high latency
Provider suppose to provide `ProviderOptimization` to improve latency of traversal, and followings are currently available optimizations.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/bin/s2graph-daemon.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph-daemon.sh b/bin/s2graph-daemon.sh
index d0ffe80..2566c30 100755
--- a/bin/s2graph-daemon.sh
+++ b/bin/s2graph-daemon.sh
@@ -20,7 +20,7 @@
usage="This script is intended to be used by other scripts in this directory."
usage=$"$usage\n Please refer to start-s2graph.sh and stop-s2graph.sh"
usage=$"$usage\n Usage: s2graph-daemon.sh (start|stop|restart|run|status)"
-usage="$usage (s2rest_play|s2rest_netty|s2graphql|...) <args...>"
+usage="$usage s2http <args...>"
bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
@@ -62,11 +62,8 @@ classpath="$S2GRAPH_CONF_DIR:$S2GRAPH_LIB_DIR/*"
# determine the main class
case $service in
-s2rest_play)
- main="play.core.server.NettyServer"
- ;;
-s2graphql)
- main="org.apache.s2graph.graphql.Server"
+s2http)
+ main="org.apache.s2graph.http.Server"
;;
hbase)
main="org.apache.hadoop.hbase.master.HMaster"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/bin/s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph.sh b/bin/s2graph.sh
index bef373a..9a1ff3c 100755
--- a/bin/s2graph.sh
+++ b/bin/s2graph.sh
@@ -17,7 +17,7 @@
# starts/stops/restarts an S2Graph server
usage="Usage: s2graph.sh (start|stop|restart|run|status)"
-usage="$usage (s2rest_play|s2rest_netty|s2graphql|...)"
+usage="$usage (s2http)"
bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/bin/start-s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/start-s2graph.sh b/bin/start-s2graph.sh
index 1347a1a..cb582fe 100755
--- a/bin/start-s2graph.sh
+++ b/bin/start-s2graph.sh
@@ -26,7 +26,7 @@ bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
$bin/hbase-standalone.sh start
$bin/s2graph-daemon.sh start h2
-service="s2rest_play"
+service="s2http"
[ $# -gt 0 ] && { service=$1; }
$bin/s2graph.sh start ${service}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/bin/stop-s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/stop-s2graph.sh b/bin/stop-s2graph.sh
index 09f7426..2a56a81 100755
--- a/bin/stop-s2graph.sh
+++ b/bin/stop-s2graph.sh
@@ -23,7 +23,7 @@ bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
. $bin/s2graph-env.sh
. $bin/s2graph-common.sh
-service="s2rest_play"
+service="s2http"
[ $# -gt 0 ] && { service=$1; }
$bin/s2graph.sh stop ${service}
$bin/s2graph-daemon.sh stop h2
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 1052ac6..8bf212d 100755
--- a/build.sbt
+++ b/build.sbt
@@ -39,13 +39,6 @@ lazy val commonSettings = Seq(
Revolver.settings
-lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogback)
- .dependsOn(s2core, s2counter_core)
- .settings(commonSettings: _*)
- .settings(testOptions in Test += Tests.Argument("sequential"))
- .settings(dockerSettings: _*)
- .enablePlugins(sbtdocker.DockerPlugin, JavaAppPackaging)
-
lazy val s2http = project
.dependsOn(s2core, s2graphql)
.settings(commonSettings: _*)
@@ -78,8 +71,8 @@ lazy val s2graph_gremlin = project.dependsOn(s2core)
.settings(commonSettings: _*)
lazy val root = (project in file("."))
- .aggregate(s2core, s2rest_play, s2jobs, s2http)
- .dependsOn(s2rest_play, s2http, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project
+ .aggregate(s2core, s2jobs, s2http)
+ .dependsOn(s2http, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project
.settings(commonSettings: _*)
lazy val dockerSettings = Seq(
@@ -123,7 +116,6 @@ runRatTask := {
Packager.defaultSettings
publishSigned := {
- (publishSigned in s2rest_play).value
(publishSigned in s2http).value
(publishSigned in s2core).value
(publishSigned in spark).value
@@ -148,4 +140,4 @@ releasePublishArtifactsAction := publishSigned.value
mainClass in Compile := None
-parallelExecution in ThisBuild := false
\ No newline at end of file
+parallelExecution in ThisBuild := false
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ac630bf6/dev_support/README.md
----------------------------------------------------------------------
diff --git a/dev_support/README.md b/dev_support/README.md
index 9c274e9..f4b7820 100644
--- a/dev_support/README.md
+++ b/dev_support/README.md
@@ -7,9 +7,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,30 +22,24 @@
# Run S2Graph using Docker
1. Build a docker image of the s2graph in the project's root directory
- - you can build images for each type of API Server
- ```
- // s2graphql
- sbt "project s2graphql" 'set version := "latest"' docker
-
- // s2rest_play
- sbt "project s2rest_play" 'set version := "latest"' docker
-
- // s2rest_netty
- sbt "project s2rest_netty" 'set version := "latest"' docker
- ```
-
- - find local image is created correctly by using `docker images`
-
- - (optional) If you need to add extra jars in classpath, use environment variable 'EXTRA_JARS'
- ```
+ - you can build images for each type of API Server
+ ```
+ // s2http
+ sbt "project s2http" 'set version := "latest"' docker
+ ```
+
+ - find local image is created correctly by using `docker images`
+
+ - (optional) If you need to add extra jars in classpath, use environment variable 'EXTRA_JARS'
+ ```
docker run --name s2graph -v /LocalJarsDir:/extraJars -e EXTRA_JARS=/extraJars -dit s2graph/s2graphql:latest ...
```
-
+
2. Run MySQL and HBase container first.
- - change directory to dev-support. `cd dev_support`
- - `docker-compose build`
+ - change directory to dev-support. `cd dev_support`
+ - `docker-compose build`
3. Run graph container
- - `docker-compose up -d`
+ - `docker-compose up -d`
> S2Graph should be connected with MySQL at initial state. Therefore you have to run MySQL and HBase before running it.
@@ -73,7 +67,7 @@ In order to develop and test S2Graph. You might be want to run S2Graph as `dev`
- Run s2graph as 'dev' mode
```
-# sbt "project s2rest_play" run -Dhost=default
+# sbt "project s2http" run -Dhost=default
```
- or run test cases
[06/20] incubator-s2graph git commit: add graphql route in s2http
Posted by st...@apache.org.
add graphql route in s2http
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/dcaa1f34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/dcaa1f34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/dcaa1f34
Branch: refs/heads/master
Commit: dcaa1f34c4dd9bba863d5e532b9b0893e6b9a287
Parents: f41ab56
Author: daewon <da...@apache.org>
Authored: Fri Nov 30 16:46:03 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Fri Nov 30 16:46:03 2018 +0900
----------------------------------------------------------------------
s2graphql/build.sbt | 12 +-
s2graphql/src/main/resources/application.conf | 11 +-
s2graphql/src/main/resources/assets/.gitignore | 1 -
.../src/main/resources/assets/graphiql.html | 151 ++++++++++++++++++
.../apache/s2graph/graphql/GraphQLServer.scala | 81 ++++------
.../org/apache/s2graph/graphql/HttpServer.scala | 154 -------------------
s2graphql/src/test/resources/application.conf | 11 +-
.../apache/s2graph/http/PlayJsonSupport.scala | 4 +-
.../s2graph/http/S2GraphMutateRoute.scala | 1 -
.../apache/s2graph/http/S2GraphQLRoute.scala | 105 +++++++++++++
.../s2graph/http/SangriaGraphQLSupport.scala | 38 +++++
.../scala/org/apache/s2graph/http/Server.scala | 14 +-
12 files changed, 351 insertions(+), 232 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/build.sbt
----------------------------------------------------------------------
diff --git a/s2graphql/build.sbt b/s2graphql/build.sbt
index bc71f98..dbf38c3 100644
--- a/s2graphql/build.sbt
+++ b/s2graphql/build.sbt
@@ -29,15 +29,11 @@ libraryDependencies ++= Seq(
"org.scala-lang" % "scala-compiler" % scalaVersion.value,
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
- "org.sangria-graphql" %% "sangria" % "1.4.0",
- "org.sangria-graphql" %% "sangria-spray-json" % "1.0.0",
- "org.sangria-graphql" %% "sangria-play-json" % "1.0.1" % Test,
+ "org.sangria-graphql" %% "sangria" % "1.4.2",
+ "org.sangria-graphql" %% "sangria-spray-json" % "1.0.1",
+ "org.sangria-graphql" %% "sangria-play-json" % "1.0.5" % Test,
- "com.typesafe.akka" %% "akka-http" % "10.0.10",
- "com.typesafe.akka" %% "akka-http-spray-json" % "10.0.10",
- "com.typesafe.akka" %% "akka-slf4j" % "2.4.6",
-
- "org.scalatest" %% "scalatest" % "3.0.4" % Test
+ "org.scalatest" %% "scalatest" % "3.0.5" % Test
)
Revolver.settings
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/resources/application.conf b/s2graphql/src/main/resources/application.conf
index aac21d1..b956031 100644
--- a/s2graphql/src/main/resources/application.conf
+++ b/s2graphql/src/main/resources/application.conf
@@ -16,11 +16,12 @@
# specific language governing permissions and limitations
# under the License.
#
-akka {
- loggers = ["akka.event.slf4j.Slf4jLogger"]
- event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
- loglevel = "INFO"
-}
+
+//akka {
+// loggers = ["akka.event.slf4j.Slf4jLogger"]
+// event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
+// loglevel = "INFO"
+//}
//db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL",
//db.default.password = sa
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/main/resources/assets/.gitignore
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/resources/assets/.gitignore b/s2graphql/src/main/resources/assets/.gitignore
deleted file mode 100644
index 8b13789..0000000
--- a/s2graphql/src/main/resources/assets/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/main/resources/assets/graphiql.html
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/resources/assets/graphiql.html b/s2graphql/src/main/resources/assets/graphiql.html
new file mode 100644
index 0000000..3fe0083
--- /dev/null
+++ b/s2graphql/src/main/resources/assets/graphiql.html
@@ -0,0 +1,151 @@
+<!--
+ * LICENSE AGREEMENT For GraphiQL software
+ *
+ * Facebook, Inc. (“Facebook”) owns all right, title and interest, including all
+ * intellectual property and other proprietary rights, in and to the GraphiQL
+ * software. Subject to your compliance with these terms, you are hereby granted a
+ * non-exclusive, worldwide, royalty-free copyright license to (1) use and copy the
+ * GraphiQL software; and (2) reproduce and distribute the GraphiQL software as
+ * part of your own software (“Your Software”). Facebook reserves all rights not
+ * expressly granted to you in this license agreement.
+ *
+ * THE SOFTWARE AND DOCUMENTATION, IF ANY, ARE PROVIDED "AS IS" AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES (INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE) ARE DISCLAIMED. IN NO
+ * EVENT SHALL FACEBOOK OR ITS AFFILIATES, OFFICES, DIRECTORS OR EMPLOYEES BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THE SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You will include in Your Software (e.g., in the file(s), documentation or other
+ * materials accompanying your software): (1) the disclaimer set forth above; (2)
+ * this sentence; and (3) the following copyright notice:
+ *
+ * Copyright (c) 2015, Facebook, Inc. All rights reserved.
+-->
+<!DOCTYPE html>
+<html>
+ <head>
+ <style>
+ body {
+ height: 100%;
+ margin: 0;
+ width: 100%;
+ overflow: hidden;
+ }
+
+ #graphiql {
+ height: 100vh;
+ }
+ </style>
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/graphiql@0.11.11/graphiql.css" />
+ <script src="//cdn.jsdelivr.net/es6-promise/4.0.5/es6-promise.auto.min.js"></script>
+ <script src="//cdn.jsdelivr.net/fetch/0.9.0/fetch.min.js"></script>
+ <script src="//cdn.jsdelivr.net/react/15.4.2/react.min.js"></script>
+ <script src="//cdn.jsdelivr.net/react/15.4.2/react-dom.min.js"></script>
+ <script src="//cdn.jsdelivr.net/npm/graphiql@0.11.11/graphiql.min.js"></script>
+ </head>
+ <body>
+ <div id="graphiql">Loading...</div>
+
+ <script>
+
+ /**
+ * This GraphiQL example illustrates how to use some of GraphiQL's props
+ * in order to enable reading and updating the URL parameters, making
+ * link sharing of queries a little bit easier.
+ *
+ * This is only one example of this kind of feature, GraphiQL exposes
+ * various React params to enable interesting integrations.
+ */
+
+ // Parse the search string to get url parameters.
+ var search = window.location.search;
+ var parameters = {};
+ search.substr(1).split('&').forEach(function (entry) {
+ var eq = entry.indexOf('=');
+ if (eq >= 0) {
+ parameters[decodeURIComponent(entry.slice(0, eq))] =
+ decodeURIComponent(entry.slice(eq + 1));
+ }
+ });
+
+ // if variables was provided, try to format it.
+ if (parameters.variables) {
+ try {
+ parameters.variables =
+ JSON.stringify(JSON.parse(parameters.variables), null, 2);
+ } catch (e) {
+ // Do nothing, we want to display the invalid JSON as a string, rather
+ // than present an error.
+ }
+ }
+
+ // When the query and variables string is edited, update the URL bar so
+ // that it can be easily shared
+ function onEditQuery(newQuery) {
+ parameters.query = newQuery;
+ updateURL();
+ }
+
+ function onEditVariables(newVariables) {
+ parameters.variables = newVariables;
+ updateURL();
+ }
+
+ function onEditOperationName(newOperationName) {
+ parameters.operationName = newOperationName;
+ updateURL();
+ }
+
+ function updateURL() {
+ var newSearch = '?' + Object.keys(parameters).filter(function (key) {
+ return Boolean(parameters[key]);
+ }).map(function (key) {
+ return encodeURIComponent(key) + '=' +
+ encodeURIComponent(parameters[key]);
+ }).join('&');
+ history.replaceState(null, null, newSearch);
+ }
+
+ // Defines a GraphQL fetcher using the fetch API.
+ function graphQLFetcher(graphQLParams) {
+ return fetch('/graphql', {
+ method: 'post',
+ headers: {
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify(graphQLParams),
+ credentials: 'include',
+ }).then(function (response) {
+ return response.text();
+ }).then(function (responseBody) {
+ try {
+ return JSON.parse(responseBody);
+ } catch (error) {
+ return responseBody;
+ }
+ });
+ }
+
+ // Render <GraphiQL /> into the body.
+ ReactDOM.render(
+ React.createElement(GraphiQL, {
+ fetcher: graphQLFetcher,
+ query: parameters.query,
+ variables: parameters.variables,
+ operationName: parameters.operationName,
+ onEditQuery: onEditQuery,
+ onEditVariables: onEditVariables,
+ onEditOperationName: onEditOperationName
+ }),
+ document.getElementById('graphiql')
+ );
+ </script>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
index b650714..a013b2d 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
@@ -19,15 +19,9 @@
package org.apache.s2graph.graphql
-import java.util.concurrent.Executors
-
-import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
-import akka.http.scaladsl.model.StatusCodes._
-import akka.http.scaladsl.server.Directives._
-import akka.http.scaladsl.server._
import com.typesafe.config.ConfigFactory
-import org.apache.s2graph.graphql.middleware.{GraphFormatted, Transform}
-import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.graphql.middleware.{GraphFormatted}
+import org.apache.s2graph.core.{S2GraphLike}
import org.apache.s2graph.core.utils.SafeUpdateCache
import org.apache.s2graph.graphql.repository.GraphRepository
import org.apache.s2graph.graphql.types.SchemaDef
@@ -36,36 +30,42 @@ import sangria.ast.Document
import sangria.execution._
import sangria.execution.deferred.DeferredResolver
import sangria.marshalling.sprayJson._
-import sangria.parser.{QueryParser, SyntaxError}
+import sangria.parser.{SyntaxError}
import sangria.schema.Schema
import spray.json._
import scala.collection.JavaConverters._
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext}
import scala.util.control.NonFatal
-import scala.util.{Failure, Success, Try}
+import scala.util._
-class GraphQLServer() {
- val className = Schema.getClass.getName
- val logger = LoggerFactory.getLogger(this.getClass)
+object GraphQLServer {
+ def formatError(error: Throwable): JsValue = error match {
+ case syntaxError: SyntaxError ⇒
+ JsObject("errors" → JsArray(
+ JsObject(
+ "message" → JsString(syntaxError.getMessage),
+ "locations" → JsArray(JsObject(
+ "line" → JsNumber(syntaxError.originalError.position.line),
+ "column" → JsNumber(syntaxError.originalError.position.column))))))
- // Init s2graph
- val numOfThread = Runtime.getRuntime.availableProcessors()
- val threadPool = Executors.newFixedThreadPool(numOfThread * 2)
+ case NonFatal(e) ⇒ formatError(e.toString)
+ case e ⇒ throw e
+ }
- implicit val ec = ExecutionContext.fromExecutor(threadPool)
+ def formatError(message: String): JsObject =
+ JsObject("errors" → JsArray(JsObject("message" → JsString(message))))
+}
- val config = ConfigFactory.load()
- val s2graph = new S2Graph(config)
- val schemaCacheTTL = Try(config.getInt("schemaCacheTTL")).getOrElse(3000)
- val enableMutation = Try(config.getBoolean("enableMutation")).getOrElse(false)
+class GraphQLServer(s2graph: S2GraphLike, schemaCacheTTL: Int = 60) {
+ val logger = LoggerFactory.getLogger(this.getClass)
val schemaConfig = ConfigFactory.parseMap(Map(
SafeUpdateCache.MaxSizeKey -> 1, SafeUpdateCache.TtlKey -> schemaCacheTTL
).asJava)
// Manage schema instance lifecycle
- val schemaCache = new SafeUpdateCache(schemaConfig)
+ val schemaCache = new SafeUpdateCache(schemaConfig)(s2graph.ec)
def updateEdgeFetcher(requestJSON: spray.json.JsValue)(implicit e: ExecutionContext): Try[Unit] = {
val ret = Try {
@@ -79,9 +79,9 @@ class GraphQLServer() {
ret
}
- val schemaCacheKey = className + "s2Schema"
+ val schemaCacheKey = Schema.getClass.getName + "s2Schema"
- schemaCache.put(schemaCacheKey, createNewSchema(enableMutation))
+ schemaCache.put(schemaCacheKey, createNewSchema(true))
/**
* In development mode(schemaCacheTTL = 1),
@@ -101,33 +101,17 @@ class GraphQLServer() {
newSchema -> s2Repository
}
- def formatError(error: Throwable): JsValue = error match {
- case syntaxError: SyntaxError ⇒
- JsObject("errors" → JsArray(
- JsObject(
- "message" → JsString(syntaxError.getMessage),
- "locations" → JsArray(JsObject(
- "line" → JsNumber(syntaxError.originalError.position.line),
- "column" → JsNumber(syntaxError.originalError.position.column))))))
-
- case NonFatal(e) ⇒ formatError(e.toString)
- case e ⇒ throw e
- }
-
- def formatError(message: String): JsObject =
- JsObject("errors" → JsArray(JsObject("message" → JsString(message))))
-
def onEvictSchema(o: AnyRef): Unit = {
logger.info("Schema Evicted")
}
val TransformMiddleWare = List(org.apache.s2graph.graphql.middleware.Transform())
- def executeGraphQLQuery(query: Document, op: Option[String], vars: JsObject)(implicit e: ExecutionContext) = {
+ def executeQuery(query: Document, op: Option[String], vars: JsObject)(implicit e: ExecutionContext) = {
import GraphRepository._
val (schemaDef, s2Repository) =
- schemaCache.withCache(schemaCacheKey, broadcast = false, onEvict = onEvictSchema)(createNewSchema(enableMutation))
+ schemaCache.withCache(schemaCacheKey, broadcast = false, onEvict = onEvictSchema)(createNewSchema(true))
val resolver: DeferredResolver[GraphRepository] = DeferredResolver.fetchers(vertexFetcher, edgeFetcher)
@@ -142,15 +126,8 @@ class GraphQLServer() {
operationName = op,
deferredResolver = resolver,
middleware = middleWares
- ).map((res: spray.json.JsValue) => OK -> res)
- .recover {
- case error: QueryAnalysisError =>
- logger.error("Error on execute", error)
- BadRequest -> error.resolveError
- case error: ErrorWithResolver =>
- logger.error("Error on execute", error)
- InternalServerError -> error.resolveError
- }
+ )
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
deleted file mode 100644
index 8b89c73..0000000
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.s2graph.graphql
-
-import java.nio.charset.Charset
-
-import akka.actor.ActorSystem
-import akka.http.scaladsl.Http
-import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
-import akka.http.scaladsl.model._
-import akka.http.scaladsl.server.Directives._
-import akka.http.scaladsl.server.{Route, StandardRoute}
-import akka.stream.ActorMaterializer
-import org.slf4j.LoggerFactory
-import sangria.parser.QueryParser
-import spray.json._
-
-import scala.concurrent.Await
-import scala.language.postfixOps
-import scala.util._
-import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller, ToResponseMarshallable}
-import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
-import akka.util.ByteString
-import sangria.ast.Document
-import sangria.renderer.{QueryRenderer, QueryRendererConfig}
-
-import scala.collection.immutable.Seq
-
-object Server extends App {
- val logger = LoggerFactory.getLogger(this.getClass)
-
- implicit val system = ActorSystem("s2graphql-server")
- implicit val materializer = ActorMaterializer()
-
- import system.dispatcher
- import scala.concurrent.duration._
-
- import spray.json.DefaultJsonProtocol._
-
- val graphQLServer = new GraphQLServer()
-
- val route: Route =
- get {
- getFromResource("assets/graphiql.html")
- } ~ (post & path("updateEdgeFetcher")) {
- entity(as[JsValue]) { body =>
- graphQLServer.updateEdgeFetcher(body) match {
- case Success(_) => complete(StatusCodes.OK -> JsString("Update fetcher finished"))
- case Failure(e) =>
- logger.error("Error on execute", e)
- complete(StatusCodes.InternalServerError -> spray.json.JsObject("message" -> JsString(e.toString)))
- }
- }
- } ~ (post & path("graphql")) {
- parameters('operationName.?, 'variables.?) { (operationNameParam, variablesParam) =>
- entity(as[Document]) { document ⇒
- variablesParam.map(parseJson) match {
- case None ⇒ complete(graphQLServer.executeGraphQLQuery(document, operationNameParam, JsObject()))
- case Some(Right(js)) ⇒ complete(graphQLServer.executeGraphQLQuery(document, operationNameParam, js.asJsObject))
- case Some(Left(e)) ⇒
- logger.error("Error on execute", e)
- complete(StatusCodes.BadRequest -> graphQLServer.formatError(e))
- }
- } ~ entity(as[JsValue]) { body ⇒
- val fields = body.asJsObject.fields
-
- val query = fields.get("query").map(js => js.convertTo[String])
- val operationName = fields.get("operationName").filterNot(_ == JsNull).map(_.convertTo[String])
- val variables = fields.get("variables").filterNot(_ == JsNull)
-
- query.map(QueryParser.parse(_)) match {
- case None ⇒ complete(StatusCodes.BadRequest -> graphQLServer.formatError("No query to execute"))
- case Some(Failure(error)) ⇒
- logger.error("Error on execute", error)
- complete(StatusCodes.BadRequest -> graphQLServer.formatError(error))
- case Some(Success(document)) => variables match {
- case Some(js) ⇒ complete(graphQLServer.executeGraphQLQuery(document, operationName, js.asJsObject))
- case None ⇒ complete(graphQLServer.executeGraphQLQuery(document, operationName, JsObject()))
- }
- }
- }
- }
- }
-
- val port = sys.props.get("http.port").fold(8000)(_.toInt)
-
- logger.info(s"Starting GraphQL server... $port")
-
- Http().bindAndHandle(route, "0.0.0.0", port).foreach { binding =>
- logger.info(s"GraphQL server ready for connect")
- }
-
- def shutdown(): Unit = {
- logger.info("Terminating...")
-
- system.terminate()
- Await.result(system.whenTerminated, 30 seconds)
-
- logger.info("Terminated.")
- }
-
- // Unmarshaller
-
- def unmarshallerContentTypes: Seq[ContentTypeRange] = mediaTypes.map(ContentTypeRange.apply)
-
- def mediaTypes: Seq[MediaType.WithFixedCharset] =
- Seq(MediaType.applicationWithFixedCharset("graphql", HttpCharsets.`UTF-8`, "graphql"))
-
- implicit def documentMarshaller(implicit config: QueryRendererConfig = QueryRenderer.Compact): ToEntityMarshaller[Document] = {
- Marshaller.oneOf(mediaTypes: _*) {
- mediaType ⇒
- Marshaller.withFixedContentType(ContentType(mediaType)) {
- json ⇒ HttpEntity(mediaType, QueryRenderer.render(json, config))
- }
- }
- }
-
- implicit val documentUnmarshaller: FromEntityUnmarshaller[Document] = {
- Unmarshaller.byteStringUnmarshaller
- .forContentTypes(unmarshallerContentTypes: _*)
- .map {
- case ByteString.empty ⇒ throw Unmarshaller.NoContentException
- case data ⇒
- import sangria.parser.DeliveryScheme.Throw
- QueryParser.parse(data.decodeString(Charset.forName("UTF-8")))
- }
- }
-
- def parseJson(jsStr: String): Either[Throwable, JsValue] = {
- val parsed = Try(jsStr.parseJson)
- parsed match {
- case Success(js) => Right(js)
- case Failure(e) => Left(e)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2graphql/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2graphql/src/test/resources/application.conf b/s2graphql/src/test/resources/application.conf
index 74821e4..31d9cbd 100644
--- a/s2graphql/src/test/resources/application.conf
+++ b/s2graphql/src/test/resources/application.conf
@@ -16,11 +16,12 @@
# specific language governing permissions and limitations
# under the License.
#
-akka {
- loggers = ["akka.event.slf4j.Slf4jLogger"]
- event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
- loglevel = "INFO"
-}
+
+//akka {
+// loggers = ["akka.event.slf4j.Slf4jLogger"]
+// event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
+// loglevel = "INFO"
+//}
//db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL",
//db.default.password = sa
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
index 244e588..b41ebd8 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
@@ -10,10 +10,10 @@ import play.api.libs.json._
trait PlayJsonSupport {
- val mediaTypes: Seq[MediaType.WithFixedCharset] =
+ private val mediaTypes: Seq[MediaType.WithFixedCharset] =
Seq(MediaType.applicationWithFixedCharset("json", HttpCharsets.`UTF-8`, "js"))
- val unmarshallerContentTypes: Seq[ContentTypeRange] = mediaTypes.map(ContentTypeRange.apply)
+ private val unmarshallerContentTypes: Seq[ContentTypeRange] = mediaTypes.map(ContentTypeRange.apply)
implicit val playJsonMarshaller: ToEntityMarshaller[JsValue] = {
Marshaller.oneOf(mediaTypes: _*) { mediaType =>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
index a65db5a..e7d1b88 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
@@ -19,7 +19,6 @@ trait S2GraphMutateRoute extends PlayJsonSupport {
lazy val parser = new RequestParser(s2graph)
- // lazy val requestParser = new RequestParser(s2graph)
lazy val exceptionHandler = ExceptionHandler {
case ex: JsonParseException => complete(StatusCodes.BadRequest -> ex.getMessage)
case ex: java.lang.IllegalArgumentException => complete(StatusCodes.BadRequest -> ex.getMessage)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2http/src/main/scala/org/apache/s2graph/http/S2GraphQLRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphQLRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphQLRoute.scala
new file mode 100644
index 0000000..bf7c92e
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphQLRoute.scala
@@ -0,0 +1,105 @@
+package org.apache.s2graph.http
+
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server._
+import org.apache.s2graph.core.S2Graph
+import org.apache.s2graph.graphql.GraphQLServer
+import org.slf4j.LoggerFactory
+import sangria.ast.Document
+import sangria.execution.{ErrorWithResolver, QueryAnalysisError}
+import sangria.parser.QueryParser
+import spray.json.{JsNull, JsObject, JsString, JsValue}
+
+import scala.util.{Failure, Left, Right, Success, Try}
+
+object S2GraphQLRoute {
+ def parseJson(jsStr: String): Either[Throwable, JsValue] = {
+ import spray.json._
+ val parsed = Try(jsStr.parseJson)
+
+ parsed match {
+ case Success(js) => Right(js)
+ case Failure(e) => Left(e)
+ }
+ }
+}
+
+trait S2GraphQLRoute extends SprayJsonSupport with SangriaGraphQLSupport {
+
+ import S2GraphQLRoute._
+ import spray.json.DefaultJsonProtocol._
+ import sangria.marshalling.sprayJson._
+
+ val s2graph: S2Graph
+ val logger = LoggerFactory.getLogger(this.getClass)
+
+ lazy val graphQLServer = new GraphQLServer(s2graph)
+
+ private val exceptionHandler = ExceptionHandler {
+ case error: QueryAnalysisError =>
+ logger.error("Error on execute", error)
+ complete(StatusCodes.BadRequest -> error.resolveError)
+ case error: ErrorWithResolver =>
+ logger.error("Error on execute", error)
+ complete(StatusCodes.InternalServerError -> error.resolveError)
+ }
+
+ lazy val updateEdgeFetcher = path("updateEdgeFetcher") {
+ entity(as[spray.json.JsValue]) { body =>
+ graphQLServer.updateEdgeFetcher(body)(s2graph.ec) match {
+ case Success(_) => complete(StatusCodes.OK -> JsString("Update fetcher finished"))
+ case Failure(e) =>
+ logger.error("Error on execute", e)
+ complete(StatusCodes.InternalServerError -> spray.json.JsObject("message" -> JsString(e.toString)))
+ }
+ }
+ }
+
+ lazy val graphql = parameters('operationName.?, 'variables.?) { (operationNameParam, variablesParam) =>
+ implicit val ec = s2graph.ec
+
+ entity(as[Document]) { document ⇒
+ variablesParam.map(parseJson) match {
+ case None ⇒ complete(graphQLServer.executeQuery(document, operationNameParam, JsObject()))
+ case Some(Right(js)) ⇒ complete(graphQLServer.executeQuery(document, operationNameParam, js.asJsObject))
+ case Some(Left(e)) ⇒
+ logger.error("Error on execute", e)
+ complete(StatusCodes.BadRequest -> GraphQLServer.formatError(e))
+ }
+ } ~ entity(as[spray.json.JsValue]) { body ⇒
+ val fields = body.asJsObject.fields
+
+ val query = fields.get("query").map(js => js.convertTo[String])
+ val operationName = fields.get("operationName").filterNot(_ == JsNull).map(_.convertTo[String])
+ val variables = fields.get("variables").filterNot(_ == JsNull)
+
+ query.map(QueryParser.parse(_)) match {
+ case None ⇒ complete(StatusCodes.BadRequest -> GraphQLServer.formatError("No query to execute"))
+ case Some(Failure(error)) ⇒
+ logger.error("Error on execute", error)
+ complete(StatusCodes.BadRequest -> GraphQLServer.formatError(error))
+ case Some(Success(document)) => variables match {
+ case Some(js) ⇒ complete(graphQLServer.executeQuery(document, operationName, js.asJsObject))
+ case None ⇒ complete(graphQLServer.executeQuery(document, operationName, JsObject()))
+ }
+ }
+ }
+ }
+
+ // expose routes
+ lazy val graphqlRoute: Route =
+ get {
+ getFromResource("assets/graphiql.html")
+ } ~
+ post {
+ handleExceptions(exceptionHandler) {
+ concat(
+ updateEdgeFetcher,
+ graphql
+ )
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2http/src/main/scala/org/apache/s2graph/http/SangriaGraphQLSupport.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/SangriaGraphQLSupport.scala b/s2http/src/main/scala/org/apache/s2graph/http/SangriaGraphQLSupport.scala
new file mode 100644
index 0000000..965e17a
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/SangriaGraphQLSupport.scala
@@ -0,0 +1,38 @@
+package org.apache.s2graph.http
+
+import java.nio.charset.Charset
+
+import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller}
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
+import akka.util.ByteString
+import sangria.ast.Document
+import sangria.parser.QueryParser
+import sangria.renderer.{QueryRenderer, QueryRendererConfig}
+
+trait SangriaGraphQLSupport {
+ private val mediaTypes: Seq[MediaType.WithFixedCharset] =
+ Seq(MediaType.applicationWithFixedCharset("graphql", HttpCharsets.`UTF-8`, "graphql"))
+
+ private val unmarshallerContentTypes: Seq[ContentTypeRange] = mediaTypes.map(ContentTypeRange.apply)
+
+ implicit def documentMarshaller(implicit config: QueryRendererConfig = QueryRenderer.Compact): ToEntityMarshaller[Document] = {
+ Marshaller.oneOf(mediaTypes: _*) {
+ mediaType ⇒
+ Marshaller.withFixedContentType(ContentType(mediaType)) {
+ json ⇒ HttpEntity(mediaType, QueryRenderer.render(json, config))
+ }
+ }
+ }
+
+ implicit val documentUnmarshaller: FromEntityUnmarshaller[Document] = {
+ Unmarshaller.byteStringUnmarshaller
+ .forContentTypes(unmarshallerContentTypes: _*)
+ .map {
+ case ByteString.empty ⇒ throw Unmarshaller.NoContentException
+ case data ⇒
+ import sangria.parser.DeliveryScheme.Throw
+ QueryParser.parse(data.decodeString(Charset.forName("UTF-8")))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/dcaa1f34/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
index c26e314..00146f6 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
@@ -36,7 +36,8 @@ import org.slf4j.LoggerFactory
object Server extends App
with S2GraphTraversalRoute
with S2GraphAdminRoute
- with S2GraphMutateRoute {
+ with S2GraphMutateRoute
+ with S2GraphQLRoute {
implicit val system: ActorSystem = ActorSystem("S2GraphHttpServer")
implicit val materializer: ActorMaterializer = ActorMaterializer()
@@ -57,15 +58,20 @@ object Server extends App
pathPrefix("graphs")(traversalRoute),
pathPrefix("mutate")(mutateRoute),
pathPrefix("admin")(adminRoute),
+ pathPrefix("graphql")(graphqlRoute),
get(complete(health))
)
val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, "localhost", port)
binding.onComplete {
case Success(bound) => logger.info(s"Server online at http://${bound.localAddress.getHostString}:${bound.localAddress.getPort}/")
- case Failure(e) =>
- logger.error(s"Server could not start!", e)
- system.terminate()
+ case Failure(e) => logger.error(s"Server could not start!", e)
+ }
+
+ scala.sys.addShutdownHook { () =>
+ s2graph.shutdown()
+ system.terminate()
+ logger.info("System terminated")
}
Await.result(system.whenTerminated, Duration.Inf)
[20/20] incubator-s2graph git commit: [S2GRAPH-248] HTTP interface
integration.
Posted by st...@apache.org.
[S2GRAPH-248] HTTP interface integration.
JIRA:
[S2GRAPH-248] https://issues.apache.org/jira/browse/S2GRAPH-248
Pull Request:
Closes #193
Author
daewon <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/19254301
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/19254301
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/19254301
Branch: refs/heads/master
Commit: 19254301d4cea7e976754fe1b1f6491144b51173
Parents: 53ff5cd
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Jan 26 07:48:00 2019 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Jan 26 07:48:00 2019 +0900
----------------------------------------------------------------------
CHANGES | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/19254301/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 26388ab..6b7d679 100644
--- a/CHANGES
+++ b/CHANGES
@@ -115,6 +115,7 @@ Release Notes - S2Graph - Version 0.2.0
* [S2GRAPH-229] - 'Step' abstraction for combinable queries
* [S2GRAPH-245] - Remove install hbase step on travis CI.
* [S2GRAPH-246] - Integration of documents into the S2Graph project.
+ * [S2GRAPH-248] - HTTP interface integration.
Release 0.1.0 - Released
[14/20] incubator-s2graph git commit: update edge mutate
Posted by st...@apache.org.
update edge mutate
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/aa309b22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/aa309b22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/aa309b22
Branch: refs/heads/master
Commit: aa309b220844280429841784a61e06cccafcd89d
Parents: c131990
Author: daewon <da...@apache.org>
Authored: Thu Dec 27 17:39:13 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Dec 27 17:39:13 2018 +0900
----------------------------------------------------------------------
doc/source/api/index.rst | 2 +-
doc/source/api/mutate/mutate_edge.rst | 54 +++++++++++++++++++-----------
2 files changed, 35 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa309b22/doc/source/api/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst
index df495ab..e74650a 100644
--- a/doc/source/api/index.rst
+++ b/doc/source/api/index.rst
@@ -3,7 +3,7 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
-API Glossary
+HTTP API Glossary
==============================================
The following is a non-exhaustive list of commonly used S2Graph APIs and their examples
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/aa309b22/doc/source/api/mutate/mutate_edge.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/mutate/mutate_edge.rst b/doc/source/api/mutate/mutate_edge.rst
index 1e21079..553c8a1 100644
--- a/doc/source/api/mutate/mutate_edge.rst
+++ b/doc/source/api/mutate/mutate_edge.rst
@@ -13,19 +13,20 @@ The following fields need to be specified when inserting an edge, and are return
:header: "Field Name", "Definition", "Data Type", "Example", "Note"
:widths: 15, 30, 30, 30, 30
- "timestamp", "Issue time of request", "Long", "1430116731156", "Required. Unix Epoch time in milliseconds. S2Graph TTL and timestamp unit is milliseconds."
+ "**timestamp**", "Issue time of request", "Long", "1430116731156", "Required. Unix Epoch time in milliseconds. S2Graph TTL and timestamp unit is **milliseconds**"
"operation", "One of insert, delete, update, or increment", "String", "'i', 'insert'", "Required only for bulk operations. Aliases are also available: i (insert), d (delete), u (update), in (increment). Default is insert"
- "from", "Id of source vertex", "Long/String", "1", "Required. Use long if possible. Maximum string byte-size is 249"
- "to", "Id of target vertex", "Long/String", "101", "Required. Use long if possible. Maximum string byte-size is 249"
+ "from", "Id of source vertex", "Long/String", "1", "Required. Use long if possible. **Maximum string byte-size is 249**"
+ "to", "Id of target vertex", "Long/String", "101", "Required. Use long if possible. **Maximum string byte-size is 249**"
"label", "Label name", "String", "graph_test", "Required"
- "direction", "Direction of the edge. Should be one of out/ in/ undirected", "String", "out", "Required. Alias are also available: o (out), i (in), u (undirected)"
- "props", "Additional properties of the edge", "JSON (dictionary)", "{""timestamp"": 1417616431, ""affinity_score"":10, ""is_hidden"": false, ""is_valid"": true}", "Required. If in indexed properties isn't given, default values will be added"
+ "direction", "Direction of the edge. Should be one of **out/ in/ undirected**", "String", "out", "Required. Alias are also available: o (out), i (in), u (undirected)"
+ "props", "Additional properties of the edge", "JSON (dictionary)", "{""timestamp"": 1417616431, ""affinity_score"":10, ""is_hidden"": false, ""is_valid"": true}", "Required. **If in indexed properties isn't given, default values will be added**"
Basic Edge Operations
--------------------------
In S2Graph, an Edge supports five different operations.
+
- ``insert``: Create new edge.
- ``delete``: Delete existing edge.
- ``update``: Update existing edge`s state.
@@ -33,6 +34,7 @@ In S2Graph, an Edge supports five different operations.
- ``deleteAll``: Delete all adjacent edges from certain source vertex. (Available for strong consistency only)
Edge operations work differently depending on the target label`s consistency level.
+
For a better understanding, please take a look at the following test cases.
Create 2 different labels, one of each consistencyLevels.
@@ -56,7 +58,6 @@ Then insert a same set of edges to each labels and query them as follows.
Note that only one edge exist between (101, 10, s2graph_label_test, out).
-
.. code:: json
{
@@ -176,10 +177,15 @@ This time there are ``three edges`` between (101, 10, s2graph_label_test_weak, o
Strong Consistency
---------------------
-**Insert** - ``POST /mutate/edge/insert``
+Insert - ``POST /mutate/edge/insert``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
A unique edge is identified by a combination of (from, to, label, direction). For insert operations, S2Graph first checks if an edge with same (from, to, label, direction) information exists. If there is an existing edge, then insert will work as ``update``. See above example.
-**Delete** - ``POST /mutate/edge/delete``
+Delete - ``POST /mutate/edge/delete``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
For edge deletion, again, S2Graph looks for a unique edge with (from, to, label, direction). However, this time it checks the timestamp of the delete request and the existing edge. The timestamp on the delete request ``must be larger than that on the existing edge`` or else the request will be ignored. If everything is well, the edge will be deleted. Also note that no props information is necessary for a delete request on a strongly consistent label since there will be only one edge with edge`s unique id(from, to, label, direction).
.. code:: bash
@@ -189,7 +195,8 @@ For edge deletion, again, S2Graph looks for a unique edge with (from, to, label,
{"timestamp": 10, "from": 101, "to": 10, "label": "s2graph_label_test"}
]'
-**Update** - ``POST /mutate/edge/update``
+Update - ``POST /mutate/edge/update``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
What an update operation does to a strongly consistent label is identical to an insert.
@@ -202,7 +209,8 @@ What an update operation does to a strongly consistent label is identical to an
]'
-**Increment** - ``POST /mutate/edge/increment``
+Increment - ``POST /mutate/edge/increment``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Works like update, other than it returns the incremented value and not the old value.
@@ -213,9 +221,10 @@ Works like update, other than it returns the incremented value and not the old v
{"timestamp": 10, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": 100, "weight": -10}}
]'
-**Delete All** - ``POST /mutate/edge/deleteAll``
+Delete All - ``POST /mutate/edge/deleteAll``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Delete all adjacent edges to the source vertex. **Please note that edges with both in and out directions will be deleted**
+Delete all adjacent edges to the source vertex. ``Please note that edges with both in and out directions will be deleted``
.. code:: bash
@@ -229,11 +238,13 @@ Weak Consistency
-----------------
-**Insert** ``POST /mutate/edge/insert``
+Insert ``POST /mutate/edge/insert``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-S2Graph **does not look** for a unique edge defined by (from, to, label, direction). It simply stores a new edge according to the request. No read, no consistency check. Note that this difference allows multiple edges with same (from, to, label, direction) id.
+S2Graph ``does not look`` for a unique edge defined by (from, to, label, direction). It simply stores a new edge according to the request. No read, no consistency check. Note that this difference allows multiple edges with same (from, to, label, direction) id.
-**Delete** - ``POST /graphs/edges/delete``
+Delete - ``POST /graphs/edges/delete``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
For deletion on weakly consistent edges, first, S2Graph fetches existing edges from storage. Then, on each resulting edges, fires the actual delete operations.
@@ -294,15 +305,18 @@ For deletion on weakly consistent edges, first, S2Graph fetches existing edges f
}
]'
-**Update** - ``POST /mutate/edge/update``
+Update - ``POST /mutate/edge/update``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Like insert, S2Graph **does not check** for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be updated.
+Like insert, S2Graph ``does not check`` for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be updated.
-**Increment** - ``POST /mutate/edge/increment``
+Increment - ``POST /mutate/edge/increment``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-For increment, S2Graph also **does not check** for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be incremented.
+For increment, S2Graph also ``does not check`` for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be incremented.
-**Delete All** - ``POST /mutate/edge/deleteAll``
+Delete All - ``POST /mutate/edge/deleteAll``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Identical to strong consistency.
[11/20] incubator-s2graph git commit: complete management
Posted by st...@apache.org.
complete management
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/0a2707d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/0a2707d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/0a2707d8
Branch: refs/heads/master
Commit: 0a2707d8ad250c3144661b2addee182478aaf3c6
Parents: 7e4f6c8
Author: daewon <da...@apache.org>
Authored: Wed Dec 26 21:32:21 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Dec 26 21:32:21 2018 +0900
----------------------------------------------------------------------
doc/source/_static/css/s2graph_style.css | 1 +
doc/source/api/index.rst | 18 ++
doc/source/api/management.rst | 444 ++++++++++++++++++++++++++
doc/source/api/read.rst | 0
doc/source/api/write.rst | 0
5 files changed, 463 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0a2707d8/doc/source/_static/css/s2graph_style.css
----------------------------------------------------------------------
diff --git a/doc/source/_static/css/s2graph_style.css b/doc/source/_static/css/s2graph_style.css
new file mode 100644
index 0000000..edf43da
--- /dev/null
+++ b/doc/source/_static/css/s2graph_style.css
@@ -0,0 +1 @@
+@import "theme.css";
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0a2707d8/doc/source/api/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst
new file mode 100644
index 0000000..63276bc
--- /dev/null
+++ b/doc/source/api/index.rst
@@ -0,0 +1,18 @@
+.. Sphinx API Example documentation master file, created by
+ sphinx-quickstart on Wed May 2 12:55:56 2012.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+API Glossary
+==============================================
+
+The following is a non-exhaustive list of commonly used S2Graph APIs and their examples
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ management
+ write
+ read
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0a2707d8/doc/source/api/management.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/management.rst b/doc/source/api/management.rst
new file mode 100644
index 0000000..516651c
--- /dev/null
+++ b/doc/source/api/management.rst
@@ -0,0 +1,444 @@
+Management APIs
+==================
+Admin Apis for Management Service, Label, Index ..
+
+****************
+Create a Service
+****************
+
+``Service`` is the top level abstraction in S2Graph which could be considered as a database in MySQL.
+
+.. code:: bash
+
+ POST /admin/createService
+
+Service Fields
+---------------
+
+In order to create a Service, the following fields should be specified in the request.
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "serviceName", "User defined namespace", "String", "talk_friendship", "Required"
+ "cluster", "Zookeeper quorum address", "String", "abc.com:2181,abd.com:2181", "Optional"
+ "hTableName", "HBase table name", "String", "test", "Optional"
+ "hTableTTL", "Time to live setting for data","Integer", "86000", "Optional"
+ "preSplitSize", "Factor for the table pre-split size", "Integer", "1", "Optional"
+
+.. list
+ - By default, S2Graph looks for "hbase.zookeeper.quorum" in your application.conf. If "hbase.zookeeper.quorum" is undefined, this value is set as "localhost".
+
+
+Basic Service Operations
+--------------------------
+
+You can create a service using the following API:
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createService -H 'Content-Type: Application/json' -d '
+ {
+ "serviceName": "s2graph",
+ "cluster": "address for zookeeper",
+ "hTableName": "hbase table name",
+ "hTableTTL": 86000,
+ "preSplitSize": 2
+ }'
+
+
+****************
+Create a Label
+****************
+
+A ``Label`` represents a relation between two serviceColumns. Labels are to S2Graph what tables are to RDBMS since they contain the schema information, i.e. descriptive information of the data being managed or indices used for efficient retrieval.
+In most scenarios, defining an edge schema (in other words, label) requires a little more care compared to a vertex schema (which is pretty straightforward).
+First, think about the kind of queries you will be using, then, model user actions or relations into ``edges`` and design a label accordingly.
+
+.. code:: bash
+
+ POST /admin/createLabel
+
+Label Fields
+---------------
+
+A Label creation request includes the following information.
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "label", "Name of the relation", "String", "talk_friendship", "Required"
+ "srcServiceName", "Source column's service", "String", "kakaotalk", "Required"
+ "srcColumnName", "Source column's name", "String", "user_id", "Required"
+ "srcColumnType", "Source column's data type","Long/Integer/String", "string", "Required"
+ "tgtServiceName", "Target column's service", "String", "kakaotalk/kakaomusic", "Optional"
+ "tgtColumnName", "Target column's name", "String", "item_id", "Required"
+ "tgtColumnType", "Target column's data type", "Long/Integer/String", "string", "Required"
+ "isDirected", "Wether the label is directed or undirected", "True/False", "true/false", "Optional. default is true"
+ "serviceName", "Which service the label belongs to", "String", "kakaotalk", "Optional. tgtServiceName is used by default"
+ "hTableName", "A dedicated HBase table to your Label", "String", "s2graph-batch", "Optional. Service hTableName is used by default"
+ "hTableTTL", "Data time to live setting", "Integer", "86000", "Optional. Service hTableTTL is used by default"
+ "consistencyLevel", "If set to 'strong', only one edge is alowed between a pair of source/ target vertices. Set to 'weak', and multiple-edge is supported", "String", "strong/weak", "Optional. default is 'weak'"
+
+
+Props & Indices
+----------------
+
+A couple of key elements of a Label are its Properties (props) and indices.
+Supplementary information of a Vertex or Edge can be stored as props. A single property can be defined in a simple key-value JSON as follows:
+
+.. code:: json
+
+ {
+ "name": "name of property",
+ "dataType": "data type of property value",
+ "defaultValue": "default value in string"
+ }
+
+In a scenario where user - video playback history is stored in a Label, a typical example for props would look like this:
+
+.. code:: json
+
+ [
+ {"name": "play_count", "defaultValue": 0, "dataType": "integer"},
+ {"name": "is_hidden","defaultValue": false,"dataType": "boolean"},
+ {"name": "category","defaultValue": "jazz","dataType": "string"},
+ {"name": "score","defaultValue": 0,"dataType": "float"}
+ ]
+
+Props can have data types of ``numeric`` (byte/ short/ integer/ float/ double), ``boolean`` or ``string``.
+In order to achieve efficient data retrieval, a Label can be indexed using the "indices" option.
+Default value for indices is ``_timestamp``, a hidden label property.
+
+All labels have ``_timestamp`` in their props under the hood
+
+The first index in indices array will be the primary index ``(Think of PRIMARY INDEX idx_xxx(p1, p2) in MySQL)``
+S2Graph will automatically store edges according to the primary index.
+Trailing indices are used for multiple ordering on edges. ``(Think of ALTER TABLE ADD INDEX idx_xxx(p2, p1) in MySQL)``
+
+props define meta datas that will not be affect the order of edges.
+Please avoid using S2Graph-reserved property names:
+
+
+- ``_timestamp`` is reserved for system wise timestamp. this can be interpreted as last_modified_at
+- ``_from`` is reserved for label's start vertex.
+- ``_to`` is reserved for label's target vertex.
+
+
+Basic Label Operations
+--------------------------
+
+Here is an sample request that creates a label ``user_article_liked`` between column ``user_id`` of service ``s2graph`` and column ``article_id`` of service ``s2graph_news``.
+Note that the default indexed property ``_timestamp`` will be created since the ``indexedProps`` field is empty.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
+ {
+ "label": "user_article_liked",
+ "srcServiceName": "s2graph",
+ "srcColumnName": "user_id",
+ "srcColumnType": "long",
+ "tgtServiceName": "s2graph_news",
+ "tgtColumnName": "article_id",
+ "tgtColumnType": "string",
+ "indices": [], // _timestamp will be used as default
+ "props": [],
+ "serviceName": "s2graph_news"
+ }'
+
+
+The created label ``user_article_liked`` will manage edges in a timestamp-descending order (which seems to be the common requirement for most services).
+Here is another example that creates a label ``friends``, which represents the friend relation between ``users`` in service ``s2graph``.
+This time, edges are managed by both affinity_score and ``_timestamp``.
+
+Friends with higher affinity_scores come first and if affinity_score is a tie, recently added friends comes first.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
+ {
+ "label": "friends",
+ "srcServiceName": "s2graph",
+ "srcColumnName": "user_id",
+ "srcColumnType": "long",
+ "tgtServiceName": "s2graph",
+ "tgtColumnName": "user_id",
+ "tgtColumnType": "long",
+ "indices": [
+ {"name": "idx_affinity_timestamp", "propNames": ["affinity_score", "_timestamp"]}
+ ],
+ "props": [
+ {"name": "affinity_score", "dataType": "float", "defaultValue": 0.0},
+ {"name": "_timestamp", "dataType": "long", "defaultValue": 0},
+ {"name": "is_hidden", "dataType": "boolean", "defaultValue": false},
+ {"name": "is_blocked", "dataType": "boolean", "defaultValue": true},
+ {"name": "error_code", "dataType": "integer", "defaultValue": 500}
+ ],
+ "serviceName": "s2graph",
+ "consistencyLevel": "strong"
+ }'
+
+S2Graph supports **multiple indices** on a label which means you can add separate ordering options for edges.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
+ {
+ "label": "friends",
+ "indices": [
+ {"name": "idx_3rd", "propNames": ["is_blocked", "_timestamp"]}
+ ]
+ }'
+
+In order to get general information on a label, make a GET request to ``/admin/getLabel/{label name}``
+
+.. code:: bash
+
+ curl -XGET localhost:9000/admin/getLabel/friends
+
+
+Delete a label with a PUT request to ``/admin/deleteLabel/{label name}``
+
+.. code:: bash
+
+ curl -XPUT localhost:9000/admin/deleteLabel/friends
+
+
+Label updates are not supported (except when you are adding an index). Instead, you can delete the label and re-create it.
+
+Adding Extra Properties to Labels
+----------------------------------
+
+To add a new property, use ``/admin/addProp/{label name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addProp/friend -H 'Content-Type: Application/json' -d '
+ {
+ "name": "is_blocked",
+ "defaultValue": false,
+ "dataType": "boolean"
+ }'
+
+
+Consistency Level
+------------------
+
+Simply put, the consistency level of your label will determine how the edges are stored at storage level.
+First, note that S2Graph identifies a unique edge by combining its from, label, to values as a key.
+
+Now, let's consider inserting the following two edges that have same keys (1, graph_test, 101) and different timestamps (1418950524721 and 1418950524723).
+
+.. code:: bash
+
+ 1418950524721 insert e 1 101 graph_test {"weight": 10} = (1, graph_test, 101)
+ 1418950524723 insert e 1 101 graph_test {"weight": 20} = (1, graph_test, 101)
+
+
+**Each consistency levels handle the case differently.**
+
+- strong
+
+ - The strong option makes sure that there is only one edge record stored in the HBase table for edge key (1, graph_test, 101). With strong consistency level, the later insertion will overwrite the previous one.
+
+- weak
+
+ - The weak option will allow two different edges stored in the table with different timestamps and weight values.
+
+
+For a better understanding, let's simplify the notation for an edge that connects two vertices u - v at time t as u -> (t, v), and assume that we are inserting these four edges into two different labels with each consistency configuration (both indexed by timestamp only).
+
+.. code:: bash
+
+ u1 -> (t1, v1)
+ u1 -> (t2, v2)
+ u1 -> (t3, v2)
+ u1 -> (t4, v1)
+
+With a strong consistencyLevel, your Label contents will be:
+
+.. code:: bash
+
+ u1 -> (t4, v1)
+ u1 -> (t3, v2)
+
+Note that edges with same vertices and earlier timestamp (u1 -> (t1, v1) and u1 -> (t2, v2)) were overwritten and do not exist.
+On the other hand, with consistencyLevel weak.
+
+.. code:: bash
+
+ u1 -> (t1, v1)
+ u1 -> (t2, v2)
+ u1 -> (t3, v2)
+ u1 -> (t4, v1)
+
+**It is recommended to set consistencyLevel to weak unless you are expecting concurrent updates on same edge.**
+
+In real world systems, it is not guaranteed that operation requests arrive at S2Graph in the order of their timestamp. Depending on the environment (network conditions, client making asynchronous calls, use of a message que, and so on) request that were made earlier can arrive later. Consistency level also determines how S2Graph handles these cases.
+Strong consistencyLevel promises a final result consistent to the timestamp.
+For example, consider a set of operation requests on edge (1, graph_test, 101) were made in the following order;
+
+
+.. code:: bash
+
+ 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
+ 1418950524722 delete e 1 101 graph_test
+ 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
+ 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
+ 1418950524726 update e 1 101 graph_test {"is_blocked": true}
+
+
+and actually arrived in a shuffled order due to complications
+
+
+.. code:: bash
+
+ 1418950524726 update e 1 101 graph_test {"is_blocked": true}
+ 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
+ 1418950524722 delete e 1 101 graph_test
+ 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
+ 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
+
+Strong consistency still makes sure that you get the same eventual state on (1, graph_test, 101).
+Here is pseudocode of what S2Graph does to provide a strong consistency level.
+
+.. code:: bash
+
+ complexity = O(one read) + O(one delete) + O(2 put)
+
+ fetchedEdge = fetch edge with (1, graph_test, 101) from lookup table.
+
+ if fetchedEdge is not exist:
+ create new edge same as current insert operation
+ update lookup table as current insert operation
+ else:
+ valid = compare fetchedEdge vs current insert operation.
+ if valid:
+ delete fetchedEdge
+ create new edge after comparing fetchedEdge and current insert.
+ update lookup table
+
+Limitations Since S2Graph makes asynchronous writes to HBase via Asynchbase, there is no consistency guaranteed on same edge within its flushInterval (1 second).
+
+Adding Extra Indices (Optional)
+---------------------------------
+
+.. code:: bash
+
+ POST /admin/addIndex
+
+A label can have multiple properties set as indexes. When edges are queried, the ordering will determined according to indexes, therefore, deciding which edges will be included in the top-K results.
+
+**Edge retrieval queries in S2Graph by default returns top-K edges. Clients must issue another query to fetch the next K edges, i.e., top-K ~ 2 x top-K**
+
+Edges sorted according to the indices in order to limit the number of edges being fetched by a query. If no ordering property is given, S2Graph will use the timestamp as an index, thus resulting in the most recent data.
+
+**It would be extremely difficult to fetch millions of edges and sort them at request time and return a top-K in a reasonable amount of time. Instead, S2Graph uses vertex-centric indexes to avoid this.
+Using a vertex-centric index, having millions of edges is fine as long as size K of the top-K values is reasonable (under 1K) Note that indexes must be created prior to inserting any data on the label (which is the same case with the conventional RDBMS).**
+
+New indexes can be dynamically added, but will not be applied to pre-existing data (support for this is planned for future versions). Currently, a label can have up to eight indices.
+The following is an example of adding index ``play_count`` to a label ``graph_test``.
+
+.. code:: bash
+
+ // add prop first
+ curl -XPOST localhost:9000/admin/addProp/graph_test -H 'Content-Type: Application/json' -d '
+ { "name": "play_count", "defaultValue": 0, "dataType": "integer" }'
+
+ // then add index
+ curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
+ {
+ "label": "graph_test",
+ "indices": [
+ { name: "idx_play_count", propNames: ["play-count"] }
+ ]
+ }'
+
+
+**********************************
+Create a ServiceColumn (Optional)
+**********************************
+
+.. code:: bash
+
+ POST /admin/createServiceColumn
+
+If your use case requires props assigned to vertices instead of edges, what you need is a Service Column
+
+**Remark: If it is only the vertex id that you need and not additional props, there's no need to create a Service Column explicitly. At label creation, by default, S2Graph creates column space with empty properties according to the label schema.**
+
+
+Service Column Fields
+----------------------
+
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "Field Name", "Definition" "Data Type", "Example", "Remarks"
+ "serviceName", "Which service the Service Column belongs to", "String", "kakaotalk", "Required"
+ "columnName", "Service Column`s name", "String", "talk_user_id", "Required"
+ "props", "Optional properties of Service Column", "JSON (array dictionaries)", "Please refer to the examples", "Optional"
+
+
+Basic Service Column Operations
+-------------------------------
+
+Here are some sample requests for Service Column creation as well as vertex insertion and selection.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createServiceColumn -H 'Content-Type: Application/json' -d '
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id",
+ "columnType": "long",
+ "props": [
+ {"name": "is_active", "dataType": "boolean", "defaultValue": true},
+ {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
+ {"name": "nickname", "dataType": "string", "defaultValue": ".."},
+ {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
+ {"name": "age", "dataType": "integer", "defaultValue": 0}
+ ]
+ }'
+
+General information on a vertex schema can be retrieved with ``/admin/getServiceColumn/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XGET localhost:9000/admin/getServiceColumn/s2graph/user_id
+
+This will give all properties on serviceName ``s2graph`` and columnName ``user_id`` serviceColumn.
+Properties can be added to a Service Column with ``/admin/addServiceColumnProps/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addServiceColumnProps/s2graph/user_id -H 'Content-Type: Application/json' -d '
+ [
+ {"name": "home_address", "defaultValue": "korea", "dataType": "string"}
+ ]'
+
+Vertices can be inserted to a Service Column using ``/admin/vertices/insert/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/insert/s2graph/user_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"props":{"is_active":true}, "timestamp":1417616431},
+ {"id":2,"props":{},"timestamp":1417616431}
+ ]'
+
+Finally, query your vertex via ``/graphs/getVertices``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getVertices -H 'Content-Type: Application/json' -d '
+ [
+ {"serviceName": "s2graph", "columnName": "user_id", "ids": [1, 2, 3]}
+ ]'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0a2707d8/doc/source/api/read.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/read.rst b/doc/source/api/read.rst
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/0a2707d8/doc/source/api/write.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/write.rst b/doc/source/api/write.rst
new file mode 100644
index 0000000..e69de29
[12/20] incubator-s2graph git commit: add vertex manage
Posted by st...@apache.org.
add vertex manage
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/6414cb77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/6414cb77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/6414cb77
Branch: refs/heads/master
Commit: 6414cb777cc951c049843a0032fa100abdc21f2a
Parents: 0a2707d
Author: daewon <da...@apache.org>
Authored: Thu Dec 27 16:00:08 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Dec 27 16:00:08 2018 +0900
----------------------------------------------------------------------
doc/source/api/index.rst | 5 +-
doc/source/api/management.rst | 444 -------------------------------------
doc/source/api/read.rst | 0
doc/source/api/write.rst | 0
4 files changed, 2 insertions(+), 447 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6414cb77/doc/source/api/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst
index 63276bc..df495ab 100644
--- a/doc/source/api/index.rst
+++ b/doc/source/api/index.rst
@@ -13,6 +13,5 @@ Contents:
.. toctree::
:maxdepth: 2
- management
- write
- read
+ management/index
+ mutate/index
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6414cb77/doc/source/api/management.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/management.rst b/doc/source/api/management.rst
deleted file mode 100644
index 516651c..0000000
--- a/doc/source/api/management.rst
+++ /dev/null
@@ -1,444 +0,0 @@
-Management APIs
-==================
-Admin Apis for Management Service, Label, Index ..
-
-****************
-Create a Service
-****************
-
-``Service`` is the top level abstraction in S2Graph which could be considered as a database in MySQL.
-
-.. code:: bash
-
- POST /admin/createService
-
-Service Fields
----------------
-
-In order to create a Service, the following fields should be specified in the request.
-
-.. csv-table:: Option
- :header: "Field Name", "Definition", "Data Type", "Example", "Note"
- :widths: 15, 30, 30, 30, 30
-
- "serviceName", "User defined namespace", "String", "talk_friendship", "Required"
- "cluster", "Zookeeper quorum address", "String", "abc.com:2181,abd.com:2181", "Optional"
- "hTableName", "HBase table name", "String", "test", "Optional"
- "hTableTTL", "Time to live setting for data","Integer", "86000", "Optional"
- "preSplitSize", "Factor for the table pre-split size", "Integer", "1", "Optional"
-
-.. list
- - By default, S2Graph looks for "hbase.zookeeper.quorum" in your application.conf. If "hbase.zookeeper.quorum" is undefined, this value is set as "localhost".
-
-
-Basic Service Operations
---------------------------
-
-You can create a service using the following API:
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/createService -H 'Content-Type: Application/json' -d '
- {
- "serviceName": "s2graph",
- "cluster": "address for zookeeper",
- "hTableName": "hbase table name",
- "hTableTTL": 86000,
- "preSplitSize": 2
- }'
-
-
-****************
-Create a Label
-****************
-
-A ``Label`` represents a relation between two serviceColumns. Labels are to S2Graph what tables are to RDBMS since they contain the schema information, i.e. descriptive information of the data being managed or indices used for efficient retrieval.
-In most scenarios, defining an edge schema (in other words, label) requires a little more care compared to a vertex schema (which is pretty straightforward).
-First, think about the kind of queries you will be using, then, model user actions or relations into ``edges`` and design a label accordingly.
-
-.. code:: bash
-
- POST /admin/createLabel
-
-Label Fields
----------------
-
-A Label creation request includes the following information.
-
-.. csv-table:: Option
- :header: "Field Name", "Definition", "Data Type", "Example", "Note"
- :widths: 15, 30, 30, 30, 30
-
- "label", "Name of the relation", "String", "talk_friendship", "Required"
- "srcServiceName", "Source column's service", "String", "kakaotalk", "Required"
- "srcColumnName", "Source column's name", "String", "user_id", "Required"
- "srcColumnType", "Source column's data type","Long/Integer/String", "string", "Required"
- "tgtServiceName", "Target column's service", "String", "kakaotalk/kakaomusic", "Optional"
- "tgtColumnName", "Target column's name", "String", "item_id", "Required"
- "tgtColumnType", "Target column's data type", "Long/Integer/String", "string", "Required"
- "isDirected", "Wether the label is directed or undirected", "True/False", "true/false", "Optional. default is true"
- "serviceName", "Which service the label belongs to", "String", "kakaotalk", "Optional. tgtServiceName is used by default"
- "hTableName", "A dedicated HBase table to your Label", "String", "s2graph-batch", "Optional. Service hTableName is used by default"
- "hTableTTL", "Data time to live setting", "Integer", "86000", "Optional. Service hTableTTL is used by default"
- "consistencyLevel", "If set to 'strong', only one edge is alowed between a pair of source/ target vertices. Set to 'weak', and multiple-edge is supported", "String", "strong/weak", "Optional. default is 'weak'"
-
-
-Props & Indices
-----------------
-
-A couple of key elements of a Label are its Properties (props) and indices.
-Supplementary information of a Vertex or Edge can be stored as props. A single property can be defined in a simple key-value JSON as follows:
-
-.. code:: json
-
- {
- "name": "name of property",
- "dataType": "data type of property value",
- "defaultValue": "default value in string"
- }
-
-In a scenario where user - video playback history is stored in a Label, a typical example for props would look like this:
-
-.. code:: json
-
- [
- {"name": "play_count", "defaultValue": 0, "dataType": "integer"},
- {"name": "is_hidden","defaultValue": false,"dataType": "boolean"},
- {"name": "category","defaultValue": "jazz","dataType": "string"},
- {"name": "score","defaultValue": 0,"dataType": "float"}
- ]
-
-Props can have data types of ``numeric`` (byte/ short/ integer/ float/ double), ``boolean`` or ``string``.
-In order to achieve efficient data retrieval, a Label can be indexed using the "indices" option.
-Default value for indices is ``_timestamp``, a hidden label property.
-
-All labels have ``_timestamp`` in their props under the hood
-
-The first index in indices array will be the primary index ``(Think of PRIMARY INDEX idx_xxx(p1, p2) in MySQL)``
-S2Graph will automatically store edges according to the primary index.
-Trailing indices are used for multiple ordering on edges. ``(Think of ALTER TABLE ADD INDEX idx_xxx(p2, p1) in MySQL)``
-
-props define meta datas that will not be affect the order of edges.
-Please avoid using S2Graph-reserved property names:
-
-
-- ``_timestamp`` is reserved for system wise timestamp. this can be interpreted as last_modified_at
-- ``_from`` is reserved for label's start vertex.
-- ``_to`` is reserved for label's target vertex.
-
-
-Basic Label Operations
---------------------------
-
-Here is an sample request that creates a label ``user_article_liked`` between column ``user_id`` of service ``s2graph`` and column ``article_id`` of service ``s2graph_news``.
-Note that the default indexed property ``_timestamp`` will be created since the ``indexedProps`` field is empty.
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
- {
- "label": "user_article_liked",
- "srcServiceName": "s2graph",
- "srcColumnName": "user_id",
- "srcColumnType": "long",
- "tgtServiceName": "s2graph_news",
- "tgtColumnName": "article_id",
- "tgtColumnType": "string",
- "indices": [], // _timestamp will be used as default
- "props": [],
- "serviceName": "s2graph_news"
- }'
-
-
-The created label ``user_article_liked`` will manage edges in a timestamp-descending order (which seems to be the common requirement for most services).
-Here is another example that creates a label ``friends``, which represents the friend relation between ``users`` in service ``s2graph``.
-This time, edges are managed by both affinity_score and ``_timestamp``.
-
-Friends with higher affinity_scores come first and if affinity_score is a tie, recently added friends comes first.
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
- {
- "label": "friends",
- "srcServiceName": "s2graph",
- "srcColumnName": "user_id",
- "srcColumnType": "long",
- "tgtServiceName": "s2graph",
- "tgtColumnName": "user_id",
- "tgtColumnType": "long",
- "indices": [
- {"name": "idx_affinity_timestamp", "propNames": ["affinity_score", "_timestamp"]}
- ],
- "props": [
- {"name": "affinity_score", "dataType": "float", "defaultValue": 0.0},
- {"name": "_timestamp", "dataType": "long", "defaultValue": 0},
- {"name": "is_hidden", "dataType": "boolean", "defaultValue": false},
- {"name": "is_blocked", "dataType": "boolean", "defaultValue": true},
- {"name": "error_code", "dataType": "integer", "defaultValue": 500}
- ],
- "serviceName": "s2graph",
- "consistencyLevel": "strong"
- }'
-
-S2Graph supports **multiple indices** on a label which means you can add separate ordering options for edges.
-
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
- {
- "label": "friends",
- "indices": [
- {"name": "idx_3rd", "propNames": ["is_blocked", "_timestamp"]}
- ]
- }'
-
-In order to get general information on a label, make a GET request to ``/admin/getLabel/{label name}``
-
-.. code:: bash
-
- curl -XGET localhost:9000/admin/getLabel/friends
-
-
-Delete a label with a PUT request to ``/admin/deleteLabel/{label name}``
-
-.. code:: bash
-
- curl -XPUT localhost:9000/admin/deleteLabel/friends
-
-
-Label updates are not supported (except when you are adding an index). Instead, you can delete the label and re-create it.
-
-Adding Extra Properties to Labels
-----------------------------------
-
-To add a new property, use ``/admin/addProp/{label name}``
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/addProp/friend -H 'Content-Type: Application/json' -d '
- {
- "name": "is_blocked",
- "defaultValue": false,
- "dataType": "boolean"
- }'
-
-
-Consistency Level
-------------------
-
-Simply put, the consistency level of your label will determine how the edges are stored at storage level.
-First, note that S2Graph identifies a unique edge by combining its from, label, to values as a key.
-
-Now, let's consider inserting the following two edges that have same keys (1, graph_test, 101) and different timestamps (1418950524721 and 1418950524723).
-
-.. code:: bash
-
- 1418950524721 insert e 1 101 graph_test {"weight": 10} = (1, graph_test, 101)
- 1418950524723 insert e 1 101 graph_test {"weight": 20} = (1, graph_test, 101)
-
-
-**Each consistency levels handle the case differently.**
-
-- strong
-
- - The strong option makes sure that there is only one edge record stored in the HBase table for edge key (1, graph_test, 101). With strong consistency level, the later insertion will overwrite the previous one.
-
-- weak
-
- - The weak option will allow two different edges stored in the table with different timestamps and weight values.
-
-
-For a better understanding, let's simplify the notation for an edge that connects two vertices u - v at time t as u -> (t, v), and assume that we are inserting these four edges into two different labels with each consistency configuration (both indexed by timestamp only).
-
-.. code:: bash
-
- u1 -> (t1, v1)
- u1 -> (t2, v2)
- u1 -> (t3, v2)
- u1 -> (t4, v1)
-
-With a strong consistencyLevel, your Label contents will be:
-
-.. code:: bash
-
- u1 -> (t4, v1)
- u1 -> (t3, v2)
-
-Note that edges with same vertices and earlier timestamp (u1 -> (t1, v1) and u1 -> (t2, v2)) were overwritten and do not exist.
-On the other hand, with consistencyLevel weak.
-
-.. code:: bash
-
- u1 -> (t1, v1)
- u1 -> (t2, v2)
- u1 -> (t3, v2)
- u1 -> (t4, v1)
-
-**It is recommended to set consistencyLevel to weak unless you are expecting concurrent updates on same edge.**
-
-In real world systems, it is not guaranteed that operation requests arrive at S2Graph in the order of their timestamp. Depending on the environment (network conditions, client making asynchronous calls, use of a message que, and so on) request that were made earlier can arrive later. Consistency level also determines how S2Graph handles these cases.
-Strong consistencyLevel promises a final result consistent to the timestamp.
-For example, consider a set of operation requests on edge (1, graph_test, 101) were made in the following order;
-
-
-.. code:: bash
-
- 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
- 1418950524722 delete e 1 101 graph_test
- 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
- 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
- 1418950524726 update e 1 101 graph_test {"is_blocked": true}
-
-
-and actually arrived in a shuffled order due to complications
-
-
-.. code:: bash
-
- 1418950524726 update e 1 101 graph_test {"is_blocked": true}
- 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
- 1418950524722 delete e 1 101 graph_test
- 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
- 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
-
-Strong consistency still makes sure that you get the same eventual state on (1, graph_test, 101).
-Here is pseudocode of what S2Graph does to provide a strong consistency level.
-
-.. code:: bash
-
- complexity = O(one read) + O(one delete) + O(2 put)
-
- fetchedEdge = fetch edge with (1, graph_test, 101) from lookup table.
-
- if fetchedEdge is not exist:
- create new edge same as current insert operation
- update lookup table as current insert operation
- else:
- valid = compare fetchedEdge vs current insert operation.
- if valid:
- delete fetchedEdge
- create new edge after comparing fetchedEdge and current insert.
- update lookup table
-
-Limitations Since S2Graph makes asynchronous writes to HBase via Asynchbase, there is no consistency guaranteed on same edge within its flushInterval (1 second).
-
-Adding Extra Indices (Optional)
----------------------------------
-
-.. code:: bash
-
- POST /admin/addIndex
-
-A label can have multiple properties set as indexes. When edges are queried, the ordering will determined according to indexes, therefore, deciding which edges will be included in the top-K results.
-
-**Edge retrieval queries in S2Graph by default returns top-K edges. Clients must issue another query to fetch the next K edges, i.e., top-K ~ 2 x top-K**
-
-Edges sorted according to the indices in order to limit the number of edges being fetched by a query. If no ordering property is given, S2Graph will use the timestamp as an index, thus resulting in the most recent data.
-
-**It would be extremely difficult to fetch millions of edges and sort them at request time and return a top-K in a reasonable amount of time. Instead, S2Graph uses vertex-centric indexes to avoid this.
-Using a vertex-centric index, having millions of edges is fine as long as size K of the top-K values is reasonable (under 1K) Note that indexes must be created prior to inserting any data on the label (which is the same case with the conventional RDBMS).**
-
-New indexes can be dynamically added, but will not be applied to pre-existing data (support for this is planned for future versions). Currently, a label can have up to eight indices.
-The following is an example of adding index ``play_count`` to a label ``graph_test``.
-
-.. code:: bash
-
- // add prop first
- curl -XPOST localhost:9000/admin/addProp/graph_test -H 'Content-Type: Application/json' -d '
- { "name": "play_count", "defaultValue": 0, "dataType": "integer" }'
-
- // then add index
- curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
- {
- "label": "graph_test",
- "indices": [
- { name: "idx_play_count", propNames: ["play-count"] }
- ]
- }'
-
-
-**********************************
-Create a ServiceColumn (Optional)
-**********************************
-
-.. code:: bash
-
- POST /admin/createServiceColumn
-
-If your use case requires props assigned to vertices instead of edges, what you need is a Service Column
-
-**Remark: If it is only the vertex id that you need and not additional props, there's no need to create a Service Column explicitly. At label creation, by default, S2Graph creates column space with empty properties according to the label schema.**
-
-
-Service Column Fields
-----------------------
-
-
-.. csv-table:: Option
- :header: "Field Name", "Definition", "Data Type", "Example", "Note"
- :widths: 15, 30, 30, 30, 30
-
- "Field Name", "Definition" "Data Type", "Example", "Remarks"
- "serviceName", "Which service the Service Column belongs to", "String", "kakaotalk", "Required"
- "columnName", "Service Column`s name", "String", "talk_user_id", "Required"
- "props", "Optional properties of Service Column", "JSON (array dictionaries)", "Please refer to the examples", "Optional"
-
-
-Basic Service Column Operations
--------------------------------
-
-Here are some sample requests for Service Column creation as well as vertex insertion and selection.
-
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/createServiceColumn -H 'Content-Type: Application/json' -d '
- {
- "serviceName": "s2graph",
- "columnName": "user_id",
- "columnType": "long",
- "props": [
- {"name": "is_active", "dataType": "boolean", "defaultValue": true},
- {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
- {"name": "nickname", "dataType": "string", "defaultValue": ".."},
- {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
- {"name": "age", "dataType": "integer", "defaultValue": 0}
- ]
- }'
-
-General information on a vertex schema can be retrieved with ``/admin/getServiceColumn/{service name}/{column name}``
-
-.. code:: bash
-
- curl -XGET localhost:9000/admin/getServiceColumn/s2graph/user_id
-
-This will give all properties on serviceName ``s2graph`` and columnName ``user_id`` serviceColumn.
-Properties can be added to a Service Column with ``/admin/addServiceColumnProps/{service name}/{column name}``
-
-.. code:: bash
-
- curl -XPOST localhost:9000/admin/addServiceColumnProps/s2graph/user_id -H 'Content-Type: Application/json' -d '
- [
- {"name": "home_address", "defaultValue": "korea", "dataType": "string"}
- ]'
-
-Vertices can be inserted to a Service Column using ``/admin/vertices/insert/{service name}/{column name}``
-
-.. code:: bash
-
- curl -XPOST localhost:9000/mutate/vertex/insert/s2graph/user_id -H 'Content-Type: Application/json' -d '
- [
- {"id":1,"props":{"is_active":true}, "timestamp":1417616431},
- {"id":2,"props":{},"timestamp":1417616431}
- ]'
-
-Finally, query your vertex via ``/graphs/getVertices``
-
-.. code:: bash
-
- curl -XPOST localhost:9000/graphs/getVertices -H 'Content-Type: Application/json' -d '
- [
- {"serviceName": "s2graph", "columnName": "user_id", "ids": [1, 2, 3]}
- ]'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6414cb77/doc/source/api/read.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/read.rst b/doc/source/api/read.rst
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/6414cb77/doc/source/api/write.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/write.rst b/doc/source/api/write.rst
deleted file mode 100644
index e69de29..0000000
[16/20] incubator-s2graph git commit: add data model
Posted by st...@apache.org.
add data model
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/bbdc6618
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/bbdc6618
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/bbdc6618
Branch: refs/heads/master
Commit: bbdc6618553a42dafba7d227ae31edc78a8c2b01
Parents: 3bb9f02
Author: daewon <da...@apache.org>
Authored: Thu Dec 27 19:03:16 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Dec 27 19:03:16 2018 +0900
----------------------------------------------------------------------
doc/source/api/index.rst | 6 +-----
doc/source/api/query/index.rst | 4 +---
doc/source/api/query/query_edge.rst | 8 ++------
doc/source/api/query/query_vertex.rst | 13 +++++++++++++
doc/source/index.rst | 6 +-----
5 files changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bbdc6618/doc/source/api/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst
index e74650a..ed45c97 100644
--- a/doc/source/api/index.rst
+++ b/doc/source/api/index.rst
@@ -1,8 +1,3 @@
-.. Sphinx API Example documentation master file, created by
- sphinx-quickstart on Wed May 2 12:55:56 2012.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
HTTP API Glossary
==============================================
@@ -15,3 +10,4 @@ Contents:
management/index
mutate/index
+ query/index
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bbdc6618/doc/source/api/query/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/index.rst b/doc/source/api/query/index.rst
index 26e2b01..6d5b3fb 100644
--- a/doc/source/api/query/index.rst
+++ b/doc/source/api/query/index.rst
@@ -12,6 +12,4 @@ Contents:
query_options
query_edge
-
- ..
- query_vertex
+ query_vertex
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bbdc6618/doc/source/api/query/query_edge.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/query_edge.rst b/doc/source/api/query/query_edge.rst
index 2b74daf..b7bfb21 100644
--- a/doc/source/api/query/query_edge.rst
+++ b/doc/source/api/query/query_edge.rst
@@ -15,12 +15,8 @@ Another tip is to not be shy to ask! Ask any questions on our `mailing list`_. l
.. _github: https://github.com/apache/incubator-s2graph
-APIs
--------------------------------------
-
-
checkEdges - ``POST /graphs/checkEdges``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+------------------------------------------
return edge for given vertex pair only if edge exist.
This is more ``general`` way to check edge existence between any given vertex pairs comparing using ``_to`` on query parameter
@@ -36,7 +32,7 @@ This is more ``general`` way to check edge existence between any given vertex pa
getEdges - ``POST /graphs/getEdges``
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------------------------
Select edges with query.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bbdc6618/doc/source/api/query/query_vertex.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/query/query_vertex.rst b/doc/source/api/query/query_vertex.rst
index 15f5e3e..1ff0b2d 100644
--- a/doc/source/api/query/query_vertex.rst
+++ b/doc/source/api/query/query_vertex.rst
@@ -2,3 +2,16 @@
Query Vertices
****************
+
+POST - ``/graphs/getVertices``
+--------------------------------
+
+Selecting all vertices from serviceColumn account_id of a service s2graph.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getVertices -H 'Content-Type: Application/json' -d '
+ [
+ {"serviceName": "s2graph", "columnName": "account_id", "ids": [1, 2, 3]},
+ {"serviceName": "agit", "columnName": "user_id", "ids": [1, 2, 3]}
+ ]'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/bbdc6618/doc/source/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/index.rst b/doc/source/index.rst
index fd4e755..7f4455e 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -1,8 +1,3 @@
-.. S2Graph documentation master file, created by
- sphinx-quickstart on Mon Nov 12 15:53:14 2018.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
Introduction
===================================
@@ -20,6 +15,7 @@ dev@s2graph.incubator.apache.org is for people who want to contribute to S2Graph
getting_started/index
getting_started/your_first_graph
+ design/data_model
api/index
..
[18/20] incubator-s2graph git commit: fix the bug: broken test
Posted by st...@apache.org.
fix the bug: broken test
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/8c3f4a28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/8c3f4a28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/8c3f4a28
Branch: refs/heads/master
Commit: 8c3f4a28d4681999b04575eec250d02102d3bbd6
Parents: 4bcf9d9
Author: daewon <da...@apache.org>
Authored: Thu Jan 24 20:59:10 2019 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Jan 24 20:59:10 2019 +0900
----------------------------------------------------------------------
.../scala/org/apache/s2graph/graphql/TestGraph.scala | 2 +-
.../scala/org/apache/s2graph/http/AdminRouteSpec.scala | 10 +++++++---
.../org/apache/s2graph/http/MutateRouteSpec.scala | 13 ++++++++++---
3 files changed, 18 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8c3f4a28/s2graphql/src/test/scala/org/apache/s2graph/graphql/TestGraph.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/test/scala/org/apache/s2graph/graphql/TestGraph.scala b/s2graphql/src/test/scala/org/apache/s2graph/graphql/TestGraph.scala
index ef01d52..dc578bb 100644
--- a/s2graphql/src/test/scala/org/apache/s2graph/graphql/TestGraph.scala
+++ b/s2graphql/src/test/scala/org/apache/s2graph/graphql/TestGraph.scala
@@ -85,7 +85,7 @@ class EmptyGraph(config: Config) extends TestGraph {
override def cleanup(): Unit = graph.shutdown(true)
- override def schema: Schema[GraphRepository, Any] = new SchemaDef(s2Repository).S2GraphSchema
+ override def schema: Schema[GraphRepository, Any] = new SchemaDef(s2Repository).schema
override def showSchema: String = SchemaRenderer.renderSchema(schema)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8c3f4a28/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
index 26a7045..42ddda0 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
@@ -12,12 +12,16 @@ import org.slf4j.LoggerFactory
import play.api.libs.json.{JsString, JsValue, Json}
class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll {
- val config = ConfigFactory.load()
- val s2graph = new S2Graph(config)
+ import scala.collection.JavaConverters._
+
+ val dbUrl = "jdbc:h2:file:./var/metastore_admin_route;MODE=MYSQL;AUTO_SERVER=true"
+ val config =
+ ConfigFactory.parseMap(Map("db.default.url" -> dbUrl).asJava)
+ lazy val s2graph = new S2Graph(config.withFallback(ConfigFactory.load()))
override val logger = LoggerFactory.getLogger(this.getClass)
override def afterAll(): Unit = {
- s2graph.shutdown()
+ s2graph.shutdown(true)
}
lazy val routes = adminRoute
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/8c3f4a28/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
index 943db98..7a3d2ab 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
@@ -4,6 +4,7 @@ import akka.http.scaladsl.marshalling.Marshal
import akka.http.scaladsl.model._
import akka.http.scaladsl.testkit.ScalatestRouteTest
import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.Management.JsonModel.Prop
import org.apache.s2graph.core.S2Graph
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec}
@@ -11,12 +12,17 @@ import org.slf4j.LoggerFactory
import play.api.libs.json.{JsValue, Json}
class MutateRouteSpec extends WordSpec with Matchers with PlayJsonSupport with ScalaFutures with ScalatestRouteTest with S2GraphMutateRoute with BeforeAndAfterAll {
- val config = ConfigFactory.load()
- val s2graph = new S2Graph(config)
+
+ import scala.collection.JavaConverters._
+
+ val dbUrl = "jdbc:h2:file:./var/metastore_mutate_route;MODE=MYSQL;AUTO_SERVER=true"
+ val config =
+ ConfigFactory.parseMap(Map("db.default.url" -> dbUrl).asJava)
+ lazy val s2graph = new S2Graph(config.withFallback(ConfigFactory.load()))
override val logger = LoggerFactory.getLogger(this.getClass)
override def afterAll(): Unit = {
- s2graph.shutdown()
+ s2graph.shutdown(true)
}
lazy val routes = mutateRoute
@@ -28,6 +34,7 @@ class MutateRouteSpec extends WordSpec with Matchers with PlayJsonSupport with S
"be able to insert vertex (POST /mutate/vertex/insert)" in {
s2graph.management.createService(serviceName, "localhost", s"${serviceName}-dev", 1, None)
+ s2graph.management.createServiceColumn(serviceName, columnName, "string", Seq(Prop("age", "0", "integer")))
// {"timestamp": 10, "serviceName": "s2graph", "columnName": "user", "id": 1, "props": {}}
val param = Json.obj(
[05/20] incubator-s2graph git commit: merge S2GRAPH-249
Posted by st...@apache.org.
merge S2GRAPH-249
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/f41ab569
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/f41ab569
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/f41ab569
Branch: refs/heads/master
Commit: f41ab569345e9180e8f66355422bbeb7548f0670
Parents: 1dda954
Author: daewon <da...@apache.org>
Authored: Thu Nov 29 18:55:39 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Nov 29 18:55:39 2018 +0900
----------------------------------------------------------------------
.../apache/s2graph/http/PlayJsonSupport.scala | 42 ++++++++-
.../apache/s2graph/http/S2GraphAdminRoute.scala | 95 ++++++--------------
.../s2graph/http/S2GraphMutateRoute.scala | 55 ++++--------
.../apache/s2graph/http/AdminRouteSpec.scala | 4 +-
.../apache/s2graph/http/MutateRouteSpec.scala | 8 +-
5 files changed, 91 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f41ab569/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
index 8b4e91c..244e588 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
@@ -6,7 +6,7 @@ import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller}
import akka.http.scaladsl.model._
import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
import akka.util.ByteString
-import play.api.libs.json.{JsValue, Json}
+import play.api.libs.json._
trait PlayJsonSupport {
@@ -31,4 +31,44 @@ trait PlayJsonSupport {
case data => Json.parse(data.decodeString(Charset.forName("UTF-8")))
}
}
+
+ trait ToPlayJson[T] {
+ def toJson(msg: T): JsValue
+ }
+
+ import scala.language.reflectiveCalls
+
+ object ToPlayJson {
+ type ToPlayJsonReflective = {
+ def toJson: JsValue
+ }
+
+ implicit def forToJson[A <: ToPlayJsonReflective] = new ToPlayJson[A] {
+ def toJson(js: A) = js.toJson
+ }
+
+ implicit def forPlayJson[A <: JsValue] = new ToPlayJson[A] {
+ def toJson(js: A) = js
+ }
+ }
+
+ implicit object JsErrorJsonWriter extends Writes[JsError] {
+ def writes(o: JsError): JsValue = Json.obj(
+ "errors" -> JsArray(
+ o.errors.map {
+ case (path, validationErrors) => Json.obj(
+ "path" -> Json.toJson(path.toString()),
+ "validationErrors" -> JsArray(validationErrors.map(validationError => Json.obj(
+ "message" -> JsString(validationError.message),
+ "args" -> JsArray(validationError.args.map {
+ case x: Int => JsNumber(x)
+ case x => JsString(x.toString)
+ })
+ )))
+ )
+ }
+ )
+ )
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f41ab569/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
index 9bf7eb4..47ac86a 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
@@ -12,30 +12,9 @@ import play.api.libs.json._
import scala.util._
-object S2GraphAdminRoute {
-
- trait AdminMessageFormatter[T] {
- def toJson(msg: T): JsValue
- }
-
- import scala.language.reflectiveCalls
-
- object AdminMessageFormatter {
- type ToPlayJson = {
- def toJson: JsValue
- }
-
- implicit def toPlayJson[A <: ToPlayJson] = new AdminMessageFormatter[A] {
- def toJson(js: A) = js.toJson
- }
-
- implicit def fromPlayJson[T <: JsValue] = new AdminMessageFormatter[T] {
- def toJson(js: T) = js
- }
- }
-
- def toHttpEntity[A: AdminMessageFormatter](opt: Option[A], status: StatusCode = StatusCodes.OK, message: String = ""): HttpResponse = {
- val ev = implicitly[AdminMessageFormatter[A]]
+object S2GraphAdminRoute extends PlayJsonSupport {
+ def toHttpEntity[A: ToPlayJson](opt: Option[A], status: StatusCode = StatusCodes.OK, message: String = ""): HttpResponse = {
+ val ev = implicitly[ToPlayJson[A]]
val res = opt.map(ev.toJson).getOrElse(Json.obj("message" -> message))
HttpResponse(
@@ -44,9 +23,9 @@ object S2GraphAdminRoute {
)
}
- def toHttpEntity[A: AdminMessageFormatter](opt: Try[A]): HttpResponse = {
- val ev = implicitly[AdminMessageFormatter[A]]
- val (status, res) = opt match {
+ def toHttpEntity[A: ToPlayJson](_try: Try[A]): HttpResponse = {
+ val ev = implicitly[ToPlayJson[A]]
+ val (status, res) = _try match {
case Success(m) => StatusCodes.Created -> Json.obj("status" -> "ok", "message" -> ev.toJson(m))
case Failure(e) => StatusCodes.OK -> Json.obj("status" -> "failure", "message" -> e.toString)
}
@@ -54,9 +33,9 @@ object S2GraphAdminRoute {
toHttpEntity(Option(res), status = status)
}
- def toHttpEntity[A: AdminMessageFormatter](ls: Seq[A], status: StatusCode = StatusCodes.OK): HttpResponse = {
- val ev = implicitly[AdminMessageFormatter[A]]
- val res = ls.map(ev.toJson)
+ def toHttpEntity[A: ToPlayJson](ls: Seq[A], status: StatusCode): HttpResponse = {
+ val ev = implicitly[ToPlayJson[A]]
+ val res = JsArray(ls.map(ev.toJson))
HttpResponse(
status = status,
@@ -85,11 +64,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
// GET /graphs/getServiceColumn/:serviceName/:columnName
- lazy val getServiceColumn = path("getServiceColumn" / Segments) { params =>
- val (serviceName, columnName) = params match {
- case s :: c :: Nil => (s, c)
- }
-
+ lazy val getServiceColumn = path("getServiceColumn" / Segment / Segment) { (serviceName, columnName) =>
val ret = Management.findServiceColumn(serviceName, columnName)
complete(toHttpEntity(ret, message = s"ServiceColumn not found: ${serviceName}, ${columnName}"))
}
@@ -105,7 +80,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
lazy val getLabels = path("getLabels" / Segment) { serviceName =>
val ret = Management.findLabels(serviceName)
- complete(toHttpEntity(ret))
+ complete(toHttpEntity(ret, StatusCodes.OK))
}
/* POST */
@@ -175,6 +150,7 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
val (serviceName, columnName, storeInGlobalIndex) = params match {
case s :: c :: Nil => (s, c, false)
case s :: c :: i :: Nil => (s, c, i.toBoolean)
+ case _ => throw new RuntimeException("Invalid Params")
}
entity(as[JsValue]) { params =>
@@ -199,45 +175,33 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
hTableParams.preSplitSize, hTableParams.hTableTTL,
hTableParams.compressionAlgorithm.getOrElse(Management.DefaultCompressionAlgorithm))
- complete(toHttpEntity(None, status = StatusCodes.OK, message = "created"))
+ complete(toHttpEntity(None: Option[JsValue], status = StatusCodes.OK, message = "created"))
}
- case err@JsError(_) => complete(toHttpEntity(None, status = StatusCodes.BadRequest, message = Json.toJson(err).toString))
+ case err@JsError(_) => complete(toHttpEntity(None: Option[JsValue], status = StatusCodes.BadRequest, message = Json.toJson(err).toString))
}
}
}
// POST /graphs/copyLabel/:oldLabelName/:newLabelName
- lazy val copyLabel = path("copyLabel" / Segments) { params =>
- val (oldLabelName, newLabelName) = params match {
- case oldLabel :: newLabel :: Nil => (oldLabel, newLabel)
- }
-
+ lazy val copyLabel = path("copyLabel" / Segment / Segment) { (oldLabelName, newLabelName) =>
val copyTry = management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
complete(toHttpEntity(copyTry))
}
// POST /graphs/renameLabel/:oldLabelName/:newLabelName
- lazy val renameLabel = path("renameLabel" / Segments) { params =>
- val (oldLabelName, newLabelName) = params match {
- case oldLabel :: newLabel :: Nil => (oldLabel, newLabel)
- }
-
+ lazy val renameLabel = path("renameLabel" / Segment / Segment) { (oldLabelName, newLabelName) =>
Label.findByName(oldLabelName) match {
- case None => complete(toHttpEntity(None, status = StatusCodes.NotFound, message = s"Label $oldLabelName not found."))
+ case None => complete(toHttpEntity(None: Option[JsValue], status = StatusCodes.NotFound, message = s"Label $oldLabelName not found."))
case Some(label) =>
Management.updateLabelName(oldLabelName, newLabelName)
- complete(toHttpEntity(None, message = s"${label} was updated."))
+ complete(toHttpEntity(None: Option[JsValue], message = s"${label} was updated."))
}
}
// POST /graphs/swapLabels/:leftLabelName/:rightLabelName
- lazy val swapLabel = path("swapLabel" / Segments) { params =>
- val (leftLabelName, rightLabelName) = params match {
- case left :: right :: Nil => (left, right)
- }
-
+ lazy val swapLabel = path("swapLabel" / Segment / Segment) { (leftLabelName, rightLabelName) =>
val left = Label.findByName(leftLabelName, useCache = false)
val right = Label.findByName(rightLabelName, useCache = false)
// verify same schema
@@ -245,20 +209,15 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
(left, right) match {
case (Some(l), Some(r)) =>
Management.swapLabelNames(leftLabelName, rightLabelName)
-
- complete(toHttpEntity(None, message = s"Labels were swapped."))
+ complete(toHttpEntity(None: Option[JsValue], message = s"Labels were swapped."))
case _ =>
- complete(toHttpEntity(None, status = StatusCodes.NotFound, message = s"Label ${leftLabelName} or ${rightLabelName} not found."))
+ complete(toHttpEntity(None: Option[JsValue], status = StatusCodes.NotFound, message = s"Label ${leftLabelName} or ${rightLabelName} not found."))
}
}
// POST /graphs/updateHTable/:labelName/:newHTableName
- lazy val updateHTable = path("updateHTable" / Segments) { params =>
- val (labelName, newHTableName) = params match {
- case l :: h :: Nil => (l, h)
- }
-
- val updateTry = Management.updateHTable(labelName, newHTableName)
+ lazy val updateHTable = path("updateHTable" / Segment / Segment) { (labelName, newHTableName) =>
+ val updateTry = Management.updateHTable(labelName, newHTableName).map(Json.toJson(_))
complete(toHttpEntity(updateTry))
}
@@ -273,17 +232,13 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
// PUT /graphs/markDeletedLabel/:labelName
lazy val markDeletedLabel = path("markDeletedLabel" / Segment) { labelName =>
- val ret = Management.markDeletedLabel(labelName).toOption
+ val ret = Management.markDeletedLabel(labelName).toOption.map(Json.toJson(_))
complete(toHttpEntity(ret, message = s"Label not found: ${labelName}"))
}
// PUT /graphs/deleteServiceColumn/:serviceName/:columnName
- lazy val deleteServiceColumn = path("deleteServiceColumn" / Segments) { params =>
- val (serviceName, columnName) = params match {
- case s :: c :: Nil => (s, c)
- }
-
+ lazy val deleteServiceColumn = path("deleteServiceColumn" / Segment / Segment) { (serviceName, columnName) =>
val ret = Management.deleteColumn(serviceName, columnName).toOption
complete(toHttpEntity(ret, message = s"ServiceColumn not found: ${serviceName}, ${columnName}"))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f41ab569/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
index fc3b768..a65db5a 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
@@ -12,56 +12,40 @@ import play.api.libs.json.{JsValue, Json}
import scala.concurrent.{ExecutionContext, Future}
-trait S2GraphMutateRoute {
+trait S2GraphMutateRoute extends PlayJsonSupport {
val s2graph: S2Graph
val logger = LoggerFactory.getLogger(this.getClass)
+
lazy val parser = new RequestParser(s2graph)
// lazy val requestParser = new RequestParser(s2graph)
lazy val exceptionHandler = ExceptionHandler {
- case ex: JsonParseException =>
- complete(StatusCodes.BadRequest -> ex.getMessage)
- case ex: java.lang.IllegalArgumentException =>
- complete(StatusCodes.BadRequest -> ex.getMessage)
+ case ex: JsonParseException => complete(StatusCodes.BadRequest -> ex.getMessage)
+ case ex: java.lang.IllegalArgumentException => complete(StatusCodes.BadRequest -> ex.getMessage)
}
lazy val mutateVertex = path("vertex" / Segments) { params =>
+ implicit val ec = s2graph.ec
+
val (operation, serviceNameOpt, columnNameOpt) = params match {
- case operation :: serviceName :: columnName :: Nil =>
- (operation, Option(serviceName), Option(columnName))
- case operation :: Nil =>
- (operation, None, None)
+ case operation :: serviceName :: columnName :: Nil => (operation, Option(serviceName), Option(columnName))
+ case operation :: Nil => (operation, None, None)
+ case _ => throw new RuntimeException("invalid params")
}
- entity(as[String]) { body =>
- val payload = Json.parse(body)
-
- implicit val ec = s2graph.ec
-
- val future = vertexMutate(payload, operation, serviceNameOpt, columnNameOpt).map { mutateResponses =>
- HttpResponse(
- status = StatusCodes.OK,
- entity = HttpEntity(ContentTypes.`application/json`, Json.toJson(mutateResponses).toString)
- )
- }
+ entity(as[JsValue]) { payload =>
+ val future = vertexMutate(payload, operation, serviceNameOpt, columnNameOpt).map(Json.toJson(_))
complete(future)
}
}
lazy val mutateEdge = path("edge" / Segment) { operation =>
- entity(as[String]) { body =>
- val payload = Json.parse(body)
-
- implicit val ec = s2graph.ec
+ implicit val ec = s2graph.ec
- val future = edgeMutate(payload, operation, withWait = true).map { mutateResponses =>
- HttpResponse(
- status = StatusCodes.OK,
- entity = HttpEntity(ContentTypes.`application/json`, Json.toJson(mutateResponses).toString)
- )
- }
+ entity(as[JsValue]) { payload =>
+ val future = edgeMutate(payload, operation, withWait = true).map(Json.toJson(_))
complete(future)
}
@@ -79,13 +63,10 @@ trait S2GraphMutateRoute {
s2graph.mutateVertices(verticesToStore, withWait).map(_.map(_.isSuccess))
}
- def edgeMutate(elementsWithTsv: Seq[(GraphElement, String)],
- withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ def edgeMutate(elementsWithTsv: Seq[(GraphElement, String)], withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
val elementWithIdxs = elementsWithTsv.zipWithIndex
+ val (elementSync, elementAsync) = elementWithIdxs.partition { case ((element, tsv), idx) => !element.isAsync }
- val (elementSync, elementAsync) = elementWithIdxs.partition { case ((element, tsv), idx) =>
- !element.isAsync
- }
val retToSkip = elementAsync.map(_._2 -> MutateResponse.Success)
val (elementsToStore, _) = elementSync.map(_._1).unzip
val elementsIdxToStore = elementSync.map(_._2)
@@ -95,9 +76,7 @@ trait S2GraphMutateRoute {
}.map(_.sortBy(_._1).map(_._2.isSuccess))
}
- def edgeMutate(jsValue: JsValue,
- operation: String,
- withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ def edgeMutate(jsValue: JsValue, operation: String, withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
val edgesWithTsv = parser.parseJsonFormat(jsValue, operation)
edgeMutate(edgesWithTsv, withWait)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f41ab569/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
index eade7e6..26a7045 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
@@ -32,7 +32,7 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
"compressionAlgorithm" -> "gz"
)
- val serviceEntity = Marshal(serviceParam.toString).to[MessageEntity].futureValue
+ val serviceEntity = Marshal(serviceParam).to[MessageEntity].futureValue
val request = Post("/createService").withEntity(serviceEntity)
request ~> routes ~> check {
@@ -71,7 +71,7 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
)
)
- val serviceColumnEntity = Marshal(serviceColumnParam.toString).to[MessageEntity].futureValue
+ val serviceColumnEntity = Marshal(serviceColumnParam).to[MessageEntity].futureValue
val request = Post("/createServiceColumn").withEntity(serviceColumnEntity)
request ~> routes ~> check {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/f41ab569/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
index f823cd5..943db98 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
@@ -25,8 +25,11 @@ class MutateRouteSpec extends WordSpec with Matchers with PlayJsonSupport with S
val columnName = "userName"
"MutateRoute" should {
+
"be able to insert vertex (POST /mutate/vertex/insert)" in {
- // {"timestamp": 10, "serviceName": "s2graph", "columnName": "user", "id": 1, "props": {}}
+ s2graph.management.createService(serviceName, "localhost", s"${serviceName}-dev", 1, None)
+
+ // {"timestamp": 10, "serviceName": "s2graph", "columnName": "user", "id": 1, "props": {}}
val param = Json.obj(
"timestamp" -> 10,
"serviceName" -> serviceName,
@@ -36,7 +39,8 @@ class MutateRouteSpec extends WordSpec with Matchers with PlayJsonSupport with S
"age" -> 20
)
)
- val entity = Marshal(param.toString).to[MessageEntity].futureValue
+
+ val entity = Marshal(param).to[MessageEntity].futureValue
val request = Post("/vertex/insert").withEntity(entity)
request ~> routes ~> check {
[03/20] incubator-s2graph git commit: merge S2GRAPH-249
Posted by st...@apache.org.
merge S2GRAPH-249
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b8ab86dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b8ab86dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b8ab86dd
Branch: refs/heads/master
Commit: b8ab86dd39fd3810afcf44160324fa2dc6f5ee3f
Parents: 4154bbe
Author: DO YUNG YOON <st...@apache.org>
Authored: Thu Nov 29 10:51:20 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Thu Nov 29 10:51:20 2018 +0900
----------------------------------------------------------------------
.../org/apache/s2graph/core/Management.scala | 44 ++++
.../org/apache/s2graph/core/schema/Label.scala | 38 ++--
.../apache/s2graph/http/S2GraphAdminRoute.scala | 224 ++++++++++++++++++-
.../s2graph/http/S2GraphMutateRoute.scala | 119 ++++++++++
.../scala/org/apache/s2graph/http/Server.scala | 4 +-
.../apache/s2graph/http/AdminRouteSpec.scala | 41 +++-
.../apache/s2graph/http/MutateRouteSpec.scala | 52 +++++
.../rest/play/controllers/AdminController.scala | 32 +--
8 files changed, 492 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index 0c41ee3..78edf80 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -66,18 +66,56 @@ object Management {
}
object JsonModel {
+ import play.api.libs.functional.syntax._
case class Prop(name: String, defaultValue: String, dataType: String, storeInGlobalIndex: Boolean = false)
object Prop extends ((String, String, String, Boolean) => Prop)
case class Index(name: String, propNames: Seq[String], direction: Option[Int] = None, options: Option[String] = None)
+
+ case class HTableParams(cluster: String, hTableName: String,
+ preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
+
+ override def toString(): String = {
+ s"""HtableParams
+ |-- cluster : $cluster
+ |-- hTableName : $hTableName
+ |-- preSplitSize : $preSplitSize
+ |-- hTableTTL : $hTableTTL
+ |-- compressionAlgorithm : $compressionAlgorithm
+ |""".stripMargin
+ }
+ }
+
+ implicit object HTableParamsJsonConverter extends Format[HTableParams] {
+ def reads(json: JsValue): JsResult[HTableParams] = (
+ (__ \ "cluster").read[String] and
+ (__ \ "hTableName").read[String] and
+ (__ \ "preSplitSize").read[Int] and
+ (__ \ "hTableTTL").readNullable[Int] and
+ (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
+
+ def writes(o: HTableParams): JsValue = Json.obj(
+ "cluster" -> o.cluster,
+ "hTableName" -> o.hTableName,
+ "preSplitSize" -> o.preSplitSize,
+ "hTableTTL" -> o.hTableTTL,
+ "compressionAlgorithm" -> o.compressionAlgorithm
+ )
+ }
}
def findService(serviceName: String) = {
Service.findByName(serviceName, useCache = false)
}
+ def findServiceColumn(serviceName: String, columnName: String): Option[ServiceColumn] = {
+ Service.findByName(serviceName, useCache = false).flatMap { service =>
+ ServiceColumn.find(service.id.get, columnName, useCache = false)
+ }
+ }
+
def deleteService(serviceName: String) = {
Service.findByName(serviceName).foreach { service =>
// service.deleteAll()
@@ -133,6 +171,12 @@ object Management {
Label.findByName(labelName, useCache = useCache)
}
+ def findLabels(serviceName: String, useCache: Boolean = false): Seq[Label] = {
+ Service.findByName(serviceName, useCache = useCache).map { service =>
+ Label.findBySrcServiceId(service.id.get, useCache = useCache)
+ }.getOrElse(Nil)
+ }
+
def deleteLabel(labelName: String): Try[Label] = {
Schema withTx { implicit session =>
val label = Label.findByName(labelName, useCache = false).getOrElse(throw GraphExceptions.LabelNotExistException(labelName))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
index a359958..2780475 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/schema/Label.scala
@@ -126,44 +126,50 @@ object Label extends SQLSyntaxSupport[Label] {
else sql.get
}
- def findByTgtColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ def findByTgtColumnId(columnId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[Label] = {
val cacheKey = className + "tgtColumnId=" + columnId
val col = ServiceColumn.findById(columnId)
- withCaches(cacheKey)(
- sql"""
+ lazy val sql = sql"""
select *
from labels
where tgt_column_name = ${col.columnName}
and service_id = ${col.serviceId}
and deleted_at is null
- """.map { rs => Label(rs) }.list().apply())
+ """.map { rs => Label(rs) }.list().apply()
+
+ if (useCache) withCaches(cacheKey)(sql)
+ else sql
}
- def findBySrcColumnId(columnId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ def findBySrcColumnId(columnId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[Label] = {
val cacheKey = className + "srcColumnId=" + columnId
val col = ServiceColumn.findById(columnId)
- withCaches(cacheKey)(
- sql"""
+ lazy val sql = sql"""
select *
from labels
where src_column_name = ${col.columnName}
and service_id = ${col.serviceId}
and deleted_at is null
- """.map { rs => Label(rs) }.list().apply())
+ """.map { rs => Label(rs) }.list().apply()
+
+ if (useCache) withCaches(cacheKey)(sql)
+ else sql
}
- def findBySrcServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ def findBySrcServiceId(serviceId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[Label] = {
val cacheKey = className + "srcServiceId=" + serviceId
- withCaches(cacheKey)(
- sql"""select * from labels where src_service_id = ${serviceId} and deleted_at is null""".map { rs => Label(rs) }.list().apply
- )
+ lazy val sql = sql"""select * from labels where src_service_id = ${serviceId} and deleted_at is null""".map { rs => Label(rs) }.list().apply
+
+ if (useCache) withCaches(cacheKey)(sql)
+ else sql
}
- def findByTgtServiceId(serviceId: Int)(implicit session: DBSession = AutoSession): List[Label] = {
+ def findByTgtServiceId(serviceId: Int, useCache: Boolean = true)(implicit session: DBSession = AutoSession): List[Label] = {
val cacheKey = className + "tgtServiceId=" + serviceId
- withCaches(cacheKey)(
- sql"""select * from labels where tgt_service_id = ${serviceId} and deleted_at is null""".map { rs => Label(rs) }.list().apply
- )
+ lazy val sql = sql"""select * from labels where tgt_service_id = ${serviceId} and deleted_at is null""".map { rs => Label(rs) }.list().apply
+
+ if (useCache) withCaches(cacheKey)(sql)
+ else sql
}
def insertAll(labelName: String, srcServiceName: String, srcColumnName: String, srcColumnType: String,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
index 8efd2ab..9bf7eb4 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
@@ -3,10 +3,10 @@ package org.apache.s2graph.http
import akka.http.scaladsl.model._
import org.apache.s2graph.core.rest.RequestParser
import org.apache.s2graph.core.{Management, S2Graph}
-
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.Directives._
-
+import org.apache.s2graph.core.Management.JsonModel.HTableParams
+import org.apache.s2graph.core.schema._
import org.slf4j.LoggerFactory
import play.api.libs.json._
@@ -53,6 +53,16 @@ object S2GraphAdminRoute {
toHttpEntity(Option(res), status = status)
}
+
+ def toHttpEntity[A: AdminMessageFormatter](ls: Seq[A], status: StatusCode = StatusCodes.OK): HttpResponse = {
+ val ev = implicitly[AdminMessageFormatter[A]]
+ val res = ls.map(ev.toJson)
+
+ HttpResponse(
+ status = status,
+ entity = HttpEntity(ContentTypes.`application/json`, res.toString)
+ )
+ }
}
trait S2GraphAdminRoute extends PlayJsonSupport {
@@ -66,18 +76,40 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
lazy val requestParser: RequestParser = new RequestParser(s2graph)
// routes impl
+ /* GET */
+ // GET /graphs/getService/:serviceName
+ lazy val getService = path("getService" / Segment) { serviceName =>
+ val serviceOpt = Management.findService(serviceName)
+
+ complete(toHttpEntity(serviceOpt, message = s"Service not found: ${serviceName}"))
+ }
+
+ // GET /graphs/getServiceColumn/:serviceName/:columnName
+ lazy val getServiceColumn = path("getServiceColumn" / Segments) { params =>
+ val (serviceName, columnName) = params match {
+ case s :: c :: Nil => (s, c)
+ }
+
+ val ret = Management.findServiceColumn(serviceName, columnName)
+ complete(toHttpEntity(ret, message = s"ServiceColumn not found: ${serviceName}, ${columnName}"))
+ }
+
+ // GET /graphs/getLabel/:labelName
lazy val getLabel = path("getLabel" / Segment) { labelName =>
val labelOpt = Management.findLabel(labelName)
complete(toHttpEntity(labelOpt, message = s"Label not found: ${labelName}"))
}
- lazy val getService = path("getService" / Segment) { serviceName =>
- val serviceOpt = Management.findService(serviceName)
+ // GET /graphs/getLabels/:serviceName
+ lazy val getLabels = path("getLabels" / Segment) { serviceName =>
+ val ret = Management.findLabels(serviceName)
- complete(toHttpEntity(serviceOpt, message = s"Service not found: ${serviceName}"))
+ complete(toHttpEntity(ret))
}
+ /* POST */
+ // POST /graphs/createService
lazy val createService = path("createService") {
entity(as[JsValue]) { params =>
@@ -91,28 +123,204 @@ trait S2GraphAdminRoute extends PlayJsonSupport {
}
}
+ // POST /graphs/createServiceColumn
+ lazy val createServiceColumn = path("createServiceColumn") {
+ entity(as[JsValue]) { params =>
+
+ val parseTry = requestParser.toServiceColumnElements(params)
+ val serviceColumnTry = for {
+ (serviceName, columnName, columnType, props) <- parseTry
+ serviceColumn <- Try(management.createServiceColumn(serviceName, columnName, columnType, props))
+ } yield serviceColumn
+
+ complete(toHttpEntity(serviceColumnTry))
+ }
+ }
+
+ // POST /graphs/createLabel
lazy val createLabel = path("createLabel") {
entity(as[JsValue]) { params =>
+ val labelTry = requestParser.toLabelElements(params)
+
+ complete(toHttpEntity(labelTry))
+ }
+ }
+ // POST /graphs/addIndex
+ lazy val addIndex = path("addIndex") {
+ entity(as[JsValue]) { params =>
val labelTry = for {
- label <- requestParser.toLabelElements(params)
+ (labelName, indices) <- requestParser.toIndexElements(params)
+ label <- Management.addIndex(labelName, indices)
} yield label
complete(toHttpEntity(labelTry))
}
}
+ // POST /graphs/addProp/:labelName
+ lazy val addProp = path("addProp" / Segment) { labelName =>
+ entity(as[JsValue]) { params =>
+ val labelMetaTry = for {
+ prop <- requestParser.toPropElements(params)
+ labelMeta <- Management.addProp(labelName, prop)
+ } yield labelMeta
+
+ complete(toHttpEntity(labelMetaTry))
+ }
+ }
+
+ // POST /graphs/addServiceColumnProp/:serviceName/:columnName
+ lazy val addServiceColumnProp = path("addServiceColumnProp" / Segments) { params =>
+ val (serviceName, columnName, storeInGlobalIndex) = params match {
+ case s :: c :: Nil => (s, c, false)
+ case s :: c :: i :: Nil => (s, c, i.toBoolean)
+ }
+
+ entity(as[JsValue]) { params =>
+ val columnMetaOpt = for {
+ service <- Service.findByName(serviceName)
+ serviceColumn <- ServiceColumn.find(service.id.get, columnName)
+ prop <- requestParser.toPropElements(params).toOption
+ } yield {
+ ColumnMeta.findOrInsert(serviceColumn.id.get, prop.name, prop.dataType, prop.defaultValue, storeInGlobalIndex)
+ }
+
+ complete(toHttpEntity(columnMetaOpt, message = s"can`t find service with $serviceName or can`t find serviceColumn with $columnName"))
+ }
+ }
+
+ // POST /graphs/createHTable
+ lazy val createHTable = path("createHTable") {
+ entity(as[JsValue]) { params =>
+ params.validate[HTableParams] match {
+ case JsSuccess(hTableParams, _) => {
+ management.createStorageTable(hTableParams.cluster, hTableParams.hTableName, List("e", "v"),
+ hTableParams.preSplitSize, hTableParams.hTableTTL,
+ hTableParams.compressionAlgorithm.getOrElse(Management.DefaultCompressionAlgorithm))
+
+ complete(toHttpEntity(None, status = StatusCodes.OK, message = "created"))
+ }
+ case err@JsError(_) => complete(toHttpEntity(None, status = StatusCodes.BadRequest, message = Json.toJson(err).toString))
+ }
+ }
+ }
+
+ // POST /graphs/copyLabel/:oldLabelName/:newLabelName
+ lazy val copyLabel = path("copyLabel" / Segments) { params =>
+ val (oldLabelName, newLabelName) = params match {
+ case oldLabel :: newLabel :: Nil => (oldLabel, newLabel)
+ }
+
+ val copyTry = management.copyLabel(oldLabelName, newLabelName, Some(newLabelName))
+
+ complete(toHttpEntity(copyTry))
+ }
+
+ // POST /graphs/renameLabel/:oldLabelName/:newLabelName
+ lazy val renameLabel = path("renameLabel" / Segments) { params =>
+ val (oldLabelName, newLabelName) = params match {
+ case oldLabel :: newLabel :: Nil => (oldLabel, newLabel)
+ }
+
+ Label.findByName(oldLabelName) match {
+ case None => complete(toHttpEntity(None, status = StatusCodes.NotFound, message = s"Label $oldLabelName not found."))
+ case Some(label) =>
+ Management.updateLabelName(oldLabelName, newLabelName)
+
+ complete(toHttpEntity(None, message = s"${label} was updated."))
+ }
+ }
+
+ // POST /graphs/swapLabels/:leftLabelName/:rightLabelName
+ lazy val swapLabel = path("swapLabel" / Segments) { params =>
+ val (leftLabelName, rightLabelName) = params match {
+ case left :: right :: Nil => (left, right)
+ }
+
+ val left = Label.findByName(leftLabelName, useCache = false)
+ val right = Label.findByName(rightLabelName, useCache = false)
+ // verify same schema
+
+ (left, right) match {
+ case (Some(l), Some(r)) =>
+ Management.swapLabelNames(leftLabelName, rightLabelName)
+
+ complete(toHttpEntity(None, message = s"Labels were swapped."))
+ case _ =>
+ complete(toHttpEntity(None, status = StatusCodes.NotFound, message = s"Label ${leftLabelName} or ${rightLabelName} not found."))
+ }
+ }
+
+ // POST /graphs/updateHTable/:labelName/:newHTableName
+ lazy val updateHTable = path("updateHTable" / Segments) { params =>
+ val (labelName, newHTableName) = params match {
+ case l :: h :: Nil => (l, h)
+ }
+
+ val updateTry = Management.updateHTable(labelName, newHTableName)
+
+ complete(toHttpEntity(updateTry))
+ }
+
+ /* PUT */
+ // PUT /graphs/deleteLabelReally/:labelName
+ lazy val deleteLabelReally = path("deleteLabelReally" / Segment) { labelName =>
+ val ret = Management.deleteLabel(labelName).toOption
+
+ complete(toHttpEntity(ret, message = s"Label not found: ${labelName}"))
+ }
+
+ // PUT /graphs/markDeletedLabel/:labelName
+ lazy val markDeletedLabel = path("markDeletedLabel" / Segment) { labelName =>
+ val ret = Management.markDeletedLabel(labelName).toOption
+
+ complete(toHttpEntity(ret, message = s"Label not found: ${labelName}"))
+ }
+
+ // PUT /graphs/deleteServiceColumn/:serviceName/:columnName
+ lazy val deleteServiceColumn = path("deleteServiceColumn" / Segments) { params =>
+ val (serviceName, columnName) = params match {
+ case s :: c :: Nil => (s, c)
+ }
+
+ val ret = Management.deleteColumn(serviceName, columnName).toOption
+
+ complete(toHttpEntity(ret, message = s"ServiceColumn not found: ${serviceName}, ${columnName}"))
+ }
+
+ // TODO:
+ // delete service?
+ // PUT /graphs/loadCache
+
// expose routes
lazy val adminRoute: Route =
get {
concat(
getService,
- getLabel
+ getServiceColumn,
+ getLabel,
+ getLabels
)
} ~ post {
concat(
createService,
- createLabel
+ createServiceColumn,
+ createLabel,
+ addIndex,
+ addProp,
+ addServiceColumnProp,
+ createHTable,
+ copyLabel,
+ renameLabel,
+ swapLabel,
+ updateHTable
+ )
+ } ~ put {
+ concat(
+ deleteLabelReally,
+ markDeletedLabel,
+ deleteServiceColumn
)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
new file mode 100644
index 0000000..fc3b768
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphMutateRoute.scala
@@ -0,0 +1,119 @@
+package org.apache.s2graph.http
+
+import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes}
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server.{ExceptionHandler, Route}
+import com.fasterxml.jackson.core.JsonParseException
+import org.apache.s2graph.core.rest.RequestParser
+import org.apache.s2graph.core.storage.MutateResponse
+import org.apache.s2graph.core.{GraphElement, S2Graph}
+import org.slf4j.LoggerFactory
+import play.api.libs.json.{JsValue, Json}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait S2GraphMutateRoute {
+
+ val s2graph: S2Graph
+ val logger = LoggerFactory.getLogger(this.getClass)
+ lazy val parser = new RequestParser(s2graph)
+
+ // lazy val requestParser = new RequestParser(s2graph)
+ lazy val exceptionHandler = ExceptionHandler {
+ case ex: JsonParseException =>
+ complete(StatusCodes.BadRequest -> ex.getMessage)
+ case ex: java.lang.IllegalArgumentException =>
+ complete(StatusCodes.BadRequest -> ex.getMessage)
+ }
+
+ lazy val mutateVertex = path("vertex" / Segments) { params =>
+ val (operation, serviceNameOpt, columnNameOpt) = params match {
+ case operation :: serviceName :: columnName :: Nil =>
+ (operation, Option(serviceName), Option(columnName))
+ case operation :: Nil =>
+ (operation, None, None)
+ }
+
+ entity(as[String]) { body =>
+ val payload = Json.parse(body)
+
+ implicit val ec = s2graph.ec
+
+ val future = vertexMutate(payload, operation, serviceNameOpt, columnNameOpt).map { mutateResponses =>
+ HttpResponse(
+ status = StatusCodes.OK,
+ entity = HttpEntity(ContentTypes.`application/json`, Json.toJson(mutateResponses).toString)
+ )
+ }
+
+ complete(future)
+ }
+ }
+
+ lazy val mutateEdge = path("edge" / Segment) { operation =>
+ entity(as[String]) { body =>
+ val payload = Json.parse(body)
+
+ implicit val ec = s2graph.ec
+
+ val future = edgeMutate(payload, operation, withWait = true).map { mutateResponses =>
+ HttpResponse(
+ status = StatusCodes.OK,
+ entity = HttpEntity(ContentTypes.`application/json`, Json.toJson(mutateResponses).toString)
+ )
+ }
+
+ complete(future)
+ }
+ }
+
+ def vertexMutate(jsValue: JsValue,
+ operation: String,
+ serviceNameOpt: Option[String] = None,
+ columnNameOpt: Option[String] = None,
+ withWait: Boolean = true)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ val vertices = parser.toVertices(jsValue, operation, serviceNameOpt, columnNameOpt)
+
+ val verticesToStore = vertices.filterNot(_.isAsync)
+
+ s2graph.mutateVertices(verticesToStore, withWait).map(_.map(_.isSuccess))
+ }
+
+ def edgeMutate(elementsWithTsv: Seq[(GraphElement, String)],
+ withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ val elementWithIdxs = elementsWithTsv.zipWithIndex
+
+ val (elementSync, elementAsync) = elementWithIdxs.partition { case ((element, tsv), idx) =>
+ !element.isAsync
+ }
+ val retToSkip = elementAsync.map(_._2 -> MutateResponse.Success)
+ val (elementsToStore, _) = elementSync.map(_._1).unzip
+ val elementsIdxToStore = elementSync.map(_._2)
+
+ s2graph.mutateElements(elementsToStore, withWait).map { mutateResponses =>
+ elementsIdxToStore.zip(mutateResponses) ++ retToSkip
+ }.map(_.sortBy(_._1).map(_._2.isSuccess))
+ }
+
+ def edgeMutate(jsValue: JsValue,
+ operation: String,
+ withWait: Boolean)(implicit ec: ExecutionContext): Future[Seq[Boolean]] = {
+ val edgesWithTsv = parser.parseJsonFormat(jsValue, operation)
+ edgeMutate(edgesWithTsv, withWait)
+ }
+
+ // expose routes
+ lazy val mutateRoute: Route =
+ post {
+ concat(
+ handleExceptions(exceptionHandler) {
+ mutateVertex
+ },
+ handleExceptions(exceptionHandler) {
+ mutateEdge
+ }
+ )
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
index 82d2343..c26e314 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala
@@ -35,7 +35,8 @@ import org.slf4j.LoggerFactory
object Server extends App
with S2GraphTraversalRoute
- with S2GraphAdminRoute {
+ with S2GraphAdminRoute
+ with S2GraphMutateRoute {
implicit val system: ActorSystem = ActorSystem("S2GraphHttpServer")
implicit val materializer: ActorMaterializer = ActorMaterializer()
@@ -54,6 +55,7 @@ object Server extends App
// Allows you to determine routes to expose according to external settings.
lazy val routes: Route = concat(
pathPrefix("graphs")(traversalRoute),
+ pathPrefix("mutate")(mutateRoute),
pathPrefix("admin")(adminRoute),
get(complete(health))
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
index 9239598..eade7e6 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
@@ -4,17 +4,16 @@ import akka.http.scaladsl.marshalling.Marshal
import akka.http.scaladsl.model._
import akka.http.scaladsl.testkit.ScalatestRouteTest
import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.Management.JsonModel.Prop
import org.apache.s2graph.core.S2Graph
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec}
import org.slf4j.LoggerFactory
import play.api.libs.json.{JsString, JsValue, Json}
-class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll with PlayJsonSupport {
-
+class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll {
val config = ConfigFactory.load()
val s2graph = new S2Graph(config)
-
override val logger = LoggerFactory.getLogger(this.getClass)
override def afterAll(): Unit = {
@@ -23,14 +22,17 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
lazy val routes = adminRoute
+ val serviceName = "kakaoFavorites"
+ val columnName = "userName"
+
"AdminRoute" should {
"be able to create service (POST /createService)" in {
val serviceParam = Json.obj(
- "serviceName" -> "kakaoFavorites",
+ "serviceName" -> serviceName,
"compressionAlgorithm" -> "gz"
)
- val serviceEntity = Marshal(serviceParam).to[MessageEntity].futureValue
+ val serviceEntity = Marshal(serviceParam.toString).to[MessageEntity].futureValue
val request = Post("/createService").withEntity(serviceEntity)
request ~> routes ~> check {
@@ -45,7 +47,7 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
}
"return service if present (GET /getService/{serviceName})" in {
- val request = HttpRequest(uri = "/getService/kakaoFavorites")
+ val request = HttpRequest(uri = s"/getService/$serviceName")
request ~> routes ~> check {
status should ===(StatusCodes.OK)
@@ -56,6 +58,33 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
(response \\ "name").head should ===(JsString("kakaoFavorites"))
}
}
+
+ "be able to create serviceColumn (POST /createServiceColumn)" in {
+ val serviceColumnParam = Json.obj(
+ "serviceName" -> serviceName,
+ "columnName" -> columnName,
+ "columnType" -> "string",
+ "props" -> Json.toJson(
+ Seq(
+ Json.obj("name" -> "age", "defaultValue" -> "-1", "dataType" -> "integer")
+ )
+ )
+ )
+
+ val serviceColumnEntity = Marshal(serviceColumnParam.toString).to[MessageEntity].futureValue
+ val request = Post("/createServiceColumn").withEntity(serviceColumnEntity)
+
+ request ~> routes ~> check {
+ status should ===(StatusCodes.Created)
+ contentType should ===(ContentTypes.`application/json`)
+
+ val response = entityAs[JsValue]
+
+ (response \\ "serviceName").head should ===(JsString("kakaoFavorites"))
+ (response \\ "columnName").head should ===(JsString("userName"))
+ (response \\ "status").head should ===(JsString("ok"))
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
new file mode 100644
index 0000000..f823cd5
--- /dev/null
+++ b/s2http/src/test/scala/org/apache/s2graph/http/MutateRouteSpec.scala
@@ -0,0 +1,52 @@
+package org.apache.s2graph.http
+
+import akka.http.scaladsl.marshalling.Marshal
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.testkit.ScalatestRouteTest
+import com.typesafe.config.ConfigFactory
+import org.apache.s2graph.core.S2Graph
+import org.scalatest.concurrent.ScalaFutures
+import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec}
+import org.slf4j.LoggerFactory
+import play.api.libs.json.{JsValue, Json}
+
+class MutateRouteSpec extends WordSpec with Matchers with PlayJsonSupport with ScalaFutures with ScalatestRouteTest with S2GraphMutateRoute with BeforeAndAfterAll {
+ val config = ConfigFactory.load()
+ val s2graph = new S2Graph(config)
+ override val logger = LoggerFactory.getLogger(this.getClass)
+
+ override def afterAll(): Unit = {
+ s2graph.shutdown()
+ }
+
+ lazy val routes = mutateRoute
+
+ val serviceName = "kakaoFavorites"
+ val columnName = "userName"
+
+ "MutateRoute" should {
+ "be able to insert vertex (POST /mutate/vertex/insert)" in {
+ // {"timestamp": 10, "serviceName": "s2graph", "columnName": "user", "id": 1, "props": {}}
+ val param = Json.obj(
+ "timestamp" -> 10,
+ "serviceName" -> serviceName,
+ "columnName" -> columnName,
+ "id" -> "user_a",
+ "props" -> Json.obj(
+ "age" -> 20
+ )
+ )
+ val entity = Marshal(param.toString).to[MessageEntity].futureValue
+ val request = Post("/vertex/insert").withEntity(entity)
+
+ request ~> routes ~> check {
+ status should ===(StatusCodes.OK)
+ contentType should ===(ContentTypes.`application/json`)
+
+ val response = entityAs[JsValue]
+ response should ===(Json.toJson(Seq(true)))
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b8ab86dd/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
index 00702cf..fdd9b63 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/AdminController.scala
@@ -20,6 +20,7 @@
package org.apache.s2graph.rest.play.controllers
import org.apache.s2graph.core.Management
+import org.apache.s2graph.core.Management.JsonModel.HTableParams
import org.apache.s2graph.core.schema._
import org.apache.s2graph.core.rest.RequestParser
import org.apache.s2graph.core.utils.logger
@@ -392,37 +393,6 @@ object AdminController extends Controller {
}
}
- case class HTableParams(cluster: String, hTableName: String,
- preSplitSize: Int, hTableTTL: Option[Int], compressionAlgorithm: Option[String]) {
-
- override def toString(): String = {
- s"""HtableParams
- |-- cluster : $cluster
- |-- hTableName : $hTableName
- |-- preSplitSize : $preSplitSize
- |-- hTableTTL : $hTableTTL
- |-- compressionAlgorithm : $compressionAlgorithm
- |""".stripMargin
- }
- }
-
- implicit object HTableParamsJsonConverter extends Format[HTableParams] {
- def reads(json: JsValue): JsResult[HTableParams] = (
- (__ \ "cluster").read[String] and
- (__ \ "hTableName").read[String] and
- (__ \ "preSplitSize").read[Int] and
- (__ \ "hTableTTL").readNullable[Int] and
- (__ \ "compressionAlgorithm").readNullable[String])(HTableParams.apply _).reads(json)
-
- def writes(o: HTableParams): JsValue = Json.obj(
- "cluster" -> o.cluster,
- "hTableName" -> o.hTableName,
- "preSplitSize" -> o.preSplitSize,
- "hTableTTL" -> o.hTableTTL,
- "compressionAlgorithm" -> o.compressionAlgorithm
- )
- }
-
implicit object JsErrorJsonWriter extends Writes[JsError] {
def writes(o: JsError): JsValue = Json.obj(
"errors" -> JsArray(
[13/20] incubator-s2graph git commit: fix csv-table warning
Posted by st...@apache.org.
fix csv-table warning
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c131990e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c131990e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c131990e
Branch: refs/heads/master
Commit: c131990e6d5c0a4b342d0c46f7d17727600f0f36
Parents: 6414cb7
Author: daewon <da...@apache.org>
Authored: Thu Dec 27 16:18:48 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Dec 27 16:18:48 2018 +0900
----------------------------------------------------------------------
doc/source/api/management/index.rst | 444 +++++++++++++++++++++++++++
doc/source/api/mutate/index.rst | 10 +
doc/source/api/mutate/mutate_edge.rst | 314 +++++++++++++++++++
doc/source/api/mutate/mutate_vertex.rst | 139 +++++++++
4 files changed, 907 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c131990e/doc/source/api/management/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/management/index.rst b/doc/source/api/management/index.rst
new file mode 100644
index 0000000..17061b0
--- /dev/null
+++ b/doc/source/api/management/index.rst
@@ -0,0 +1,444 @@
+Management APIs
+==================
+Admin Apis for Management Service, Label, Index ..
+
+****************
+Create a Service
+****************
+
+``Service`` is the top level abstraction in S2Graph which could be considered as a database in MySQL.
+
+.. code:: bash
+
+ POST /admin/createService
+
+Service Fields
+---------------
+
+In order to create a Service, the following fields should be specified in the request.
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "serviceName", "User defined namespace", "String", "talk_friendship", "Required"
+ "cluster", "Zookeeper quorum address", "String", "abc.com:2181,abd.com:2181", "Optional"
+ "hTableName", "HBase table name", "String", "test", "Optional"
+ "hTableTTL", "Time to live setting for data","Integer", "86000", "Optional"
+ "preSplitSize", "Factor for the table pre-split size", "Integer", "1", "Optional"
+
+.. list
+ - By default, S2Graph looks for "hbase.zookeeper.quorum" in your application.conf. If "hbase.zookeeper.quorum" is undefined, this value is set as "localhost".
+
+
+Basic Service Operations
+--------------------------
+
+You can create a service using the following API:
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createService -H 'Content-Type: Application/json' -d '
+ {
+ "serviceName": "s2graph",
+ "cluster": "address for zookeeper",
+ "hTableName": "hbase table name",
+ "hTableTTL": 86000,
+ "preSplitSize": 2
+ }'
+
+
+****************
+Create a Label
+****************
+
+A ``Label`` represents a relation between two serviceColumns. Labels are to S2Graph what tables are to RDBMS since they contain the schema information, i.e. descriptive information of the data being managed or indices used for efficient retrieval.
+In most scenarios, defining an edge schema (in other words, label) requires a little more care compared to a vertex schema (which is pretty straightforward).
+First, think about the kind of queries you will be using, then, model user actions or relations into ``edges`` and design a label accordingly.
+
+.. code:: bash
+
+ POST /admin/createLabel
+
+Label Fields
+---------------
+
+A Label creation request includes the following information.
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "label", "Name of the relation", "String", "talk_friendship", "Required"
+ "srcServiceName", "Source column's service", "String", "kakaotalk", "Required"
+ "srcColumnName", "Source column's name", "String", "user_id", "Required"
+ "srcColumnType", "Source column's data type","Long/Integer/String", "string", "Required"
+ "tgtServiceName", "Target column's service", "String", "kakaotalk/kakaomusic", "Optional"
+ "tgtColumnName", "Target column's name", "String", "item_id", "Required"
+ "tgtColumnType", "Target column's data type", "Long/Integer/String", "string", "Required"
+ "isDirected", "Wether the label is directed or undirected", "True/False", "true/false", "Optional. default is true"
+ "serviceName", "Which service the label belongs to", "String", "kakaotalk", "Optional. tgtServiceName is used by default"
+ "hTableName", "A dedicated HBase table to your Label", "String", "s2graph-batch", "Optional. Service hTableName is used by default"
+ "hTableTTL", "Data time to live setting", "Integer", "86000", "Optional. Service hTableTTL is used by default"
+ "consistencyLevel", "If set to 'strong', only one edge is alowed between a pair of source/ target vertices. Set to 'weak', and multiple-edge is supported", "String", "strong/weak", "Optional. default is 'weak'"
+
+
+Props & Indices
+----------------
+
+A couple of key elements of a Label are its Properties (props) and indices.
+Supplementary information of a Vertex or Edge can be stored as props. A single property can be defined in a simple key-value JSON as follows:
+
+.. code:: json
+
+ {
+ "name": "name of property",
+ "dataType": "data type of property value",
+ "defaultValue": "default value in string"
+ }
+
+In a scenario where user - video playback history is stored in a Label, a typical example for props would look like this:
+
+.. code:: json
+
+ [
+ {"name": "play_count", "defaultValue": 0, "dataType": "integer"},
+ {"name": "is_hidden","defaultValue": false,"dataType": "boolean"},
+ {"name": "category","defaultValue": "jazz","dataType": "string"},
+ {"name": "score","defaultValue": 0,"dataType": "float"}
+ ]
+
+Props can have data types of ``numeric`` (byte/ short/ integer/ float/ double), ``boolean`` or ``string``.
+In order to achieve efficient data retrieval, a Label can be indexed using the "indices" option.
+Default value for indices is ``_timestamp``, a hidden label property.
+
+All labels have ``_timestamp`` in their props under the hood
+
+The first index in indices array will be the primary index ``(Think of PRIMARY INDEX idx_xxx(p1, p2) in MySQL)``
+S2Graph will automatically store edges according to the primary index.
+Trailing indices are used for multiple ordering on edges. ``(Think of ALTER TABLE ADD INDEX idx_xxx(p2, p1) in MySQL)``
+
+props define meta datas that will not be affect the order of edges.
+Please avoid using S2Graph-reserved property names:
+
+
+- ``_timestamp`` is reserved for system wise timestamp. this can be interpreted as last_modified_at
+- ``_from`` is reserved for label's start vertex.
+- ``_to`` is reserved for label's target vertex.
+
+
+Basic Label Operations
+--------------------------
+
+Here is an sample request that creates a label ``user_article_liked`` between column ``user_id`` of service ``s2graph`` and column ``article_id`` of service ``s2graph_news``.
+Note that the default indexed property ``_timestamp`` will be created since the ``indexedProps`` field is empty.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
+ {
+ "label": "user_article_liked",
+ "srcServiceName": "s2graph",
+ "srcColumnName": "user_id",
+ "srcColumnType": "long",
+ "tgtServiceName": "s2graph_news",
+ "tgtColumnName": "article_id",
+ "tgtColumnType": "string",
+ "indices": [], // _timestamp will be used as default
+ "props": [],
+ "serviceName": "s2graph_news"
+ }'
+
+
+The created label ``user_article_liked`` will manage edges in a timestamp-descending order (which seems to be the common requirement for most services).
+Here is another example that creates a label ``friends``, which represents the friend relation between ``users`` in service ``s2graph``.
+This time, edges are managed by both affinity_score and ``_timestamp``.
+
+Friends with higher affinity_scores come first and if affinity_score is a tie, recently added friends comes first.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createLabel -H 'Content-Type: Application/json' -d '
+ {
+ "label": "friends",
+ "srcServiceName": "s2graph",
+ "srcColumnName": "user_id",
+ "srcColumnType": "long",
+ "tgtServiceName": "s2graph",
+ "tgtColumnName": "user_id",
+ "tgtColumnType": "long",
+ "indices": [
+ {"name": "idx_affinity_timestamp", "propNames": ["affinity_score", "_timestamp"]}
+ ],
+ "props": [
+ {"name": "affinity_score", "dataType": "float", "defaultValue": 0.0},
+ {"name": "_timestamp", "dataType": "long", "defaultValue": 0},
+ {"name": "is_hidden", "dataType": "boolean", "defaultValue": false},
+ {"name": "is_blocked", "dataType": "boolean", "defaultValue": true},
+ {"name": "error_code", "dataType": "integer", "defaultValue": 500}
+ ],
+ "serviceName": "s2graph",
+ "consistencyLevel": "strong"
+ }'
+
+S2Graph supports **multiple indices** on a label which means you can add separate ordering options for edges.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
+ {
+ "label": "friends",
+ "indices": [
+ {"name": "idx_3rd", "propNames": ["is_blocked", "_timestamp"]}
+ ]
+ }'
+
+In order to get general information on a label, make a GET request to ``/admin/getLabel/{label name}``
+
+.. code:: bash
+
+ curl -XGET localhost:9000/admin/getLabel/friends
+
+
+Delete a label with a PUT request to ``/admin/deleteLabel/{label name}``
+
+.. code:: bash
+
+ curl -XPUT localhost:9000/admin/deleteLabel/friends
+
+
+Label updates are not supported (except when you are adding an index). Instead, you can delete the label and re-create it.
+
+Adding Extra Properties to Labels
+----------------------------------
+
+To add a new property, use ``/admin/addProp/{label name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addProp/friend -H 'Content-Type: Application/json' -d '
+ {
+ "name": "is_blocked",
+ "defaultValue": false,
+ "dataType": "boolean"
+ }'
+
+
+Consistency Level
+------------------
+
+Simply put, the consistency level of your label will determine how the edges are stored at storage level.
+First, note that S2Graph identifies a unique edge by combining its from, label, to values as a key.
+
+Now, let's consider inserting the following two edges that have same keys (1, graph_test, 101) and different timestamps (1418950524721 and 1418950524723).
+
+.. code:: bash
+
+ 1418950524721 insert e 1 101 graph_test {"weight": 10} = (1, graph_test, 101)
+ 1418950524723 insert e 1 101 graph_test {"weight": 20} = (1, graph_test, 101)
+
+
+**Each consistency levels handle the case differently.**
+
+- strong
+
+ - The strong option makes sure that there is only one edge record stored in the HBase table for edge key (1, graph_test, 101). With strong consistency level, the later insertion will overwrite the previous one.
+
+- weak
+
+ - The weak option will allow two different edges stored in the table with different timestamps and weight values.
+
+
+For a better understanding, let's simplify the notation for an edge that connects two vertices u - v at time t as u -> (t, v), and assume that we are inserting these four edges into two different labels with each consistency configuration (both indexed by timestamp only).
+
+.. code:: bash
+
+ u1 -> (t1, v1)
+ u1 -> (t2, v2)
+ u1 -> (t3, v2)
+ u1 -> (t4, v1)
+
+With a strong consistencyLevel, your Label contents will be:
+
+.. code:: bash
+
+ u1 -> (t4, v1)
+ u1 -> (t3, v2)
+
+Note that edges with same vertices and earlier timestamp (u1 -> (t1, v1) and u1 -> (t2, v2)) were overwritten and do not exist.
+On the other hand, with consistencyLevel weak.
+
+.. code:: bash
+
+ u1 -> (t1, v1)
+ u1 -> (t2, v2)
+ u1 -> (t3, v2)
+ u1 -> (t4, v1)
+
+**It is recommended to set consistencyLevel to weak unless you are expecting concurrent updates on same edge.**
+
+In real world systems, it is not guaranteed that operation requests arrive at S2Graph in the order of their timestamp. Depending on the environment (network conditions, client making asynchronous calls, use of a message que, and so on) request that were made earlier can arrive later. Consistency level also determines how S2Graph handles these cases.
+Strong consistencyLevel promises a final result consistent to the timestamp.
+For example, consider a set of operation requests on edge (1, graph_test, 101) were made in the following order;
+
+
+.. code:: bash
+
+ 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
+ 1418950524722 delete e 1 101 graph_test
+ 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
+ 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
+ 1418950524726 update e 1 101 graph_test {"is_blocked": true}
+
+
+and actually arrived in a shuffled order due to complications
+
+
+.. code:: bash
+
+ 1418950524726 update e 1 101 graph_test {"is_blocked": true}
+ 1418950524723 insert e 1 101 graph_test {"is_hidden": false, "weight": 10}
+ 1418950524722 delete e 1 101 graph_test
+ 1418950524721 insert e 1 101 graph_test {"is_blocked": false}
+ 1418950524724 update e 1 101 graph_test {"time": 1, "weight": -10}
+
+Strong consistency still makes sure that you get the same eventual state on (1, graph_test, 101).
+Here is pseudocode of what S2Graph does to provide a strong consistency level.
+
+.. code:: bash
+
+ complexity = O(one read) + O(one delete) + O(2 put)
+
+ fetchedEdge = fetch edge with (1, graph_test, 101) from lookup table.
+
+ if fetchedEdge is not exist:
+ create new edge same as current insert operation
+ update lookup table as current insert operation
+ else:
+ valid = compare fetchedEdge vs current insert operation.
+ if valid:
+ delete fetchedEdge
+ create new edge after comparing fetchedEdge and current insert.
+ update lookup table
+
+Limitations Since S2Graph makes asynchronous writes to HBase via Asynchbase, there is no consistency guaranteed on same edge within its flushInterval (1 second).
+
+Adding Extra Indices (Optional)
+---------------------------------
+
+.. code:: bash
+
+ POST /admin/addIndex
+
+A label can have multiple properties set as indexes. When edges are queried, the ordering will determined according to indexes, therefore, deciding which edges will be included in the top-K results.
+
+**Edge retrieval queries in S2Graph by default returns top-K edges. Clients must issue another query to fetch the next K edges, i.e., top-K ~ 2 x top-K**
+
+Edges sorted according to the indices in order to limit the number of edges being fetched by a query. If no ordering property is given, S2Graph will use the timestamp as an index, thus resulting in the most recent data.
+
+**It would be extremely difficult to fetch millions of edges and sort them at request time and return a top-K in a reasonable amount of time. Instead, S2Graph uses vertex-centric indexes to avoid this.
+Using a vertex-centric index, having millions of edges is fine as long as size K of the top-K values is reasonable (under 1K) Note that indexes must be created prior to inserting any data on the label (which is the same case with the conventional RDBMS).**
+
+New indexes can be dynamically added, but will not be applied to pre-existing data (support for this is planned for future versions). Currently, a label can have up to eight indices.
+The following is an example of adding index ``play_count`` to a label ``graph_test``.
+
+.. code:: bash
+
+ // add prop first
+ curl -XPOST localhost:9000/admin/addProp/graph_test -H 'Content-Type: Application/json' -d '
+ { "name": "play_count", "defaultValue": 0, "dataType": "integer" }'
+
+ // then add index
+ curl -XPOST localhost:9000/admin/addIndex -H 'Content-Type: Application/json' -d '
+ {
+ "label": "graph_test",
+ "indices": [
+ { name: "idx_play_count", propNames: ["play-count"] }
+ ]
+ }'
+
+
+**********************************
+Create a ServiceColumn (Optional)
+**********************************
+
+.. code:: bash
+
+ POST /admin/createServiceColumn
+
+If your use case requires props assigned to vertices instead of edges, what you need is a Service Column
+
+**Remark: If it is only the vertex id that you need and not additional props, there's no need to create a Service Column explicitly. At label creation, by default, S2Graph creates column space with empty properties according to the label schema.**
+
+
+Service Column Fields
+----------------------
+
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "Field Name", "Definition", "Data Type", "Example", "Remarks"
+ "serviceName", "Which service the Service Column belongs to", "String", "kakaotalk", "Required"
+ "columnName", "Service Column`s name", "String", "talk_user_id", "Required"
+ "props", "Optional properties of Service Column", "JSON (array dictionaries)", "Please refer to the examples", "Optional"
+
+
+Basic Service Column Operations
+-------------------------------
+
+Here are some sample requests for Service Column creation as well as vertex insertion and selection.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/createServiceColumn -H 'Content-Type: Application/json' -d '
+ {
+ "serviceName": "s2graph",
+ "columnName": "user_id",
+ "columnType": "long",
+ "props": [
+ {"name": "is_active", "dataType": "boolean", "defaultValue": true},
+ {"name": "phone_number", "dataType": "string", "defaultValue": "-"},
+ {"name": "nickname", "dataType": "string", "defaultValue": ".."},
+ {"name": "activity_score", "dataType": "float", "defaultValue": 0.0},
+ {"name": "age", "dataType": "integer", "defaultValue": 0}
+ ]
+ }'
+
+General information on a vertex schema can be retrieved with ``/admin/getServiceColumn/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XGET localhost:9000/admin/getServiceColumn/s2graph/user_id
+
+This will give all properties on serviceName ``s2graph`` and columnName ``user_id`` serviceColumn.
+Properties can be added to a Service Column with ``/admin/addServiceColumnProps/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/admin/addServiceColumnProps/s2graph/user_id -H 'Content-Type: Application/json' -d '
+ [
+ {"name": "home_address", "defaultValue": "korea", "dataType": "string"}
+ ]'
+
+Vertices can be inserted to a Service Column using ``/admin/vertices/insert/{service name}/{column name}``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/insert/s2graph/user_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"props":{"is_active":true}, "timestamp":1417616431},
+ {"id":2,"props":{},"timestamp":1417616431}
+ ]'
+
+Finally, query your vertex via ``/graphs/getVertices``
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getVertices -H 'Content-Type: Application/json' -d '
+ [
+ {"serviceName": "s2graph", "columnName": "user_id", "ids": [1, 2, 3]}
+ ]'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c131990e/doc/source/api/mutate/index.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/mutate/index.rst b/doc/source/api/mutate/index.rst
new file mode 100644
index 0000000..c85c55d
--- /dev/null
+++ b/doc/source/api/mutate/index.rst
@@ -0,0 +1,10 @@
+Mutate APIs
+==============================================
+
+Contents:
+
+.. toctree::
+ :maxdepth: 2
+
+ mutate_edge
+ mutate_vertex
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c131990e/doc/source/api/mutate/mutate_edge.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/mutate/mutate_edge.rst b/doc/source/api/mutate/mutate_edge.rst
new file mode 100644
index 0000000..1e21079
--- /dev/null
+++ b/doc/source/api/mutate/mutate_edge.rst
@@ -0,0 +1,314 @@
+****************
+Manage Edges
+****************
+
+An ``Edge`` represents a relation between two vertices, with properties according to the schema defined in its label.
+
+Edge Fields
+---------------
+
+The following fields need to be specified when inserting an edge, and are returned when queried on edges.
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "timestamp", "Issue time of request", "Long", "1430116731156", "Required. Unix Epoch time in milliseconds. S2Graph TTL and timestamp unit is milliseconds."
+ "operation", "One of insert, delete, update, or increment", "String", "'i', 'insert'", "Required only for bulk operations. Aliases are also available: i (insert), d (delete), u (update), in (increment). Default is insert"
+ "from", "Id of source vertex", "Long/String", "1", "Required. Use long if possible. Maximum string byte-size is 249"
+ "to", "Id of target vertex", "Long/String", "101", "Required. Use long if possible. Maximum string byte-size is 249"
+ "label", "Label name", "String", "graph_test", "Required"
+ "direction", "Direction of the edge. Should be one of out/ in/ undirected", "String", "out", "Required. Alias are also available: o (out), i (in), u (undirected)"
+ "props", "Additional properties of the edge", "JSON (dictionary)", "{""timestamp"": 1417616431, ""affinity_score"":10, ""is_hidden"": false, ""is_valid"": true}", "Required. If in indexed properties isn't given, default values will be added"
+
+
+Basic Edge Operations
+--------------------------
+
+In S2Graph, an Edge supports five different operations.
+- ``insert``: Create new edge.
+- ``delete``: Delete existing edge.
+- ``update``: Update existing edge`s state.
+- ``increment``: Increment existing edge`s state.
+- ``deleteAll``: Delete all adjacent edges from certain source vertex. (Available for strong consistency only)
+
+Edge operations work differently depending on the target label`s consistency level.
+For a better understanding, please take a look at the following test cases.
+
+Create 2 different labels, one of each consistencyLevels.
+
+- s2graph_label_test (strong)
+- s2graph_label_test_weak (weak)
+
+Then insert a same set of edges to each labels and query them as follows.
+
+**strong consistency**
+
+.. code:: bash
+
+ curl -XPOST localhosnt:9000/graphs/edges/insert -H 'Content-Type: Application/json' -d '
+ [
+ {"timestamp": 1, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": 0}},
+ {"timestamp": 2, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": -10}},
+ {"timestamp": 3, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": -30}}
+ ]'
+
+
+Note that only one edge exist between (101, 10, s2graph_label_test, out).
+
+
+.. code:: json
+
+ {
+ "size": 1,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test",
+ "direction": "out",
+ "_degree": 1
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -20,
+ "from": 101,
+ "to": 10,
+ "label": "s2graph_label_test",
+ "direction": "out",
+ "_timestamp": 3,
+ "timestamp": 3,
+ "score": 1,
+ "props": {
+ "_timestamp": 3,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ],
+ "impressionId": -1650835965
+ }
+
+**weak consistency**
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/edges/insert -H 'Content-Type: Application/json' -d '
+ [
+ {"timestamp": 1, "from": 101, "to": 10, "label": "s2graph_label_test_weak", "props": {"time": 0}},
+ {"timestamp": 2, "from": 101, "to": 10, "label": "s2graph_label_test_weak", "props": {"time": -10}},
+ {"timestamp": 3, "from": 101, "to": 10, "label": "s2graph_label_test_weak", "props": {"time": -30}}
+ ]'
+
+This time there are ``three edges`` between (101, 10, s2graph_label_test_weak, out).
+
+.. code:: json
+
+ {
+ "size": 3,
+ "degrees": [
+ {
+ "from": 101,
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_degree": 3
+ }
+ ],
+ "results": [
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 3,
+ "timestamp": 3,
+ "score": 1,
+ "props": {
+ "_timestamp": 3,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 2,
+ "timestamp": 2,
+ "score": 1,
+ "props": {
+ "_timestamp": 2,
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 1,
+ "timestamp": 1,
+ "score": 1,
+ "props": {
+ "_timestamp": 1,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ],
+ "impressionId": 1972178414
+ }
+
+
+Strong Consistency
+---------------------
+
+**Insert** - ``POST /mutate/edge/insert``
+A unique edge is identified by a combination of (from, to, label, direction). For insert operations, S2Graph first checks if an edge with same (from, to, label, direction) information exists. If there is an existing edge, then insert will work as ``update``. See above example.
+
+**Delete** - ``POST /mutate/edge/delete``
+For edge deletion, again, S2Graph looks for a unique edge with (from, to, label, direction). However, this time it checks the timestamp of the delete request and the existing edge. The timestamp on the delete request ``must be larger than that on the existing edge`` or else the request will be ignored. If everything is well, the edge will be deleted. Also note that no props information is necessary for a delete request on a strongly consistent label since there will be only one edge with edge`s unique id(from, to, label, direction).
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/edge/delete -H 'Content-Type: Application/json' -d '
+ [
+ {"timestamp": 10, "from": 101, "to": 10, "label": "s2graph_label_test"}
+ ]'
+
+**Update** - ``POST /mutate/edge/update``
+
+What an update operation does to a strongly consistent label is identical to an insert.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/edge/update -H 'Content-Type: Application/json' -d '
+ [
+ {"timestamp": 10, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": 100, "weight": -10}}
+ ]'
+
+
+**Increment** - ``POST /mutate/edge/increment``
+
+Works like update, other than it returns the incremented value and not the old value.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/edge/increment -H 'Content-Type: Application/json' -d '
+ [
+ {"timestamp": 10, "from": 101, "to": 10, "label": "s2graph_label_test", "props": {"time": 100, "weight": -10}}
+ ]'
+
+**Delete All** - ``POST /mutate/edge/deleteAll``
+
+Delete all adjacent edges to the source vertex. **Please note that edges with both in and out directions will be deleted**
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/edge/deleteAll -H 'Content-Type: Application/json' -d '
+ [
+ {"ids" : [101], "label":"s2graph_label_test", "direction": "out", "timestamp":1417616441000}
+ ]'
+
+
+Weak Consistency
+-----------------
+
+
+**Insert** ``POST /mutate/edge/insert``
+
+S2Graph **does not look** for a unique edge defined by (from, to, label, direction). It simply stores a new edge according to the request. No read, no consistency check. Note that this difference allows multiple edges with same (from, to, label, direction) id.
+
+**Delete** - ``POST /graphs/edges/delete``
+
+For deletion on weakly consistent edges, first, S2Graph fetches existing edges from storage. Then, on each resulting edges, fires the actual delete operations.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/edges/delete -H 'Content-Type: Application/json' -d '
+ [
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 3,
+ "timestamp": 3,
+ "score": 1,
+ "props": {
+ "_timestamp": 3,
+ "time": -30,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 2,
+ "timestamp": 2,
+ "score": 1,
+ "props": {
+ "_timestamp": 2,
+ "time": -10,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ },
+ {
+ "cacheRemain": -148,
+ "from": 101,
+ "to": "10",
+ "label": "s2graph_label_test_weak",
+ "direction": "out",
+ "_timestamp": 1,
+ "timestamp": 1,
+ "score": 1,
+ "props": {
+ "_timestamp": 1,
+ "time": 0,
+ "weight": 0,
+ "is_hidden": false,
+ "is_blocked": false
+ }
+ }
+ ]'
+
+**Update** - ``POST /mutate/edge/update``
+
+Like insert, S2Graph **does not check** for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be updated.
+
+**Increment** - ``POST /mutate/edge/increment``
+
+For increment, S2Graph also **does not check** for uniqueness. Update requires a pre-fetch of existing edges, similar to delete. Props of the resulting edges will be incremented.
+
+**Delete All** - ``POST /mutate/edge/deleteAll``
+
+Identical to strong consistency.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/edge/deleteAll -H 'Content-Type: Application/json' -d '
+ [
+ {"ids" : [101], "label":"s2graph_label_test", "direction": "out", "timestamp":1417616441}
+ ]'
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c131990e/doc/source/api/mutate/mutate_vertex.rst
----------------------------------------------------------------------
diff --git a/doc/source/api/mutate/mutate_vertex.rst b/doc/source/api/mutate/mutate_vertex.rst
new file mode 100644
index 0000000..06cc61e
--- /dev/null
+++ b/doc/source/api/mutate/mutate_vertex.rst
@@ -0,0 +1,139 @@
+****************************
+Manage Vertices (Optional)
+****************************
+
+Vertices are the two end points of an edge, and logically stored in columns of a service. If your use case requires storing metadata corresponding to vertices rather than edges, there are operations available on vertices as well.
+
+
+Vertex Fields
+----------------
+
+Unlike edges and their labels, properties of a vertex are not indexed nor require a predefined schema. The following fields are used when operating on vertices.
+
+
+.. csv-table:: Option
+ :header: "Field Name", "Definition", "Data Type", "Example", "Note"
+ :widths: 15, 30, 30, 30, 30
+
+ "timestamp", "Issue time of request", "Long", "1430116731156", "Required. Unix Epoch time in **milliseconds**"
+ "operation", "One of insert, delete, update, increment", "String", "i, insert", "Required only for bulk operations. Alias are also available: i (insert), d (delete), u (update), in (increment). Default is insert."
+ "**serviceName**", "Corresponding service name", "String", "kakaotalk/kakaogroup", "Required"
+ "**columnName**", "Corresponding column name", "String", "user_id", "Required"
+ "id", "Unique identifier of vertex", "Long/String", "101", "Required. Use Long if possible"
+ "**props**", "Additional properties of vertex", "JSON (dictionary)", "{""is_active_user"": true, ""age"":10, ""gender"": ""F"", ""country_iso"": ""kr""}", "Required"
+
+
+
+
+Basic Vertex Operations
+--------------------------
+
+Insert - ``POST /mutate/vertex/insert/:serviceName/:columnName``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/insert/s2graph/account_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"props":{"is_active":true, "talk_user_id":10},"timestamp":1417616431000},
+ {"id":2,"props":{"is_active":true, "talk_user_id":12},"timestamp":1417616431000},
+ {"id":3,"props":{"is_active":false, "talk_user_id":13},"timestamp":1417616431000},
+ {"id":4,"props":{"is_active":true, "talk_user_id":14},"timestamp":1417616431000},
+ {"id":5,"props":{"is_active":true, "talk_user_id":15},"timestamp":1417616431000}
+ ]'
+
+
+Delete - ``POST /mutate/vertex/delete/:serviceName/:columnName``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/delete/s2graph/account_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"timestamp":1417616431001},
+ {"id":2,"timestamp":1417616431002}
+ ]'
+
+
+This operation will delete only the vertex data of a specified column and will not delete any edges connected to those vertices.
+Also important thing is timestamp. vertex will be deleted only if delete requests ``timestamp is larger than all of vertexs`` property`s timestamp.
+
+following example shows the difference.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/insert/s2graph_test/account_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"props":{"is_active":true, "talk_user_id":10},"timestamp":1417616431000},
+ {"id":1,"props":{"talk_user_id":20},"timestamp":1417616431002}
+ ]'
+
+if user request delete(ts) on vertex like below then vertex would not be deleted, but only properties on this vertex that is updated before ts will be deleted.
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/mutate/vertex/delete/s2graph_test/account_id -H 'Content-Type: Application/json' -d '
+ [
+ {"id":1,"timestamp":1417616431001}
+ ]'
+
+
+then result still have vertex with property that is updated with larger timestamp.
+
+
+.. code:: bash
+
+ curl -XPOST localhost:9000/graphs/getVertices -H 'Content-Type: Application/json' -d '
+ [
+ {"serviceName": "s2graph_test", "columnName": "account_id", "ids": [1]}
+ ]'
+
+
+ # result
+ {
+ "serviceName": "s2graph_test",
+ "columnName": "account_id",
+ "id": 1,
+ "props": {
+ "talk_user_id": 20
+ },
+ "timestamp": 1417616431002
+ }
+
+
+**Important notes**
+
+.. note::
+ This means that edges returned by a query can contain deleted vertices. Clients are responsible for checking validity of the vertices.
+
+Delete All - ``POST /mutate/vertex/deleteAll/:serviceName/:columnName``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a **very expensive** operation. If you're interested in what goes on under the hood, please refer to the following pseudocode:
+
+.. code:: python
+
+ vertices = vertex list to delete
+ for vertex in vertices
+ labels = fetch all labels that this vertex is included.
+ for label in labels
+ for index in label.indices
+ edges = G.read with limit 50K
+ for edge in edges
+ edge.delete
+
+
+
+The total complexity is O(L L.I) reads + O(L L.I 50K) writes, worst case. **If the vertex you're trying to delete has more than 50K edges, the deletion will not be consistent**.
+
+
+Update - POST /mutate/vertex/insert/:serviceName/:columnName
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Basically update on Vertex is same with insert overwrite so use insert for update.
+
+Increment
+~~~~~~~~~~~
+
+Not yet implemented; stay tuned.
[02/20] incubator-s2graph git commit: add PlayJsonSupport
Posted by st...@apache.org.
add PlayJsonSupport
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/4154bbe1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/4154bbe1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/4154bbe1
Branch: refs/heads/master
Commit: 4154bbe1605a1fc7f4b8cae6fda71ecf8b317319
Parents: a16ecb0
Author: daewon <da...@apache.org>
Authored: Wed Nov 28 13:51:43 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Wed Nov 28 14:28:53 2018 +0900
----------------------------------------------------------------------
build.sbt | 2 +-
.../apache/s2graph/http/PlayJsonSupport.scala | 34 ++++++++++++++++++++
.../apache/s2graph/http/S2GraphAdminRoute.scala | 17 ++++------
.../s2graph/http/S2GraphTraversalRoute.scala | 19 +++++------
.../apache/s2graph/http/AdminRouteSpec.scala | 14 ++++----
5 files changed, 57 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4154bbe1/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index c4d2c35..1052ac6 100755
--- a/build.sbt
+++ b/build.sbt
@@ -47,7 +47,7 @@ lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogba
.enablePlugins(sbtdocker.DockerPlugin, JavaAppPackaging)
lazy val s2http = project
- .dependsOn(s2core)
+ .dependsOn(s2core, s2graphql)
.settings(commonSettings: _*)
.settings(dockerSettings: _*)
.enablePlugins(sbtdocker.DockerPlugin, JavaAppPackaging)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4154bbe1/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
new file mode 100644
index 0000000..8b4e91c
--- /dev/null
+++ b/s2http/src/main/scala/org/apache/s2graph/http/PlayJsonSupport.scala
@@ -0,0 +1,34 @@
+package org.apache.s2graph.http
+
+import java.nio.charset.Charset
+
+import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller}
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
+import akka.util.ByteString
+import play.api.libs.json.{JsValue, Json}
+
+trait PlayJsonSupport {
+
+ val mediaTypes: Seq[MediaType.WithFixedCharset] =
+ Seq(MediaType.applicationWithFixedCharset("json", HttpCharsets.`UTF-8`, "js"))
+
+ val unmarshallerContentTypes: Seq[ContentTypeRange] = mediaTypes.map(ContentTypeRange.apply)
+
+ implicit val playJsonMarshaller: ToEntityMarshaller[JsValue] = {
+ Marshaller.oneOf(mediaTypes: _*) { mediaType =>
+ Marshaller.withFixedContentType(ContentType(mediaType)) {
+ json => HttpEntity(mediaType, json.toString)
+ }
+ }
+ }
+
+ implicit val playJsonUnmarshaller: FromEntityUnmarshaller[JsValue] = {
+ Unmarshaller.byteStringUnmarshaller
+ .forContentTypes(unmarshallerContentTypes: _*)
+ .map {
+ case ByteString.empty => throw Unmarshaller.NoContentException
+ case data => Json.parse(data.decodeString(Charset.forName("UTF-8")))
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4154bbe1/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
index c2b832e..8efd2ab 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala
@@ -10,21 +10,20 @@ import akka.http.scaladsl.server.Directives._
import org.slf4j.LoggerFactory
import play.api.libs.json._
-import scala.util.Try
+import scala.util._
object S2GraphAdminRoute {
- import scala.util._
-
trait AdminMessageFormatter[T] {
def toJson(msg: T): JsValue
}
import scala.language.reflectiveCalls
- type ToPlayJson = {def toJson: JsValue}
-
object AdminMessageFormatter {
+ type ToPlayJson = {
+ def toJson: JsValue
+ }
implicit def toPlayJson[A <: ToPlayJson] = new AdminMessageFormatter[A] {
def toJson(js: A) = js.toJson
@@ -56,7 +55,7 @@ object S2GraphAdminRoute {
}
}
-trait S2GraphAdminRoute {
+trait S2GraphAdminRoute extends PlayJsonSupport {
import S2GraphAdminRoute._
@@ -80,8 +79,7 @@ trait S2GraphAdminRoute {
}
lazy val createService = path("createService") {
- entity(as[String]) { body =>
- val params = Json.parse(body)
+ entity(as[JsValue]) { params =>
val parseTry = Try(requestParser.toServiceElements(params))
val serviceTry = for {
@@ -94,8 +92,7 @@ trait S2GraphAdminRoute {
}
lazy val createLabel = path("createLabel") {
- entity(as[String]) { body =>
- val params = Json.parse(body)
+ entity(as[JsValue]) { params =>
val labelTry = for {
label <- requestParser.toLabelElements(params)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4154bbe1/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
----------------------------------------------------------------------
diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
index 1814436..01b7a76 100644
--- a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
+++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala
@@ -6,10 +6,10 @@ import org.slf4j.LoggerFactory
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.model.headers.RawHeader
-import akka.http.scaladsl.model.{HttpHeader, StatusCodes}
+import akka.http.scaladsl.model._
import org.apache.s2graph.core.GraphExceptions.{BadQueryException, JsonParseException}
import org.apache.s2graph.core.rest.RestHandler
-
+import play.api.libs.json._
object S2GraphTraversalRoute {
@@ -20,7 +20,7 @@ object S2GraphTraversalRoute {
}
}
-trait S2GraphTraversalRoute {
+trait S2GraphTraversalRoute extends PlayJsonSupport {
import S2GraphTraversalRoute._
@@ -40,15 +40,13 @@ trait S2GraphTraversalRoute {
val result = restHandler.doPost(request.uri.toRelative.toString(), body, request.headers)
val responseHeaders = result.headers.toList.map { case (k, v) => RawHeader(k, v) }
- val f = result.body.map(StatusCodes.OK -> _.toString).recover {
- case BadQueryException(msg, _) => StatusCodes.BadRequest -> msg
- case JsonParseException(msg) => StatusCodes.BadRequest -> msg
- case e: Exception => StatusCodes.InternalServerError -> e.toString
+ val f = result.body.map(StatusCodes.OK -> _).recover {
+ case BadQueryException(msg, _) => StatusCodes.BadRequest -> Json.obj("error" -> msg)
+ case JsonParseException(msg) => StatusCodes.BadRequest -> Json.obj("error" -> msg)
+ case e: Exception => StatusCodes.InternalServerError -> Json.obj("error" -> e.toString)
}
- respondWithHeaders(responseHeaders) {
- complete(f)
- }
+ respondWithHeaders(responseHeaders)(complete(f))
}
}
}
@@ -60,6 +58,5 @@ trait S2GraphTraversalRoute {
delegated // getEdges, experiments, etc.
)
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/4154bbe1/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
----------------------------------------------------------------------
diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
index 7e5d794..9239598 100644
--- a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
+++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala
@@ -8,19 +8,19 @@ import org.apache.s2graph.core.S2Graph
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec}
import org.slf4j.LoggerFactory
-import play.api.libs.json.{JsString, Json}
+import play.api.libs.json.{JsString, JsValue, Json}
+
+class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll with PlayJsonSupport {
-class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll {
val config = ConfigFactory.load()
val s2graph = new S2Graph(config)
+
override val logger = LoggerFactory.getLogger(this.getClass)
override def afterAll(): Unit = {
s2graph.shutdown()
}
- import akka.http.scaladsl.server.Directives._
-
lazy val routes = adminRoute
"AdminRoute" should {
@@ -30,14 +30,14 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
"compressionAlgorithm" -> "gz"
)
- val serviceEntity = Marshal(serviceParam.toString).to[MessageEntity].futureValue
+ val serviceEntity = Marshal(serviceParam).to[MessageEntity].futureValue
val request = Post("/createService").withEntity(serviceEntity)
request ~> routes ~> check {
status should ===(StatusCodes.Created)
contentType should ===(ContentTypes.`application/json`)
- val response = Json.parse(entityAs[String])
+ val response = entityAs[JsValue]
(response \\ "name").head should ===(JsString("kakaoFavorites"))
(response \\ "status").head should ===(JsString("ok"))
@@ -51,7 +51,7 @@ class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with Scal
status should ===(StatusCodes.OK)
contentType should ===(ContentTypes.`application/json`)
- val response = Json.parse(entityAs[String])
+ val response = entityAs[JsValue]
(response \\ "name").head should ===(JsString("kakaoFavorites"))
}