You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by da...@apache.org on 2018/03/30 06:27:28 UTC
[1/3] incubator-s2graph git commit: add logging for Management API
and HTTP Request/Reponse
Repository: incubator-s2graph
Updated Branches:
refs/heads/master e0b2dc0bc -> 488ee0f48
add logging for Management API and HTTP Request/Reponse
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/d137bb30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/d137bb30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/d137bb30
Branch: refs/heads/master
Commit: d137bb3056d79cfe5df22baf109b873117d19629
Parents: 653deee
Author: daewon <da...@apache.org>
Authored: Thu Mar 29 16:36:36 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Thu Mar 29 16:36:36 2018 +0900
----------------------------------------------------------------------
project/Common.scala | 3 +-
.../apache/s2graph/graphql/GraphQLServer.scala | 28 ++++++++-----
.../org/apache/s2graph/graphql/HttpServer.scala | 32 +++++++--------
.../graphql/repository/GraphRepository.scala | 41 ++++++++++++++++----
4 files changed, 70 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d137bb30/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index e2323a2..94b3589 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -48,7 +48,8 @@ object Common {
ExclusionRule("org.slf4j", "jcl-over-slf4j"),
ExclusionRule("org.slf4j", "log4j-over-slf4j"),
ExclusionRule("org.slf4j", "slf4j-log4j12"),
- ExclusionRule("org.slf4j", "jul-to-slf4j")
+ ExclusionRule("org.slf4j", "jul-to-slf4j"),
+ ExclusionRule("org.apache.logging.log4j", "log4j-slf4j-impl")
)
implicit class LoggingExcluder(moduleId: ModuleID) {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d137bb30/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
index 3365593..e4b43cc 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/GraphQLServer.scala
@@ -29,7 +29,7 @@ import com.typesafe.config.ConfigFactory
import org.apache.s2graph.core.S2Graph
import org.apache.s2graph.core.utils.SafeUpdateCache
import org.apache.s2graph.graphql.repository.GraphRepository
-import org.apache.s2graph.graphql.types._
+import org.slf4j.LoggerFactory
import sangria.ast.Document
import sangria.execution._
import sangria.marshalling.sprayJson._
@@ -42,6 +42,8 @@ import scala.util.{Failure, Success, Try}
object GraphQLServer {
+ val logger = LoggerFactory.getLogger(this.getClass)
+
// Init s2graph
val numOfThread = Runtime.getRuntime.availableProcessors()
val threadPool = Executors.newFixedThreadPool(numOfThread * 2)
@@ -69,8 +71,12 @@ object GraphQLServer {
}
QueryParser.parse(query) match {
- case Success(queryAst) => complete(executeGraphQLQuery(queryAst, operation, vars))
- case Failure(error) => complete(BadRequest -> spray.json.JsObject("error" -> JsString(error.getMessage)))
+ case Success(queryAst) =>
+ logger.info(queryAst.renderCompact)
+ complete(executeGraphQLQuery(queryAst, operation, vars))
+ case Failure(error) =>
+ logger.warn(error.getMessage, error)
+ complete(BadRequest -> spray.json.JsObject("error" -> JsString(error.getMessage)))
}
}
@@ -78,13 +84,13 @@ object GraphQLServer {
* In development mode(schemaCacheTTL = -1),
* a new schema is created for each request.
*/
- println(s"schemaCacheTTL: ${schemaCacheTTL}")
+ logger.info(s"schemaCacheTTL: ${schemaCacheTTL}")
private def createNewSchema(): Schema[GraphRepository, Any] = {
- println(s"Schema updated: ${System.currentTimeMillis()}")
+ logger.info(s"Schema updated: ${System.currentTimeMillis()}")
val newSchema = new SchemaDef(s2Repository).S2GraphSchema
- // println(SchemaRenderer.renderSchema(newSchema))
- println("-" * 80)
+ logger.info("-" * 80)
+
newSchema
}
@@ -100,8 +106,12 @@ object GraphQLServer {
)
.map((res: spray.json.JsValue) => OK -> res)
.recover {
- case error: QueryAnalysisError => BadRequest -> error.resolveError
- case error: ErrorWithResolver => InternalServerError -> error.resolveError
+ case error: QueryAnalysisError =>
+ logger.warn(error.getMessage, error)
+ BadRequest -> error.resolveError
+ case error: ErrorWithResolver =>
+ logger.error(error.getMessage, error)
+ InternalServerError -> error.resolveError
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d137bb30/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
index 9535b0e..7d3771a 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/HttpServer.scala
@@ -25,38 +25,38 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server._
import akka.stream.ActorMaterializer
+import org.slf4j.LoggerFactory
-import scala.Console._
import scala.concurrent.Await
import scala.language.postfixOps
object Server extends App {
+ val logger = LoggerFactory.getLogger(this.getClass)
- implicit val actorSystem = ActorSystem("s2graphql-server")
+ implicit val system = ActorSystem("s2graphql-server")
implicit val materializer = ActorMaterializer()
- import actorSystem.dispatcher
+ import system.dispatcher
import scala.concurrent.duration._
- println("Starting GRAPHQL server...")
-
- val route: Route =
- (post & path("graphql")) {
- entity(as[spray.json.JsValue])(GraphQLServer.endpoint)
- } ~ {
- getFromResource("assets/graphiql.html")
- }
+ val route: Route = (post & path("graphql")) {
+ entity(as[spray.json.JsValue])(GraphQLServer.endpoint)
+ } ~ {
+ getFromResource("assets/graphiql.html")
+ }
val port = sys.props.get("http.port").fold(8000)(_.toInt)
- Http().bindAndHandle(route, "0.0.0.0", port)
+ logger.info(s"Starting GraphQL server... ${port}")
+ Http().bindAndHandle(route, "0.0.0.0", port)
def shutdown(): Unit = {
- println("Terminating...")
- actorSystem.terminate()
- Await.result(actorSystem.whenTerminated, 10 seconds)
+ logger.info("Terminating...")
+
+ system.terminate()
+ Await.result(system.whenTerminated, 30 seconds)
- println("Terminated.")
+ logger.info("Terminated.")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/d137bb30/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
----------------------------------------------------------------------
diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
index c934fd5..a29d0fb 100644
--- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
+++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala
@@ -26,12 +26,23 @@ import org.apache.s2graph.core.rest.RequestParser
import org.apache.s2graph.core.storage.MutateResponse
import org.apache.s2graph.core.types._
import org.apache.s2graph.graphql.types.S2Type._
+import org.slf4j.{Logger, LoggerFactory}
import sangria.schema._
import scala.concurrent._
-import scala.util.{Failure, Try}
+import scala.util.{Failure, Success, Try}
object GraphRepository {
+
+ def withLogTryResponse[A](opName: String, tryObj: Try[A])(implicit logger: Logger): Try[A] = {
+ tryObj match {
+ case Success(v) => logger.info(s"${opName} Success:", v)
+ case Failure(e) => logger.warn(s"${opName} Failed:", e)
+ }
+
+ tryObj
+ }
+
}
/**
@@ -39,6 +50,9 @@ object GraphRepository {
* @param graph
*/
class GraphRepository(val graph: S2GraphLike) {
+ implicit val logger = LoggerFactory.getLogger(this.getClass)
+
+ import GraphRepository._
val management = graph.management
val parser = new RequestParser(graph)
@@ -92,7 +106,7 @@ class GraphRepository(val graph: S2GraphLike) {
def createService(args: Args): Try[Service] = {
val serviceName = args.arg[String]("name")
- Service.findByName(serviceName) match {
+ val serviceTry = Service.findByName(serviceName) match {
case Some(_) => Failure(new RuntimeException(s"Service (${serviceName}) already exists"))
case None =>
val cluster = args.argOpt[String]("cluster").getOrElse(parser.DefaultCluster)
@@ -111,6 +125,8 @@ class GraphRepository(val graph: S2GraphLike) {
serviceTry
}
+
+ withLogTryResponse("createService", serviceTry)
}
def createServiceColumn(args: Args): Try[ServiceColumn] = {
@@ -119,9 +135,11 @@ class GraphRepository(val graph: S2GraphLike) {
val columnType = args.arg[String]("columnType")
val props = args.argOpt[Vector[Prop]]("props").getOrElse(Vector.empty)
- Try {
+ val tryColumn = Try {
management.createServiceColumn(serviceName, columnName, columnType, props)
}
+
+ withLogTryResponse("createServiceColumn", tryColumn)
}
def deleteServiceColumn(args: Args): Try[ServiceColumn] = {
@@ -130,11 +148,13 @@ class GraphRepository(val graph: S2GraphLike) {
val serviceName = serviceColumnParam.serviceName
val columnName = serviceColumnParam.columnName
- Management.deleteColumn(serviceName, columnName)
+ val deleteTry = Management.deleteColumn(serviceName, columnName)
+
+ withLogTryResponse("deleteServiceColumn", deleteTry)
}
def addPropsToLabel(args: Args): Try[Label] = {
- Try {
+ val addPropToLabelTry = Try {
val labelName = args.arg[String]("labelName")
val props = args.arg[Vector[Prop]]("props").toList
@@ -144,10 +164,12 @@ class GraphRepository(val graph: S2GraphLike) {
Label.findByName(labelName, false).get
}
+
+ withLogTryResponse("addPropToLabel", addPropToLabelTry)
}
def addPropsToServiceColumn(args: Args): Try[ServiceColumn] = {
- Try {
+ val addPropsToServiceColumnTry = Try {
val serviceColumnParam = args.arg[ServiceColumnParam]("service")
val serviceName = serviceColumnParam.serviceName
@@ -160,6 +182,8 @@ class GraphRepository(val graph: S2GraphLike) {
val src = Service.findByName(serviceName)
ServiceColumn.find(src.get.id.get, columnName, false).get
}
+
+ withLogTryResponse("addPropsToServiceColumn", addPropsToServiceColumnTry)
}
def createLabel(args: Args): Try[Label] = {
@@ -205,13 +229,14 @@ class GraphRepository(val graph: S2GraphLike) {
options
)
- labelTry
+ withLogTryResponse("createLabel", labelTry)
}
def deleteLabel(args: Args): Try[Label] = {
val labelName = args.arg[String]("name")
- Management.deleteLabel(labelName)
+ val deleteLabelTry = Management.deleteLabel(labelName)
+ withLogTryResponse("deleteLabel", deleteLabelTry)
}
def allServices(): List[Service] = Service.findAll()
[3/3] incubator-s2graph git commit: [S2GRAPH-191] Providing request /
response logging to GraphQL server and removing println
Posted by da...@apache.org.
[S2GRAPH-191] Providing request / response logging to GraphQL server and removing println
JIRA:
[S2GRAPH-191] https://issues.apache.org/jira/browse/S2GRAPH-191
Pull Request:
Closes #146
Author
daewon <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/488ee0f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/488ee0f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/488ee0f4
Branch: refs/heads/master
Commit: 488ee0f48a3ba6e99c816a06e473f3591df33d8d
Parents: 5c81ac8
Author: daewon <da...@apache.org>
Authored: Fri Mar 30 15:26:27 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Fri Mar 30 15:26:27 2018 +0900
----------------------------------------------------------------------
CHANGES | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/488ee0f4/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index e593b95..0a65b60 100644
--- a/CHANGES
+++ b/CHANGES
@@ -47,6 +47,7 @@ Release Notes - S2Graph - Version 0.2.0
* [S2GRAPH-188] - migrate log4j.properties to log4j2.xml
* [S2GRAPH-189] - graphiql.html file not found
* [S2GRAPH-190] - Remove unnecessary TC JsonBenchmarkSpec
+ * [S2GRAPH-191] - Providing request / response logging to GraphQL server and removing println
* [S2GRAPH-192] - could not find service column when creating the label
** Improvement
[2/3] incubator-s2graph git commit: Merge branch 'master' into
S2GRAPH-191
Posted by da...@apache.org.
Merge branch 'master' into S2GRAPH-191
* master:
[S2GRAPH-192] could not find service column when creating the label
[S2GRAPH-190] Remove unnecessary TC JsonBenchmarkSpec
fix name conflict
remove broken import
remove unused benchmark
[S2GRAPH-185]: Support Spark Structured Streaming to work with data in streaming and batch
remove count operation
wait termination for multiple streaming query
shade protobuf to resolve confilct with spark lib
add s2graph sink for spark structured streaming
add simple etl job interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5c81ac8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5c81ac8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5c81ac8b
Branch: refs/heads/master
Commit: 5c81ac8bf6195646efdc7e3abdea529791db8c4d
Parents: d137bb3 e0b2dc0
Author: daewon <da...@apache.org>
Authored: Fri Mar 30 15:24:01 2018 +0900
Committer: daewon <da...@apache.org>
Committed: Fri Mar 30 15:24:01 2018 +0900
----------------------------------------------------------------------
CHANGES | 3 +
project/Common.scala | 2 +-
.../core/benchmark/BenchmarkCommon.scala | 1 -
.../core/benchmark/JsonBenchmarkSpec.scala | 78 --------
.../core/benchmark/SamplingBenchmarkSpec.scala | 105 ----------
.../loader/core/CounterEtlFunctionsSpec.scala | 2 +-
.../graphql/types/S2ManagementType.scala | 4 +-
.../org/apache/s2graph/graphql/SchemaTest.scala | 22 ++-
s2jobs/build.sbt | 12 +-
.../scala/org/apache/s2graph/s2jobs/Job.scala | 70 +++++++
.../apache/s2graph/s2jobs/JobDescription.scala | 88 +++++++++
.../org/apache/s2graph/s2jobs/JobLauncher.scala | 88 +++++++++
.../org/apache/s2graph/s2jobs/Logger.scala | 26 +++
.../org/apache/s2graph/s2jobs/Schema.scala | 35 ++++
.../apache/s2graph/s2jobs/task/Process.scala | 53 +++++
.../org/apache/s2graph/s2jobs/task/Sink.scala | 195 +++++++++++++++++++
.../org/apache/s2graph/s2jobs/task/Source.scala | 104 ++++++++++
.../org/apache/s2graph/s2jobs/task/Task.scala | 38 ++++
.../s2graph/spark/sql/streaming/Logger.scala | 26 +++
.../spark/sql/streaming/S2CommitProtocol.scala | 67 +++++++
.../spark/sql/streaming/S2SinkConfigs.scala | 47 +++++
.../spark/sql/streaming/S2SinkContext.scala | 44 +++++
.../spark/sql/streaming/S2SinkMetadataLog.scala | 44 +++++
.../spark/sql/streaming/S2SinkProvider.scala | 45 +++++
.../spark/sql/streaming/S2SinkStatus.scala | 28 +++
.../sql/streaming/S2SparkSqlStreamingSink.scala | 89 +++++++++
.../sql/streaming/S2StreamQueryWriter.scala | 133 +++++++++++++
.../s2graph/s2jobs/task/ProcessTest.scala | 48 +++++
28 files changed, 1297 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c81ac8b/project/Common.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5c81ac8b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
----------------------------------------------------------------------
diff --cc s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
index 6985758,6985758..35575fb
--- a/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
+++ b/s2counter_loader/src/test/scala/org/apache/s2graph/counter/loader/core/CounterEtlFunctionsSpec.scala
@@@ -22,7 -22,7 +22,7 @@@ package org.apache.s2graph.counter.load
import com.typesafe.config.ConfigFactory
import org.apache.s2graph.core.mysqls.{Label, Service}
import org.apache.s2graph.core.types.HBaseType
--import org.apache.s2graph.core.{S2Graph$, Management}
++import org.apache.s2graph.core.{S2Graph, Management}
import org.apache.s2graph.counter.models.DBModel
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}