You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2016/10/10 08:38:52 UTC
[3/5] incubator-s2graph git commit: [S2GRAPH-117] migrated to play
2.5.9
[S2GRAPH-117] migrated to play 2.5.9
- explicitly set to not using dependency injection
- converted some Iteratees to Acumulators
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/fcb041b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/fcb041b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/fcb041b3
Branch: refs/heads/master
Commit: fcb041b3cd14db591cd0cb868b118758ad70ad89
Parents: c7967ff
Author: Jong Wook Kim <jo...@nyu.edu>
Authored: Mon Oct 10 00:13:12 2016 -0400
Committer: Jong Wook Kim <jo...@nyu.edu>
Committed: Mon Oct 10 01:44:29 2016 -0400
----------------------------------------------------------------------
build.sbt | 2 +-
project/Common.scala | 2 +-
project/plugins.sbt | 2 +-
.../counter/core/v2/ExactStorageGraph.scala | 6 ++++-
.../counter/core/v2/GraphOperation.scala | 7 ++++-
.../counter/core/v2/RankingStorageGraph.scala | 7 ++++-
.../counter/loader/core/DimensionProps.scala | 6 ++++-
.../controllers/ApplicationController.scala | 5 ++--
.../rest/play/controllers/JsonBodyParser.scala | 20 ++++++++------
.../org/apache/s2graph/rest/play/package.scala | 28 ++++++++++++++++++++
s2rest_play/build.sbt | 3 ++-
11 files changed, 70 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 23ad8ef..d2df8a6 100755
--- a/build.sbt
+++ b/build.sbt
@@ -38,7 +38,7 @@ lazy val commonSettings = Seq(
Revolver.settings
-lazy val s2rest_play = project.enablePlugins(PlayScala)
+lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogback)
.dependsOn(s2core, s2counter_core)
.settings(commonSettings: _*)
.settings(testOptions in Test += Tests.Argument("sequential"))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index 0e92f05..fb4af84 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -21,7 +21,7 @@ import sbt._
object Common {
val sparkVersion = "1.4.1"
- val playVersion = "2.4.8"
+ val playVersion = "2.5.9"
val hbaseVersion = "1.2.2"
val hadoopVersion = "2.7.3"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 3efaa7d..a6e5381 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -19,7 +19,7 @@
// use the Play sbt plugin for Play projects
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.8")
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.5.9")
// http://www.scalastyle.org/sbt.html
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
index 18cfeb2..caddf8d 100644
--- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/ExactStorageGraph.scala
@@ -19,6 +19,8 @@
package org.apache.s2graph.counter.core.v2
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
import com.typesafe.config.Config
import org.apache.http.HttpStatus
import org.apache.s2graph.core.mysqls.Label
@@ -28,6 +30,7 @@ import org.apache.s2graph.counter.core.ExactCounter.ExactValueMap
import org.apache.s2graph.counter.core._
import org.apache.s2graph.counter.models.Counter
import org.apache.s2graph.counter.util.CartesianProduct
+import org.asynchttpclient.DefaultAsyncHttpClientConfig
import org.slf4j.LoggerFactory
import play.api.libs.json._
import scala.concurrent.duration._
@@ -38,7 +41,8 @@ object ExactStorageGraph {
implicit val respGraphFormat = Json.format[RespGraph]
// using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName))
+ private val builder = new DefaultAsyncHttpClientConfig.Builder()
private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
index 29a509b..329f3d0 100644
--- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/GraphOperation.scala
@@ -19,9 +19,13 @@
package org.apache.s2graph.counter.core.v2
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
import com.typesafe.config.Config
import org.apache.http.HttpStatus
import org.apache.s2graph.counter.config.S2CounterConfig
+import org.apache.s2graph.counter.core.v2.ExactStorageGraph._
+import org.asynchttpclient.DefaultAsyncHttpClientConfig
import org.slf4j.LoggerFactory
import play.api.libs.json.{JsObject, JsValue, Json}
import scala.concurrent.Await
@@ -29,7 +33,8 @@ import scala.concurrent.duration._
class GraphOperation(config: Config) {
// using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName))
+ private val builder = new DefaultAsyncHttpClientConfig.Builder()
private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
private val s2config = new S2CounterConfig(config)
val s2graphUrl = s2config.GRAPH_URL
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
index 55800f2..b9ca9a4 100644
--- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
+++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/core/v2/RankingStorageGraph.scala
@@ -19,15 +19,19 @@
package org.apache.s2graph.counter.core.v2
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
import com.typesafe.config.Config
import org.apache.commons.httpclient.HttpStatus
import org.apache.s2graph.core.GraphUtil
import org.apache.s2graph.core.mysqls.Label
import org.apache.s2graph.counter.config.S2CounterConfig
import org.apache.s2graph.counter.core.RankingCounter.RankingValueMap
+import org.apache.s2graph.counter.core.v2.ExactStorageGraph._
import org.apache.s2graph.counter.core.{RankingResult, RankingKey, RankingStorage}
import org.apache.s2graph.counter.models.{Counter, CounterModel}
import org.apache.s2graph.counter.util.{CollectionCacheConfig, CollectionCache}
+import org.asynchttpclient.DefaultAsyncHttpClientConfig
import org.slf4j.LoggerFactory
import play.api.libs.json.{JsObject, JsString, JsValue, Json}
import scala.concurrent.duration._
@@ -36,7 +40,8 @@ import scala.util.hashing.MurmurHash3
object RankingStorageGraph {
// using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName))
+ private val builder = new DefaultAsyncHttpClientConfig.Builder()
private val wsClient = new play.api.libs.ws.ning.NingWSClient(builder.build)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
index a1b9903..f941224 100644
--- a/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
+++ b/s2counter_loader/src/main/scala/org/apache/s2graph/counter/loader/core/DimensionProps.scala
@@ -19,11 +19,14 @@
package org.apache.s2graph.counter.loader.core
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
import org.apache.commons.httpclient.HttpStatus
import org.apache.s2graph.core.mysqls.{Bucket, Experiment, Service}
import org.apache.s2graph.counter.loader.config.StreamingConfig
import org.apache.s2graph.counter.models.Counter
import org.apache.s2graph.counter.util.{RetryAsync, CollectionCache, CollectionCacheConfig}
+import org.asynchttpclient.DefaultAsyncHttpClientConfig
import org.slf4j.LoggerFactory
import play.api.libs.json._
import scala.annotation.tailrec
@@ -34,7 +37,8 @@ import scala.util.Try
object DimensionProps {
// using play-ws without play app
- private val builder = new com.ning.http.client.AsyncHttpClientConfig.Builder()
+ implicit val materializer = ActorMaterializer.create(ActorSystem(getClass.getSimpleName))
+ private val builder = new DefaultAsyncHttpClientConfig.Builder()
private val client = new play.api.libs.ws.ning.NingWSClient(builder.build)
private val log = LoggerFactory.getLogger(this.getClass)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
index 4170eec..707b98c 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/ApplicationController.scala
@@ -19,9 +19,11 @@
package org.apache.s2graph.rest.play.controllers
+import akka.util.ByteString
import org.apache.s2graph.core.GraphExceptions.BadQueryException
import org.apache.s2graph.core.PostProcess
import org.apache.s2graph.core.utils.logger
+import play.api.http.HttpEntity
import play.api.libs.iteratee.Enumerator
import play.api.libs.json.{JsString, JsValue}
import play.api.mvc._
@@ -69,8 +71,7 @@ object ApplicationController extends Controller {
} else {
Result(
header = ResponseHeader(OK),
- body = Enumerator(json.toString.getBytes()),
- connection = HttpConnection.Close
+ body = HttpEntity.Strict(ByteString(json.toString.getBytes()), Some(applicationJsonHeader))
).as(applicationJsonHeader).withHeaders((CONNECTION -> "close") +: headers: _*)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
index 3245bb6..aa507d9 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/controllers/JsonBodyParser.scala
@@ -19,10 +19,12 @@
package org.apache.s2graph.rest.play.controllers
+import akka.util.ByteString
import org.apache.s2graph.core.utils.logger
import play.api.Play
import play.api.libs.iteratee.Iteratee
import play.api.libs.json.{JsValue, Json}
+import play.api.libs.streams.Streams
import play.api.mvc._
import scala.concurrent.Future
@@ -50,9 +52,11 @@ object s2parse extends BodyParsers {
import play.api.libs.iteratee.Execution.Implicits.trampoline
import play.api.libs.iteratee.Traversable
- Traversable.takeUpTo[Array[Byte]](maxLength)
- .transform(Iteratee.consume[Array[Byte]]().map(c => new String(c, "UTF-8")))
+ val iteratee = Traversable.takeUpTo[ByteString](maxLength)
+ .transform(Iteratee.consume[ByteString]().map(_.utf8String))
.flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
+
+ Streams.iterateeToAccumulator(iteratee)
}
def json(maxLength: Int): BodyParser[JsValue] = when(
@@ -76,14 +80,14 @@ object s2parse extends BodyParsers {
import scala.util.control.Exception._
- val bodyParser: Iteratee[Array[Byte], Either[Result, Either[Future[Result], A]]] =
- Traversable.takeUpTo[Array[Byte]](maxLength).transform(
- Iteratee.consume[Array[Byte]]().map { bytes =>
+ val bodyParser: Iteratee[ByteString, Either[Result, Either[Future[Result], A]]] =
+ Traversable.takeUpTo[ByteString](maxLength).transform(
+ Iteratee.consume[ByteString]().map { bytes =>
allCatch[A].either {
- parser(request, bytes)
+ parser(request, bytes.toByteBuffer.array())
}.left.map {
case NonFatal(e) =>
- val txt = new String(bytes)
+ val txt = bytes.utf8String
logger.error(s"$errorMessage: $txt", e)
createBadResult(s"$errorMessage: $e")(request)
case t => throw t
@@ -91,7 +95,7 @@ object s2parse extends BodyParsers {
}
).flatMap(Iteratee.eofOrElse(Results.EntityTooLarge))
- bodyParser.mapM {
+ Streams.iterateeToAccumulator(bodyParser).mapFuture {
case Left(tooLarge) => Future.successful(Left(tooLarge))
case Right(Left(badResult)) => badResult.map(Left.apply)
case Right(Right(body)) => Future.successful(Right(body))
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/app/org/apache/s2graph/rest/play/package.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/package.scala b/s2rest_play/app/org/apache/s2graph/rest/play/package.scala
new file mode 100644
index 0000000..be4175a
--- /dev/null
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/package.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.rest
+
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+
+package object play {
+ implicit val actorSystem = ActorSystem("s2graph")
+ implicit val materializer = ActorMaterializer.create(actorSystem)
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fcb041b3/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
index 01f4c85..77e13d4 100644
--- a/s2rest_play/build.sbt
+++ b/s2rest_play/build.sbt
@@ -27,5 +27,6 @@ libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, f
"org.specs2" %% "specs2-core" % "3.8.5" % "test"
)
-enablePlugins(JavaServerAppPackaging)
+routesGenerator := StaticRoutesGenerator
+enablePlugins(JavaServerAppPackaging)