You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by da...@apache.org on 2016/01/04 07:40:37 UTC
[04/46] incubator-s2graph git commit: add config
's2graph.read-only.url' for counter
add config 's2graph.read-only.url' for counter
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/b159a7fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/b159a7fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/b159a7fb
Branch: refs/heads/feature/test_daewon
Commit: b159a7fbf5f39b335c7709e0eb64f23a6cbfda7f
Parents: 5acead8
Author: Jaesang Kim <ho...@gmail.com>
Authored: Mon Dec 21 14:21:05 2015 +0900
Committer: Jaesang Kim <ho...@gmail.com>
Committed: Mon Dec 21 14:21:05 2015 +0900
----------------------------------------------------------------------
.../main/scala/s2/config/S2CounterConfig.scala | 5 ++--
.../s2/counter/core/v2/ExactStorageGraph.scala | 25 +++-----------------
.../counter/core/v2/RankingStorageGraph.scala | 5 ++--
.../main/scala/s2/config/StreamingConfig.scala | 1 +
.../src/test/resources/application.conf | 1 +
5 files changed, 11 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
index d0db73c..fcd0e6a 100644
--- a/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
+++ b/s2counter_core/src/main/scala/s2/config/S2CounterConfig.scala
@@ -28,7 +28,7 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
lazy val DB_DEFAULT_USER = getOrElse("db.default.user", "graph")
lazy val DB_DEFAULT_PASSWORD = getOrElse("db.default.password", "graph")
- // REDIS
+ // Redis
lazy val REDIS_INSTANCES = (for {
s <- config.getStringList("redis.instances")
} yield {
@@ -36,8 +36,9 @@ class S2CounterConfig(config: Config) extends ConfigFunctions(config) {
(sp(0), if (sp.length > 1) sp(1).toInt else 6379)
}).toList
- // graph
+ // Graph
lazy val GRAPH_URL = getOrElse("s2graph.url", "http://localhost:9000")
+ lazy val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
// Cache
lazy val CACHE_TTL_SECONDS = getOrElse("cache.ttl.seconds", 600)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
index add7c61..7dcff6c 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/ExactStorageGraph.scala
@@ -35,6 +35,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
private val labelPostfix = "_counts"
val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
val graphOp = new GraphOperation(config)
import ExactStorageGraph._
@@ -74,26 +75,6 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
seqOfSeq.flatten.toSeq.groupBy(_._1).mapValues { seq => seq.map(_._2).toMap }
}
Await.result(future, 10 second)
-// val (keyWithEq, reqJsLs) = toIncrementCountRequests(policy, counts).unzip(x => ((x._1, x._2), x._3))
-//
-// val future = wsClient.url(s"$s2graphUrl/graphs/edges/incrementCount").post(Json.toJson(reqJsLs)).map { resp =>
-// resp.status match {
-// case HttpStatus.SC_OK =>
-// val respSeq = resp.json.as[Seq[RespGraph]]
-//
-// val keyWithEqResult = {
-// for {
-// ((key, eq), RespGraph(success, result)) <- keyWithEq.zip(respSeq)
-// } yield {
-// (key, (eq, result))
-// }
-// }.groupBy(_._1).mapValues{ seq => seq.map(_._2).toMap }
-// keyWithEqResult
-// case _ =>
-// throw new RuntimeException(s"update failed: $policy $counts")
-// }
-// }
-// Await.result(future, 10 second)
}
def delete(policy: Counter, keys: Seq[ExactKeyTrait]): Unit = {
@@ -201,7 +182,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
val reqJs = Json.parse(reqJsStr)
// log.warn(s"query: ${reqJs.toString()}")
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(reqJs).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(reqJs).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val respJs = resp.json
@@ -261,7 +242,7 @@ case class ExactStorageGraph(config: Config) extends ExactStorage {
val query = Json.obj("srcVertices" -> Json.arr(src), "steps" -> Json.arr(step))
// println(s"query: ${query.toString()}")
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(query).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(query).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val respJs = resp.json
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
----------------------------------------------------------------------
diff --git a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
index b0c0a41..9ada8d4 100644
--- a/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
+++ b/s2counter_core/src/main/scala/s2/counter/core/v2/RankingStorageGraph.scala
@@ -39,6 +39,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
private val labelPostfix = "_topK"
val s2graphUrl = s2config.GRAPH_URL
+ val s2graphReadOnlyUrl = s2config.GRAPH_READONLY_URL
val prepareCache = new CollectionCache[Option[Boolean]](CollectionCacheConfig(10000, 600))
val graphOp = new GraphOperation(config)
@@ -246,7 +247,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
log.debug(strJs)
val payload = Json.parse(strJs)
- wsClient.url(s"$s2graphUrl/graphs/getEdges").post(payload).map { resp =>
+ wsClient.url(s"$s2graphReadOnlyUrl/graphs/getEdges").post(payload).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
(resp.json \ "results").asOpt[List[JsValue]].getOrElse(Nil)
@@ -278,7 +279,7 @@ class RankingStorageGraph(config: Config) extends RankingStorage {
)
)
- val future = wsClient.url(s"$s2graphUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
+ val future = wsClient.url(s"$s2graphReadOnlyUrl/graphs/checkEdges").post(checkReqJs).map { resp =>
resp.status match {
case HttpStatus.SC_OK =>
val checkRespJs = resp.json
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
index 0880bdd..ba5e863 100644
--- a/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
+++ b/s2counter_loader/src/main/scala/s2/config/StreamingConfig.scala
@@ -20,4 +20,5 @@ object StreamingConfig extends ConfigFunctions(S2ConfigFactory.config) {
// graph url
val GRAPH_URL = getOrElse("s2graph.url", "")
+ val GRAPH_READONLY_URL = getOrElse("s2graph.read-only.url", GRAPH_URL)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/b159a7fb/s2counter_loader/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/s2counter_loader/src/test/resources/application.conf b/s2counter_loader/src/test/resources/application.conf
index 369a6fa..02d68b2 100644
--- a/s2counter_loader/src/test/resources/application.conf
+++ b/s2counter_loader/src/test/resources/application.conf
@@ -75,3 +75,4 @@ profile.prefetch.size=100
# s2graph
s2graph.url = "http://"${host}":9000"
+s2graph.real-only.url = ${s2graph.url}