You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2018/06/21 01:23:35 UTC
[3/5] incubator-s2graph git commit: commit missing changes on s2core.
commit missing changes on s2core.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/da668ace
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/da668ace
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/da668ace
Branch: refs/heads/master
Commit: da668ace749adf243d43c95413b460bba218731d
Parents: f90a7f5
Author: DO YUNG YOON <st...@apache.org>
Authored: Fri Jun 15 10:09:53 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Fri Jun 15 10:12:12 2018 +0900
----------------------------------------------------------------------
.../scala/org/apache/s2graph/core/S2Graph.scala | 44 +++------------
.../apache/s2graph/core/S2GraphConfigs.scala | 57 ++++++++++++++++++++
2 files changed, 63 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/da668ace/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
index dabbf80..95880f7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala
@@ -51,40 +51,8 @@ object S2Graph {
val DefaultScore = 1.0
val FetchAllLimit = 10000000
val DefaultFetchLimit = 1000
- import S2GraphConfigs._
-
- private val DefaultConfigs: Map[String, AnyRef] = Map(
- S2GRAPH_STORE_BACKEND -> DEFAULT_S2GRAPH_STORE_BACKEND,
- PHASE -> DEFAULT_PHASE,
- HBaseConfigs.HBASE_ZOOKEEPER_QUORUM -> HBaseConfigs.DEFAULT_HBASE_ZOOKEEPER_QUORUM,
- HBaseConfigs.HBASE_ZOOKEEPER_ZNODE_PARENT -> HBaseConfigs.DEFAULT_HBASE_ZOOKEEPER_ZNODE_PARENT,
- HBaseConfigs.HBASE_TABLE_NAME -> HBaseConfigs.DEFAULT_HBASE_TABLE_NAME,
- HBaseConfigs.HBASE_TABLE_COMPRESSION_ALGORITHM -> HBaseConfigs.DEFAULT_HBASE_TABLE_COMPRESSION_ALGORITHM,
- HBaseConfigs.HBASE_CLIENT_RETRIES_NUMBER -> HBaseConfigs.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER,
- HBaseConfigs.HBASE_RPCS_BUFFERED_FLUSH_INTERVAL -> HBaseConfigs.DEFAULT_HBASE_RPCS_BUFFERED_FLUSH_INTERVAL,
- HBaseConfigs.HBASE_RPC_TIMEOUT -> HBaseConfigs.DEFAULT_HBASE_RPC_TIMEOUT,
- DBConfigs.DB_DEFAULT_DRIVER -> DBConfigs.DEFAULT_DB_DEFAULT_DRIVER,
- DBConfigs.DB_DEFAULT_URL -> DBConfigs.DEFAULT_DB_DEFAULT_URL,
- DBConfigs.DB_DEFAULT_PASSWORD -> DBConfigs.DEFAULT_DB_DEFAULT_PASSWORD,
- DBConfigs.DB_DEFAULT_USER -> DBConfigs.DEFAULT_DB_DEFAULT_USER,
- CacheConfigs.CACHE_MAX_SIZE -> CacheConfigs.DEFAULT_CACHE_MAX_SIZE,
- CacheConfigs.CACHE_TTL_SECONDS -> CacheConfigs.DEFAULT_CACHE_TTL_SECONDS,
- ResourceCacheConfigs.RESOURCE_CACHE_MAX_SIZE -> ResourceCacheConfigs.DEFAULT_RESOURCE_CACHE_MAX_SIZE,
- ResourceCacheConfigs.RESOURCE_CACHE_TTL_SECONDS -> ResourceCacheConfigs.DEFAULT_RESOURCE_CACHE_TTL_SECONDS,
- MutatorConfigs.MAX_RETRY_NUMBER -> MutatorConfigs.DEFAULT_MAX_RETRY_NUMBER,
- MutatorConfigs.LOCK_EXPIRE_TIME -> MutatorConfigs.DEFAULT_LOCK_EXPIRE_TIME,
- MutatorConfigs.MAX_BACK_OFF -> MutatorConfigs.DEFAULT_MAX_BACK_OFF,
- MutatorConfigs.BACK_OFF_TIMEOUT -> MutatorConfigs.DEFAULT_BACK_OFF_TIMEOUT,
- MutatorConfigs.HBASE_FAIL_PROB -> MutatorConfigs.DEFAULT_HBASE_FAIL_PROB,
- MutatorConfigs.DELETE_ALL_FETCH_SIZE -> MutatorConfigs.DEFAULT_DELETE_ALL_FETCH_SIZE,
- MutatorConfigs.DELETE_ALL_FETCH_COUNT -> MutatorConfigs.DEFAULT_DELETE_ALL_FETCH_COUNT,
- FutureCacheConfigs.FUTURE_CACHE_MAX_SIZE -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_MAX_SIZE,
- FutureCacheConfigs.FUTURE_CACHE_EXPIRE_AFTER_WRITE -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_WRITE,
- FutureCacheConfigs.FUTURE_CACHE_EXPIRE_AFTER_ACCESS -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_ACCESS,
- FutureCacheConfigs.FUTURE_CACHE_METRIC_INTERVAL -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_METRIC_INTERVAL,
- QueryConfigs.QUERY_HARDLIMIT -> QueryConfigs.DEFAULT_QUERY_HARDLIMIT,
- LogConfigs.QUERY_LOG_SAMPLE_RATE -> LogConfigs.DEFAULT_QUERY_LOG_SAMPLE_RATE
- )
+
+ private val DefaultConfigs = S2GraphConfigs.DEFAULT_CONFIGS
var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs)
val numOfThread = Runtime.getRuntime.availableProcessors()
@@ -128,13 +96,13 @@ object S2Graph {
}
def initStorage(graph: S2GraphLike, config: Config)(ec: ExecutionContext): Storage = {
- val storageBackend = config.getString("s2graph.storage.backend")
+ val storageBackend = config.getString(S2GraphConfigs.S2GRAPH_STORE_BACKEND)
logger.info(s"[InitStorage]: $storageBackend")
storageBackend match {
case "hbase" =>
hbaseExecutor =
- if (config.getString("hbase.zookeeper.quorum") == "localhost")
+ if (config.getString(S2GraphConfigs.HBaseConfigs.HBASE_ZOOKEEPER_QUORUM) == "localhost")
AsynchbaseStorage.initLocalHBase(config)
else
null
@@ -205,8 +173,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap
override val config = _config.withFallback(S2Graph.DefaultConfig)
val storageBackend = Try {
- config.getString("s2graph.storage.backend")
- }.getOrElse("hbase")
+ config.getString(S2GraphConfigs.S2GRAPH_STORE_BACKEND)
+ }.getOrElse(S2GraphConfigs.DEFAULT_S2GRAPH_STORE_BACKEND)
Schema.apply(config)
Schema.loadCache()
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/da668ace/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala
index 445b9f6..f262ea3 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala
@@ -1,6 +1,19 @@
package org.apache.s2graph.core
object S2GraphConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ S2GRAPH_STORE_BACKEND -> DEFAULT_S2GRAPH_STORE_BACKEND,
+ PHASE -> DEFAULT_PHASE
+ )
+ lazy val DEFAULT_CONFIGS = DEFAULTS ++
+ S2GraphConfigs.HBaseConfigs.DEFAULTS ++
+ S2GraphConfigs.DBConfigs.DEFAULTS ++
+ S2GraphConfigs.CacheConfigs.DEFAULTS ++
+ S2GraphConfigs.ResourceCacheConfigs.DEFAULTS ++
+ S2GraphConfigs.MutatorConfigs.DEFAULTS ++
+ S2GraphConfigs.QueryConfigs.DEFAULTS ++
+ S2GraphConfigs.FutureCacheConfigs.DEFAULTS ++
+ S2GraphConfigs.LogConfigs.DEFAULTS
val S2GRAPH_STORE_BACKEND = "s2graph.storage.backend"
val DEFAULT_S2GRAPH_STORE_BACKEND = "hbase"
@@ -9,6 +22,15 @@ object S2GraphConfigs {
val DEFAULT_PHASE = "dev"
object HBaseConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ HBASE_ZOOKEEPER_QUORUM -> DEFAULT_HBASE_ZOOKEEPER_QUORUM,
+ HBASE_ZOOKEEPER_ZNODE_PARENT -> DEFAULT_HBASE_ZOOKEEPER_ZNODE_PARENT,
+ HBASE_TABLE_NAME -> DEFAULT_HBASE_TABLE_NAME,
+ HBASE_TABLE_COMPRESSION_ALGORITHM -> DEFAULT_HBASE_TABLE_COMPRESSION_ALGORITHM,
+ HBASE_CLIENT_RETRIES_NUMBER -> DEFAULT_HBASE_CLIENT_RETRIES_NUMBER,
+ HBASE_RPCS_BUFFERED_FLUSH_INTERVAL -> DEFAULT_HBASE_RPCS_BUFFERED_FLUSH_INTERVAL,
+ HBASE_RPC_TIMEOUT -> DEFAULT_HBASE_RPC_TIMEOUT
+ )
val HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"
val DEFAULT_HBASE_ZOOKEEPER_QUORUM = "localhost"
@@ -31,6 +53,12 @@ object S2GraphConfigs {
val DEFAULT_HBASE_RPC_TIMEOUT = java.lang.Integer.valueOf(600000)
}
object DBConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ DB_DEFAULT_DRIVER -> DEFAULT_DB_DEFAULT_DRIVER,
+ DB_DEFAULT_URL -> DEFAULT_DB_DEFAULT_URL,
+ DB_DEFAULT_PASSWORD -> DEFAULT_DB_DEFAULT_PASSWORD,
+ DB_DEFAULT_USER -> DEFAULT_DB_DEFAULT_USER
+ )
val DB_DEFAULT_DRIVER = "db.default.driver"
val DEFAULT_DB_DEFAULT_DRIVER = "org.h2.Driver"
@@ -44,6 +72,10 @@ object S2GraphConfigs {
val DEFAULT_DB_DEFAULT_USER = "graph"
}
object CacheConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ CACHE_MAX_SIZE -> DEFAULT_CACHE_MAX_SIZE,
+ CACHE_TTL_SECONDS -> DEFAULT_CACHE_TTL_SECONDS
+ )
val CACHE_MAX_SIZE = "cache.max.size"
val DEFAULT_CACHE_MAX_SIZE = java.lang.Integer.valueOf(0)
@@ -51,6 +83,10 @@ object S2GraphConfigs {
val DEFAULT_CACHE_TTL_SECONDS = java.lang.Integer.valueOf(-1)
}
object ResourceCacheConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ RESOURCE_CACHE_MAX_SIZE -> DEFAULT_RESOURCE_CACHE_MAX_SIZE,
+ RESOURCE_CACHE_TTL_SECONDS -> DEFAULT_RESOURCE_CACHE_TTL_SECONDS
+ )
val RESOURCE_CACHE_MAX_SIZE = "resource.cache.max.size"
val DEFAULT_RESOURCE_CACHE_MAX_SIZE = java.lang.Integer.valueOf(1000)
@@ -58,6 +94,15 @@ object S2GraphConfigs {
val DEFAULT_RESOURCE_CACHE_TTL_SECONDS = java.lang.Integer.valueOf(-1)
}
object MutatorConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ MAX_RETRY_NUMBER -> DEFAULT_MAX_RETRY_NUMBER,
+ LOCK_EXPIRE_TIME -> DEFAULT_LOCK_EXPIRE_TIME,
+ MAX_BACK_OFF -> DEFAULT_MAX_BACK_OFF,
+ BACK_OFF_TIMEOUT -> DEFAULT_BACK_OFF_TIMEOUT,
+ HBASE_FAIL_PROB -> DEFAULT_HBASE_FAIL_PROB,
+ DELETE_ALL_FETCH_SIZE -> DEFAULT_DELETE_ALL_FETCH_SIZE,
+ DELETE_ALL_FETCH_COUNT -> DEFAULT_DELETE_ALL_FETCH_COUNT
+ )
val MAX_RETRY_NUMBER = "max.retry.number"
val DEFAULT_MAX_RETRY_NUMBER = java.lang.Integer.valueOf(100)
@@ -80,10 +125,19 @@ object S2GraphConfigs {
val DEFAULT_DELETE_ALL_FETCH_COUNT = java.lang.Integer.valueOf(200)
}
object QueryConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ QUERY_HARDLIMIT -> DEFAULT_QUERY_HARDLIMIT
+ )
val QUERY_HARDLIMIT = "query.hardlimit"
val DEFAULT_QUERY_HARDLIMIT = java.lang.Integer.valueOf(100000)
}
object FutureCacheConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ FUTURE_CACHE_MAX_SIZE -> DEFAULT_FUTURE_CACHE_MAX_SIZE,
+ FUTURE_CACHE_EXPIRE_AFTER_WRITE -> DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_WRITE,
+ FUTURE_CACHE_EXPIRE_AFTER_ACCESS -> DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_ACCESS,
+ FUTURE_CACHE_METRIC_INTERVAL -> DEFAULT_FUTURE_CACHE_METRIC_INTERVAL
+ )
val FUTURE_CACHE_MAX_SIZE = "future.cache.max.size"
val DEFAULT_FUTURE_CACHE_MAX_SIZE = java.lang.Integer.valueOf(100000)
@@ -97,6 +151,9 @@ object S2GraphConfigs {
val DEFAULT_FUTURE_CACHE_METRIC_INTERVAL = java.lang.Integer.valueOf(60000)
}
object LogConfigs {
+ lazy val DEFAULTS: Map[String, AnyRef] = Map(
+ QUERY_LOG_SAMPLE_RATE -> DEFAULT_QUERY_LOG_SAMPLE_RATE
+ )
val QUERY_LOG_SAMPLE_RATE = "query.log.sample.rate"
val DEFAULT_QUERY_LOG_SAMPLE_RATE = Double.box(0.05)
}