You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/03 06:37:09 UTC
[02/50] [abbrv] kylin git commit: KYLIN-1908 refactor,
extract QueryMetricsFacade
KYLIN-1908 refactor, extract QueryMetricsFacade
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b34c38b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b34c38b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b34c38b
Branch: refs/heads/1.5.x-HBase1.x
Commit: 1b34c38b2a9ff65950dbac8229af5749ad8403ed
Parents: 0188a26
Author: Yang Li <li...@apache.org>
Authored: Sun Aug 21 19:38:11 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Aug 21 19:38:11 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../common/KylinConfigCannotInitException.java | 1 +
.../kylin/rest/controller/QueryController.java | 9 +-
.../kylin/rest/init/InitialTaskManager.java | 8 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 108 +++++++++++++++++++
.../kylin/rest/util/QueryMetricsUtil.java | 94 ----------------
.../kylin/rest/metrics/QueryMetricsTest.java | 20 ++--
7 files changed, 128 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 50205fb..de6b977 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -570,6 +570,10 @@ abstract public class KylinConfigBase implements Serializable {
return Long.valueOf(this.getOptional("kylin.query.sequence.expire.time", "86400000"));//default a day
}
+ public boolean getQueryMetricsEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false"));
+ }
+
public int[] getQueryMetricsPercentilesIntervals() {
String[] dft = { "60", "300", "3600" };
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
index 7113872..a4bbeae 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common;
+@SuppressWarnings("serial")
public class KylinConfigCannotInitException extends RuntimeException {
public KylinConfigCannotInitException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 6f8a7e1..e7847c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
@@ -33,7 +32,7 @@ import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.metrics.QueryMetrics;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.model.SelectedColumnMeta;
import org.apache.kylin.rest.model.TableMeta;
@@ -43,7 +42,6 @@ import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryMetricsUtil;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
@@ -77,9 +75,6 @@ public class QueryController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(QueryController.class);
- private ConcurrentHashMap<String, QueryMetrics> metricsMap =
- new ConcurrentHashMap<String, QueryMetrics>();
-
public static final String SUCCESS_QUERY_CACHE = "StorageCache";
public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
@@ -225,7 +220,7 @@ public class QueryController extends BasicController {
queryService.logQuery(sqlRequest, sqlResponse);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
if (sqlResponse.getIsException())
throw new InternalErrorException(sqlResponse.getExceptionMessage());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 430d8a4..5cf48cb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -19,8 +19,8 @@
package org.apache.kylin.rest.init;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@@ -37,12 +37,12 @@ public class InitialTaskManager implements InitializingBean {
logger.info("Kylin service is starting.....");
runInitialTasks();
-
- //init mrtrics system for kylin
- DefaultMetricsSystem.initialize("Kylin");
}
private void runInitialTasks() {
+ // init metrics system for kylin
+ QueryMetricsFacade.init();
+
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String initTasks = kylinConfig.getInitTasks();
if (!StringUtils.isEmpty(initTasks)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
new file mode 100644
index 0000000..91aff8b
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.metrics;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The entrance of metrics features.
+ */
+@ThreadSafe
+public class QueryMetricsFacade {
+
+ private static final Logger logger = LoggerFactory.getLogger(QueryMetricsFacade.class);
+
+ private static boolean enabled = false;
+ private static ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>();
+
+ public static void init() {
+ enabled = KylinConfig.getInstanceFromEnv().getQueryMetricsEnabled();
+ if (!enabled)
+ return;
+
+ DefaultMetricsSystem.initialize("Kylin");
+ }
+
+ public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) {
+ if (!enabled)
+ return;
+
+ String projectName = sqlRequest.getProject();
+ String cubeName = sqlResponse.getCube();
+
+ update(getQueryMetrics("Server_Total", metricsMap), sqlResponse);
+
+ update(getQueryMetrics(projectName, metricsMap), sqlResponse);
+
+ String cubeMetricName = projectName + ",sub=" + cubeName;
+ update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse);
+ }
+
+ private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ try {
+ incrQueryCount(queryMetrics, sqlResponse);
+ incrCacheHitCount(queryMetrics, sqlResponse);
+
+ if (!sqlResponse.getIsException()) {
+ queryMetrics.addQueryLatency(sqlResponse.getDuration());
+ queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount());
+ queryMetrics.addResultRowCount(sqlResponse.getResults().size());
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ }
+
+ }
+
+ private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
+ queryMetrics.incrQuerySuccessCount();
+ } else {
+ queryMetrics.incrQueryFailCount();
+ }
+ queryMetrics.incrQueryCount();
+ }
+
+ private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ if (sqlResponse.isStorageCacheUsed()) {
+ queryMetrics.addCacheHitCount(1);
+ }
+ }
+
+ private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int[] intervals = config.getQueryMetricsPercentilesIntervals();
+
+ if (metricsMap.containsKey(name)) {
+ return metricsMap.get(name);
+ } else {
+ QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name);
+ metricsMap.put(name, queryMetrics);
+ return queryMetrics;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
deleted file mode 100644
index 0c1c780..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.metrics.QueryMetrics;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.rest.response.SQLResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.ThreadSafe;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * update metrics about query
- */
-@ThreadSafe
-public class QueryMetricsUtil {
-
- private static final Logger logger = LoggerFactory.getLogger(QueryMetricsUtil.class);
-
- public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
- String projectName = sqlRequest.getProject();
- String cubeName = sqlResponse.getCube();
-
- update(getQueryMetrics("Server_Total", metricsMap), sqlResponse);
-
- update(getQueryMetrics(projectName, metricsMap), sqlResponse);
-
- String cubeMetricName = projectName + ",sub=" + cubeName;
- update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse);
- }
-
- private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- try {
- incrQueryCount(queryMetrics, sqlResponse);
- incrCacheHitCount(queryMetrics, sqlResponse);
-
- if (!sqlResponse.getIsException()) {
- queryMetrics.addQueryLatency(sqlResponse.getDuration());
- queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount());
- queryMetrics.addResultRowCount(sqlResponse.getResults().size());
- }
- } catch (Exception e) {
- logger.error(e.getMessage());
- }
-
- }
-
- private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
- queryMetrics.incrQuerySuccessCount();
- } else {
- queryMetrics.incrQueryFailCount();
- }
- queryMetrics.incrQueryCount();
- }
-
- private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- if (sqlResponse.isStorageCacheUsed()) {
- queryMetrics.addCacheHitCount(1);
- }
- }
-
- private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int[] intervals = config.getQueryMetricsPercentilesIntervals();
-
- if (metricsMap.containsKey(name)) {
- return metricsMap.get(name);
- } else {
- QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name);
- metricsMap.put(name, queryMetrics);
- return queryMetrics;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
index cd7ff7a..d413f40 100644
--- a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
@@ -18,26 +18,24 @@
package org.apache.kylin.rest.metrics;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.ServiceTestBase;
-import org.apache.kylin.rest.util.QueryMetricsUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
public class QueryMetricsTest extends ServiceTestBase {
private static MBeanServer mBeanServer;
private static ObjectName objectName;
- private ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>();
@Before
public void setup() throws Exception {
@@ -71,7 +69,7 @@ public class QueryMetricsTest extends ServiceTestBase {
sqlResponse.setResults(results);
sqlResponse.setStorageCacheUsed(true);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
Thread.sleep(2000);
@@ -100,7 +98,7 @@ public class QueryMetricsTest extends ServiceTestBase {
sqlResponse2.setCube("test_cube");
sqlResponse2.setIsException(true);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse2, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse2);
Thread.sleep(2000);