You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/09/22 05:36:20 UTC
[22/50] [abbrv] incubator-carbondata git commit: clear query
statistics map
clear query statistics map
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/60fde412
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/60fde412
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/60fde412
Branch: refs/heads/branch-0.1
Commit: 60fde4123514c46883f8d1f131d9c0e518b1e981
Parents: e78106a
Author: foryou2030 <fo...@126.com>
Authored: Fri Sep 2 18:22:03 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Sep 22 09:57:11 2016 +0530
----------------------------------------------------------------------
.../DriverQueryStatisticsRecorder.java | 234 +++++++++++--------
.../QueryStatisticsConstants.java | 3 +
.../apache/carbondata/core/util/CarbonUtil.java | 14 ++
3 files changed, 158 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
index 9e83c22..534852f 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
@@ -19,15 +19,19 @@
package org.apache.carbondata.core.carbon.querystatistics;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import static org.apache.carbondata.core.util.CarbonUtil.add;
import static org.apache.carbondata.core.util.CarbonUtil.printLine;
+import org.apache.commons.lang3.StringUtils;
+
/**
* Class will be used to record and log the query statistics
*/
@@ -47,7 +51,8 @@ public class DriverQueryStatisticsRecorder {
private static final Object lock = new Object();
private DriverQueryStatisticsRecorder() {
- queryStatisticsMap = new HashMap<String, List<QueryStatistic>>();
+ // use ConcurrentHashMap, it is thread-safe
+ queryStatisticsMap = new ConcurrentHashMap<String, List<QueryStatistic>>();
}
private static DriverQueryStatisticsRecorder carbonLoadStatisticsImplInstance =
@@ -78,9 +83,32 @@ public class DriverQueryStatisticsRecorder {
*/
public void logStatisticsAsTableDriver() {
synchronized (lock) {
- String tableInfo = collectDriverStatistics();
- if (null != tableInfo) {
- LOGGER.statistic(tableInfo);
+ Iterator<Map.Entry<String, List<QueryStatistic>>> entries =
+ queryStatisticsMap.entrySet().iterator();
+ while (entries.hasNext()) {
+ Map.Entry<String, List<QueryStatistic>> entry = entries.next();
+ String queryId = entry.getKey();
+ // clear the unknown query statistics
+ if(StringUtils.isEmpty(queryId)) {
+ entries.remove();
+ } else {
+ // clear the timeout query statistics
+ long interval = System.nanoTime() - Long.parseLong(queryId);
+ if (interval > QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
+ entries.remove();
+ } else {
+ // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+ // or just print block_allocation_t,block_identification_t
+ if (entry.getValue().size() >= 2) {
+ String tableInfo = collectDriverStatistics(entry.getValue(), queryId);
+ if (null != tableInfo) {
+ LOGGER.statistic(tableInfo);
+ // clear the statistics that has been printed
+ entries.remove();
+ }
+ }
+ }
+ }
}
}
}
@@ -88,96 +116,116 @@ public class DriverQueryStatisticsRecorder {
/**
* Below method will parse queryStatisticsMap and put time into table
*/
- public String collectDriverStatistics() {
- for (String key: queryStatisticsMap.keySet()) {
- try {
- // TODO: get the finished query, and print Statistics
- if (queryStatisticsMap.get(key).size() > 3) {
- String sql_parse_time = "";
- String load_meta_time = "";
- String block_allocation_time = "";
- String block_identification_time = "";
- Double driver_part_time_tmp = 0.0;
- String splitChar = " ";
- // get statistic time from the QueryStatistic
- for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
- switch (statistic.getMessage()) {
- case QueryStatisticsConstants.SQL_PARSE:
- sql_parse_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.LOAD_META:
- load_meta_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_ALLOCATION:
- block_allocation_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
- block_identification_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- default:
- break;
- }
- }
- String driver_part_time = driver_part_time_tmp + splitChar;
- // structure the query statistics info table
- StringBuilder tableInfo = new StringBuilder();
- int len1 = 8;
- int len2 = 20;
- int len3 = 21;
- int len4 = 22;
- String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
- printLine("-", len3) + "+" + printLine("-", len4) + "+";
- String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
- printLine(" ", len3) + "+" + printLine("-", len4) + "+";
- // table header
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" +
- printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
- printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
- "Query Cost" + "|" + "\n");
- // driver part
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
- printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
- printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
- printLine(" ", (len4 - load_meta_time.length())) +
- load_meta_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
- printLine(" ", (len2 - "Block allocation".length())) +
- "Block allocation" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_allocation_time.length())) +
- block_allocation_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "Block identification".length())) +
- "Block identification" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_identification_time.length())) +
- block_identification_time + "|" + "\n");
- tableInfo.append(line).append("\n");
-
- // once the statistics be printed, remove it from the map
- queryStatisticsMap.remove(key);
- // show query statistic as "query id" + "table"
- return "Print query statistic for query id: " + key + "\n" + tableInfo.toString();
+ public String collectDriverStatistics(List<QueryStatistic> statisticsList, String queryId) {
+ String sql_parse_time = "";
+ String load_meta_time = "";
+ String block_allocation_time = "";
+ String block_identification_time = "";
+ Double driver_part_time_tmp = 0.0;
+ Double driver_part_time_tmp2 = 0.0;
+ String splitChar = " ";
+ try {
+ // get statistic time from the QueryStatistic
+ for (QueryStatistic statistic : statisticsList) {
+ switch (statistic.getMessage()) {
+ case QueryStatisticsConstants.SQL_PARSE:
+ sql_parse_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+ break;
+ case QueryStatisticsConstants.LOAD_META:
+ load_meta_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+ break;
+ case QueryStatisticsConstants.BLOCK_ALLOCATION:
+ block_allocation_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+ driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken());
+ break;
+ case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
+ block_identification_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+ driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken());
+ break;
+ default:
+ break;
}
- } catch (Exception ex) {
- return "Put statistics into table failed, catch exception: " + ex.getMessage();
}
+ String driver_part_time = driver_part_time_tmp + splitChar;
+ // structure the query statistics info table
+ StringBuilder tableInfo = new StringBuilder();
+ int len1 = 8;
+ int len2 = 20;
+ int len3 = 21;
+ int len4 = 22;
+ String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
+ printLine("-", len3) + "+" + printLine("-", len4) + "+";
+ String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
+ printLine(" ", len3) + "+" + printLine("-", len4) + "+";
+ // table header
+ tableInfo.append(line).append("\n");
+ tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" +
+ printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
+ printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) + "Query Cost" + "|" + "\n");
+ tableInfo.append(line).append("\n");
+ // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+ if (!StringUtils.isEmpty(sql_parse_time) &&
+ !StringUtils.isEmpty(load_meta_time) &&
+ !StringUtils.isEmpty(block_allocation_time) &&
+ !StringUtils.isEmpty(block_identification_time)) {
+ tableInfo.append("|" + printLine(" ", len1) + "|" +
+ printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
+ printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
+ printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
+ printLine(" ", (len4 - load_meta_time.length())) +
+ load_meta_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" + printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
+ printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - block_allocation_time.length())) +
+ block_allocation_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" +
+ printLine(" ", len1) + "|" +
+ printLine(" ", (len2 - "Block identification".length())) +
+ "Block identification" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - block_identification_time.length())) +
+ block_identification_time + "|" + "\n");
+ tableInfo.append(line).append("\n");
+
+ // show query statistic as "query id" + "table"
+ return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
+ } else if (!StringUtils.isEmpty(block_allocation_time) &&
+ !StringUtils.isEmpty(block_identification_time)) {
+ // when we can't get sql parse time, we only print the last two
+ driver_part_time = driver_part_time_tmp2 + splitChar;
+ tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
+ printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" +
+ printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
+ printLine(" ", (len4 - block_allocation_time.length())) +
+ block_allocation_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" +
+ printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
+ printLine(" ", (len2 - "Block identification".length())) +
+ "Block identification" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - block_identification_time.length())) +
+ block_identification_time + "|" + "\n");
+ tableInfo.append(line).append("\n");
+
+ // show query statistic as "query id" + "table"
+ return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
+ }
+
+ return null;
+ } catch (Exception ex) {
+ return "Put statistics into table failed, catch exception: " + ex.getMessage();
}
- return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
index 540cf1e..118dadb 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
@@ -48,6 +48,9 @@ public interface QueryStatisticsConstants {
String RESULT_SIZE = "The size of query result";
+ // clear no-use statistics timeout
+ long CLEAR_STATISTICS_TIMEOUT = 60 * 1000 * 1000000L;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/60fde412/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 77d041c..c98456e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -28,6 +28,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
@@ -1430,5 +1431,18 @@ public final class CarbonUtil {
}
return builder.toString();
}
+
+ /**
+ * Below method will for double plus double
+ *
+ * @param v1
+ * @param v2
+ */
+ public static double add(double v1, double v2)
+ {
+ BigDecimal b1 = new BigDecimal(Double.toString(v1));
+ BigDecimal b2 = new BigDecimal(Double.toString(v2));
+ return b1.add(b2).doubleValue();
+ }
}