You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:07 UTC
[09/47] incubator-carbondata git commit: [CARBONDATA-68] Added query
statistic for detail query case measuring carbon executor time (#852)
[CARBONDATA-68] Added query statistic for detail query case measuring carbon executor time (#852)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a81ed981
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a81ed981
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a81ed981
Branch: refs/heads/master
Commit: a81ed981230e3fddd8e1c29d774266c171a54643
Parents: 584402e
Author: Kumar Vishal <ku...@gmail.com>
Authored: Sat Jul 23 12:21:02 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 12:21:02 2016 +0530
----------------------------------------------------------------------
.../carbon/querystatistics/QueryStatistic.java | 12 ++++++++++
.../AbstractDetailQueryResultIterator.java | 25 +++++++++++++-------
.../iterator/DetailQueryResultIterator.java | 8 ++++++-
.../carbondata/spark/rdd/CarbonQueryRDD.scala | 18 +++++++++++---
4 files changed, 51 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
index 41f3d22..80398c1 100644
--- a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
+++ b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -61,6 +61,18 @@ public class QueryStatistic implements Serializable {
}
/**
+ * Below method will be used to add fixed time statistic.
+ * For example total time taken for scan or result preparation
+ *
+ * @param message statistic message
+ * @param timetaken
+ */
+ public void addFixedTimeStatistic(String message, long timetaken) {
+ this.timeTaken = timetaken;
+ this.message = message;
+ }
+
+ /**
* Below method will be used to get the statistic message, which will
* be used to log
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
index 01e94c8..dc94704 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
@@ -66,14 +66,18 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
*/
protected int[] blockIndexToBeExecuted;
/**
- * to store the statistics
- */
- protected QueryStatistic statistic;
- /**
* total number of records processed
*/
protected long totalNumberOfOutputRecords;
/**
+ * total scan time of the query
+ */
+ protected long totalScanTime;
+ /**
+ * total time taken to prepare the result
+ */
+ protected long totalResultPreparationTime;
+ /**
* number of cores which can be used
*/
private long numberOfCores;
@@ -113,7 +117,6 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
executor = queryExecutor;
this.blockExecutionInfos = infos;
this.blockIndexToBeExecuted = new int[(int) numberOfCores];
- statistic = new QueryStatistic();
intialiseInfos();
}
@@ -142,9 +145,15 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
if (currentCounter < totalNumberOfNode) {
return true;
}
- statistic.addStatistics("Time taken to processed " + blockExecutionInfos.size()
- + " block(s) of output record size: " + totalNumberOfOutputRecords,
- System.currentTimeMillis());
+ QueryStatistic statistic = new QueryStatistic();
+ statistic
+ .addFixedTimeStatistic("Time taken to scan " + blockExecutionInfos.size() + " block(s) ",
+ totalScanTime);
+ blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
+ statistic = new QueryStatistic();
+ statistic.addFixedTimeStatistic(
+ "Time take to prepare query result of size " + totalNumberOfOutputRecords,
+ totalResultPreparationTime);
blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
index db6e9aa..e2f54ee 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
@@ -55,8 +55,11 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
@Override public BatchResult next() {
currentCounter += updateSliceIndexToBeExecuted();
CarbonIterator<Result> result = null;
+ long startTime = System.currentTimeMillis();
+ BatchResult batchResult = null;
try {
result = executor.executeQuery(blockExecutionInfos, blockIndexToBeExecuted);
+ totalScanTime+=System.currentTimeMillis()-startTime;
} catch (QueryExecutionException e) {
throw new RuntimeException(e.getCause().getMessage());
}
@@ -71,7 +74,10 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
Result next = result.next();
if (next.size() > 0) {
totalNumberOfOutputRecords += next.size();
- return queryResultPreparator.prepareQueryResult(next);
+ startTime=System.currentTimeMillis();
+ batchResult = queryResultPreparator.prepareQueryResult(next);
+ totalResultPreparationTime+=System.currentTimeMillis()-startTime;
+ return batchResult;
} else {
return new BatchResult();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
index e0ec5fe..97f1993 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
@@ -18,7 +18,6 @@
package org.carbondata.spark.rdd
-
import java.util
import scala.collection.JavaConverters._
@@ -44,6 +43,7 @@ import org.carbondata.spark.Value
import org.carbondata.spark.load.CarbonLoaderUtil
import org.carbondata.spark.util.QueryPlanUtil
+
class CarbonSparkPartition(rddId: Int, val idx: Int,
val locations: Array[String],
val tableBlockInfos: util.List[TableBlockInfo])
@@ -216,7 +216,13 @@ class CarbonQueryRDD[V: ClassTag](
}
if (finished) {
clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
+ if (null != queryModel.getStatisticsRecorder) {
+ val queryStatistic = new QueryStatistic
+ queryStatistic
+ .addStatistics("Total Time taken to execute the query in executor Side",
+ System.currentTimeMillis - queryStartTime
+ )
+ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
queryModel.getStatisticsRecorder.logStatistics();
}
}
@@ -231,7 +237,13 @@ class CarbonQueryRDD[V: ClassTag](
recordCount += 1
if (queryModel.getLimit != -1 && recordCount >= queryModel.getLimit) {
clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
+ if (null != queryModel.getStatisticsRecorder) {
+ val queryStatistic = new QueryStatistic
+ queryStatistic
+ .addStatistics("Total Time taken to execute the query in executor Side",
+ System.currentTimeMillis - queryStartTime
+ )
+ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
queryModel.getStatisticsRecorder.logStatistics();
}
}