You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/03/18 07:39:47 UTC
[1/2] incubator-carbondata git commit: Added statistics for scan and
read Time
Repository: incubator-carbondata
Updated Branches:
refs/heads/master acd5f1b48 -> 41ccba115
Added statistics for scan and read Time
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3eedfa01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3eedfa01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3eedfa01
Branch: refs/heads/master
Commit: 3eedfa015933f984d2d7c41fd7844949df612d7a
Parents: acd5f1b
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Mar 17 19:31:59 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Sat Mar 18 12:57:55 2017 +0530
----------------------------------------------------------------------
.../AbstractDetailQueryResultIterator.java | 23 ++++++--------------
.../iterator/DetailQueryResultIterator.java | 2 --
.../scan/scanner/AbstractBlockletScanner.java | 14 ++++++++++++
.../core/scan/scanner/impl/FilterScanner.java | 21 +++++++++++++++++-
.../core/stats/QueryStatisticsConstants.java | 4 +++-
.../core/stats/QueryStatisticsRecorderImpl.java | 16 +++++++++-----
.../stats/QueryStasticsRecorderImplTest.java | 6 ++---
7 files changed, 58 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 1e308e2..a3d2fd2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -65,14 +65,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
protected FileHolder fileReader;
protected AbstractDataBlockIterator dataBlockIterator;
- /**
- * total time scan the blocks
- */
- protected long totalScanTime;
- /**
- * is the statistic recorded
- */
- protected boolean isStatisticsRecorded;
+
/**
* QueryStatisticsRecorder
*/
@@ -110,7 +103,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
private void intialiseInfos() {
- totalScanTime = System.currentTimeMillis();
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
DataRefNode startDataBlock = finder
@@ -137,13 +129,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
} else if (blockExecutionInfos.size() > 0) {
return true;
} else {
- if (!isStatisticsRecorded) {
- QueryStatistic statistic = new QueryStatistic();
- statistic.addFixedTimeStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
- System.currentTimeMillis() - totalScanTime);
- recorder.recordStatistics(statistic);
- isStatisticsRecorded = true;
- }
return false;
}
}
@@ -182,6 +167,12 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
QueryStatistic validPages = new QueryStatistic();
queryStatisticsModel.getStatisticsTypeAndObjMap()
.put(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages);
+ QueryStatistic scanTime = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime);
+ QueryStatistic readTime = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime);
}
public void processNextBatch(CarbonColumnarBatch columnarBatch) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index 1fa2f4c..e39b93f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -38,9 +38,7 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
}
@Override public BatchResult next() {
- long startTime = System.currentTimeMillis();
BatchResult batchResult = getBatchResult();
- totalScanTime += System.currentTimeMillis() - startTime;
return batchResult;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 8980fa7..e8bfc74 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -54,6 +54,7 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
@Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
throws IOException, FilterUnsupportedException {
+ long startTime = System.currentTimeMillis();
AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
@@ -126,10 +127,17 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
scannedResult
.setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ queryStatisticsModel.getRecorder().recordStatistics(scanTime);
return scannedResult;
}
@Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
+ long startTime = System.currentTimeMillis();
DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock()
.getDimensionChunks(blocksChunkHolder.getFileReader(),
blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
@@ -138,6 +146,12 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
.getMeasureChunks(blocksChunkHolder.getFileReader(),
blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ queryStatisticsModel.getRecorder().recordStatistics(readTime);
}
@Override public AbstractScannedResult createEmptyResult() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 915ed37..32b7bff 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -114,7 +114,14 @@ public class FilterScanner extends AbstractBlockletScanner {
}
@Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
+ long startTime = System.currentTimeMillis();
this.filterExecuter.readBlocks(blocksChunkHolder);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ queryStatisticsModel.getRecorder().recordStatistics(readTime);
}
/**
@@ -135,6 +142,7 @@ public class FilterScanner extends AbstractBlockletScanner {
*/
private AbstractScannedResult fillScannedResult(BlocksChunkHolder blocksChunkHolder)
throws FilterUnsupportedException, IOException {
+ long startTime = System.currentTimeMillis();
// apply filter on actual data
BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder);
// if indexes is empty then return with empty result
@@ -161,7 +169,11 @@ public class FilterScanner extends AbstractBlockletScanner {
validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
validPages.getCount() + bitSetGroup.getValidPages());
queryStatisticsModel.getRecorder().recordStatistics(validPages);
-
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ queryStatisticsModel.getRecorder().recordStatistics(totalBlockletStatistic);
int[] rowCount = new int[bitSetGroup.getNumberOfPages()];
// get the row indexes from bot set
int[][] indexesGroup = new int[bitSetGroup.getNumberOfPages()][];
@@ -270,6 +282,13 @@ public class FilterScanner extends AbstractBlockletScanner {
scannedResult.setMeasureChunks(measureColumnDataChunks);
scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
scannedResult.setNumberOfRows(rowCount);
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ queryStatisticsModel.getRecorder().recordStatistics(scanTime);
+
return scannedResult;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
index bf6b548..c8fa4a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
@@ -37,7 +37,9 @@ public interface QueryStatisticsConstants {
String SCAN_BLOCKS_NUM = "The num of blocks scanned";
- String SCAN_BLOCKS_TIME = "Time taken to scan blocks";
+ String SCAN_BLOCKlET_TIME = "Time taken to scan blocks";
+
+ String READ_BLOCKlET_TIME = "Time taken to read blocks";
String LOAD_DICTIONARY = "Time taken to load the Dictionary In Executor";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
index 177a8a7..3f048c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsRecorderImpl.java
@@ -98,14 +98,15 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder, Ser
String valid_scan_blocklet = "";
String valid_pages_blocklet = "";
String total_pages = "";
+ String readTime = "";
try {
for (QueryStatistic statistic : queryStatistics) {
switch (statistic.getMessage()) {
case QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR:
load_blocks_time += statistic.getTimeTaken() + splitChar;
break;
- case QueryStatisticsConstants.SCAN_BLOCKS_TIME:
- scan_blocks_time += statistic.getTimeTaken() + splitChar;
+ case QueryStatisticsConstants.SCAN_BLOCKlET_TIME:
+ scan_blocks_time += statistic.getCount() + splitChar;
break;
case QueryStatisticsConstants.SCAN_BLOCKS_NUM:
scan_blocks_num += statistic.getCount() + splitChar;
@@ -131,18 +132,23 @@ public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder, Ser
case QueryStatisticsConstants.TOTAL_PAGE_SCANNED:
total_pages = statistic.getCount() + splitChar;
break;
+ case QueryStatisticsConstants.READ_BLOCKlET_TIME:
+ readTime = statistic.getCount() + splitChar;
+ break;
default:
break;
}
}
- String headers = "task_id,load_blocks_time,load_dictionary_time,scan_blocks_time,"
- + "total_executor_time,scan_blocks_num,total_blocklet,"
- + "valid_scan_blocklet,total_pages,valid_pages,result_size";
+ String headers =
+ "task_id,load_blocks_time,load_dictionary_time,carbon_scan_time,carbon_IO_time, "
+ + "total_executor_time,scan_blocks_num,total_blocklets,"
+ + "valid_blocklets,total_pages,valid_pages,result_size";
List<String> values = new ArrayList<String>();
values.add(queryIWthTask);
values.add(load_blocks_time);
values.add(load_dictionary_time);
values.add(scan_blocks_time);
+ values.add(readTime);
values.add(total_executor_time);
values.add(scan_blocks_num);
values.add(total_blocklet);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3eedfa01/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
index fd2e0fd..39195b3 100644
--- a/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
@@ -48,10 +48,10 @@ public class QueryStasticsRecorderImplTest {
queryStasticsRecorderImpl.logStatistics();
queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_BLOCKS_EXECUTOR);
queryStatisticWithSCAN_BLOCKS_TIME = new QueryStatistic();
- queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
+ queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, 5L);
queryStatisticWithSCAN_BLOCKS_TIME
- .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
- queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
+ .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, 5L);
+ queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, 5L);
queryStasticsRecorderImpl.logStatistics();
queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_TIME);
queryStatisticWithSCAN_BLOCKS_NUM = new QueryStatistic();
[2/2] incubator-carbondata git commit: [CARBONDATA-790] Added
statistics for exclusive carbon scan and IO Time.This closes #668
Posted by gv...@apache.org.
[CARBONDATA-790] Added statistics for exclusive carbon scan and IO Time.This closes #668
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/41ccba11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/41ccba11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/41ccba11
Branch: refs/heads/master
Commit: 41ccba1155974e405f21c6cff0a72e7e1f775202
Parents: acd5f1b 3eedfa0
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Sat Mar 18 13:08:27 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Sat Mar 18 13:08:27 2017 +0530
----------------------------------------------------------------------
.../AbstractDetailQueryResultIterator.java | 23 ++++++--------------
.../iterator/DetailQueryResultIterator.java | 2 --
.../scan/scanner/AbstractBlockletScanner.java | 14 ++++++++++++
.../core/scan/scanner/impl/FilterScanner.java | 21 +++++++++++++++++-
.../core/stats/QueryStatisticsConstants.java | 4 +++-
.../core/stats/QueryStatisticsRecorderImpl.java | 16 +++++++++-----
.../stats/QueryStasticsRecorderImplTest.java | 6 ++---
7 files changed, 58 insertions(+), 28 deletions(-)
----------------------------------------------------------------------