You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2020/04/05 10:33:55 UTC
[carbondata] branch master updated: [CARBONDATA-3764] Reduce
Reusable buffer creation when few projections selected out of many columns
This is an automated email from the ASF dual-hosted git repository.
jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 7ed7cf1 [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of many columns
7ed7cf1 is described below
commit 7ed7cf1f69b9b852c8f3a5bf41f6d2a9b5f175e7
Author: ajantha-bhat <aj...@gmail.com>
AuthorDate: Fri Apr 3 12:31:34 2020 +0530
[CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of many columns
Why is this PR needed?
If few projections selected out of many columns, reusable buffer is based on columns count instead of projections count. Hence many unused objects were created.
What changes were proposed in this PR?
Create reusable buffers only as per the projection count.
Fix spelling error from Resusable to Reusable
Does this PR introduce any user interface change?
No
Is any new testcase added?
No. Internal change
This closes #3693
---
.../scan/executor/impl/AbstractQueryExecutor.java | 25 ++++++++++------------
.../scan/executor/infos/BlockExecutionInfo.java | 20 ++++++++---------
.../core/scan/result/BlockletScannedResult.java | 4 ++--
3 files changed, 23 insertions(+), 26 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 05d3876..dcb2c0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -382,16 +382,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(),
dataRefNode.getBlockInfos().get(0).getSegment());
if (null == dimensionReusableDataBuffers || null == measureReusableDataBuffers) {
- dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionResusableDataBuffer();
- measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureResusableDataBuffer();
+ dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionReusableDataBuffer();
+ measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureReusableDataBuffer();
} else {
if (dimensionReusableDataBuffers.length == blockExecutionInfoForBlock
- .getDimensionResusableDataBuffer().length) {
- blockExecutionInfoForBlock.setDimensionResusableDataBuffer(dimensionReusableDataBuffers);
+ .getDimensionReusableDataBuffer().length) {
+ blockExecutionInfoForBlock.setDimensionReusableDataBuffer(dimensionReusableDataBuffers);
}
if (measureReusableDataBuffers.length == blockExecutionInfoForBlock
- .getMeasureResusableDataBuffer().length) {
- blockExecutionInfoForBlock.setMeasureResusableDataBuffer(measureReusableDataBuffers);
+ .getMeasureReusableDataBuffer().length) {
+ blockExecutionInfoForBlock.setMeasureReusableDataBuffer(measureReusableDataBuffers);
}
}
blockExecutionInfoList.add(blockExecutionInfoForBlock);
@@ -510,13 +510,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions,
segmentProperties.getDimensionOrdinalToChunkMapping(),
currentBlockFilterDimensions, allProjectionListDimensionIdexes);
- int reusableBufferSize = Math.max(segmentProperties.getDimensionOrdinalToChunkMapping().size(),
- projectDimensions.size());
- ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[reusableBufferSize];
+ ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[projectDimensions.size()];
for (int i = 0; i < dimensionBuffer.length; i++) {
dimensionBuffer[i] = new ReusableDataBuffer();
}
- blockExecutionInfo.setDimensionResusableDataBuffer(dimensionBuffer);
+ blockExecutionInfo.setDimensionReusableDataBuffer(dimensionBuffer);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
@@ -541,13 +539,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
projectionMeasures, expressionMeasures,
segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures,
allProjectionListMeasureIndexes);
- reusableBufferSize = Math.max(segmentProperties.getMeasuresOrdinalToChunkMapping().size(),
- allProjectionListMeasureIndexes.size());
- ReusableDataBuffer[] measureBuffer = new ReusableDataBuffer[reusableBufferSize];
+ ReusableDataBuffer[] measureBuffer =
+ new ReusableDataBuffer[allProjectionListMeasureIndexes.size()];
for (int i = 0; i < measureBuffer.length; i++) {
measureBuffer[i] = new ReusableDataBuffer();
}
- blockExecutionInfo.setMeasureResusableDataBuffer(measureBuffer);
+ blockExecutionInfo.setMeasureReusableDataBuffer(measureBuffer);
if (measureChunkIndexes.length > 0) {
numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1]
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index eb19852..e3b4cd9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -203,9 +203,9 @@ public class BlockExecutionInfo {
*/
private boolean isDirectVectorFill;
- private ReusableDataBuffer[] dimensionResusableDataBuffer;
+ private ReusableDataBuffer[] dimensionReusableDataBuffer;
- private ReusableDataBuffer[] measureResusableDataBuffer;
+ private ReusableDataBuffer[] measureReusableDataBuffer;
/**
* It is used to read only the deleted data of a particular version. It will be used to get the
@@ -588,20 +588,20 @@ public class BlockExecutionInfo {
isDirectVectorFill = directVectorFill;
}
- public ReusableDataBuffer[] getDimensionResusableDataBuffer() {
- return dimensionResusableDataBuffer;
+ public ReusableDataBuffer[] getDimensionReusableDataBuffer() {
+ return dimensionReusableDataBuffer;
}
- public void setDimensionResusableDataBuffer(ReusableDataBuffer[] dimensionResusableDataBuffer) {
- this.dimensionResusableDataBuffer = dimensionResusableDataBuffer;
+ public void setDimensionReusableDataBuffer(ReusableDataBuffer[] dimensionReusableDataBuffer) {
+ this.dimensionReusableDataBuffer = dimensionReusableDataBuffer;
}
- public ReusableDataBuffer[] getMeasureResusableDataBuffer() {
- return measureResusableDataBuffer;
+ public ReusableDataBuffer[] getMeasureReusableDataBuffer() {
+ return measureReusableDataBuffer;
}
- public void setMeasureResusableDataBuffer(ReusableDataBuffer[] measureResusableDataBuffer) {
- this.measureResusableDataBuffer = measureResusableDataBuffer;
+ public void setMeasureReusableDataBuffer(ReusableDataBuffer[] measureReusableDataBuffer) {
+ this.measureReusableDataBuffer = measureReusableDataBuffer;
}
public boolean isReadOnlyDelta() {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index 6172e09..5f333ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -157,8 +157,8 @@ public abstract class BlockletScannedResult {
public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo,
QueryStatisticsModel queryStatisticsModel) {
- this.dimensionReusableBuffer = blockExecutionInfo.getDimensionResusableDataBuffer();
- this.measureReusableBuffer = blockExecutionInfo.getMeasureResusableDataBuffer();
+ this.dimensionReusableBuffer = blockExecutionInfo.getDimensionReusableDataBuffer();
+ this.measureReusableBuffer = blockExecutionInfo.getMeasureReusableDataBuffer();
this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();