You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/10/31 07:00:17 UTC
[19/22] carbondata git commit: [CARBONDATA-1653] Rename aggType to
measureDataType
[CARBONDATA-1653] Rename aggType to measureDataType
This closes #1444
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d70a211
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d70a211
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d70a211
Branch: refs/heads/fgdatamap
Commit: 4d70a2118fb26936e0b7eee9a0ce2147cb0501c9
Parents: 93a604d
Author: Jacky Li <ja...@qq.com>
Authored: Sat Oct 28 16:26:38 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Sat Oct 28 23:39:26 2017 +0530
----------------------------------------------------------------------
.../scan/executor/impl/AbstractQueryExecutor.java | 4 ++--
.../core/scan/executor/util/QueryUtil.java | 17 -----------------
.../core/scan/executor/util/QueryUtilTest.java | 9 ---------
.../merger/CompactionResultSortProcessor.java | 8 +-------
.../sort/sortdata/IntermediateFileMerger.java | 6 +++---
.../sort/sortdata/SortTempFileChunkHolder.java | 12 ++++++------
.../store/CarbonFactDataHandlerModel.java | 6 +++---
7 files changed, 15 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index b2c1a2b..56ed575 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -161,10 +161,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
// calculating the total number of aggeragted columns
- int aggTypeCount = queryModel.getQueryMeasures().size();
+ int measureCount = queryModel.getQueryMeasures().size();
int currentIndex = 0;
- DataType[] dataTypes = new DataType[aggTypeCount];
+ DataType[] dataTypes = new DataType[measureCount];
for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
// adding the data type and aggregation type of all the measure this
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 59387b3..6aab8a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -678,23 +678,6 @@ public class QueryUtil {
}
/**
- * below method will be used to get the actual type aggregator
- *
- * @param aggType
- * @return index in aggrgetor
- */
- public static int[] getActualTypeIndex(List<String> aggType) {
- List<Integer> indexList = new ArrayList<Integer>();
- for (int i = 0; i < aggType.size(); i++) {
- if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE
- .equals(aggType.get(i))) {
- indexList.add(i);
- }
- }
- return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
- }
-
- /**
* Below method will be used to get the key structure for the column group
*
* @param segmentProperties segment properties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 23504d3..b764569 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -221,15 +221,6 @@ public class QueryUtilTest extends TestCase {
assertArrayEquals(expectedValue, actualValue);
}
- @Test public void testGetActualTypeIndex() {
- List<String> dummyList = new ArrayList<>();
- dummyList.add("test1");
- dummyList.add("test2");
- int[] actualValue = QueryUtil.getActualTypeIndex(dummyList);
- int[] expectedValue = { 0, 1 };
- assertArrayEquals(expectedValue, actualValue);
- }
-
@AfterClass public void tearDown() {
segmentProperties = null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 0e20ef5..a361f3a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -155,7 +155,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
try {
initTempStoreLocation();
initSortDataRows();
- initAggType();
+ dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
processResult(resultIteratorList);
// After delete command, if no records are fetched from one split,
// below steps are not required to be initialized.
@@ -406,10 +406,4 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
true, false);
}
- /**
- * initialise aggregation type for measures for their storage format
- */
- private void initAggType() {
- dataTypes = CarbonDataProcessorUtil.initDataType(carbonTable, tableName, measureCount);
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index fa50f1c..ebc811c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -326,7 +326,7 @@ public class IntermediateFileMerger implements Callable<Void> {
return;
}
try {
- DataType[] aggType = mergerParameters.getMeasureDataType();
+ DataType[] measureDataType = mergerParameters.getMeasureDataType();
int[] mdkArray = (int[]) row[0];
byte[][] nonDictArray = (byte[][]) row[1];
int mdkIndex = 0;
@@ -346,7 +346,7 @@ public class IntermediateFileMerger implements Callable<Void> {
for (int counter = 0; counter < mergerParameters.getMeasureColCount(); counter++) {
if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
stream.write((byte) 1);
- DataType dataType = aggType[counter];
+ DataType dataType = measureDataType[counter];
if (dataType == DataTypes.BOOLEAN) {
stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
} else if (dataType == DataTypes.SHORT) {
@@ -362,7 +362,7 @@ public class IntermediateFileMerger implements Callable<Void> {
stream.writeInt(bigDecimalInBytes.length);
stream.write(bigDecimalInBytes);
} else {
- throw new IllegalArgumentException("unsupported data type:" + aggType[counter]);
+ throw new IllegalArgumentException("unsupported data type:" + measureDataType[counter]);
}
} else {
stream.write((byte) 0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index afa4cd7..91bc83c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -129,7 +129,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
private int noDictionaryCount;
- private DataType[] aggType;
+ private DataType[] measureDataTypes;
/**
* to store whether dimension is of dictionary type or not
@@ -150,11 +150,11 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
* @param measureCount
* @param fileBufferSize
* @param noDictionaryCount
- * @param aggType
+ * @param measureDataTypes
* @param isNoDictionaryDimensionColumn
*/
public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount,
- int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] aggType,
+ int measureCount, int fileBufferSize, int noDictionaryCount, DataType[] measureDataTypes,
boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn,
String tableName) {
// set temp file
@@ -170,7 +170,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
this.fileBufferSize = fileBufferSize;
this.executorService = Executors
.newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName));
- this.aggType = aggType;
+ this.measureDataTypes = measureDataTypes;
this.isNoDictionaryDimensionColumn = isNoDictionaryDimensionColumn;
this.isNoDictionarySortColumn = isNoDictionarySortColumn;
@@ -344,7 +344,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
// read measure values
for (int i = 0; i < this.measureCount; i++) {
if (stream.readByte() == 1) {
- DataType dataType = aggType[i];
+ DataType dataType = measureDataTypes[i];
if (dataType == DataTypes.BOOLEAN) {
measures[index++] = stream.readBoolean();
} else if (dataType == DataTypes.SHORT) {
@@ -361,7 +361,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
stream.readFully(buff);
measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
} else {
- throw new IllegalArgumentException("unsupported data type:" + aggType[i]);
+ throw new IllegalArgumentException("unsupported data type:" + measureDataTypes[i]);
}
} else {
measures[index++] = null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d70a211/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 2c346b2..b6ac19d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -299,12 +299,12 @@ public class CarbonFactDataHandlerModel {
Map<Integer, GenericDataType> complexIndexMap =
new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
- DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
+ DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()];
int i = 0;
for (CarbonMeasure msr : segmentProperties.getMeasures()) {
- aggType[i++] = msr.getDataType();
+ measureDataTypes[i++] = msr.getDataType();
}
- carbonFactDataHandlerModel.setMeasureDataType(aggType);
+ carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(),
tableName, loadModel.getPartitionId(), loadModel.getSegmentId());