You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/02 01:11:56 UTC
[1/3] incubator-carbondata git commit: Refactored code to improve
performance by using removing unnecessary conversion.
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 645eafadb -> 29f9cf226
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/carbondata/scan/result/iterator/RawResultIterator.java
index 0f9c339..e58e830 100644
--- a/core/src/main/java/org/carbondata/scan/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/carbondata/scan/result/iterator/RawResultIterator.java
@@ -23,7 +23,7 @@ import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.core.keygenerator.KeyGenException;
-import org.carbondata.scan.result.BatchRawResult;
+import org.carbondata.scan.result.BatchResult;
import org.carbondata.scan.wrappers.ByteArrayWrapper;
/**
@@ -39,7 +39,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* Iterator of the Batch raw result.
*/
- private CarbonIterator<BatchRawResult> detailRawQueryResultIterator;
+ private CarbonIterator<BatchResult> detailRawQueryResultIterator;
/**
* Counter to maintain the row counter.
@@ -57,9 +57,9 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* batch of the result.
*/
- private BatchRawResult batch;
+ private BatchResult batch;
- public RawResultIterator(CarbonIterator<BatchRawResult> detailRawQueryResultIterator,
+ public RawResultIterator(CarbonIterator<BatchResult> detailRawQueryResultIterator,
SegmentProperties sourceSegProperties, SegmentProperties destinationSegProperties) {
this.detailRawQueryResultIterator = detailRawQueryResultIterator;
this.sourceSegProperties = sourceSegProperties;
@@ -157,7 +157,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
* @param batch
* @return
*/
- private boolean checkIfBatchIsProcessedCompletely(BatchRawResult batch){
+ private boolean checkIfBatchIsProcessedCompletely(BatchResult batch){
if(counter < batch.getSize())
{
return false;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/preparator/QueryResultPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/preparator/QueryResultPreparator.java b/core/src/main/java/org/carbondata/scan/result/preparator/QueryResultPreparator.java
deleted file mode 100644
index 7ef5b6d..0000000
--- a/core/src/main/java/org/carbondata/scan/result/preparator/QueryResultPreparator.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.carbondata.scan.result.preparator;
-
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.Result;
-
-public interface QueryResultPreparator<K, V> {
-
- public BatchResult prepareQueryResult(Result<K, V> scannedResult);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/preparator/impl/AbstractQueryResultPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/preparator/impl/AbstractQueryResultPreparator.java b/core/src/main/java/org/carbondata/scan/result/preparator/impl/AbstractQueryResultPreparator.java
deleted file mode 100644
index fd6f1c8..0000000
--- a/core/src/main/java/org/carbondata/scan/result/preparator/impl/AbstractQueryResultPreparator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package org.carbondata.scan.result.preparator.impl;
-
-import java.util.List;
-
-import org.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.util.DataTypeUtil;
-import org.carbondata.scan.executor.impl.QueryExecutorProperties;
-import org.carbondata.scan.model.QueryDimension;
-import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.preparator.QueryResultPreparator;
-
-public abstract class AbstractQueryResultPreparator<K, V> implements QueryResultPreparator<K, V> {
-
- /**
- * query properties
- */
- protected QueryExecutorProperties queryExecuterProperties;
-
- /**
- * query model
- */
- protected QueryModel queryModel;
-
- public AbstractQueryResultPreparator(QueryExecutorProperties executerProperties,
- QueryModel queryModel) {
- this.queryExecuterProperties = executerProperties;
- this.queryModel = queryModel;
- }
-
- protected void fillDimensionData(Object[][] convertedResult, List<QueryDimension> queryDimensions,
- int dimensionCount, Object[] row, int rowIndex) {
- QueryDimension queryDimension;
- for (int i = 0; i < dimensionCount; i++) {
- queryDimension = queryDimensions.get(i);
- if (!CarbonUtil
- .hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)) {
- row[queryDimension.getQueryOrder()] = convertedResult[i][rowIndex];
- } else if (CarbonUtil
- .hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(queryDimension.getDimension().getDataType());
- row[queryDimension.getQueryOrder()] = directDictionaryGenerator
- .getValueFromSurrogate((Integer) convertedResult[i][rowIndex]);
- } else {
- if (queryExecuterProperties.sortDimIndexes[i] == 1) {
- row[queryDimension.getQueryOrder()] = DataTypeUtil.getDataBasedOnDataType(
- queryExecuterProperties.columnToDictionayMapping
- .get(queryDimension.getDimension().getColumnId())
- .getDictionaryValueFromSortedIndex((Integer) convertedResult[i][rowIndex]),
- queryDimension.getDimension().getDataType());
- } else {
- row[queryDimension.getQueryOrder()] = DataTypeUtil.getDataBasedOnDataType(
- queryExecuterProperties.columnToDictionayMapping
- .get(queryDimension.getDimension().getColumnId())
- .getDictionaryValueForKey((Integer) convertedResult[i][rowIndex]),
- queryDimension.getDimension().getDataType());
- }
- }
- }
- }
-
- protected Object[][] encodeToRows(Object[][] data) {
- if (data.length == 0) {
- return data;
- }
- Object[][] rData = new Object[data[0].length][data.length];
- int len = data.length;
- for (int i = 0; i < rData.length; i++) {
- for (int j = 0; j < len; j++) {
- rData[i][j] = data[j][i];
- }
- }
- return rData;
- }
-
- protected BatchResult getEmptyChunkResult(int size) {
- Object[][] row = new Object[size][1];
- BatchResult chunkResult = new BatchResult();
- chunkResult.setRows(row);
- return chunkResult;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/preparator/impl/DetailQueryResultPreparatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/preparator/impl/DetailQueryResultPreparatorImpl.java b/core/src/main/java/org/carbondata/scan/result/preparator/impl/DetailQueryResultPreparatorImpl.java
deleted file mode 100644
index 8a7d610..0000000
--- a/core/src/main/java/org/carbondata/scan/result/preparator/impl/DetailQueryResultPreparatorImpl.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.scan.result.preparator.impl;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.metadata.datatype.DataType;
-import org.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.util.DataTypeUtil;
-import org.carbondata.scan.executor.impl.QueryExecutorProperties;
-import org.carbondata.scan.model.QueryDimension;
-import org.carbondata.scan.model.QueryMeasure;
-import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * Below class will be used to get the result by converting to actual data
- * Actual data conversion can be converting the surrogate key to actual data
- *
- * @TODO there are many things in class which is very confusing, need to check
- * why it was handled like that and how we can handle that in a better
- * way.Need to revisit this class. IF aggregation is push down to spark
- * layer and if we can process the data in byte array format then this
- * class wont be useful so in future we can delete this class.
- * @TODO need to expose one interface which will return the result based on required type
- * for example its implementation case return converted result or directly result with out
- * converting to actual value
- */
-public class DetailQueryResultPreparatorImpl
- extends AbstractQueryResultPreparator<List<ListBasedResultWrapper>, Object> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DetailQueryResultPreparatorImpl.class.getName());
-
- public DetailQueryResultPreparatorImpl(QueryExecutorProperties executerProperties,
- QueryModel queryModel) {
- super(executerProperties, queryModel);
- }
-
- @Override public BatchResult prepareQueryResult(
- Result<List<ListBasedResultWrapper>, Object> scannedResult) {
- if ((null == scannedResult || scannedResult.size() < 1)) {
- return new BatchResult();
- }
- List<QueryDimension> queryDimension = queryModel.getQueryDimension();
- int dimensionCount = queryDimension.size();
- int totalNumberOfColumn = dimensionCount + queryExecuterProperties.measureDataTypes.length;
- Object[][] resultData = new Object[scannedResult.size()][totalNumberOfColumn];
- if (!queryExecuterProperties.isFunctionQuery && totalNumberOfColumn == 0
- && scannedResult.size() > 0) {
- return getEmptyChunkResult(scannedResult.size());
- }
- int currentRow = 0;
- long[] surrogateResult = null;
- int noDictionaryColumnIndex = 0;
- int complexTypeColumnIndex = 0;
- ByteArrayWrapper key = null;
- Object[] value = null;
- while (scannedResult.hasNext()) {
- key = scannedResult.getKey();
- value = scannedResult.getValue();
- if (key != null) {
- surrogateResult = queryExecuterProperties.keyStructureInfo.getKeyGenerator()
- .getKeyArray(key.getDictionaryKey(),
- queryExecuterProperties.keyStructureInfo.getMaskedBytes());
- for (int i = 0; i < dimensionCount; i++) {
- if (!CarbonUtil.hasEncoding(queryDimension.get(i).getDimension().getEncoder(),
- Encoding.DICTIONARY)) {
- resultData[currentRow][i] = DataTypeUtil.getDataBasedOnDataType(
- new String(key.getNoDictionaryKeyByIndex(noDictionaryColumnIndex++),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
- queryDimension.get(i).getDimension().getDataType());
- } else if (CarbonUtil.hasDataType(queryDimension.get(i).getDimension().getDataType(),
- new DataType[] { DataType.ARRAY, DataType.STRUCT })) {
- resultData[currentRow][i] = queryExecuterProperties.complexDimensionInfoMap
- .get(queryDimension.get(i).getDimension().getOrdinal())
- .getDataBasedOnDataTypeFromSurrogates(
- ByteBuffer.wrap(key.getComplexTypeByIndex(complexTypeColumnIndex++)));
- } else {
- resultData[currentRow][i] =
- (int) surrogateResult[queryDimension.get(i).getDimension().getKeyOrdinal()];
- }
- }
- }
- if (value != null) {
- System.arraycopy(value, 0, resultData[currentRow], dimensionCount,
- queryExecuterProperties.measureDataTypes.length);
- }
- currentRow++;
- noDictionaryColumnIndex = 0;
- }
- if (resultData.length > 0) {
- resultData = encodeToRows(resultData);
- }
- return getResult(queryModel, resultData);
- }
-
- private BatchResult getResult(QueryModel queryModel, Object[][] convertedResult) {
-
- int rowSize = convertedResult[0].length;
- Object[][] rows = new Object[rowSize][];
- List<QueryDimension> queryDimensions = queryModel.getQueryDimension();
- int dimensionCount = queryDimensions.size();
- int msrCount = queryExecuterProperties.measureDataTypes.length;
- Object[] row;
- for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) {
- row = new Object[dimensionCount + msrCount];
- fillDimensionData(convertedResult, queryDimensions, dimensionCount, row, rowIndex);
-
- QueryMeasure msr;
- for (int i = 0; i < queryModel.getQueryMeasures().size(); i++) {
- msr = queryModel.getQueryMeasures().get(i);
- row[msr.getQueryOrder()] = convertedResult[dimensionCount + i][rowIndex];
- }
- rows[rowIndex] = row;
- }
- LOGGER.info(
- "###########################################------ Total Number of records" + rowSize);
- BatchResult chunkResult = new BatchResult();
- chunkResult.setRows(rows);
- return chunkResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/preparator/impl/RawQueryResultPreparatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/preparator/impl/RawQueryResultPreparatorImpl.java b/core/src/main/java/org/carbondata/scan/result/preparator/impl/RawQueryResultPreparatorImpl.java
deleted file mode 100644
index 161b58e..0000000
--- a/core/src/main/java/org/carbondata/scan/result/preparator/impl/RawQueryResultPreparatorImpl.java
+++ /dev/null
@@ -1,136 +0,0 @@
-package org.carbondata.scan.result.preparator.impl;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.util.DataTypeUtil;
-import org.carbondata.scan.executor.impl.QueryExecutorProperties;
-import org.carbondata.scan.model.QueryDimension;
-import org.carbondata.scan.model.QueryMeasure;
-import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.model.QuerySchemaInfo;
-import org.carbondata.scan.result.BatchRawResult;
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * It does not decode the dictionary.
- */
-public class RawQueryResultPreparatorImpl
- extends AbstractQueryResultPreparator<List<ListBasedResultWrapper>, Object> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RawQueryResultPreparatorImpl.class.getName());
-
- private QuerySchemaInfo querySchemaInfo;
-
- public RawQueryResultPreparatorImpl(QueryExecutorProperties executerProperties,
- QueryModel queryModel) {
- super(executerProperties, queryModel);
- querySchemaInfo = new QuerySchemaInfo();
- querySchemaInfo.setKeyGenerator(queryExecuterProperties.keyStructureInfo.getKeyGenerator());
- querySchemaInfo.setMaskedByteIndexes(queryExecuterProperties.keyStructureInfo.getMaskedBytes());
- querySchemaInfo.setQueryDimensions(queryModel.getQueryDimension()
- .toArray(new QueryDimension[queryModel.getQueryDimension().size()]));
- querySchemaInfo.setQueryMeasures(queryModel.getQueryMeasures()
- .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
- int msrSize = queryExecuterProperties.measureDataTypes.length;
- int dimensionCount = queryModel.getQueryDimension().size();
- int[] queryOrder = new int[dimensionCount + msrSize];
- int[] queryReverseOrder = new int[dimensionCount + msrSize];
- for (int i = 0; i < dimensionCount; i++) {
- queryOrder[queryModel.getQueryDimension().get(i).getQueryOrder()] = i;
- queryReverseOrder[i] = queryModel.getQueryDimension().get(i).getQueryOrder();
- }
- for (int i = 0; i < msrSize; i++) {
- queryOrder[queryModel.getQueryMeasures().get(i).getQueryOrder()] = i + dimensionCount;
- queryReverseOrder[i + dimensionCount] = queryModel.getQueryMeasures().get(i).getQueryOrder();
- }
- querySchemaInfo.setQueryOrder(queryOrder);
- querySchemaInfo.setQueryReverseOrder(queryReverseOrder);
- }
-
- @Override public BatchResult prepareQueryResult(
- Result<List<ListBasedResultWrapper>, Object> scannedResult) {
- if ((null == scannedResult || scannedResult.size() < 1)) {
- return new BatchRawResult();
- }
- QueryDimension[] queryDimensions = querySchemaInfo.getQueryDimensions();
- int msrSize = queryExecuterProperties.measureDataTypes.length;
- int dimSize = queryDimensions.length;
- int[] order = querySchemaInfo.getQueryReverseOrder();
- Object[][] resultData = new Object[scannedResult.size()][];
- Object[] value;
- Object[] row;
- int counter = 0;
- if (queryModel.isRawBytesDetailQuery()) {
- while (scannedResult.hasNext()) {
- value = scannedResult.getValue();
- row = new Object[msrSize + 1];
- row[0] = scannedResult.getKey();
- if (value != null) {
- assert (value.length == msrSize);
- System.arraycopy(value, 0, row, 1, msrSize);
- }
- resultData[counter] = row;
- counter++;
- }
- } else {
- while (scannedResult.hasNext()) {
- value = scannedResult.getValue();
- row = new Object[msrSize + dimSize];
- ByteArrayWrapper key = scannedResult.getKey();
- if (key != null) {
- long[] surrogateResult = querySchemaInfo.getKeyGenerator()
- .getKeyArray(key.getDictionaryKey(), querySchemaInfo.getMaskedByteIndexes());
- int noDictionaryColumnIndex = 0;
- int complexTypeColumnIndex = 0;
- for (int i = 0; i < dimSize; i++) {
- if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
- row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
- new String(key.getNoDictionaryKeyByIndex(noDictionaryColumnIndex++)),
- queryDimensions[i].getDimension().getDataType());
- } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator directDictionaryGenerator =
- DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
- queryDimensions[i].getDimension().getDataType());
- if (directDictionaryGenerator != null) {
- row[order[i]] = directDictionaryGenerator.getValueFromSurrogate(
- (int) surrogateResult[queryDimensions[i].getDimension().getKeyOrdinal()]);
- }
- } else if (CarbonUtil
- .hasComplexDataType(queryDimensions[i].getDimension().getDataType())) {
- row[order[i]] = queryExecuterProperties.complexDimensionInfoMap
- .get(queryDimensions[i].getDimension().getOrdinal())
- .getDataBasedOnDataTypeFromSurrogates(
- ByteBuffer.wrap(key.getComplexTypeByIndex(complexTypeColumnIndex++)));
- } else {
- row[order[i]] =
- (int) surrogateResult[queryDimensions[i].getDimension().getKeyOrdinal()];
- }
- }
- }
- for (int i = 0; i < msrSize; i++) {
- row[order[i + queryDimensions.length]] = value[i];
- }
- resultData[counter] = row;
- counter++;
- }
- }
-
- LOGGER.info("###########################---- Total Number of records" + scannedResult.size());
- BatchRawResult result = new BatchRawResult();
- result.setRows(resultData);
- return result;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/hadoop/src/main/java/org/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 279c8ab..0934862 100644
--- a/hadoop/src/main/java/org/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -77,7 +77,6 @@ public class CarbonInputFormatUtil {
}
plan.setLimit(-1);
- plan.setRawDetailQuery(true);
plan.setQueryId(System.nanoTime() + "");
return plan;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CarbonCompactionExecutor.java b/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CarbonCompactionExecutor.java
index 43921f4..5c853c0 100644
--- a/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CarbonCompactionExecutor.java
+++ b/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CarbonCompactionExecutor.java
@@ -43,7 +43,7 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.model.QueryDimension;
import org.carbondata.scan.model.QueryMeasure;
import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.BatchRawResult;
+import org.carbondata.scan.result.BatchResult;
import org.carbondata.scan.result.iterator.RawResultIterator;
/**
@@ -143,12 +143,12 @@ public class CarbonCompactionExecutor {
* @param blockList
* @return
*/
- private CarbonIterator<BatchRawResult> executeBlockList(List<TableBlockInfo> blockList)
+ private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException {
queryModel.setTableBlockInfos(blockList);
this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
- CarbonIterator<BatchRawResult> iter = null;
+ CarbonIterator<BatchResult> iter = null;
try {
iter = queryExecutor.execute(queryModel);
} catch (QueryExecutionException e) {
@@ -187,7 +187,6 @@ public class CarbonCompactionExecutor {
model.setCountStarQuery(false);
model.setDetailQuery(true);
model.setForcedDetailRawQuery(true);
- model.setRawBytesDetailQuery(true);
model.setFilterExpressionResolverTree(null);
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
index 21aded4..8265a1b 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
@@ -46,61 +46,14 @@ case class CarbonScan(
val selectedMsrs = scala.collection.mutable.MutableList[QueryMeasure]()
@transient val carbonCatalog = ocRaw.catalog.asInstanceOf[CarbonMetastoreCatalog]
- val attributesNeedToDecode = new java.util.HashSet[AttributeReference]()
+ val attributesNeedToDecode = new java.util.LinkedHashSet[AttributeReference]()
val unprocessedExprs = new ArrayBuffer[Expression]()
val buildCarbonPlan: CarbonQueryPlan = {
val plan: CarbonQueryPlan = new CarbonQueryPlan(relationRaw.databaseName, relationRaw.tableName)
- val dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
- val measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
- val dimAttr = new Array[Attribute](dimensions.size())
- val msrAttr = new Array[Attribute](measures.size())
- attributesRaw.foreach { attr =>
- val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if(carbonDimension != null) {
- dimAttr(dimensions.indexOf(carbonDimension)) = attr
- } else {
- val carbonMeasure =
- carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
- if(carbonMeasure != null) {
- msrAttr(measures.indexOf(carbonMeasure)) = attr
- }
- }
- }
-
- attributesRaw = dimAttr.filter(f => f != null) ++ msrAttr.filter(f => f != null)
-
- var queryOrder: Integer = 0
- attributesRaw.foreach { attr =>
- val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if (carbonDimension != null) {
- val dim = new QueryDimension(attr.name)
- dim.setQueryOrder(queryOrder)
- queryOrder = queryOrder + 1
- selectedDims += dim
- } else {
- val carbonMeasure =
- carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
- if (carbonMeasure != null) {
- val m1 = new QueryMeasure(attr.name)
- m1.setQueryOrder(queryOrder)
- queryOrder = queryOrder + 1
- selectedMsrs += m1
- }
- }
- }
-
- // Fill the selected dimensions & measures obtained from
- // attributes to query plan for detailed query
- selectedDims.foreach(plan.addDimension)
- selectedMsrs.foreach(plan.addMeasure)
-
plan.setSortedDimemsions(new ArrayList[QueryDimension])
- plan.setRawDetailQuery(true)
plan.setOutLocationPath(
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS))
plan.setQueryId(System.nanoTime() + "")
@@ -129,32 +82,59 @@ case class CarbonScan(
if (attributesNeedToDecode.size() > 0) {
val attributeOut = new ArrayBuffer[Attribute]() ++ attributesRaw
- attributesNeedToDecode.asScala.map { attr =>
- val dims = plan.getDimensions.asScala.filter(f => f.getColumnName.equals(attr.name))
- val msrs = plan.getMeasures.asScala.filter(f => f.getColumnName.equals(attr.name))
- var order = plan.getDimensions.size() + plan.getMeasures.size()
- if (dims.isEmpty && msrs.isEmpty) {
- val dimension = carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if (dimension != null) {
- val qDim = new QueryDimension(dimension.getColName)
- qDim.setQueryOrder(order)
- plan.addDimension(qDim)
- attributeOut += attr
- order += 1
- } else {
- val measure = carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
- if (measure != null) {
- val qMsr = new QueryMeasure(measure.getColName)
- qMsr.setQueryOrder(order)
- plan.addMeasure(qMsr)
- order += 1
- attributeOut += attr
- }
- }
+ attributesNeedToDecode.asScala.foreach { attr =>
+ if (!attributesRaw.exists(_.name.equalsIgnoreCase(attr.name))) {
+ attributeOut += attr
}
}
attributesRaw = attributeOut
}
+
+ val dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
+ val measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
+ val dimAttr = new Array[Attribute](dimensions.size())
+ val msrAttr = new Array[Attribute](measures.size())
+ attributesRaw.foreach { attr =>
+ val carbonDimension =
+ carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
+ if(carbonDimension != null) {
+ dimAttr(dimensions.indexOf(carbonDimension)) = attr
+ } else {
+ val carbonMeasure =
+ carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
+ if(carbonMeasure != null) {
+ msrAttr(measures.indexOf(carbonMeasure)) = attr
+ }
+ }
+ }
+
+ attributesRaw = dimAttr.filter(f => f != null) ++ msrAttr.filter(f => f != null)
+
+ var queryOrder: Integer = 0
+ attributesRaw.foreach { attr =>
+ val carbonDimension =
+ carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
+ if (carbonDimension != null) {
+ val dim = new QueryDimension(attr.name)
+ dim.setQueryOrder(queryOrder)
+ queryOrder = queryOrder + 1
+ selectedDims += dim
+ } else {
+ val carbonMeasure =
+ carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
+ if (carbonMeasure != null) {
+ val m1 = new QueryMeasure(attr.name)
+ m1.setQueryOrder(queryOrder)
+ queryOrder = queryOrder + 1
+ selectedMsrs += m1
+ }
+ }
+ }
+
+ // Fill the selected dimensions & measures obtained from
+ // attributes to query plan for detailed query
+ selectedDims.foreach(plan.addDimension)
+ selectedMsrs.foreach(plan.addMeasure)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
index 995c109..2d20b73 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.optimizer.Optimizer
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.RunnableCommand
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.types.{IntegerType, StringType}
@@ -82,6 +83,9 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
*/
def transformCarbonPlan(plan: LogicalPlan,
relations: Seq[CarbonDecoderRelation]): LogicalPlan = {
+ if (plan.isInstanceOf[RunnableCommand]) {
+ return plan
+ }
var decoder = false
val aliasMap = CarbonAliasDecoderRelation()
// collect alias information before hand.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
index fe56a58..95f36bb 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
@@ -40,7 +40,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- sql("create table Carbon_automation_test (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADParti
tionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Latest_MONTH,Latest_DAY,deviceInformationId')");
+ sql("create table if not exists Carbon_automation_test (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active
_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Latest_MONTH,Latest_DAY,deviceInformationId')");
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
sql("LOAD DATA LOCAL INPATH '"+currentDirectory+"/src/test/resources/100_olap.csv' INTO table Carbon_automation_test options('DELIMITER'= ',' ,'QUOTECHAR'= '\"', 'FILEHEADER'= 'imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Late
st_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')");
@@ -48,7 +48,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("LOAD DATA LOCAL INPATH '"+currentDirectory+"/src/test/resources/100_olap.csv' INTO table Carbon_automation_hive ");
//hive table
- sql("create table Carbon_automation_test_hive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string,contractNumber int, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion stri
ng, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointId int,gamePointDescription string)row format delimited fields terminated by ','");
+ sql("create table if not exists Carbon_automation_test_hive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string,contractNumber int, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operato
rsVersion string, Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointId int,gamePointDescription string)row format delimited fields terminated by ','");
sql("LOAD DATA LOCAL INPATH '"+currentDirectory+"/src/test/resources/100_olap.csv' INTO table Carbon_automation_test_hive");
}
@@ -56,6 +56,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
override def afterAll {
sql("drop table Carbon_automation_test")
sql("drop table Carbon_automation_hive")
+ sql("drop table Carbon_automation_test_hive")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
@@ -68,7 +69,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
Seq(Row("1", 132), Row("2", 110), Row("3", 176), Row("4", 132), Row("5", 132), Row("6", 209), Row("7", 198)))
}
-
+
test("select channelsId, Latest_DAY from Carbon_automation_test where count(channelsId) = 1") {
try {
sql("select channelsId, Latest_DAY from Carbon_automation_test where count(channelsId) = 1").collect
@@ -77,7 +78,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
case ce: Exception => ce.getMessage
}
}
-
+
//Test-24
test("select channelsId, sum(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total") {
checkAnswer(
@@ -109,7 +110,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
Seq(Row("1", 11), Row("2", 11), Row("3", 11), Row("4", 11), Row("5", 11), Row("6", 11), Row("7", 11)))
}
-
+
//Test-29
test("select channelsId, avg(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total") {
checkAnswer(
@@ -134,8 +135,8 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
Seq(Row("1", 12), Row("2", 10), Row("3", 16), Row("4", 12), Row("5", 12), Row("6", 19), Row("7", 18)))
}
-
-
+
+
//Test-33
test("select channelsId, count(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by channelsId") {
checkAnswer(
@@ -159,8 +160,8 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
Seq(Row("1", 11), Row("2", 11), Row("3", 11), Row("4", 11), Row("5", 11), Row("6", 11), Row("7", 11)))
}
-
-
+
+
//Test-37
test("select channelsId, min(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total") {
checkAnswer(
@@ -184,8 +185,8 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
Seq(Row("1", 11), Row("2", 11), Row("3", 11), Row("4", 11), Row("5", 11), Row("6", 11), Row("7", 11)))
}
-
-
+
+
//Test-41
test("select channelsId, max(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total")({
@@ -218,15 +219,15 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select Latest_YEAR ,sum(distinct Latest_YEAR)+10 from Carbon_automation_test group by Latest_YEAR"),
Seq(Row(2015, 2025)))
})
-
+
//Test-47
test("select sum(gamepointid) +10 as a ,series from Carbon_automation_test group by series")({
checkAnswer(
sql("select sum(gamepointid) +10 as a ,series from Carbon_automation_test group by series"),
sql("select sum(gamepointid) +10 as a ,series from Carbon_automation_test_hive group by series"))
- })
-
+ })
+
//Test-50
test("select sum(gamepointid) +10.36 as a ,series from Carbon_automation_test group by series")({
@@ -234,7 +235,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select sum(gamepointid) +10.36 as a ,series from Carbon_automation_test group by series"),
sql("select sum(gamepointid) +10.36 as a ,series from Carbon_automation_test_hive group by series"))
})
-
+
//TC_055
test("select count(deviceinformationid)+10.32 as a ,series from Carbon_automation_test group by series")({
checkAnswer(
@@ -348,7 +349,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select sum( DISTINCT Latest_MONTH) a from Carbon_automation_test"),
Seq(Row(7)))
})
-
+
//TC_088
test("select avg (if(deviceInformationId>100,NULL,deviceInformationId)) a from Carbon_automation_test")({
checkAnswer(
@@ -382,7 +383,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select min(Latest_MONTH) a from Carbon_automation_test"),
Seq(Row(7)))
})
-
+
//TC_095
test("select min (if(deviceInformationId>100,NULL,deviceInformationId)) a from Carbon_automation_test")({
checkAnswer(
@@ -416,7 +417,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select max(Latest_MONTH) a from Carbon_automation_test"),
Seq(Row(7)))
})
-
+
//TC_102
test("select max (if(deviceInformationId>100,NULL,deviceInformationId)) a from Carbon_automation_test")({
checkAnswer(
@@ -503,7 +504,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select series,count(imei) a from Carbon_automation_test group by series order by series"),
Seq(Row("0Series", 15), Row("1Series", 3), Row("2Series", 9), Row("3Series", 8), Row("4Series", 8), Row("5Series", 17), Row("6Series", 9), Row("7Series", 11), Row("8Series", 11), Row("9Series", 8)))
})
-
+
//TC_138
test("select series,ActiveProvince,count(imei) a from Carbon_automation_test group by ActiveProvince,series order by series,ActiveProvince")({
checkAnswer(
@@ -517,14 +518,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select count(distinct deviceColor) a,deliveryProvince from Carbon_automation_test group by deliveryProvince"),
Seq(Row(10, "Hunan Province"), Row(10, "Guangdong Province"), Row(10, "Hubei Province")))
})
-
+
//TC_141
test("select series,sum(gamePointId) a from Carbon_automation_test group by series order by series")({
checkAnswer(
sql("select series,sum(gamePointId) a from Carbon_automation_test group by series order by series"),
sql("select series,sum(gamePointId) a from Carbon_automation_test_hive group by series order by series"))
})
-
+
//TC_162
test("select imei,series from Carbon_automation_test where Carbon_automation_test.series IN ('1Series','7Series')")({
checkAnswer(
@@ -538,7 +539,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select imei,series from Carbon_automation_test where Carbon_automation_test.series NOT IN ('1Series','7Series')"),
Seq(Row("1AA100", "5Series"), Row("1AA1000", "5Series"), Row("1AA100000", "9Series"), Row("1AA100001", "0Series"), Row("1AA100002", "0Series"), Row("1AA100003", "5Series"), Row("1AA100004", "4Series"), Row("1AA100006", "6Series"), Row("1AA100007", "9Series"), Row("1AA100008", "8Series"), Row("1AA100009", "0Series"), Row("1AA10001", "2Series"), Row("1AA100010", "3Series"), Row("1AA100011", "0Series"), Row("1AA100012", "4Series"), Row("1AA100014", "5Series"), Row("1AA100015", "4Series"), Row("1AA100016", "3Series"), Row("1AA100017", "9Series"), Row("1AA100018", "8Series"), Row("1AA100019", "5Series"), Row("1AA10002", "0Series"), Row("1AA100020", "5Series"), Row("1AA100021", "0Series"), Row("1AA100022", "5Series"), Row("1AA100023", "5Series"), Row("1AA100024", "6Series"), Row("1AA100025", "0Series"), Row("1AA100027", "0Series"), Row("1AA100028", "5Series"), Row("1AA100029", "2Series"), Row("1AA100033", "8Series"), Row("1AA100034", "2Series"), Row("1AA100035", "5Series"), Row("1AA
100036", "5Series"), Row("1AA100038", "6Series"), Row("1AA100039", "8Series"), Row("1AA10004", "5Series"), Row("1AA100040", "8Series"), Row("1AA100041", "5Series"), Row("1AA100042", "3Series"), Row("1AA100043", "9Series"), Row("1AA100044", "8Series"), Row("1AA100045", "2Series"), Row("1AA100046", "3Series"), Row("1AA100047", "9Series"), Row("1AA100048", "3Series"), Row("1AA100049", "0Series"), Row("1AA10005", "8Series"), Row("1AA100050", "2Series"), Row("1AA100051", "2Series"), Row("1AA100052", "6Series"), Row("1AA100053", "2Series"), Row("1AA100056", "6Series"), Row("1AA100057", "9Series"), Row("1AA100058", "5Series"), Row("1AA100059", "4Series"), Row("1AA10006", "3Series"), Row("1AA100060", "8Series"), Row("1AA100061", "6Series"), Row("1AA100062", "9Series"), Row("1AA100063", "2Series"), Row("1AA100064", "6Series"), Row("1AA100065", "0Series"), Row("1AA100066", "6Series"), Row("1AA100067", "4Series"), Row("1AA100068", "8Series"), Row("1AA100069", "8Series"), Row("1AA10007", "8Seri
es"), Row("1AA100070", "0Series"), Row("1AA100071", "0Series"), Row("1AA100072", "4Series"), Row("1AA100073", "4Series"), Row("1AA100074", "6Series"), Row("1AA100075", "3Series"), Row("1AA100076", "0Series"), Row("1AA100077", "3Series"), Row("1AA100078", "2Series"), Row("1AA100079", "4Series"), Row("1AA10008", "5Series"), Row("1AA100080", "9Series"), Row("1AA100081", "5Series"), Row("1AA100082", "5Series"), Row("1AA100083", "0Series"), Row("1AA100084", "0Series")))
})
-
+
//TC_166
test("select Upper(series) a from Carbon_automation_test")({
checkAnswer(
@@ -559,7 +560,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select imei,series from Carbon_automation_test limit 10"),
Seq(Row("1AA1", "7Series"), Row("1AA10", "7Series"), Row("1AA100", "5Series"), Row("1AA1000", "5Series"), Row("1AA10000", "7Series"), Row("1AA100000", "9Series"), Row("1AA1000000", "7Series"), Row("1AA100001", "0Series"), Row("1AA100002", "0Series"), Row("1AA100003", "5Series")))
})
-
+
//TC_171
test("select Lower(series) a from Carbon_automation_test")({
checkAnswer(
@@ -580,7 +581,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select distinct Latest_DAY from Carbon_automation_test"),
Seq(Row(1)))
})
-
+
//TC_175
test("select distinct channelsId from Carbon_automation_test")({
checkAnswer(
@@ -671,14 +672,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select series,ActiveProvince,sum(gamePointId) a from Carbon_automation_test group by series,ActiveProvince order by series desc,ActiveProvince asc"),
sql("select series,ActiveProvince,sum(gamePointId) a from Carbon_automation_test_hive group by series,ActiveProvince order by series desc,ActiveProvince asc"))
})
-
+
//TC_208
test("select Latest_DAY as a from Carbon_automation_test where Latest_DAY<=>Latest_areaId")({
checkAnswer(
sql("select Latest_DAY as a from Carbon_automation_test where Latest_DAY<=>Latest_areaId"),
Seq(Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1), Row(1)))
})
-
+
//TC_210
test("select Latest_DAY from Carbon_automation_test where Latest_DAY<>Latest_areaId")({
checkAnswer(
@@ -720,7 +721,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select imei, Latest_DAY from Carbon_automation_test where Latest_DAY NOT BETWEEN Latest_areaId AND Latest_HOUR"),
Seq(Row("1AA1", 1), Row("1AA10", 1), Row("1AA100", 1), Row("1AA1000", 1), Row("1AA10000", 1), Row("1AA100000", 1), Row("1AA100001", 1), Row("1AA100002", 1), Row("1AA100003", 1), Row("1AA100004", 1), Row("1AA100006", 1), Row("1AA100007", 1), Row("1AA100008", 1), Row("1AA100009", 1), Row("1AA10001", 1), Row("1AA100010", 1), Row("1AA100011", 1), Row("1AA100012", 1), Row("1AA100013", 1), Row("1AA100014", 1), Row("1AA100015", 1), Row("1AA100016", 1), Row("1AA100017", 1), Row("1AA100018", 1), Row("1AA100019", 1), Row("1AA10002", 1), Row("1AA100020", 1), Row("1AA100021", 1), Row("1AA100022", 1), Row("1AA100023", 1), Row("1AA100024", 1), Row("1AA100029", 1), Row("1AA10003", 1), Row("1AA100030", 1), Row("1AA100031", 1), Row("1AA100032", 1), Row("1AA100033", 1), Row("1AA100035", 1), Row("1AA100036", 1), Row("1AA100037", 1), Row("1AA100038", 1), Row("1AA10004", 1), Row("1AA100040", 1), Row("1AA100041", 1), Row("1AA100042", 1), Row("1AA100043", 1), Row("1AA100044", 1), Row("1AA100045", 1)
, Row("1AA100046", 1), Row("1AA100047", 1), Row("1AA100048", 1), Row("1AA100049", 1), Row("1AA10005", 1), Row("1AA100051", 1), Row("1AA100053", 1), Row("1AA100054", 1), Row("1AA100055", 1), Row("1AA100056", 1), Row("1AA100057", 1), Row("1AA100058", 1), Row("1AA100059", 1), Row("1AA10006", 1), Row("1AA100060", 1), Row("1AA100062", 1), Row("1AA100063", 1), Row("1AA100064", 1), Row("1AA100065", 1), Row("1AA100066", 1), Row("1AA100067", 1), Row("1AA100068", 1), Row("1AA100069", 1), Row("1AA10007", 1), Row("1AA100070", 1), Row("1AA100071", 1), Row("1AA100072", 1), Row("1AA100073", 1), Row("1AA100075", 1), Row("1AA100076", 1), Row("1AA100077", 1), Row("1AA100078", 1), Row("1AA100079", 1), Row("1AA10008", 1), Row("1AA100080", 1), Row("1AA100081", 1), Row("1AA100082", 1), Row("1AA100083", 1), Row("1AA100084", 1)))
})
-
+
//TC_219
test("select imei, Latest_DAY from Carbon_automation_test where Latest_DAY IS NOT NULL")({
checkAnswer(
@@ -734,14 +735,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select imei, Latest_DAY from Carbon_automation_test where imei IS NOT NULL"),
Seq(Row("1AA1", 1), Row("1AA10", 1), Row("1AA100", 1), Row("1AA1000", 1), Row("1AA10000", 1), Row("1AA100000", 1), Row("1AA1000000", 1), Row("1AA100001", 1), Row("1AA100002", 1), Row("1AA100003", 1), Row("1AA100004", 1), Row("1AA100005", 1), Row("1AA100006", 1), Row("1AA100007", 1), Row("1AA100008", 1), Row("1AA100009", 1), Row("1AA10001", 1), Row("1AA100010", 1), Row("1AA100011", 1), Row("1AA100012", 1), Row("1AA100013", 1), Row("1AA100014", 1), Row("1AA100015", 1), Row("1AA100016", 1), Row("1AA100017", 1), Row("1AA100018", 1), Row("1AA100019", 1), Row("1AA10002", 1), Row("1AA100020", 1), Row("1AA100021", 1), Row("1AA100022", 1), Row("1AA100023", 1), Row("1AA100024", 1), Row("1AA100025", 1), Row("1AA100026", 1), Row("1AA100027", 1), Row("1AA100028", 1), Row("1AA100029", 1), Row("1AA10003", 1), Row("1AA100030", 1), Row("1AA100031", 1), Row("1AA100032", 1), Row("1AA100033", 1), Row("1AA100034", 1), Row("1AA100035", 1), Row("1AA100036", 1), Row("1AA100037", 1), Row("1AA100038",
1), Row("1AA100039", 1), Row("1AA10004", 1), Row("1AA100040", 1), Row("1AA100041", 1), Row("1AA100042", 1), Row("1AA100043", 1), Row("1AA100044", 1), Row("1AA100045", 1), Row("1AA100046", 1), Row("1AA100047", 1), Row("1AA100048", 1), Row("1AA100049", 1), Row("1AA10005", 1), Row("1AA100050", 1), Row("1AA100051", 1), Row("1AA100052", 1), Row("1AA100053", 1), Row("1AA100054", 1), Row("1AA100055", 1), Row("1AA100056", 1), Row("1AA100057", 1), Row("1AA100058", 1), Row("1AA100059", 1), Row("1AA10006", 1), Row("1AA100060", 1), Row("1AA100061", 1), Row("1AA100062", 1), Row("1AA100063", 1), Row("1AA100064", 1), Row("1AA100065", 1), Row("1AA100066", 1), Row("1AA100067", 1), Row("1AA100068", 1), Row("1AA100069", 1), Row("1AA10007", 1), Row("1AA100070", 1), Row("1AA100071", 1), Row("1AA100072", 1), Row("1AA100073", 1), Row("1AA100074", 1), Row("1AA100075", 1), Row("1AA100076", 1), Row("1AA100077", 1), Row("1AA100078", 1), Row("1AA100079", 1), Row("1AA10008", 1), Row("1AA100080", 1), Row("1AA100
081", 1), Row("1AA100082", 1), Row("1AA100083", 1), Row("1AA100084", 1)))
})
-
+
//TC_223
test("select * from (select if( Latest_areaId=7,7,NULL) as babu,Latest_MONTH from Carbon_automation_test) qq where babu LIKE Latest_MONTH")({
checkAnswer(
sql("select * from (select if( Latest_areaId=7,7,NULL) as babu,Latest_MONTH from Carbon_automation_test) qq where babu LIKE Latest_MONTH"),
Seq(Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7), Row(7, 7)))
})
-
+
//TC_263
test("SELECT AMSize, ActiveAreaId, SUM(gamePointId) AS Sum_gamePointId FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY AMSize, ActiveAreaId ORDER BY AMSize ASC, ActiveAreaId ASC")({
checkAnswer(
@@ -755,7 +756,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("SELECT AMSize, ActiveAreaId, SUM(gamePointId) AS Sum_gamePointId FROM (select * from Carbon_automation_test) SUB_QRY WHERE NOT(AMSize = \"\") GROUP BY AMSize, ActiveAreaId ORDER BY AMSize ASC, ActiveAreaId ASC"),
sql("SELECT AMSize, ActiveAreaId, SUM(gamePointId) AS Sum_gamePointId FROM (select * from Carbon_automation_test_hive) SUB_QRY WHERE NOT(AMSize = \"\") GROUP BY AMSize, ActiveAreaId ORDER BY AMSize ASC, ActiveAreaId ASC"))
})
-
+
//TC_274
test("SELECT ActiveCountry, ActiveDistrict, Activecity, SUM(gamepointid) AS Sum_gamepointid FROM Carbon_automation_test group by ActiveCountry,ActiveDistrict,Activecity")({
checkAnswer(
@@ -783,14 +784,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("SELECT ActiveCountry, COUNT(DISTINCT imei) AS DistinctCount_imei FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY ActiveCountry ORDER BY ActiveCountry ASC"),
Seq(Row("Chinese", 99)))
})
-
+
//TC_282
test("SELECT ActiveCountry, ActiveDistrict, Activecity, SUM(gamepointid) AS Sum_gamepointid FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY ActiveCountry, ActiveDistrict, Activecity ORDER BY ActiveCountry ASC, ActiveDistrict ASC, Activecity ASC")({
checkAnswer(
sql("SELECT ActiveCountry, ActiveDistrict, Activecity, SUM(gamepointid) AS Sum_gamepointid FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY ActiveCountry, ActiveDistrict, Activecity ORDER BY ActiveCountry ASC, ActiveDistrict ASC, Activecity ASC"),
sql("SELECT ActiveCountry, ActiveDistrict, Activecity, SUM(gamepointid) AS Sum_gamepointid FROM (select * from Carbon_automation_test_hive) SUB_QRY GROUP BY ActiveCountry, ActiveDistrict, Activecity ORDER BY ActiveCountry ASC, ActiveDistrict ASC, Activecity ASC"))
})
-
+
//TC_317
test("select channelsId from Carbon_automation_test order by channelsId")({
checkAnswer(
@@ -818,7 +819,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("SELECT ActiveCountry, ActiveDistrict, Activecity, SUM(gamePointId) AS Sum_gamePointId FROM (select * from Carbon_automation_test) SUB_QRY WHERE imei = \"1AA100000\" GROUP BY ActiveCountry, ActiveDistrict, Activecity ORDER BY ActiveCountry ASC, ActiveDistrict ASC, Activecity ASC"),
Seq(Row("Chinese", "yichang", "yichang", 136.0)))
})
-
+
//TC_384
test("SELECT series, SUM(gamePointId) AS Sum_gamePointId FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY series ORDER BY series ASC")({
checkAnswer(
@@ -846,7 +847,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("SELECT imei, channelsId, COUNT(deliveryTime) AS Count_deliveryTime FROM (select * from Carbon_automation_test) SUB_QRY GROUP BY imei, channelsId ORDER BY imei ASC, channelsId ASC"),
Seq(Row("1AA1", "4", 1), Row("1AA10", "4", 1), Row("1AA100", "6", 1), Row("1AA1000", "3", 1), Row("1AA10000", "1", 1), Row("1AA100000", "6", 1), Row("1AA1000000", "6", 1), Row("1AA100001", "7", 1), Row("1AA100002", "3", 1), Row("1AA100003", "3", 1), Row("1AA100004", "2", 1), Row("1AA100005", "1", 1), Row("1AA100006", "2", 1), Row("1AA100007", "3", 1), Row("1AA100008", "1", 1), Row("1AA100009", "3", 1), Row("1AA10001", "5", 1), Row("1AA100010", "6", 1), Row("1AA100011", "1", 1), Row("1AA100012", "2", 1), Row("1AA100013", "6", 1), Row("1AA100014", "3", 1), Row("1AA100015", "1", 1), Row("1AA100016", "3", 1), Row("1AA100017", "3", 1), Row("1AA100018", "4", 1), Row("1AA100019", "3", 1), Row("1AA10002", "5", 1), Row("1AA100020", "7", 1), Row("1AA100021", "6", 1), Row("1AA100022", "5", 1), Row("1AA100023", "3", 1), Row("1AA100024", "2", 1), Row("1AA100025", "1", 1), Row("1AA100026", "7", 1), Row("1AA100027", "4", 1), Row("1AA100028", "5", 1), Row("1AA100029", "5", 1), Row("1AA10003",
"4", 1), Row("1AA100030", "3", 1), Row("1AA100031", "1", 1), Row("1AA100032", "7", 1), Row("1AA100033", "6", 1), Row("1AA100034", "6", 1), Row("1AA100035", "7", 1), Row("1AA100036", "5", 1), Row("1AA100037", "6", 1), Row("1AA100038", "3", 1), Row("1AA100039", "1", 1), Row("1AA10004", "4", 1), Row("1AA100040", "7", 1), Row("1AA100041", "1", 1), Row("1AA100042", "5", 1), Row("1AA100043", "6", 1), Row("1AA100044", "2", 1), Row("1AA100045", "6", 1), Row("1AA100046", "4", 1), Row("1AA100047", "1", 1), Row("1AA100048", "1", 1), Row("1AA100049", "6", 1), Row("1AA10005", "2", 1), Row("1AA100050", "1", 1), Row("1AA100051", "7", 1), Row("1AA100052", "7", 1), Row("1AA100053", "3", 1), Row("1AA100054", "2", 1), Row("1AA100055", "7", 1), Row("1AA100056", "5", 1), Row("1AA100057", "6", 1), Row("1AA100058", "4", 1), Row("1AA100059", "7", 1), Row("1AA10006", "5", 1), Row("1AA100060", "4", 1), Row("1AA100061", "6", 1), Row("1AA100062", "6", 1), Row("1AA100063", "3", 1), Row("1AA100064", "7", 1), Ro
w("1AA100065", "7", 1), Row("1AA100066", "4", 1), Row("1AA100067", "7", 1), Row("1AA100068", "7", 1), Row("1AA100069", "5", 1), Row("1AA10007", "3", 1), Row("1AA100070", "3", 1), Row("1AA100071", "5", 1), Row("1AA100072", "7", 1), Row("1AA100073", "2", 1), Row("1AA100074", "7", 1), Row("1AA100075", "6", 1), Row("1AA100076", "7", 1), Row("1AA100077", "6", 1), Row("1AA100078", "5", 1), Row("1AA100079", "6", 1), Row("1AA10008", "4", 1), Row("1AA100080", "6", 1), Row("1AA100081", "2", 1), Row("1AA100082", "7", 1), Row("1AA100083", "2", 1), Row("1AA100084", "4", 1)))
})
-
+
//TC_408
test("select imei,series from Carbon_automation_test where series='7Series' order by imei limit 10")({
checkAnswer(
@@ -874,7 +875,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select avg(channelsName) from Carbon_automation_test where modelId is null"),
Seq(Row(null)))
})
-
+
//TC_424
test("SELECT count(DISTINCT gamePointId) FROM Carbon_automation_test where imei is null")({
checkAnswer(
@@ -937,14 +938,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select channelsName from Carbon_automation_test where latest_day is NOT null"),
Seq(Row("guomei"), Row("guomei"), Row("yidong"), Row("shuling"), Row("taobao"), Row("yidong"), Row("yidong"), Row("liantong"), Row("shuling"), Row("shuling"), Row("jingdong"), Row("taobao"), Row("jingdong"), Row("shuling"), Row("taobao"), Row("shuling"), Row("shishang"), Row("yidong"), Row("taobao"), Row("jingdong"), Row("yidong"), Row("shuling"), Row("taobao"), Row("shuling"), Row("shuling"), Row("guomei"), Row("shuling"), Row("shishang"), Row("liantong"), Row("yidong"), Row("shishang"), Row("shuling"), Row("jingdong"), Row("taobao"), Row("liantong"), Row("guomei"), Row("shishang"), Row("shishang"), Row("guomei"), Row("shuling"), Row("taobao"), Row("liantong"), Row("yidong"), Row("yidong"), Row("liantong"), Row("shishang"), Row("yidong"), Row("shuling"), Row("taobao"), Row("guomei"), Row("liantong"), Row("taobao"), Row("shishang"), Row("yidong"), Row("jingdong"), Row("yidong"), Row("guomei"), Row("taobao"), Row("taobao"), Row("yidong"), Row("jingdong"), Row("taobao"), Row("li
antong"), Row("liantong"), Row("shuling"), Row("jingdong"), Row("liantong"), Row("shishang"), Row("yidong"), Row("guomei"), Row("liantong"), Row("shishang"), Row("guomei"), Row("yidong"), Row("yidong"), Row("shuling"), Row("liantong"), Row("liantong"), Row("guomei"), Row("liantong"), Row("liantong"), Row("shishang"), Row("shuling"), Row("shuling"), Row("shishang"), Row("liantong"), Row("jingdong"), Row("liantong"), Row("yidong"), Row("liantong"), Row("yidong"), Row("shishang"), Row("yidong"), Row("guomei"), Row("yidong"), Row("jingdong"), Row("liantong"), Row("jingdong"), Row("guomei")))
})
-
+
//TC_439
test("SELECT min(AMSize) FROM Carbon_automation_test where imei is NOT null")({
checkAnswer(
sql("SELECT min(AMSize) FROM Carbon_automation_test where imei is NOT null"),
Seq(Row("0RAM size")))
})
-
+
//TC_448
test("select var_samp(Latest_YEAR) from Carbon_automation_test")({
checkAnswer(
@@ -958,7 +959,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select var_samp(AMSize) from Carbon_automation_test"),
Seq(Row(null)))
})
-
+
//TC_451
test("select stddev_pop(bomcode)from Carbon_automation_test")({
checkAnswer(
@@ -979,7 +980,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select stddev_pop(AMSIZE)from Carbon_automation_test"),
Seq(Row(null)))
})
-
+
//TC_457
test("select stddev_samp(deviceInformationId)from Carbon_automation_test1")({
checkAnswer(
@@ -1034,8 +1035,8 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("Select percentile(1,1.0) from Carbon_automation_test"),
Seq(Row(1.0)))
- })
-
+ })
+
//TC_481
test("select percentile_approx(1, 0.5 ,5000) from Carbon_automation_test")({
checkAnswer(
@@ -1056,7 +1057,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
sql("select percentile_approx(1, 0.5 ,700) from Carbon_automation_test"),
Seq(Row(1.0)))
})
-
+
//TC_484
test("select percentile_approx(1, 0.5 ,500) from Carbon_automation_test")({
checkAnswer(
@@ -1098,7 +1099,7 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("select c1,count(c1) from (select c1 as c1,c2 as c2 from carbonunion union all select c2 as c1,c1 as c2 from carbonunion)t where c1='200' group by c1"),
sql("select c1,count(c1) from (select c1 as c1,c2 as c2 from sparkunion union all select c2 as c1,c1 as c2 from sparkunion)t where c1='200' group by c1"))
+ sql("drop table if exists carbonunion")
})
- sql("drop table if exists carbonunion")
}
\ No newline at end of file
[2/3] incubator-carbondata git commit: Refactored code to improve
performance by using removing unnecessary conversion.
Posted by ch...@apache.org.
Refactored code to improve performance by using removing unnecessary conversion.
Refactored code
Fixed review comments
support create using select as for other formats
Fixed issues after merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9be7451e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9be7451e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9be7451e
Branch: refs/heads/master
Commit: 9be7451ec7bb5561b1f142ebdd40d6048713760e
Parents: 645eafa
Author: ravipesala <ra...@gmail.com>
Authored: Thu Jul 21 20:50:33 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Aug 2 09:09:45 2016 +0800
----------------------------------------------------------------------
.../chunk/DimensionColumnDataChunk.java | 10 +
.../impl/ColumnGroupDimensionDataChunk.java | 22 +-
.../impl/FixedLengthDimensionDataChunk.java | 22 +-
.../impl/VariableLengthDimensionDataChunk.java | 18 +-
.../core/keygenerator/KeyGenerator.java | 9 +
.../core/keygenerator/mdkey/Bits.java | 4 +-
.../mdkey/MultiDimKeyVarLengthGenerator.java | 11 +-
.../org/carbondata/core/util/CarbonUtil.java | 42 +++-
.../scan/collector/ScannedResultCollector.java | 11 +-
.../impl/AbstractScannedResultCollector.java | 157 ++++++++++++++
.../impl/DictionaryBasedResultCollector.java | 130 +++++++++++
.../impl/ListBasedResultCollector.java | 216 -------------------
.../collector/impl/RawBasedResultCollector.java | 67 ++++++
.../scan/complextypes/PrimitiveQueryType.java | 21 +-
.../scan/executor/QueryExecutorFactory.java | 7 +-
.../executor/impl/AbstractQueryExecutor.java | 37 +---
.../scan/executor/impl/DetailQueryExecutor.java | 6 +-
.../impl/DetailRawRecordQueryExecutor.java | 24 ---
.../scan/executor/infos/BlockExecutionInfo.java | 43 ++--
.../scan/executor/infos/KeyStructureInfo.java | 12 ++
.../scan/executor/util/QueryUtil.java | 5 +
.../org/carbondata/scan/model/QueryModel.java | 14 --
.../processor/AbstractDataBlockIterator.java | 18 +-
.../processor/impl/DataBlockIteratorImpl.java | 22 +-
.../scan/result/AbstractScannedResult.java | 53 +++++
.../carbondata/scan/result/BatchRawResult.java | 43 ----
.../org/carbondata/scan/result/BatchResult.java | 31 ++-
.../scan/result/ListBasedResultWrapper.java | 57 -----
.../result/impl/FilterQueryScannedResult.java | 19 ++
.../scan/result/impl/ListBasedResult.java | 153 -------------
.../impl/NonFilterQueryScannedResult.java | 19 ++
.../iterator/DetailQueryResultIterator.java | 15 +-
.../scan/result/iterator/RawResultIterator.java | 10 +-
.../preparator/QueryResultPreparator.java | 10 -
.../impl/AbstractQueryResultPreparator.java | 87 --------
.../impl/DetailQueryResultPreparatorImpl.java | 148 -------------
.../impl/RawQueryResultPreparatorImpl.java | 136 ------------
.../hadoop/util/CarbonInputFormatUtil.java | 1 -
.../spark/merger/CarbonCompactionExecutor.java | 7 +-
.../org/apache/spark/sql/CarbonOperators.scala | 120 +++++------
.../spark/sql/optimizer/CarbonOptimizer.scala | 4 +
.../AllDataTypesTestCaseAggregate.scala | 87 ++++----
42 files changed, 789 insertions(+), 1139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
index 75c29d2..6a30bc7 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
@@ -36,6 +36,16 @@ public interface DimensionColumnDataChunk<T> {
int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
/**
+ * It uses to convert column data to dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo);
+
+ /**
* Below method to get the data based in row id
*
* @param row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
index e85e17d..d8536a1 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -55,7 +55,7 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
* @param data data to filed
* @param offset offset from which data need to be filed
* @param rowId row id of the chunk
- * @param keyStructureInfo define the structure of the key
+ * @param restructuringInfo define the structure of the key
* @return how many bytes was copied
*/
@Override public int fillChunkData(byte[] data, int offset, int rowId,
@@ -67,10 +67,22 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
}
/**
- * Below method to get the data based in row id
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo info) {
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ long[] keyArray = info.getKeyGenerator().getKeyArray(dataChunk, start);
+ int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+ for (int i = 0; i < ordinal.length; i++) {
+ row[columnIndex++] = (int)keyArray[ordinal[i]];
+ }
+ return columnIndex;
+ }
+
+ /**
+ * Below method masks key
*
- * @param row id row id of the data
- * @return chunk
*/
public byte[] getMaskedKey(byte[] data, int offset, KeyStructureInfo info) {
byte[] maskedKey = new byte[info.getMaskByteRanges().length];
@@ -86,7 +98,7 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
/**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param rowId row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int rowId) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 08ae703..cbe48b2 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -54,7 +54,7 @@ public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<b
*
* @param data data to filed
* @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
+ * @param index row id of the chunk
* @param keyStructureInfo define the structure of the key
* @return how many bytes was copied
*/
@@ -69,9 +69,27 @@ public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<b
}
/**
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ if (chunkAttributes.getInvertedIndexes() != null) {
+ rowId = chunkAttributes.getInvertedIndexesReverse()[rowId];
+ }
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ int dict = 0;
+ for (int i = start; i < start + chunkAttributes.getColumnValueSize(); i++) {
+ dict <<= 8;
+ dict ^= dataChunk[i] & 0xFF;
+ }
+ row[columnIndex] = dict;
+ return columnIndex + 1;
+ }
+
+ /**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param index row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int index) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 51306b5..0aa2406 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -39,8 +39,8 @@ public class VariableLengthDimensionDataChunk implements DimensionColumnDataChun
*
* @param data data to filed
* @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
- * @param keyStructureInfo define the structure of the key
+ * @param index row id of the chunk
+ * @param restructuringInfo define the structure of the key
* @return how many bytes was copied
*/
@Override public int fillChunkData(byte[] data, int offset, int index,
@@ -51,9 +51,21 @@ public class VariableLengthDimensionDataChunk implements DimensionColumnDataChun
}
/**
+ * Converts to column dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ return columnIndex + 1;
+ }
+
+ /**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param index row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int index) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
index 5652d08..f9016f8 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
@@ -58,6 +58,15 @@ public interface KeyGenerator extends Serializable, Comparator<byte[]> {
* It gets array of keys out of single key aka byte array
*
* @param key
+ * @param offset
+ * @return array of keys.
+ */
+ long[] getKeyArray(byte[] key, int offset);
+
+ /**
+ * It gets array of keys out of single key aka byte array
+ *
+ * @param key
* @param maskedByteRanges
* @return array of keys
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
index 2515f0f..2ce64ba 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
@@ -252,7 +252,7 @@ public class Bits implements Serializable {
return getBytesVal(words);
}
- public long[] getKeyArray(byte[] key) {
+ public long[] getKeyArray(byte[] key, int offset) {
int length = 8;
int ls = byteSize;
@@ -269,7 +269,7 @@ public class Bits implements Serializable {
}
for (int j = ls; j < m; j++) {
l <<= 8;
- l ^= key[j] & 0xFF;
+ l ^= key[j + offset] & 0xFF;
}
words[i] = l;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
index 1cdc847..1aa2b81 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
@@ -52,12 +52,17 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
@Override public long[] getKeyArray(byte[] key) {
- return bits.getKeyArray(key);
+ return bits.getKeyArray(key, 0);
+ }
+
+ @Override public long[] getKeyArray(byte[] key, int offset) {
+
+ return bits.getKeyArray(key, offset);
}
@Override public long getKey(byte[] key, int index) {
- return bits.getKeyArray(key)[index];
+ return bits.getKeyArray(key, 0)[index];
}
public int getKeySizeInBytes() {
@@ -68,7 +73,7 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
if (index < 0 || size == 0) {
return null;
}
- long[] keys = bits.getKeyArray(key);
+ long[] keys = bits.getKeyArray(key, 0);
long[] rtn = new long[size];
System.arraycopy(keys, index, rtn, 0, size);
return rtn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
index 9db8f24..eb8a0fc 100644
--- a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
@@ -20,11 +20,21 @@
package org.carbondata.core.util;
-import java.io.*;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.security.PrivilegedExceptionAction;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -55,6 +65,7 @@ import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
import org.carbondata.core.datastorage.store.impl.FileFactory;
import org.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.carbondata.core.metadata.ValueEncoderMeta;
+import org.carbondata.scan.model.QueryDimension;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
@@ -1054,6 +1065,33 @@ public final class CarbonUtil {
}
}
+ public static boolean[] getDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY);
+ }
+ return dictionaryEncodingArray;
+ }
+
+ public static boolean[] getDirectDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY);
+ }
+ return dictionaryEncodingArray;
+ }
+
+ public static boolean[] getComplexDataTypeArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ CarbonUtil.hasComplexDataType(queryDimensions[i].getDimension().getDataType());
+ }
+ return dictionaryEncodingArray;
+ }
+
/**
* Below method will be used to read the data file matadata
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
index f214a9d..424bd86 100644
--- a/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
+++ b/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
@@ -18,8 +18,9 @@
*/
package org.carbondata.scan.collector;
+import java.util.List;
+
import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.Result;
/**
* Interface which will be used to aggregate the scan result
@@ -32,12 +33,6 @@ public interface ScannedResultCollector {
* @param scannedResult scanned result
* @return how many records was aggregated
*/
- int collectData(AbstractScannedResult scannedResult, int batchSize);
+ List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
- /**
- * Below method will be used to get the aggregated result
- *
- * @return
- */
- Result getCollectedResult();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
new file mode 100644
index 0000000..81d7b6a
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.util.DataTypeUtil;
+import org.carbondata.scan.collector.ScannedResultCollector;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.carbondata.scan.executor.util.QueryUtil;
+import org.carbondata.scan.result.AbstractScannedResult;
+import org.carbondata.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
+
+ /**
+ * restructuring info
+ */
+ private KeyStructureInfo restructureInfos;
+
+ /**
+ * table block execution infos
+ */
+ protected BlockExecutionInfo tableBlockExecutionInfos;
+
+ /**
+ * Measure ordinals
+ */
+ protected int[] measuresOrdinal;
+
+ /**
+ * to check whether measure exists in current table block or not this to
+ * handle restructuring scenario
+ */
+ protected boolean[] isMeasureExistsInCurrentBlock;
+
+ /**
+ * default value of the measures in case of restructuring some measure wont
+ * be present in the table so in that default value will be used to
+ * aggregate the data for that measure columns
+ */
+ private Object[] measureDefaultValue;
+
+ /**
+ * measure datatypes.
+ */
+ protected DataType[] measureDatatypes;
+
+ public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ this.tableBlockExecutionInfos = blockExecutionInfos;
+ restructureInfos = blockExecutionInfos.getKeyStructureInfo();
+ measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
+ isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
+ measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
+ this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
+ }
+
+ protected void fillMeasureData(Object[] msrValues, int offset,
+ AbstractScannedResult scannedResult) {
+ for (short i = 0; i < measuresOrdinal.length; i++) {
+ // if measure exists is block then pass measure column
+ // data chunk to the collector
+ if (isMeasureExistsInCurrentBlock[i]) {
+ msrValues[i + offset] = getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
+ scannedResult.getCurrenrRowId(), measureDatatypes[i]);
+ } else {
+ // if not then get the default value and use that value in aggregation
+ msrValues[i + offset] = measureDefaultValue[i];
+ }
+ }
+ }
+
+ private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
+ if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
+ Object msrVal;
+ switch (dataType) {
+ case INT:
+ case LONG:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
+ break;
+ case DECIMAL:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
+ break;
+ default:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
+ }
+ return DataTypeUtil.getMeasureDataBasedOnDataType(msrVal, dataType);
+ }
+ return null;
+ }
+
+ /**
+ * Below method will used to get the result
+ */
+ protected void updateData(List<Object[]> listBasedResult) {
+ if (tableBlockExecutionInfos.isFixedKeyUpdateRequired()) {
+ updateKeyWithLatestBlockKeygenerator(listBasedResult);
+ }
+ }
+
+ /**
+ * Below method will be used to update the fixed length key with the
+ * latest block key generator
+ *
+ * @return updated block
+ */
+ private void updateKeyWithLatestBlockKeygenerator(List<Object[]> listBasedResult) {
+ try {
+ long[] data = null;
+ ByteArrayWrapper key = null;
+ for (int i = 0; i < listBasedResult.size(); i++) {
+ // get the key
+ key = (ByteArrayWrapper)listBasedResult.get(i)[0];
+ // unpack the key with table block key generator
+ data = tableBlockExecutionInfos.getBlockKeyGenerator()
+ .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
+ // packed the key with latest block key generator
+ // and generate the masked key for that key
+ key.setDictionaryKey(QueryUtil
+ .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
+ restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
+ restructureInfos.getMaskByteRanges().length));
+ }
+ } catch (KeyGenException e) {
+ LOGGER.error(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
new file mode 100644
index 0000000..89a7d9d
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.util.DataTypeUtil;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.filter.GenericQueryType;
+import org.carbondata.scan.model.QueryDimension;
+import org.carbondata.scan.model.QueryMeasure;
+import org.carbondata.scan.result.AbstractScannedResult;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DictionaryBasedResultCollector.class.getName());
+
+ public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ boolean isMsrsPresent = measureDatatypes.length > 0;
+ QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ Map<Integer, GenericQueryType> comlexDimensionInfoMap =
+ tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ boolean[] dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
+ boolean[] directDictionaryEncodingArray =
+ CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
+ boolean[] complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
+ int dimSize = queryDimensions.length;
+ boolean isDimensionsExist = dimSize > 0;
+ int[] order = new int[dimSize + queryMeasures.length];
+ for (int i = 0; i < dimSize; i++) {
+ order[i] = queryDimensions[i].getQueryOrder();
+ }
+ for (int i = 0; i < queryMeasures.length; i++) {
+ order[i + dimSize] = queryMeasures[i].getQueryOrder();
+ }
+ // scan the record and add to list
+ int rowCounter = 0;
+ int dictionaryColumnIndex = 0;
+ int noDictionaryColumnIndex = 0;
+ int complexTypeColumnIndex = 0;
+ int[] surrogateResult;
+ String[] noDictionaryKeys;
+ byte[][] complexTypeKeyArray;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[dimSize + queryMeasures.length];
+ if (isDimensionsExist) {
+ surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
+ dictionaryColumnIndex = 0;
+ noDictionaryColumnIndex = 0;
+ complexTypeColumnIndex = 0;
+ for (int i = 0; i < dimSize; i++) {
+ if (!dictionaryEncodingArray[i]) {
+ row[order[i]] = DataTypeUtil
+ .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
+ } else if (directDictionaryEncodingArray[i]) {
+ DirectDictionaryGenerator directDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
+ if (directDictionaryGenerator != null) {
+ row[order[i]] = directDictionaryGenerator.getValueFromSurrogate(
+ surrogateResult[dictionaryColumnIndex++]);
+ }
+ } else if (complexDataTypeArray[i]) {
+ row[order[i]] = comlexDimensionInfoMap
+ .get(queryDimensions[i].getDimension().getOrdinal())
+ .getDataBasedOnDataTypeFromSurrogates(
+ ByteBuffer.wrap(complexTypeKeyArray[complexTypeColumnIndex++]));
+ } else {
+ row[order[i]] = surrogateResult[dictionaryColumnIndex++];
+ }
+ }
+
+ } else {
+ scannedResult.incrementCounter();
+ }
+ if (isMsrsPresent) {
+ Object[] msrValues = new Object[measureDatatypes.length];
+ fillMeasureData(msrValues, 0, scannedResult);
+ for (int i = 0; i < msrValues.length; i++) {
+ row[order[i + dimSize]] = msrValues[i];
+ }
+ }
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ return listBasedResult;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
deleted file mode 100644
index a199cad..0000000
--- a/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.scan.collector.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
-import org.carbondata.core.carbon.metadata.datatype.DataType;
-import org.carbondata.core.keygenerator.KeyGenException;
-import org.carbondata.core.util.DataTypeUtil;
-import org.carbondata.scan.collector.ScannedResultCollector;
-import org.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.carbondata.scan.executor.util.QueryUtil;
-import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.result.impl.ListBasedResult;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * It is not a collector it is just a scanned result holder.
- *
- */
-public class ListBasedResultCollector implements ScannedResultCollector {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ListBasedResultCollector.class.getName());
-
- /**
- * to keep a track of number of row processed to handle limit push down in
- * case of detail query scenario
- */
- private int rowCounter;
-
- /**
- * dimension values list
- */
- private List<ListBasedResultWrapper> listBasedResult;
-
- /**
- * restructuring info
- */
- private KeyStructureInfo restructureInfos;
-
- /**
- * table block execution infos
- */
- private BlockExecutionInfo tableBlockExecutionInfos;
-
- private int[] measuresOrdinal;
-
- /**
- * to check whether measure exists in current table block or not this to
- * handle restructuring scenario
- */
- private boolean[] isMeasureExistsInCurrentBlock;
-
- /**
- * default value of the measures in case of restructuring some measure wont
- * be present in the table so in that default value will be used to
- * aggregate the data for that measure columns
- */
- private Object[] measureDefaultValue;
-
- /**
- * measure datatypes.
- */
- private DataType[] measureDatatypes;
-
- public ListBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- this.tableBlockExecutionInfos = blockExecutionInfos;
- restructureInfos = blockExecutionInfos.getKeyStructureInfo();
- measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
- isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
- measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
- this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
- }
-
- @Override
- /**
- * This method will add a record both key and value to list object
- * it will keep track of how many record is processed, to handle limit scenario
- * @param scanned result
- *
- */
- public int collectData(AbstractScannedResult scannedResult, int batchSize) {
- this.listBasedResult =
- new ArrayList<>(batchSize);
- boolean isMsrsPresent = measureDatatypes.length > 0;
- ByteArrayWrapper wrapper = null;
- // scan the record and add to list
- ListBasedResultWrapper resultWrapper;
- int rowCounter = 0;
- while (scannedResult.hasNext() && rowCounter < batchSize) {
- resultWrapper = new ListBasedResultWrapper();
- if(tableBlockExecutionInfos.isDimensionsExistInQuery()) {
- wrapper = new ByteArrayWrapper();
- wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
- wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
- wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
- resultWrapper.setKey(wrapper);
- } else {
- scannedResult.incrementCounter();
- }
- if(isMsrsPresent) {
- Object[] msrValues = new Object[measureDatatypes.length];
- fillMeasureData(msrValues, scannedResult);
- resultWrapper.setValue(msrValues);
- }
- listBasedResult.add(resultWrapper);
- rowCounter++;
- }
- return rowCounter;
- }
-
- private void fillMeasureData(Object[] msrValues, AbstractScannedResult scannedResult) {
- for (short i = 0; i < measuresOrdinal.length; i++) {
- // if measure exists is block then pass measure column
- // data chunk to the collector
- if (isMeasureExistsInCurrentBlock[i]) {
- msrValues[i] =
- getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
- scannedResult.getCurrenrRowId(),measureDatatypes[i]);
- } else {
- // if not then get the default value and use that value in aggregation
- msrValues[i] = measureDefaultValue[i];
- }
- }
- }
-
- private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
- if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
- Object msrVal;
- switch (dataType) {
- case INT:
- case LONG:
- msrVal = dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
- break;
- case DECIMAL:
- msrVal = dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
- break;
- default:
- msrVal = dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
- }
- return DataTypeUtil.getMeasureDataBasedOnDataType(msrVal, dataType);
- }
- return null;
- }
-
- /**
- * Below method will used to get the result
- */
- @Override public Result getCollectedResult() {
- Result<List<ListBasedResultWrapper>, Object> result = new ListBasedResult();
- if (tableBlockExecutionInfos.isFixedKeyUpdateRequired() && tableBlockExecutionInfos
- .isDimensionsExistInQuery()) {
- updateKeyWithLatestBlockKeygenerator();
- result.addScannedResult(listBasedResult);
- } else {
- result.addScannedResult(listBasedResult);
- }
- return result;
- }
-
-
-
- /**
- * Below method will be used to update the fixed length key with the
- * latest block key generator
- *
- * @return updated block
- */
- private void updateKeyWithLatestBlockKeygenerator() {
- try {
- long[] data = null;
- ByteArrayWrapper key = null;
- for (int i = 0; i < listBasedResult.size(); i++) {
- // get the key
- key = listBasedResult.get(i).getKey();
- // unpack the key with table block key generator
- data = tableBlockExecutionInfos.getBlockKeyGenerator()
- .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
- // packed the key with latest block key generator
- // and generate the masked key for that key
- key.setDictionaryKey(QueryUtil
- .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
- restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
- restructureInfos.getMaskByteRanges().length));
- listBasedResult.get(i).setKey(key);
- }
- } catch (KeyGenException e) {
- LOGGER.error(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
new file mode 100644
index 0000000..5bb0567
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.model.QueryMeasure;
+import org.carbondata.scan.result.AbstractScannedResult;
+import org.carbondata.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class RawBasedResultCollector extends AbstractScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(RawBasedResultCollector.class.getName());
+
+ public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ ByteArrayWrapper wrapper = null;
+ // scan the record and add to list
+ int rowCounter = 0;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[1 + queryMeasures.length];
+ wrapper = new ByteArrayWrapper();
+ wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
+ wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
+ wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
+ row[0] = wrapper;
+ fillMeasureData(row, 1, scannedResult);
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ updateData(listBasedResult);
+ return listBasedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
index 21a1d02..33a22c7 100644
--- a/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
@@ -33,12 +33,7 @@ import org.carbondata.core.util.DataTypeUtil;
import org.carbondata.scan.filter.GenericQueryType;
import org.carbondata.scan.processor.BlocksChunkHolder;
-import org.apache.spark.sql.types.BooleanType;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.IntegerType;
-import org.apache.spark.sql.types.LongType;
-import org.apache.spark.sql.types.TimestampType;
+import org.apache.spark.sql.types.*;
public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
@@ -135,17 +130,17 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
@Override public DataType getSchemaType() {
switch (dataType) {
case INT:
- return new IntegerType();
+ return IntegerType$.MODULE$;
case DOUBLE:
- return new DoubleType();
+ return DoubleType$.MODULE$;
case LONG:
- return new LongType();
+ return LongType$.MODULE$;
case BOOLEAN:
- return new BooleanType();
+ return BooleanType$.MODULE$;
case TIMESTAMP:
- return new TimestampType();
+ return TimestampType$.MODULE$;
default:
- return new IntegerType();
+ return IntegerType$.MODULE$;
}
}
@@ -165,7 +160,7 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
byte[] data = new byte[keySize];
surrogateData.get(data);
Bits bit = new Bits(new int[]{keySize * 8});
- int surrgateValue = (int)bit.getKeyArray(data)[0];
+ int surrgateValue = (int)bit.getKeyArray(data, 0)[0];
Object actualData = null;
if (isDirectDictionary) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java b/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
index 724b8b6..7f7203c 100644
--- a/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
+++ b/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
@@ -19,7 +19,6 @@
package org.carbondata.scan.executor;
import org.carbondata.scan.executor.impl.DetailQueryExecutor;
-import org.carbondata.scan.executor.impl.DetailRawRecordQueryExecutor;
import org.carbondata.scan.model.QueryModel;
/**
@@ -29,10 +28,6 @@ import org.carbondata.scan.model.QueryModel;
public class QueryExecutorFactory {
public static QueryExecutor getQueryExecutor(QueryModel queryModel) {
- if (queryModel.isForcedDetailRawQuery()) {
- return new DetailRawRecordQueryExecutor();
- } else {
- return new DetailQueryExecutor();
- }
+ return new DetailQueryExecutor();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index dd23119..519d4e6 100644
--- a/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -48,7 +48,6 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.AggregatorInfo;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.carbondata.scan.executor.infos.SortInfo;
import org.carbondata.scan.executor.util.QueryUtil;
import org.carbondata.scan.executor.util.RestructureUtil;
import org.carbondata.scan.filter.FilterUtil;
@@ -229,7 +228,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
int[] maksedByte =
QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
- blockExecutionInfo.setDimensionsExistInQuery(updatedQueryDimension.size() > 0);
+ blockExecutionInfo.setQueryDimensions(
+ updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()]));
+ blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures()
+ .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
blockExecutionInfo.setDataBlock(blockIndex);
blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// adding aggregation info for query
@@ -373,37 +375,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
}
/**
- * Below method will be used to get the sort information which will be
- * required during sorting the data on dimension column
- *
- * @param queryModel query model
- * @return Sort infos
- * @throws QueryExecutionException if problem while
- */
- protected SortInfo getSortInfos(QueryModel queryModel) throws QueryExecutionException {
-
- // get the masked by range for order by dimension
- int[][] maskedByteRangeForSorting = QueryUtil
- .getMaskedByteRangeForSorting(queryModel.getSortDimension(),
- queryProperties.keyStructureInfo.getKeyGenerator(),
- queryProperties.keyStructureInfo.getMaskByteRanges());
- // get masked key for sorting
- byte[][] maksedKeyForSorting = QueryUtil.getMaksedKeyForSorting(queryModel.getSortDimension(),
- queryProperties.keyStructureInfo.getKeyGenerator(), maskedByteRangeForSorting,
- queryProperties.keyStructureInfo.getMaskByteRanges());
- // fill sort dimension indexes
- queryProperties.sortDimIndexes = QueryUtil
- .getSortDimensionIndexes(queryModel.getSortDimension(), queryModel.getQueryDimension());
- SortInfo sortInfos = new SortInfo();
- sortInfos.setDimensionMaskKeyForSorting(maksedKeyForSorting);
- sortInfos.setDimensionSortOrder(queryModel.getSortOrder());
- sortInfos.setMaskedByteRangeForSorting(maskedByteRangeForSorting);
- sortInfos.setSortDimensionIndex(queryProperties.sortDimIndexes);
- sortInfos.setSortDimension(queryModel.getSortDimension());
- return sortInfos;
- }
-
- /**
* Below method will be used to get the aggrgator info for the query
*
* @param queryModel query model
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
index edb7675..30eb2d2 100644
--- a/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
@@ -24,9 +24,7 @@ import org.carbondata.common.CarbonIterator;
import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.iterator.ChunkRowIterator;
import org.carbondata.scan.result.iterator.DetailQueryResultIterator;
-import org.carbondata.scan.result.preparator.impl.DetailQueryResultPreparatorImpl;
/**
* Below class will be used to execute the detail query
@@ -38,9 +36,7 @@ public class DetailQueryExecutor extends AbstractQueryExecutor {
@Override public CarbonIterator<Object[]> execute(QueryModel queryModel)
throws QueryExecutionException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new ChunkRowIterator(
- new DetailQueryResultIterator(blockExecutionInfoList, queryModel,
- new DetailQueryResultPreparatorImpl(queryProperties, queryModel)));
+ return new DetailQueryResultIterator(blockExecutionInfoList, queryModel);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
deleted file mode 100644
index 8659e82..0000000
--- a/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.carbondata.scan.executor.impl;
-
-import java.util.List;
-
-import org.carbondata.common.CarbonIterator;
-import org.carbondata.scan.executor.exception.QueryExecutionException;
-import org.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.iterator.DetailQueryResultIterator;
-import org.carbondata.scan.result.preparator.impl.RawQueryResultPreparatorImpl;
-
-/**
- * Executor for raw records, it does not parse to actual data
- */
-public class DetailRawRecordQueryExecutor extends AbstractQueryExecutor<BatchResult> {
-
- @Override public CarbonIterator<BatchResult> execute(QueryModel queryModel)
- throws QueryExecutionException {
- List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new DetailQueryResultIterator(blockExecutionInfoList, queryModel,
- new RawQueryResultPreparatorImpl(queryProperties, queryModel));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
index 7b62681..883357f 100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
@@ -29,6 +29,8 @@ import org.carbondata.core.datastorage.store.impl.FileFactory.FileType;
import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.scan.filter.GenericQueryType;
import org.carbondata.scan.filter.executer.FilterExecuter;
+import org.carbondata.scan.model.QueryDimension;
+import org.carbondata.scan.model.QueryMeasure;
/**
* Below class will have all the properties which needed during query execution
@@ -198,11 +200,6 @@ public class BlockExecutionInfo {
private boolean isRawRecordDetailQuery;
/**
- * whether dimensions exist in query.
- */
- private boolean isDimensionsExistInQuery;
-
- /**
* complexParentIndexToQueryMap
*/
private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
@@ -225,6 +222,16 @@ public class BlockExecutionInfo {
}
/**
+ * list of dimension selected for in query
+ */
+ private QueryDimension[] queryDimensions;
+
+ /**
+ * list of measure selected in query
+ */
+ private QueryMeasure[] queryMeasures;
+
+ /**
* @param blockIndex the tableBlock to set
*/
public void setDataBlock(AbstractIndex blockIndex) {
@@ -336,6 +343,8 @@ public class BlockExecutionInfo {
return maskedByteForBlock;
}
+
+
/**
* @param maskedByteForBlock the maskedByteForBlock to set
*/
@@ -618,14 +627,6 @@ public class BlockExecutionInfo {
isRawRecordDetailQuery = rawRecordDetailQuery;
}
- public boolean isDimensionsExistInQuery() {
- return isDimensionsExistInQuery;
- }
-
- public void setDimensionsExistInQuery(boolean dimensionsExistInQuery) {
- isDimensionsExistInQuery = dimensionsExistInQuery;
- }
-
/**
* @return the complexParentIndexToQueryMap
*/
@@ -661,4 +662,20 @@ public class BlockExecutionInfo {
public void setStatisticsRecorder(QueryStatisticsRecorder statisticsRecorder) {
this.statisticsRecorder = statisticsRecorder;
}
+
+ public QueryDimension[] getQueryDimensions() {
+ return queryDimensions;
+ }
+
+ public void setQueryDimensions(QueryDimension[] queryDimensions) {
+ this.queryDimensions = queryDimensions;
+ }
+
+ public QueryMeasure[] getQueryMeasures() {
+ return queryMeasures;
+ }
+
+ public void setQueryMeasures(QueryMeasure[] queryMeasures) {
+ this.queryMeasures = queryMeasures;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
index e19a3e9..a595f33 100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
@@ -49,6 +49,11 @@ public class KeyStructureInfo {
private byte[] maxKey;
/**
+ * dimension ordinals inside the column group
+ */
+ private int[] mdkeyQueryDimensionOrdinal;
+
+ /**
* @return the keyGenerator
*/
public KeyGenerator getKeyGenerator() {
@@ -104,4 +109,11 @@ public class KeyStructureInfo {
this.maxKey = maxKey;
}
+ public int[] getMdkeyQueryDimensionOrdinal() {
+ return mdkeyQueryDimensionOrdinal;
+ }
+
+ public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) {
+ this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
index 00e0f48..2b26ba0 100644
--- a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
@@ -546,8 +546,11 @@ public class QueryUtil {
// sort the ordinal
List<Integer> ordinal = next.getValue();
List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ //Un sorted
+ List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
for (Integer ord : ordinal) {
mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
+ mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
}
Collections.sort(mdKeyOrdinal);
// get the masked byte range for column group
@@ -560,6 +563,8 @@ public class QueryUtil {
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
restructureInfos.setMaskedBytes(maksedByte);
+ restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
+ .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
rowGroupToItsRSInfo
.put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
restructureInfos);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/model/QueryModel.java b/core/src/main/java/org/carbondata/scan/model/QueryModel.java
index e02c56e..e299e7b 100644
--- a/core/src/main/java/org/carbondata/scan/model/QueryModel.java
+++ b/core/src/main/java/org/carbondata/scan/model/QueryModel.java
@@ -130,12 +130,6 @@ public class QueryModel implements Serializable {
*/
private CarbonTable table;
- /**
- * This is used only whne [forcedDetailRawQuery = true]. By default forcedDetailRawQuery returns
- * dictionary values. But user wants in detail raw bytes the user set this field to true.
- */
- private boolean rawBytesDetailQuery;
-
private QueryStatisticsRecorder statisticsRecorder;
public QueryModel() {
@@ -495,14 +489,6 @@ public class QueryModel implements Serializable {
this.columnToDictionaryMapping = columnToDictionaryMapping;
}
- public boolean isRawBytesDetailQuery() {
- return rawBytesDetailQuery;
- }
-
- public void setRawBytesDetailQuery(boolean rawBytesDetailQuery) {
- this.rawBytesDetailQuery = rawBytesDetailQuery;
- }
-
public int getInMemoryRecordSize() {
return inMemoryRecordSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
index e385bec..cdf4f07 100644
--- a/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
@@ -18,17 +18,19 @@
*/
package org.carbondata.scan.processor;
+import java.util.List;
+
import org.carbondata.common.CarbonIterator;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.datastore.DataRefNode;
import org.carbondata.core.datastorage.store.FileHolder;
import org.carbondata.scan.collector.ScannedResultCollector;
-import org.carbondata.scan.collector.impl.ListBasedResultCollector;
+import org.carbondata.scan.collector.impl.DictionaryBasedResultCollector;
+import org.carbondata.scan.collector.impl.RawBasedResultCollector;
import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.Result;
import org.carbondata.scan.scanner.BlockletScanner;
import org.carbondata.scan.scanner.impl.FilterScanner;
import org.carbondata.scan.scanner.impl.NonFilterScanner;
@@ -37,7 +39,7 @@ import org.carbondata.scan.scanner.impl.NonFilterScanner;
* This abstract class provides a skeletal implementation of the
* Block iterator.
*/
-public abstract class AbstractDataBlockIterator extends CarbonIterator<Result> {
+public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
@@ -88,9 +90,13 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<Result> {
} else {
blockletScanner = new NonFilterScanner(blockExecutionInfo);
}
-
- this.scannerResultAggregator =
- new ListBasedResultCollector(blockExecutionInfo);
+ if (blockExecutionInfo.isRawRecordDetailQuery()) {
+ this.scannerResultAggregator =
+ new RawBasedResultCollector(blockExecutionInfo);
+ } else {
+ this.scannerResultAggregator =
+ new DictionaryBasedResultCollector(blockExecutionInfo);
+ }
this.batchSize = batchSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
index 0c61947..ab9c665 100644
--- a/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
@@ -18,10 +18,11 @@
*/
package org.carbondata.scan.processor.impl;
+import java.util.List;
+
import org.carbondata.core.datastorage.store.FileHolder;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.processor.AbstractDataBlockIterator;
-import org.carbondata.scan.result.Result;
/**
* Below class will be used to process the block for detail query
@@ -33,8 +34,8 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
*
* @param blockExecutionInfo execution information
*/
- public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo,
- FileHolder fileReader, int batchSize) {
+ public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
+ int batchSize) {
super(blockExecutionInfo, fileReader, batchSize);
}
@@ -43,14 +44,15 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
*
* @return Result of @batchSize
*/
- public Result next() {
- this.scannerResultAggregator.collectData(scannedResult, batchSize);
- Result result = this.scannerResultAggregator.getCollectedResult();
- while (result.size() < batchSize && hasNext()) {
- this.scannerResultAggregator.collectData(scannedResult, batchSize-result.size());
- result.merge(this.scannerResultAggregator.getCollectedResult());
+ public List<Object[]> next() {
+ List<Object[]> collectedResult =
+ this.scannerResultAggregator.collectData(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && hasNext()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectData(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
}
- return result;
+ return collectedResult;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java b/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
index 9e7f007..d917c2a 100644
--- a/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
@@ -93,6 +93,8 @@ public abstract class AbstractScannedResult {
*/
private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+ private int totalDimensionsSize;
+
/**
* parent block indexes
*/
@@ -105,6 +107,7 @@ public abstract class AbstractScannedResult {
this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
}
/**
@@ -156,6 +159,25 @@ public abstract class AbstractScannedResult {
}
/**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
* Just increment the counter incase of query only on measures.
*/
public void incrementCounter() {
@@ -193,6 +215,23 @@ public abstract class AbstractScannedResult {
}
/**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected String[] getNoDictionaryKeyStringArray(int rowId) {
+ String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ new String(dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId));
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
* Below method will be used to get the complex type keys array based
* on row id for all the complex type dimension selected in query
*
@@ -324,6 +363,12 @@ public abstract class AbstractScannedResult {
public abstract byte[] getDictionaryKeyArray();
/**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
* Return the dimension data based on dimension ordinal
*
* @param dimensionOrdinal dimension ordinal
@@ -347,6 +392,14 @@ public abstract class AbstractScannedResult {
public abstract byte[][] getNoDictionaryKeyArray();
/**
+ * Below method will be used to get the no dictionary key
+ * array in string array format for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract String[] getNoDictionaryKeyStringArray();
+
+ /**
* Below method will be used to to check whether measure value
* is null or for a measure
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java b/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
deleted file mode 100644
index c13b0f7..0000000
--- a/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result;
-
-/**
- * Below class holds the query result of batches.
- */
-public class BatchRawResult extends BatchResult {
-
- /**
- * This method will return one row at a time based on the counter given.
- * @param counter
- * @return
- */
- public Object[] getRawRow(int counter) {
- return rows[counter];
- }
-
- /**
- * For getting the total size.
- * @return
- */
- public int getSize() {
- return rows.length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/BatchResult.java b/core/src/main/java/org/carbondata/scan/result/BatchResult.java
index c5e1182..456717e 100644
--- a/core/src/main/java/org/carbondata/scan/result/BatchResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/BatchResult.java
@@ -19,6 +19,8 @@
package org.carbondata.scan.result;
+import java.util.ArrayList;
+import java.util.List;
import java.util.NoSuchElementException;
import org.carbondata.common.CarbonIterator;
@@ -31,7 +33,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
/**
* list of keys
*/
- protected Object[][] rows;
+ protected List<Object[]> rows;
/**
* counter to check whether all the records are processed or not
@@ -39,7 +41,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
protected int counter;
public BatchResult() {
- this.rows = new Object[0][];
+ this.rows = new ArrayList<>();
}
/**
@@ -47,7 +49,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
*
* @return
*/
- public Object[][] getRows() {
+ public List<Object[]> getRows() {
return rows;
}
@@ -56,10 +58,27 @@ public class BatchResult extends CarbonIterator<Object[]> {
*
* @param rows
*/
- public void setRows(Object[][] rows) {
+ public void setRows(List<Object[]> rows) {
this.rows = rows;
}
+ /**
+ * This method will return one row at a time based on the counter given.
+ * @param counter
+ * @return
+ */
+ public Object[] getRawRow(int counter) {
+ return rows.get(counter);
+ }
+
+ /**
+ * For getting the total size.
+ * @return
+ */
+ public int getSize() {
+ return rows.size();
+ }
+
/**
* Returns {@code true} if the iteration has more elements.
@@ -67,7 +86,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
* @return {@code true} if the iteration has more elements
*/
@Override public boolean hasNext() {
- return counter < rows.length;
+ return counter < rows.size();
}
/**
@@ -79,7 +98,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
if (!hasNext()) {
throw new NoSuchElementException();
}
- Object[] row = rows[counter];
+ Object[] row = rows.get(counter);
counter++;
return row;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java b/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
deleted file mode 100644
index f3085ce..0000000
--- a/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result;
-
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-public class ListBasedResultWrapper {
-
- private ByteArrayWrapper key;
-
- private Object[] value;
-
- /**
- * @return the key
- */
- public ByteArrayWrapper getKey() {
- return key;
- }
-
- /**
- * @param key the key to set
- */
- public void setKey(ByteArrayWrapper key) {
- this.key = key;
- }
-
- /**
- * @return the value
- */
- public Object[] getValue() {
- return value;
- }
-
- /**
- * @param value the value to set
- */
- public void setValue(Object[] value) {
- this.value = value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
index 962d9a3..e519f9e 100644
--- a/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
@@ -44,6 +44,15 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(rowMapping[currentRow]);
+ }
+
+ /**
* Below method will be used to get the complex type key array
*
* @return complex type key array
@@ -63,6 +72,16 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(rowMapping[currentRow]);
+ }
+
+ /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
deleted file mode 100644
index 24ebf5b..0000000
--- a/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * Below class is a holder over list based result wrapper
- */
-public class ListBasedResult implements Result<List<ListBasedResultWrapper>, Object> {
-
- /**
- * current result list
- */
- private List<ListBasedResultWrapper> currentRowPointer;
-
- /**
- * all result list , this is required because if we merger all the scanned
- * result from all the blocks in one list, that list creation will take more
- * time as every time list will create a big array and then it will do copy
- * the older element to new array, and creation of big array will also be a
- * problem if memory is fragmented then jvm in to do defragmentation to
- * create a big space, but if divide the data in multiple list than it avoid
- * copy and defragmentation
- */
- private List<List<ListBasedResultWrapper>> allRowsResult;
-
- /**
- * counter to check how many result processed
- */
- private int totalRecordCounter = -1;
-
- /**
- * number of records
- */
- private int totalNumberOfRecords;
-
- /**
- * current counter of the record in list
- */
- private int listRecordCounter = -1;
-
- /**
- * current list counter
- */
- private int currentListCounter;
-
- public ListBasedResult() {
- currentRowPointer =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- allRowsResult =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- /**
- * below method will be used to add the scan result
- */
- @Override public void addScannedResult(List<ListBasedResultWrapper> listBasedResult) {
- this.currentRowPointer = listBasedResult;
- totalNumberOfRecords = listBasedResult.size();
- allRowsResult.add(listBasedResult);
- }
-
- /**
- * Method to check more result is present
- * or not
- */
- @Override public boolean hasNext() {
- if (allRowsResult.size() == 0) {
- return false;
- }
- // As we are storing data in list of list, below code is to check whether
- // any more result is present
- // in the result.
- // first it will check list counter is zero if it is zero
- // than it will check list counter to check how many list has been processed
- // if more list are present and all the list of current list is processed
- // than it will take a new list from all row result list
- totalRecordCounter++;
- listRecordCounter++;
- if (listRecordCounter == 0 || (listRecordCounter >= currentRowPointer.size()
- && currentListCounter < allRowsResult.size())) {
- listRecordCounter = 0;
- currentRowPointer = allRowsResult.get(currentListCounter);
- currentListCounter++;
- }
- return totalRecordCounter < totalNumberOfRecords;
- }
-
- /**
- * @return key
- */
- @Override public ByteArrayWrapper getKey() {
- return currentRowPointer.get(listRecordCounter).getKey();
- }
-
- /**
- * @return will return the value
- */
- @Override public Object[] getValue() {
- return currentRowPointer.get(listRecordCounter).getValue();
- }
-
- /***
- * below method will be used to merge the
- * scanned result
- *
- * @param otherResult return to be merged
- */
- @Override public void merge(Result<List<ListBasedResultWrapper>, Object> otherResult) {
- if (otherResult.size() > 0) {
- totalNumberOfRecords += otherResult.size();
- this.allRowsResult.add(otherResult.getResult());
- }
- }
-
- /**
- * Return the size of the result
- */
- @Override public int size() {
- return totalNumberOfRecords;
- }
-
- /**
- * @return the complete result
- */
- @Override public List<ListBasedResultWrapper> getResult() {
- return currentRowPointer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
index 9782099..00b5563 100644
--- a/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
@@ -26,6 +26,15 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(currentRow);
+ }
+
+ /**
* Below method will be used to get the complex type key array
*
* @return complex type key array
@@ -45,6 +54,16 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(currentRow);
+ }
+
+ /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
index 4b2d777..0539cfa 100644
--- a/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
@@ -29,8 +29,6 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.model.QueryModel;
import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.preparator.QueryResultPreparator;
/**
* In case of detail query we cannot keep all the records in memory so for
@@ -39,19 +37,12 @@ import org.carbondata.scan.result.preparator.QueryResultPreparator;
*/
public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator {
- /**
- * to prepare the result
- */
- private QueryResultPreparator<List<ListBasedResultWrapper>, Object> queryResultPreparator;
-
private ExecutorService execService = Executors.newFixedThreadPool(1);
private Future<BatchResult> future;
- public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
- QueryResultPreparator queryResultPreparator) {
+ public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel) {
super(infos, queryModel);
- this.queryResultPreparator = queryResultPreparator;
}
@Override public BatchResult next() {
@@ -81,7 +72,9 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
private Future<BatchResult> execute() {
return execService.submit(new Callable<BatchResult>() {
@Override public BatchResult call() throws QueryExecutionException {
- return queryResultPreparator.prepareQueryResult(dataBlockIterator.next());
+ BatchResult batchResult = new BatchResult();
+ batchResult.setRows(dataBlockIterator.next());
+ return batchResult;
}
});
}
[3/3] incubator-carbondata git commit: [CARBONDATA-92]Refactored code
to improve performance by using removing unnecessary result conversion This
closes #50
Posted by ch...@apache.org.
[CARBONDATA-92]Refactored code to improve performance by using removing unnecessary result conversion This closes #50
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/29f9cf22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/29f9cf22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/29f9cf22
Branch: refs/heads/master
Commit: 29f9cf2261ab5788590c98751848da1daae44918
Parents: 645eafa 9be7451
Author: chenliang613 <ch...@apache.org>
Authored: Tue Aug 2 09:11:28 2016 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Aug 2 09:11:28 2016 +0800
----------------------------------------------------------------------
.../chunk/DimensionColumnDataChunk.java | 10 +
.../impl/ColumnGroupDimensionDataChunk.java | 22 +-
.../impl/FixedLengthDimensionDataChunk.java | 22 +-
.../impl/VariableLengthDimensionDataChunk.java | 18 +-
.../core/keygenerator/KeyGenerator.java | 9 +
.../core/keygenerator/mdkey/Bits.java | 4 +-
.../mdkey/MultiDimKeyVarLengthGenerator.java | 11 +-
.../org/carbondata/core/util/CarbonUtil.java | 42 +++-
.../scan/collector/ScannedResultCollector.java | 11 +-
.../impl/AbstractScannedResultCollector.java | 157 ++++++++++++++
.../impl/DictionaryBasedResultCollector.java | 130 +++++++++++
.../impl/ListBasedResultCollector.java | 216 -------------------
.../collector/impl/RawBasedResultCollector.java | 67 ++++++
.../scan/complextypes/PrimitiveQueryType.java | 21 +-
.../scan/executor/QueryExecutorFactory.java | 7 +-
.../executor/impl/AbstractQueryExecutor.java | 37 +---
.../scan/executor/impl/DetailQueryExecutor.java | 6 +-
.../impl/DetailRawRecordQueryExecutor.java | 24 ---
.../scan/executor/infos/BlockExecutionInfo.java | 43 ++--
.../scan/executor/infos/KeyStructureInfo.java | 12 ++
.../scan/executor/util/QueryUtil.java | 5 +
.../org/carbondata/scan/model/QueryModel.java | 14 --
.../processor/AbstractDataBlockIterator.java | 18 +-
.../processor/impl/DataBlockIteratorImpl.java | 22 +-
.../scan/result/AbstractScannedResult.java | 53 +++++
.../carbondata/scan/result/BatchRawResult.java | 43 ----
.../org/carbondata/scan/result/BatchResult.java | 31 ++-
.../scan/result/ListBasedResultWrapper.java | 57 -----
.../result/impl/FilterQueryScannedResult.java | 19 ++
.../scan/result/impl/ListBasedResult.java | 153 -------------
.../impl/NonFilterQueryScannedResult.java | 19 ++
.../iterator/DetailQueryResultIterator.java | 15 +-
.../scan/result/iterator/RawResultIterator.java | 10 +-
.../preparator/QueryResultPreparator.java | 10 -
.../impl/AbstractQueryResultPreparator.java | 87 --------
.../impl/DetailQueryResultPreparatorImpl.java | 148 -------------
.../impl/RawQueryResultPreparatorImpl.java | 136 ------------
.../hadoop/util/CarbonInputFormatUtil.java | 1 -
.../spark/merger/CarbonCompactionExecutor.java | 7 +-
.../org/apache/spark/sql/CarbonOperators.scala | 120 +++++------
.../spark/sql/optimizer/CarbonOptimizer.scala | 4 +
.../AllDataTypesTestCaseAggregate.scala | 87 ++++----
42 files changed, 789 insertions(+), 1139 deletions(-)
----------------------------------------------------------------------