You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/02 01:11:57 UTC
[2/3] incubator-carbondata git commit: Refactored code to improve
performance by using removing unnecessary conversion.
Refactored code to improve performance by using removing unnecessary conversion.
Refactored code
Fixed review comments
support create using select as for other formats
Fixed issues after merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9be7451e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9be7451e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9be7451e
Branch: refs/heads/master
Commit: 9be7451ec7bb5561b1f142ebdd40d6048713760e
Parents: 645eafa
Author: ravipesala <ra...@gmail.com>
Authored: Thu Jul 21 20:50:33 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Aug 2 09:09:45 2016 +0800
----------------------------------------------------------------------
.../chunk/DimensionColumnDataChunk.java | 10 +
.../impl/ColumnGroupDimensionDataChunk.java | 22 +-
.../impl/FixedLengthDimensionDataChunk.java | 22 +-
.../impl/VariableLengthDimensionDataChunk.java | 18 +-
.../core/keygenerator/KeyGenerator.java | 9 +
.../core/keygenerator/mdkey/Bits.java | 4 +-
.../mdkey/MultiDimKeyVarLengthGenerator.java | 11 +-
.../org/carbondata/core/util/CarbonUtil.java | 42 +++-
.../scan/collector/ScannedResultCollector.java | 11 +-
.../impl/AbstractScannedResultCollector.java | 157 ++++++++++++++
.../impl/DictionaryBasedResultCollector.java | 130 +++++++++++
.../impl/ListBasedResultCollector.java | 216 -------------------
.../collector/impl/RawBasedResultCollector.java | 67 ++++++
.../scan/complextypes/PrimitiveQueryType.java | 21 +-
.../scan/executor/QueryExecutorFactory.java | 7 +-
.../executor/impl/AbstractQueryExecutor.java | 37 +---
.../scan/executor/impl/DetailQueryExecutor.java | 6 +-
.../impl/DetailRawRecordQueryExecutor.java | 24 ---
.../scan/executor/infos/BlockExecutionInfo.java | 43 ++--
.../scan/executor/infos/KeyStructureInfo.java | 12 ++
.../scan/executor/util/QueryUtil.java | 5 +
.../org/carbondata/scan/model/QueryModel.java | 14 --
.../processor/AbstractDataBlockIterator.java | 18 +-
.../processor/impl/DataBlockIteratorImpl.java | 22 +-
.../scan/result/AbstractScannedResult.java | 53 +++++
.../carbondata/scan/result/BatchRawResult.java | 43 ----
.../org/carbondata/scan/result/BatchResult.java | 31 ++-
.../scan/result/ListBasedResultWrapper.java | 57 -----
.../result/impl/FilterQueryScannedResult.java | 19 ++
.../scan/result/impl/ListBasedResult.java | 153 -------------
.../impl/NonFilterQueryScannedResult.java | 19 ++
.../iterator/DetailQueryResultIterator.java | 15 +-
.../scan/result/iterator/RawResultIterator.java | 10 +-
.../preparator/QueryResultPreparator.java | 10 -
.../impl/AbstractQueryResultPreparator.java | 87 --------
.../impl/DetailQueryResultPreparatorImpl.java | 148 -------------
.../impl/RawQueryResultPreparatorImpl.java | 136 ------------
.../hadoop/util/CarbonInputFormatUtil.java | 1 -
.../spark/merger/CarbonCompactionExecutor.java | 7 +-
.../org/apache/spark/sql/CarbonOperators.scala | 120 +++++------
.../spark/sql/optimizer/CarbonOptimizer.scala | 4 +
.../AllDataTypesTestCaseAggregate.scala | 87 ++++----
42 files changed, 789 insertions(+), 1139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
index 75c29d2..6a30bc7 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
@@ -36,6 +36,16 @@ public interface DimensionColumnDataChunk<T> {
int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
/**
+ * It uses to convert column data to dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo);
+
+ /**
* Below method to get the data based in row id
*
* @param row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
index e85e17d..d8536a1 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -55,7 +55,7 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
* @param data data to filed
* @param offset offset from which data need to be filed
* @param rowId row id of the chunk
- * @param keyStructureInfo define the structure of the key
+ * @param restructuringInfo define the structure of the key
* @return how many bytes was copied
*/
@Override public int fillChunkData(byte[] data, int offset, int rowId,
@@ -67,10 +67,22 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
}
/**
- * Below method to get the data based in row id
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo info) {
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ long[] keyArray = info.getKeyGenerator().getKeyArray(dataChunk, start);
+ int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+ for (int i = 0; i < ordinal.length; i++) {
+ row[columnIndex++] = (int)keyArray[ordinal[i]];
+ }
+ return columnIndex;
+ }
+
+ /**
+ * Below method masks key
*
- * @param row id row id of the data
- * @return chunk
*/
public byte[] getMaskedKey(byte[] data, int offset, KeyStructureInfo info) {
byte[] maskedKey = new byte[info.getMaskByteRanges().length];
@@ -86,7 +98,7 @@ public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<b
/**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param rowId row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int rowId) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 08ae703..cbe48b2 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -54,7 +54,7 @@ public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<b
*
* @param data data to filed
* @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
+ * @param index row id of the chunk
* @param keyStructureInfo define the structure of the key
* @return how many bytes was copied
*/
@@ -69,9 +69,27 @@ public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<b
}
/**
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ if (chunkAttributes.getInvertedIndexes() != null) {
+ rowId = chunkAttributes.getInvertedIndexesReverse()[rowId];
+ }
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ int dict = 0;
+ for (int i = start; i < start + chunkAttributes.getColumnValueSize(); i++) {
+ dict <<= 8;
+ dict ^= dataChunk[i] & 0xFF;
+ }
+ row[columnIndex] = dict;
+ return columnIndex + 1;
+ }
+
+ /**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param index row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int index) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 51306b5..0aa2406 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -39,8 +39,8 @@ public class VariableLengthDimensionDataChunk implements DimensionColumnDataChun
*
* @param data data to filed
* @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
- * @param keyStructureInfo define the structure of the key
+ * @param index row id of the chunk
+ * @param restructuringInfo define the structure of the key
* @return how many bytes was copied
*/
@Override public int fillChunkData(byte[] data, int offset, int index,
@@ -51,9 +51,21 @@ public class VariableLengthDimensionDataChunk implements DimensionColumnDataChun
}
/**
+ * Converts to column dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ return columnIndex + 1;
+ }
+
+ /**
* Below method to get the data based in row id
*
- * @param row id row id of the data
+ * @param index row id of the data
* @return chunk
*/
@Override public byte[] getChunkData(int index) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
index 5652d08..f9016f8 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/KeyGenerator.java
@@ -58,6 +58,15 @@ public interface KeyGenerator extends Serializable, Comparator<byte[]> {
* It gets array of keys out of single key aka byte array
*
* @param key
+ * @param offset
+ * @return array of keys.
+ */
+ long[] getKeyArray(byte[] key, int offset);
+
+ /**
+ * It gets array of keys out of single key aka byte array
+ *
+ * @param key
* @param maskedByteRanges
* @return array of keys
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
index 2515f0f..2ce64ba 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/Bits.java
@@ -252,7 +252,7 @@ public class Bits implements Serializable {
return getBytesVal(words);
}
- public long[] getKeyArray(byte[] key) {
+ public long[] getKeyArray(byte[] key, int offset) {
int length = 8;
int ls = byteSize;
@@ -269,7 +269,7 @@ public class Bits implements Serializable {
}
for (int j = ls; j < m; j++) {
l <<= 8;
- l ^= key[j] & 0xFF;
+ l ^= key[j + offset] & 0xFF;
}
words[i] = l;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
index 1cdc847..1aa2b81 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
@@ -52,12 +52,17 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
@Override public long[] getKeyArray(byte[] key) {
- return bits.getKeyArray(key);
+ return bits.getKeyArray(key, 0);
+ }
+
+ @Override public long[] getKeyArray(byte[] key, int offset) {
+
+ return bits.getKeyArray(key, offset);
}
@Override public long getKey(byte[] key, int index) {
- return bits.getKeyArray(key)[index];
+ return bits.getKeyArray(key, 0)[index];
}
public int getKeySizeInBytes() {
@@ -68,7 +73,7 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
if (index < 0 || size == 0) {
return null;
}
- long[] keys = bits.getKeyArray(key);
+ long[] keys = bits.getKeyArray(key, 0);
long[] rtn = new long[size];
System.arraycopy(keys, index, rtn, 0, size);
return rtn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
index 9db8f24..eb8a0fc 100644
--- a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
@@ -20,11 +20,21 @@
package org.carbondata.core.util;
-import java.io.*;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.security.PrivilegedExceptionAction;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -55,6 +65,7 @@ import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
import org.carbondata.core.datastorage.store.impl.FileFactory;
import org.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.carbondata.core.metadata.ValueEncoderMeta;
+import org.carbondata.scan.model.QueryDimension;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
@@ -1054,6 +1065,33 @@ public final class CarbonUtil {
}
}
+ public static boolean[] getDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY);
+ }
+ return dictionaryEncodingArray;
+ }
+
+ public static boolean[] getDirectDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY);
+ }
+ return dictionaryEncodingArray;
+ }
+
+ public static boolean[] getComplexDataTypeArray(QueryDimension[] queryDimensions) {
+ boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
+ for (int i = 0; i < queryDimensions.length; i++) {
+ dictionaryEncodingArray[i] =
+ CarbonUtil.hasComplexDataType(queryDimensions[i].getDimension().getDataType());
+ }
+ return dictionaryEncodingArray;
+ }
+
/**
* Below method will be used to read the data file matadata
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
index f214a9d..424bd86 100644
--- a/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
+++ b/core/src/main/java/org/carbondata/scan/collector/ScannedResultCollector.java
@@ -18,8 +18,9 @@
*/
package org.carbondata.scan.collector;
+import java.util.List;
+
import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.Result;
/**
* Interface which will be used to aggregate the scan result
@@ -32,12 +33,6 @@ public interface ScannedResultCollector {
* @param scannedResult scanned result
* @return how many records was aggregated
*/
- int collectData(AbstractScannedResult scannedResult, int batchSize);
+ List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
- /**
- * Below method will be used to get the aggregated result
- *
- * @return
- */
- Result getCollectedResult();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
new file mode 100644
index 0000000..81d7b6a
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.util.DataTypeUtil;
+import org.carbondata.scan.collector.ScannedResultCollector;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.carbondata.scan.executor.util.QueryUtil;
+import org.carbondata.scan.result.AbstractScannedResult;
+import org.carbondata.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
+
+ /**
+ * restructuring info
+ */
+ private KeyStructureInfo restructureInfos;
+
+ /**
+ * table block execution infos
+ */
+ protected BlockExecutionInfo tableBlockExecutionInfos;
+
+ /**
+ * Measure ordinals
+ */
+ protected int[] measuresOrdinal;
+
+ /**
+ * to check whether measure exists in current table block or not this to
+ * handle restructuring scenario
+ */
+ protected boolean[] isMeasureExistsInCurrentBlock;
+
+ /**
+ * default value of the measures in case of restructuring some measure wont
+ * be present in the table so in that default value will be used to
+ * aggregate the data for that measure columns
+ */
+ private Object[] measureDefaultValue;
+
+ /**
+ * measure datatypes.
+ */
+ protected DataType[] measureDatatypes;
+
+ public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ this.tableBlockExecutionInfos = blockExecutionInfos;
+ restructureInfos = blockExecutionInfos.getKeyStructureInfo();
+ measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
+ isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
+ measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
+ this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
+ }
+
+ protected void fillMeasureData(Object[] msrValues, int offset,
+ AbstractScannedResult scannedResult) {
+ for (short i = 0; i < measuresOrdinal.length; i++) {
+ // if measure exists is block then pass measure column
+ // data chunk to the collector
+ if (isMeasureExistsInCurrentBlock[i]) {
+ msrValues[i + offset] = getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
+ scannedResult.getCurrenrRowId(), measureDatatypes[i]);
+ } else {
+ // if not then get the default value and use that value in aggregation
+ msrValues[i + offset] = measureDefaultValue[i];
+ }
+ }
+ }
+
+ private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
+ if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
+ Object msrVal;
+ switch (dataType) {
+ case INT:
+ case LONG:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
+ break;
+ case DECIMAL:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
+ break;
+ default:
+ msrVal = dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
+ }
+ return DataTypeUtil.getMeasureDataBasedOnDataType(msrVal, dataType);
+ }
+ return null;
+ }
+
+ /**
+ * Below method will used to get the result
+ */
+ protected void updateData(List<Object[]> listBasedResult) {
+ if (tableBlockExecutionInfos.isFixedKeyUpdateRequired()) {
+ updateKeyWithLatestBlockKeygenerator(listBasedResult);
+ }
+ }
+
+ /**
+ * Below method will be used to update the fixed length key with the
+ * latest block key generator
+ *
+ * @return updated block
+ */
+ private void updateKeyWithLatestBlockKeygenerator(List<Object[]> listBasedResult) {
+ try {
+ long[] data = null;
+ ByteArrayWrapper key = null;
+ for (int i = 0; i < listBasedResult.size(); i++) {
+ // get the key
+ key = (ByteArrayWrapper)listBasedResult.get(i)[0];
+ // unpack the key with table block key generator
+ data = tableBlockExecutionInfos.getBlockKeyGenerator()
+ .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
+ // packed the key with latest block key generator
+ // and generate the masked key for that key
+ key.setDictionaryKey(QueryUtil
+ .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
+ restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
+ restructureInfos.getMaskByteRanges().length));
+ }
+ } catch (KeyGenException e) {
+ LOGGER.error(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
new file mode 100644
index 0000000..89a7d9d
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.util.DataTypeUtil;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.filter.GenericQueryType;
+import org.carbondata.scan.model.QueryDimension;
+import org.carbondata.scan.model.QueryMeasure;
+import org.carbondata.scan.result.AbstractScannedResult;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DictionaryBasedResultCollector.class.getName());
+
+ public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ boolean isMsrsPresent = measureDatatypes.length > 0;
+ QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ Map<Integer, GenericQueryType> comlexDimensionInfoMap =
+ tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ boolean[] dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
+ boolean[] directDictionaryEncodingArray =
+ CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
+ boolean[] complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
+ int dimSize = queryDimensions.length;
+ boolean isDimensionsExist = dimSize > 0;
+ int[] order = new int[dimSize + queryMeasures.length];
+ for (int i = 0; i < dimSize; i++) {
+ order[i] = queryDimensions[i].getQueryOrder();
+ }
+ for (int i = 0; i < queryMeasures.length; i++) {
+ order[i + dimSize] = queryMeasures[i].getQueryOrder();
+ }
+ // scan the record and add to list
+ int rowCounter = 0;
+ int dictionaryColumnIndex = 0;
+ int noDictionaryColumnIndex = 0;
+ int complexTypeColumnIndex = 0;
+ int[] surrogateResult;
+ String[] noDictionaryKeys;
+ byte[][] complexTypeKeyArray;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[dimSize + queryMeasures.length];
+ if (isDimensionsExist) {
+ surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
+ dictionaryColumnIndex = 0;
+ noDictionaryColumnIndex = 0;
+ complexTypeColumnIndex = 0;
+ for (int i = 0; i < dimSize; i++) {
+ if (!dictionaryEncodingArray[i]) {
+ row[order[i]] = DataTypeUtil
+ .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
+ } else if (directDictionaryEncodingArray[i]) {
+ DirectDictionaryGenerator directDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
+ if (directDictionaryGenerator != null) {
+ row[order[i]] = directDictionaryGenerator.getValueFromSurrogate(
+ surrogateResult[dictionaryColumnIndex++]);
+ }
+ } else if (complexDataTypeArray[i]) {
+ row[order[i]] = comlexDimensionInfoMap
+ .get(queryDimensions[i].getDimension().getOrdinal())
+ .getDataBasedOnDataTypeFromSurrogates(
+ ByteBuffer.wrap(complexTypeKeyArray[complexTypeColumnIndex++]));
+ } else {
+ row[order[i]] = surrogateResult[dictionaryColumnIndex++];
+ }
+ }
+
+ } else {
+ scannedResult.incrementCounter();
+ }
+ if (isMsrsPresent) {
+ Object[] msrValues = new Object[measureDatatypes.length];
+ fillMeasureData(msrValues, 0, scannedResult);
+ for (int i = 0; i < msrValues.length; i++) {
+ row[order[i + dimSize]] = msrValues[i];
+ }
+ }
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ return listBasedResult;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
deleted file mode 100644
index a199cad..0000000
--- a/core/src/main/java/org/carbondata/scan/collector/impl/ListBasedResultCollector.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.scan.collector.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
-import org.carbondata.core.carbon.metadata.datatype.DataType;
-import org.carbondata.core.keygenerator.KeyGenException;
-import org.carbondata.core.util.DataTypeUtil;
-import org.carbondata.scan.collector.ScannedResultCollector;
-import org.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.carbondata.scan.executor.util.QueryUtil;
-import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.result.impl.ListBasedResult;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * It is not a collector it is just a scanned result holder.
- *
- */
-public class ListBasedResultCollector implements ScannedResultCollector {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ListBasedResultCollector.class.getName());
-
- /**
- * to keep a track of number of row processed to handle limit push down in
- * case of detail query scenario
- */
- private int rowCounter;
-
- /**
- * dimension values list
- */
- private List<ListBasedResultWrapper> listBasedResult;
-
- /**
- * restructuring info
- */
- private KeyStructureInfo restructureInfos;
-
- /**
- * table block execution infos
- */
- private BlockExecutionInfo tableBlockExecutionInfos;
-
- private int[] measuresOrdinal;
-
- /**
- * to check whether measure exists in current table block or not this to
- * handle restructuring scenario
- */
- private boolean[] isMeasureExistsInCurrentBlock;
-
- /**
- * default value of the measures in case of restructuring some measure wont
- * be present in the table so in that default value will be used to
- * aggregate the data for that measure columns
- */
- private Object[] measureDefaultValue;
-
- /**
- * measure datatypes.
- */
- private DataType[] measureDatatypes;
-
- public ListBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- this.tableBlockExecutionInfos = blockExecutionInfos;
- restructureInfos = blockExecutionInfos.getKeyStructureInfo();
- measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
- isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
- measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
- this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
- }
-
- @Override
- /**
- * This method will add a record both key and value to list object
- * it will keep track of how many record is processed, to handle limit scenario
- * @param scanned result
- *
- */
- public int collectData(AbstractScannedResult scannedResult, int batchSize) {
- this.listBasedResult =
- new ArrayList<>(batchSize);
- boolean isMsrsPresent = measureDatatypes.length > 0;
- ByteArrayWrapper wrapper = null;
- // scan the record and add to list
- ListBasedResultWrapper resultWrapper;
- int rowCounter = 0;
- while (scannedResult.hasNext() && rowCounter < batchSize) {
- resultWrapper = new ListBasedResultWrapper();
- if(tableBlockExecutionInfos.isDimensionsExistInQuery()) {
- wrapper = new ByteArrayWrapper();
- wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
- wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
- wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
- resultWrapper.setKey(wrapper);
- } else {
- scannedResult.incrementCounter();
- }
- if(isMsrsPresent) {
- Object[] msrValues = new Object[measureDatatypes.length];
- fillMeasureData(msrValues, scannedResult);
- resultWrapper.setValue(msrValues);
- }
- listBasedResult.add(resultWrapper);
- rowCounter++;
- }
- return rowCounter;
- }
-
- private void fillMeasureData(Object[] msrValues, AbstractScannedResult scannedResult) {
- for (short i = 0; i < measuresOrdinal.length; i++) {
- // if measure exists is block then pass measure column
- // data chunk to the collector
- if (isMeasureExistsInCurrentBlock[i]) {
- msrValues[i] =
- getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
- scannedResult.getCurrenrRowId(),measureDatatypes[i]);
- } else {
- // if not then get the default value and use that value in aggregation
- msrValues[i] = measureDefaultValue[i];
- }
- }
- }
-
- private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
- if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
- Object msrVal;
- switch (dataType) {
- case INT:
- case LONG:
- msrVal = dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
- break;
- case DECIMAL:
- msrVal = dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
- break;
- default:
- msrVal = dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
- }
- return DataTypeUtil.getMeasureDataBasedOnDataType(msrVal, dataType);
- }
- return null;
- }
-
- /**
- * Below method will used to get the result
- */
- @Override public Result getCollectedResult() {
- Result<List<ListBasedResultWrapper>, Object> result = new ListBasedResult();
- if (tableBlockExecutionInfos.isFixedKeyUpdateRequired() && tableBlockExecutionInfos
- .isDimensionsExistInQuery()) {
- updateKeyWithLatestBlockKeygenerator();
- result.addScannedResult(listBasedResult);
- } else {
- result.addScannedResult(listBasedResult);
- }
- return result;
- }
-
-
-
- /**
- * Below method will be used to update the fixed length key with the
- * latest block key generator
- *
- * @return updated block
- */
- private void updateKeyWithLatestBlockKeygenerator() {
- try {
- long[] data = null;
- ByteArrayWrapper key = null;
- for (int i = 0; i < listBasedResult.size(); i++) {
- // get the key
- key = listBasedResult.get(i).getKey();
- // unpack the key with table block key generator
- data = tableBlockExecutionInfos.getBlockKeyGenerator()
- .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
- // packed the key with latest block key generator
- // and generate the masked key for that key
- key.setDictionaryKey(QueryUtil
- .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
- restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
- restructureInfos.getMaskByteRanges().length));
- listBasedResult.get(i).setKey(key);
- }
- } catch (KeyGenException e) {
- LOGGER.error(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
new file mode 100644
index 0000000..5bb0567
--- /dev/null
+++ b/core/src/main/java/org/carbondata/scan/collector/impl/RawBasedResultCollector.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.collector.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.carbondata.scan.model.QueryMeasure;
+import org.carbondata.scan.result.AbstractScannedResult;
+import org.carbondata.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class RawBasedResultCollector extends AbstractScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(RawBasedResultCollector.class.getName());
+
+ public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ ByteArrayWrapper wrapper = null;
+ // scan the record and add to list
+ int rowCounter = 0;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[1 + queryMeasures.length];
+ wrapper = new ByteArrayWrapper();
+ wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
+ wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
+ wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
+ row[0] = wrapper;
+ fillMeasureData(row, 1, scannedResult);
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ updateData(listBasedResult);
+ return listBasedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
index 21a1d02..33a22c7 100644
--- a/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/carbondata/scan/complextypes/PrimitiveQueryType.java
@@ -33,12 +33,7 @@ import org.carbondata.core.util.DataTypeUtil;
import org.carbondata.scan.filter.GenericQueryType;
import org.carbondata.scan.processor.BlocksChunkHolder;
-import org.apache.spark.sql.types.BooleanType;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.IntegerType;
-import org.apache.spark.sql.types.LongType;
-import org.apache.spark.sql.types.TimestampType;
+import org.apache.spark.sql.types.*;
public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
@@ -135,17 +130,17 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
@Override public DataType getSchemaType() {
switch (dataType) {
case INT:
- return new IntegerType();
+ return IntegerType$.MODULE$;
case DOUBLE:
- return new DoubleType();
+ return DoubleType$.MODULE$;
case LONG:
- return new LongType();
+ return LongType$.MODULE$;
case BOOLEAN:
- return new BooleanType();
+ return BooleanType$.MODULE$;
case TIMESTAMP:
- return new TimestampType();
+ return TimestampType$.MODULE$;
default:
- return new IntegerType();
+ return IntegerType$.MODULE$;
}
}
@@ -165,7 +160,7 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
byte[] data = new byte[keySize];
surrogateData.get(data);
Bits bit = new Bits(new int[]{keySize * 8});
- int surrgateValue = (int)bit.getKeyArray(data)[0];
+ int surrgateValue = (int)bit.getKeyArray(data, 0)[0];
Object actualData = null;
if (isDirectDictionary) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java b/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
index 724b8b6..7f7203c 100644
--- a/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
+++ b/core/src/main/java/org/carbondata/scan/executor/QueryExecutorFactory.java
@@ -19,7 +19,6 @@
package org.carbondata.scan.executor;
import org.carbondata.scan.executor.impl.DetailQueryExecutor;
-import org.carbondata.scan.executor.impl.DetailRawRecordQueryExecutor;
import org.carbondata.scan.model.QueryModel;
/**
@@ -29,10 +28,6 @@ import org.carbondata.scan.model.QueryModel;
public class QueryExecutorFactory {
public static QueryExecutor getQueryExecutor(QueryModel queryModel) {
- if (queryModel.isForcedDetailRawQuery()) {
- return new DetailRawRecordQueryExecutor();
- } else {
- return new DetailQueryExecutor();
- }
+ return new DetailQueryExecutor();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index dd23119..519d4e6 100644
--- a/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -48,7 +48,6 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.AggregatorInfo;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.carbondata.scan.executor.infos.SortInfo;
import org.carbondata.scan.executor.util.QueryUtil;
import org.carbondata.scan.executor.util.RestructureUtil;
import org.carbondata.scan.filter.FilterUtil;
@@ -229,7 +228,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
int[] maksedByte =
QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
- blockExecutionInfo.setDimensionsExistInQuery(updatedQueryDimension.size() > 0);
+ blockExecutionInfo.setQueryDimensions(
+ updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()]));
+ blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures()
+ .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
blockExecutionInfo.setDataBlock(blockIndex);
blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// adding aggregation info for query
@@ -373,37 +375,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
}
/**
- * Below method will be used to get the sort information which will be
- * required during sorting the data on dimension column
- *
- * @param queryModel query model
- * @return Sort infos
- * @throws QueryExecutionException if problem while
- */
- protected SortInfo getSortInfos(QueryModel queryModel) throws QueryExecutionException {
-
- // get the masked by range for order by dimension
- int[][] maskedByteRangeForSorting = QueryUtil
- .getMaskedByteRangeForSorting(queryModel.getSortDimension(),
- queryProperties.keyStructureInfo.getKeyGenerator(),
- queryProperties.keyStructureInfo.getMaskByteRanges());
- // get masked key for sorting
- byte[][] maksedKeyForSorting = QueryUtil.getMaksedKeyForSorting(queryModel.getSortDimension(),
- queryProperties.keyStructureInfo.getKeyGenerator(), maskedByteRangeForSorting,
- queryProperties.keyStructureInfo.getMaskByteRanges());
- // fill sort dimension indexes
- queryProperties.sortDimIndexes = QueryUtil
- .getSortDimensionIndexes(queryModel.getSortDimension(), queryModel.getQueryDimension());
- SortInfo sortInfos = new SortInfo();
- sortInfos.setDimensionMaskKeyForSorting(maksedKeyForSorting);
- sortInfos.setDimensionSortOrder(queryModel.getSortOrder());
- sortInfos.setMaskedByteRangeForSorting(maskedByteRangeForSorting);
- sortInfos.setSortDimensionIndex(queryProperties.sortDimIndexes);
- sortInfos.setSortDimension(queryModel.getSortDimension());
- return sortInfos;
- }
-
- /**
* Below method will be used to get the aggrgator info for the query
*
* @param queryModel query model
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
index edb7675..30eb2d2 100644
--- a/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/scan/executor/impl/DetailQueryExecutor.java
@@ -24,9 +24,7 @@ import org.carbondata.common.CarbonIterator;
import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.iterator.ChunkRowIterator;
import org.carbondata.scan.result.iterator.DetailQueryResultIterator;
-import org.carbondata.scan.result.preparator.impl.DetailQueryResultPreparatorImpl;
/**
* Below class will be used to execute the detail query
@@ -38,9 +36,7 @@ public class DetailQueryExecutor extends AbstractQueryExecutor {
@Override public CarbonIterator<Object[]> execute(QueryModel queryModel)
throws QueryExecutionException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new ChunkRowIterator(
- new DetailQueryResultIterator(blockExecutionInfoList, queryModel,
- new DetailQueryResultPreparatorImpl(queryProperties, queryModel)));
+ return new DetailQueryResultIterator(blockExecutionInfoList, queryModel);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java b/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
deleted file mode 100644
index 8659e82..0000000
--- a/core/src/main/java/org/carbondata/scan/executor/impl/DetailRawRecordQueryExecutor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.carbondata.scan.executor.impl;
-
-import java.util.List;
-
-import org.carbondata.common.CarbonIterator;
-import org.carbondata.scan.executor.exception.QueryExecutionException;
-import org.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.carbondata.scan.model.QueryModel;
-import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.iterator.DetailQueryResultIterator;
-import org.carbondata.scan.result.preparator.impl.RawQueryResultPreparatorImpl;
-
-/**
- * Executor for raw records, it does not parse to actual data
- */
-public class DetailRawRecordQueryExecutor extends AbstractQueryExecutor<BatchResult> {
-
- @Override public CarbonIterator<BatchResult> execute(QueryModel queryModel)
- throws QueryExecutionException {
- List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new DetailQueryResultIterator(blockExecutionInfoList, queryModel,
- new RawQueryResultPreparatorImpl(queryProperties, queryModel));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
index 7b62681..883357f 100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/BlockExecutionInfo.java
@@ -29,6 +29,8 @@ import org.carbondata.core.datastorage.store.impl.FileFactory.FileType;
import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.scan.filter.GenericQueryType;
import org.carbondata.scan.filter.executer.FilterExecuter;
+import org.carbondata.scan.model.QueryDimension;
+import org.carbondata.scan.model.QueryMeasure;
/**
* Below class will have all the properties which needed during query execution
@@ -198,11 +200,6 @@ public class BlockExecutionInfo {
private boolean isRawRecordDetailQuery;
/**
- * whether dimensions exist in query.
- */
- private boolean isDimensionsExistInQuery;
-
- /**
* complexParentIndexToQueryMap
*/
private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
@@ -225,6 +222,16 @@ public class BlockExecutionInfo {
}
/**
+ * list of dimension selected for in query
+ */
+ private QueryDimension[] queryDimensions;
+
+ /**
+ * list of measure selected in query
+ */
+ private QueryMeasure[] queryMeasures;
+
+ /**
* @param blockIndex the tableBlock to set
*/
public void setDataBlock(AbstractIndex blockIndex) {
@@ -336,6 +343,8 @@ public class BlockExecutionInfo {
return maskedByteForBlock;
}
+
+
/**
* @param maskedByteForBlock the maskedByteForBlock to set
*/
@@ -618,14 +627,6 @@ public class BlockExecutionInfo {
isRawRecordDetailQuery = rawRecordDetailQuery;
}
- public boolean isDimensionsExistInQuery() {
- return isDimensionsExistInQuery;
- }
-
- public void setDimensionsExistInQuery(boolean dimensionsExistInQuery) {
- isDimensionsExistInQuery = dimensionsExistInQuery;
- }
-
/**
* @return the complexParentIndexToQueryMap
*/
@@ -661,4 +662,20 @@ public class BlockExecutionInfo {
public void setStatisticsRecorder(QueryStatisticsRecorder statisticsRecorder) {
this.statisticsRecorder = statisticsRecorder;
}
+
+ public QueryDimension[] getQueryDimensions() {
+ return queryDimensions;
+ }
+
+ public void setQueryDimensions(QueryDimension[] queryDimensions) {
+ this.queryDimensions = queryDimensions;
+ }
+
+ public QueryMeasure[] getQueryMeasures() {
+ return queryMeasures;
+ }
+
+ public void setQueryMeasures(QueryMeasure[] queryMeasures) {
+ this.queryMeasures = queryMeasures;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
index e19a3e9..a595f33 100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
@@ -49,6 +49,11 @@ public class KeyStructureInfo {
private byte[] maxKey;
/**
+ * dimension ordinals inside the column group
+ */
+ private int[] mdkeyQueryDimensionOrdinal;
+
+ /**
* @return the keyGenerator
*/
public KeyGenerator getKeyGenerator() {
@@ -104,4 +109,11 @@ public class KeyStructureInfo {
this.maxKey = maxKey;
}
+ public int[] getMdkeyQueryDimensionOrdinal() {
+ return mdkeyQueryDimensionOrdinal;
+ }
+
+ public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) {
+ this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
index 00e0f48..2b26ba0 100644
--- a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
@@ -546,8 +546,11 @@ public class QueryUtil {
// sort the ordinal
List<Integer> ordinal = next.getValue();
List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ //Un sorted
+ List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
for (Integer ord : ordinal) {
mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
+ mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
}
Collections.sort(mdKeyOrdinal);
// get the masked byte range for column group
@@ -560,6 +563,8 @@ public class QueryUtil {
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
restructureInfos.setMaskedBytes(maksedByte);
+ restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
+ .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
rowGroupToItsRSInfo
.put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
restructureInfos);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/model/QueryModel.java b/core/src/main/java/org/carbondata/scan/model/QueryModel.java
index e02c56e..e299e7b 100644
--- a/core/src/main/java/org/carbondata/scan/model/QueryModel.java
+++ b/core/src/main/java/org/carbondata/scan/model/QueryModel.java
@@ -130,12 +130,6 @@ public class QueryModel implements Serializable {
*/
private CarbonTable table;
- /**
- * This is used only whne [forcedDetailRawQuery = true]. By default forcedDetailRawQuery returns
- * dictionary values. But user wants in detail raw bytes the user set this field to true.
- */
- private boolean rawBytesDetailQuery;
-
private QueryStatisticsRecorder statisticsRecorder;
public QueryModel() {
@@ -495,14 +489,6 @@ public class QueryModel implements Serializable {
this.columnToDictionaryMapping = columnToDictionaryMapping;
}
- public boolean isRawBytesDetailQuery() {
- return rawBytesDetailQuery;
- }
-
- public void setRawBytesDetailQuery(boolean rawBytesDetailQuery) {
- this.rawBytesDetailQuery = rawBytesDetailQuery;
- }
-
public int getInMemoryRecordSize() {
return inMemoryRecordSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
index e385bec..cdf4f07 100644
--- a/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/carbondata/scan/processor/AbstractDataBlockIterator.java
@@ -18,17 +18,19 @@
*/
package org.carbondata.scan.processor;
+import java.util.List;
+
import org.carbondata.common.CarbonIterator;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.datastore.DataRefNode;
import org.carbondata.core.datastorage.store.FileHolder;
import org.carbondata.scan.collector.ScannedResultCollector;
-import org.carbondata.scan.collector.impl.ListBasedResultCollector;
+import org.carbondata.scan.collector.impl.DictionaryBasedResultCollector;
+import org.carbondata.scan.collector.impl.RawBasedResultCollector;
import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.result.AbstractScannedResult;
-import org.carbondata.scan.result.Result;
import org.carbondata.scan.scanner.BlockletScanner;
import org.carbondata.scan.scanner.impl.FilterScanner;
import org.carbondata.scan.scanner.impl.NonFilterScanner;
@@ -37,7 +39,7 @@ import org.carbondata.scan.scanner.impl.NonFilterScanner;
* This abstract class provides a skeletal implementation of the
* Block iterator.
*/
-public abstract class AbstractDataBlockIterator extends CarbonIterator<Result> {
+public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
@@ -88,9 +90,13 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<Result> {
} else {
blockletScanner = new NonFilterScanner(blockExecutionInfo);
}
-
- this.scannerResultAggregator =
- new ListBasedResultCollector(blockExecutionInfo);
+ if (blockExecutionInfo.isRawRecordDetailQuery()) {
+ this.scannerResultAggregator =
+ new RawBasedResultCollector(blockExecutionInfo);
+ } else {
+ this.scannerResultAggregator =
+ new DictionaryBasedResultCollector(blockExecutionInfo);
+ }
this.batchSize = batchSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
index 0c61947..ab9c665 100644
--- a/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
@@ -18,10 +18,11 @@
*/
package org.carbondata.scan.processor.impl;
+import java.util.List;
+
import org.carbondata.core.datastorage.store.FileHolder;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.processor.AbstractDataBlockIterator;
-import org.carbondata.scan.result.Result;
/**
* Below class will be used to process the block for detail query
@@ -33,8 +34,8 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
*
* @param blockExecutionInfo execution information
*/
- public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo,
- FileHolder fileReader, int batchSize) {
+ public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
+ int batchSize) {
super(blockExecutionInfo, fileReader, batchSize);
}
@@ -43,14 +44,15 @@ public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
*
* @return Result of @batchSize
*/
- public Result next() {
- this.scannerResultAggregator.collectData(scannedResult, batchSize);
- Result result = this.scannerResultAggregator.getCollectedResult();
- while (result.size() < batchSize && hasNext()) {
- this.scannerResultAggregator.collectData(scannedResult, batchSize-result.size());
- result.merge(this.scannerResultAggregator.getCollectedResult());
+ public List<Object[]> next() {
+ List<Object[]> collectedResult =
+ this.scannerResultAggregator.collectData(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && hasNext()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectData(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
}
- return result;
+ return collectedResult;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java b/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
index 9e7f007..d917c2a 100644
--- a/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/AbstractScannedResult.java
@@ -93,6 +93,8 @@ public abstract class AbstractScannedResult {
*/
private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+ private int totalDimensionsSize;
+
/**
* parent block indexes
*/
@@ -105,6 +107,7 @@ public abstract class AbstractScannedResult {
this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
}
/**
@@ -156,6 +159,25 @@ public abstract class AbstractScannedResult {
}
/**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
* Just increment the counter incase of query only on measures.
*/
public void incrementCounter() {
@@ -193,6 +215,23 @@ public abstract class AbstractScannedResult {
}
/**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected String[] getNoDictionaryKeyStringArray(int rowId) {
+ String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ new String(dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId));
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
* Below method will be used to get the complex type keys array based
* on row id for all the complex type dimension selected in query
*
@@ -324,6 +363,12 @@ public abstract class AbstractScannedResult {
public abstract byte[] getDictionaryKeyArray();
/**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
* Return the dimension data based on dimension ordinal
*
* @param dimensionOrdinal dimension ordinal
@@ -347,6 +392,14 @@ public abstract class AbstractScannedResult {
public abstract byte[][] getNoDictionaryKeyArray();
/**
+ * Below method will be used to get the no dictionary key
+ * array in string array format for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract String[] getNoDictionaryKeyStringArray();
+
+ /**
* Below method will be used to to check whether measure value
* is null or for a measure
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java b/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
deleted file mode 100644
index c13b0f7..0000000
--- a/core/src/main/java/org/carbondata/scan/result/BatchRawResult.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result;
-
-/**
- * Below class holds the query result of batches.
- */
-public class BatchRawResult extends BatchResult {
-
- /**
- * This method will return one row at a time based on the counter given.
- * @param counter
- * @return
- */
- public Object[] getRawRow(int counter) {
- return rows[counter];
- }
-
- /**
- * For getting the total size.
- * @return
- */
- public int getSize() {
- return rows.length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/BatchResult.java b/core/src/main/java/org/carbondata/scan/result/BatchResult.java
index c5e1182..456717e 100644
--- a/core/src/main/java/org/carbondata/scan/result/BatchResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/BatchResult.java
@@ -19,6 +19,8 @@
package org.carbondata.scan.result;
+import java.util.ArrayList;
+import java.util.List;
import java.util.NoSuchElementException;
import org.carbondata.common.CarbonIterator;
@@ -31,7 +33,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
/**
* list of keys
*/
- protected Object[][] rows;
+ protected List<Object[]> rows;
/**
* counter to check whether all the records are processed or not
@@ -39,7 +41,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
protected int counter;
public BatchResult() {
- this.rows = new Object[0][];
+ this.rows = new ArrayList<>();
}
/**
@@ -47,7 +49,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
*
* @return
*/
- public Object[][] getRows() {
+ public List<Object[]> getRows() {
return rows;
}
@@ -56,10 +58,27 @@ public class BatchResult extends CarbonIterator<Object[]> {
*
* @param rows
*/
- public void setRows(Object[][] rows) {
+ public void setRows(List<Object[]> rows) {
this.rows = rows;
}
+ /**
+ * This method will return one row at a time based on the counter given.
+ * @param counter
+ * @return
+ */
+ public Object[] getRawRow(int counter) {
+ return rows.get(counter);
+ }
+
+ /**
+ * For getting the total size.
+ * @return
+ */
+ public int getSize() {
+ return rows.size();
+ }
+
/**
* Returns {@code true} if the iteration has more elements.
@@ -67,7 +86,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
* @return {@code true} if the iteration has more elements
*/
@Override public boolean hasNext() {
- return counter < rows.length;
+ return counter < rows.size();
}
/**
@@ -79,7 +98,7 @@ public class BatchResult extends CarbonIterator<Object[]> {
if (!hasNext()) {
throw new NoSuchElementException();
}
- Object[] row = rows[counter];
+ Object[] row = rows.get(counter);
counter++;
return row;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java b/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
deleted file mode 100644
index f3085ce..0000000
--- a/core/src/main/java/org/carbondata/scan/result/ListBasedResultWrapper.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result;
-
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-public class ListBasedResultWrapper {
-
- private ByteArrayWrapper key;
-
- private Object[] value;
-
- /**
- * @return the key
- */
- public ByteArrayWrapper getKey() {
- return key;
- }
-
- /**
- * @param key the key to set
- */
- public void setKey(ByteArrayWrapper key) {
- this.key = key;
- }
-
- /**
- * @return the value
- */
- public Object[] getValue() {
- return value;
- }
-
- /**
- * @param value the value to set
- */
- public void setValue(Object[] value) {
- this.value = value;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
index 962d9a3..e519f9e 100644
--- a/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/impl/FilterQueryScannedResult.java
@@ -44,6 +44,15 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(rowMapping[currentRow]);
+ }
+
+ /**
* Below method will be used to get the complex type key array
*
* @return complex type key array
@@ -63,6 +72,16 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(rowMapping[currentRow]);
+ }
+
+ /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
deleted file mode 100644
index 24ebf5b..0000000
--- a/core/src/main/java/org/carbondata/scan/result/impl/ListBasedResult.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.scan.result.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.Result;
-import org.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * Below class is a holder over list based result wrapper
- */
-public class ListBasedResult implements Result<List<ListBasedResultWrapper>, Object> {
-
- /**
- * current result list
- */
- private List<ListBasedResultWrapper> currentRowPointer;
-
- /**
- * all result list , this is required because if we merger all the scanned
- * result from all the blocks in one list, that list creation will take more
- * time as every time list will create a big array and then it will do copy
- * the older element to new array, and creation of big array will also be a
- * problem if memory is fragmented then jvm in to do defragmentation to
- * create a big space, but if divide the data in multiple list than it avoid
- * copy and defragmentation
- */
- private List<List<ListBasedResultWrapper>> allRowsResult;
-
- /**
- * counter to check how many result processed
- */
- private int totalRecordCounter = -1;
-
- /**
- * number of records
- */
- private int totalNumberOfRecords;
-
- /**
- * current counter of the record in list
- */
- private int listRecordCounter = -1;
-
- /**
- * current list counter
- */
- private int currentListCounter;
-
- public ListBasedResult() {
- currentRowPointer =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- allRowsResult =
- new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- /**
- * below method will be used to add the scan result
- */
- @Override public void addScannedResult(List<ListBasedResultWrapper> listBasedResult) {
- this.currentRowPointer = listBasedResult;
- totalNumberOfRecords = listBasedResult.size();
- allRowsResult.add(listBasedResult);
- }
-
- /**
- * Method to check more result is present
- * or not
- */
- @Override public boolean hasNext() {
- if (allRowsResult.size() == 0) {
- return false;
- }
- // As we are storing data in list of list, below code is to check whether
- // any more result is present
- // in the result.
- // first it will check list counter is zero if it is zero
- // than it will check list counter to check how many list has been processed
- // if more list are present and all the list of current list is processed
- // than it will take a new list from all row result list
- totalRecordCounter++;
- listRecordCounter++;
- if (listRecordCounter == 0 || (listRecordCounter >= currentRowPointer.size()
- && currentListCounter < allRowsResult.size())) {
- listRecordCounter = 0;
- currentRowPointer = allRowsResult.get(currentListCounter);
- currentListCounter++;
- }
- return totalRecordCounter < totalNumberOfRecords;
- }
-
- /**
- * @return key
- */
- @Override public ByteArrayWrapper getKey() {
- return currentRowPointer.get(listRecordCounter).getKey();
- }
-
- /**
- * @return will return the value
- */
- @Override public Object[] getValue() {
- return currentRowPointer.get(listRecordCounter).getValue();
- }
-
- /***
- * below method will be used to merge the
- * scanned result
- *
- * @param otherResult return to be merged
- */
- @Override public void merge(Result<List<ListBasedResultWrapper>, Object> otherResult) {
- if (otherResult.size() > 0) {
- totalNumberOfRecords += otherResult.size();
- this.allRowsResult.add(otherResult.getResult());
- }
- }
-
- /**
- * Return the size of the result
- */
- @Override public int size() {
- return totalNumberOfRecords;
- }
-
- /**
- * @return the complete result
- */
- @Override public List<ListBasedResultWrapper> getResult() {
- return currentRowPointer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
index 9782099..00b5563 100644
--- a/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
+++ b/core/src/main/java/org/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
@@ -26,6 +26,15 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(currentRow);
+ }
+
+ /**
* Below method will be used to get the complex type key array
*
* @return complex type key array
@@ -45,6 +54,16 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(currentRow);
+ }
+
+ /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9be7451e/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
index 4b2d777..0539cfa 100644
--- a/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/scan/result/iterator/DetailQueryResultIterator.java
@@ -29,8 +29,6 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.carbondata.scan.model.QueryModel;
import org.carbondata.scan.result.BatchResult;
-import org.carbondata.scan.result.ListBasedResultWrapper;
-import org.carbondata.scan.result.preparator.QueryResultPreparator;
/**
* In case of detail query we cannot keep all the records in memory so for
@@ -39,19 +37,12 @@ import org.carbondata.scan.result.preparator.QueryResultPreparator;
*/
public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator {
- /**
- * to prepare the result
- */
- private QueryResultPreparator<List<ListBasedResultWrapper>, Object> queryResultPreparator;
-
private ExecutorService execService = Executors.newFixedThreadPool(1);
private Future<BatchResult> future;
- public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
- QueryResultPreparator queryResultPreparator) {
+ public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel) {
super(infos, queryModel);
- this.queryResultPreparator = queryResultPreparator;
}
@Override public BatchResult next() {
@@ -81,7 +72,9 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
private Future<BatchResult> execute() {
return execService.submit(new Callable<BatchResult>() {
@Override public BatchResult call() throws QueryExecutionException {
- return queryResultPreparator.prepareQueryResult(dataBlockIterator.next());
+ BatchResult batchResult = new BatchResult();
+ batchResult.setRows(dataBlockIterator.next());
+ return batchResult;
}
});
}