You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by aj...@apache.org on 2020/02/04 13:47:27 UTC
[carbondata] branch master updated: [CARBONDATA-3674] remove
Encoding.DICTIONARY and Encoding.DIRECT_DICTIONARY usage
This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new e925b42 [CARBONDATA-3674] remove Encoding.DICTIONARY and Encoding.DIRECT_DICTIONARY usage
e925b42 is described below
commit e925b425ff7408fe4193b0aba20407c367917c8f
Author: Jacky Li <ja...@qq.com>
AuthorDate: Wed Jan 29 18:24:37 2020 +0800
[CARBONDATA-3674] remove Encoding.DICTIONARY and Encoding.DIRECT_DICTIONARY usage
Why is this PR needed?
After Global Dictionary feature is deprecated, we can refactor the
Dict/NoDict usage. This PR is one of the effort for this refactory.
What changes were proposed in this PR?
This PR remove Encoding.DICTIONARY and Encoding.DIRECT_DICTIONARY usage
in if check. They are changed to check against DataTypes.DATE instead of
checking Encoding.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3595
---
.../carbondata/core/datastore/TableSpec.java | 5 +-
.../core/metadata/schema/table/CarbonTable.java | 8 +--
.../schema/table/column/CarbonDimension.java | 8 ---
.../impl/DictionaryBasedResultCollector.java | 21 ++----
.../impl/DictionaryBasedVectorResultCollector.java | 5 +-
.../RestructureBasedDictionaryResultCollector.java | 2 +-
.../impl/RestructureBasedRawResultCollector.java | 26 ++-----
.../RestructureBasedVectorResultCollector.java | 7 +-
.../core/scan/complextypes/PrimitiveQueryType.java | 8 +--
.../scan/executor/impl/AbstractQueryExecutor.java | 5 +-
.../core/scan/executor/util/QueryUtil.java | 9 ++-
.../core/scan/executor/util/RestructureUtil.java | 14 ++--
.../scan/filter/FilterExpressionProcessor.java | 82 +++++++++-------------
.../carbondata/core/scan/filter/FilterUtil.java | 18 +++--
.../filter/executer/IncludeFilterExecuterImpl.java | 9 ++-
.../executer/RangeValueFilterExecuterImpl.java | 14 ++--
.../filter/executer/RestructureEvaluatorImpl.java | 5 +-
.../executer/RowLevelFilterExecuterImpl.java | 36 +---------
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 19 +++--
...LevelRangeGrtrThanEquaToFilterExecuterImpl.java | 19 +++--
...wLevelRangeLessThanEqualFilterExecuterImpl.java | 18 +++--
.../RowLevelRangeLessThanFilterExecuterImpl.java | 18 +++--
.../resolver/ConditionalFilterResolverImpl.java | 40 ++---------
.../resolver/RowLevelRangeFilterResolverImpl.java | 28 +++-----
.../visitor/FilterInfoTypeVisitorFactory.java | 9 +--
.../core/scan/model/QueryModelBuilder.java | 4 +-
.../iterator/ColumnDriftRawResultIterator.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 48 ++++---------
.../apache/carbondata/core/util/DataTypeUtil.java | 3 +-
.../carbondata/core/util/CarbonUtilTest.java | 54 +-------------
.../datamap/bloom/AbstractBloomDataMapWriter.java | 4 +-
.../datamap/bloom/BloomCoarseGrainDataMap.java | 4 +-
.../datamap/bloom/BloomDataMapWriter.java | 2 +-
.../datamap/lucene/LuceneDataMapFactoryBase.java | 3 +-
.../hadoop/stream/StreamRecordReader.java | 3 +-
.../carbondata/presto/CarbondataPageSource.java | 13 ++--
.../presto/PrestoCarbonVectorizedRecordReader.java | 13 ++--
.../sql/commands/TestCarbonDropCacheCommand.scala | 3 +-
.../spark/load/DataLoadProcessBuilderOnSpark.scala | 2 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 3 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../vectorreader/VectorizedCarbonRecordReader.java | 11 +--
.../datamap/IndexDataMapRebuildRDD.scala | 2 +-
.../command/management/CarbonAddLoadCommand.scala | 5 --
.../command/management/CarbonLoadDataCommand.scala | 6 +-
.../schema/CarbonAlterTableDropColumnCommand.scala | 3 +-
.../command/table/CarbonCreateTableCommand.scala | 12 ----
.../datasources/SparkCarbonTableFormat.scala | 8 +--
.../processing/datatypes/ArrayDataType.java | 12 ++--
.../processing/datatypes/StructDataType.java | 12 ++--
.../loading/CarbonDataLoadConfiguration.java | 8 +--
.../carbondata/processing/loading/DataField.java | 6 +-
.../processing/loading/DataLoadProcessBuilder.java | 2 +-
.../converter/impl/FieldEncoderFactory.java | 7 +-
.../InputProcessorStepWithNoConverterImpl.java | 3 +-
.../processing/merger/CarbonCompactionUtil.java | 17 ++---
.../merger/CompactionResultSortProcessor.java | 7 +-
.../processing/sort/sortdata/TableFieldStat.java | 4 +-
.../store/CarbonFactDataHandlerModel.java | 6 +-
.../carbondata/processing/store/TablePage.java | 4 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
.../processing/util/CarbonDataProcessorUtil.java | 19 +++--
.../java/org/apache/carbondata/tool/DataFile.java | 3 +-
.../org/apache/carbondata/tool/DataSummary.java | 7 +-
64 files changed, 255 insertions(+), 511 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index 03e858f..631429e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -93,8 +93,7 @@ public class TableSpec {
dimensionSpec[dimIndex++] = spec;
noDictionaryDimensionSpec.add(spec);
noSortNoDictDimSpec.add(spec);
- } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension
- .isDirectDictionaryEncoding()) {
+ } else if (dimension.getDataType() == DataTypes.TIMESTAMP) {
spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension, noDictActualPosition++);
dimensionSpec[dimIndex++] = spec;
noDictionaryDimensionSpec.add(spec);
@@ -103,7 +102,7 @@ public class TableSpec {
} else {
noSortNoDictDimSpec.add(spec);
}
- } else if (dimension.isDirectDictionaryEncoding()) {
+ } else if (dimension.getDataType() == DataTypes.DATE) {
spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension, dictActualPosition++);
dimensionSpec[dimIndex++] = spec;
dictDimensionSpec.add(spec);
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index b2f15d2..ba1208e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -43,7 +43,7 @@ import org.apache.carbondata.core.features.TableOperation;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.DatabaseLocationProvider;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaReader;
@@ -367,14 +367,14 @@ public class CarbonTable implements Serializable, Writable {
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfSortColumns++;
}
- if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
+ if (columnSchema.getDataType() != DataTypes.DATE) {
CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
columnSchema.getSchemaOrdinal(), -1, -1);
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfNoDictSortColumns++;
}
allDimensions.add(dimension);
- } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
+ } else if (columnSchema.getDataType() == DataTypes.DATE) {
CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
columnSchema.getSchemaOrdinal(), keyOrdinal++, -1);
allDimensions.add(dimension);
@@ -824,7 +824,7 @@ public class CarbonTable implements Serializable, Writable {
for (int i = 0; i < dimensions.size(); i++) {
CarbonDimension dimension = dimensions.get(i);
if (dimension.isSortColumn() &&
- !dimension.getColumnSchema().hasEncoding(Encoding.DICTIONARY)) {
+ dimension.getDataType() != DataTypes.DATE) {
noDictSortColumns.add(dimension);
}
}
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
index fc2abf5..d220a28 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
@@ -104,14 +104,6 @@ public class CarbonDimension extends CarbonColumn {
this.complexTypeOrdinal = complexTypeOrdinal;
}
- public boolean isDirectDictionaryEncoding() {
- return getEncoder().contains(Encoding.DIRECT_DICTIONARY);
- }
-
- public boolean isGlobalDictionaryEncoding() {
- return getEncoder().contains(Encoding.DICTIONARY);
- }
-
/**
* @return is column participated in sorting or not
*/
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index 554d11a..e6d0528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -28,7 +28,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.complextypes.StructQueryType;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
@@ -59,10 +58,6 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
private int[] actualIndexInSurrogateKey;
- boolean[] dictionaryEncodingArray;
-
- boolean[] directDictionaryEncodingArray;
-
private boolean[] implicitColumnArray;
private boolean[] complexDataTypeArray;
@@ -220,7 +215,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
// TODO have to fill out for dictionary columns. Once the support for push down in
// complex dictionary columns comes.
ByteBuffer buffer;
- if (!dictionaryEncodingArray[i]) {
+ if (queryDimensions[i].getDimension().getDataType() != DataTypes.DATE) {
if (implicitColumnArray[i]) {
throw new RuntimeException("Not Supported Column Type");
} else if (complexDataTypeArray[i]) {
@@ -228,7 +223,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
} else {
buffer = ByteBuffer.wrap(noDictionaryKeys[noDictionaryComplexColumnIndex++]);
}
- } else if (directDictionaryEncodingArray[i]) {
+ } else if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
throw new RuntimeException("Direct Dictionary Column Type Not Supported Yet.");
} else if (complexDataTypeArray[i]) {
buffer = ByteBuffer.wrap(complexTypeKeyArray[complexTypeComplexColumnIndex++]);
@@ -263,7 +258,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> complexDimensionInfoMap, Object[] row, int i,
int actualOrdinal) {
- if (!dictionaryEncodingArray[i]) {
+ if (queryDimensions[i].getDimension().getDataType() != DataTypes.DATE) {
if (implicitColumnArray[i]) {
if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
.equals(queryDimensions[i].getColumnName())) {
@@ -297,7 +292,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
queryDimensions[i].getDimension().getDataType());
}
}
- } else if (directDictionaryEncodingArray[i]) {
+ } else if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
if (directDictionaryGenerators[i] != null) {
row[order[i]] = directDictionaryGenerators[i].getValueFromSurrogate(
surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]]);
@@ -355,8 +350,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
void initDimensionAndMeasureIndexesForFillingData() {
List<Integer> dictionaryIndexes = new ArrayList<Integer>();
for (int i = 0; i < queryDimensions.length; i++) {
- if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) || queryDimensions[i]
- .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
dictionaryIndexes.add(queryDimensions[i].getDimension().getOrdinal());
}
}
@@ -366,16 +360,13 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
actualIndexInSurrogateKey = new int[dictionaryIndexes.size()];
int index = 0;
- dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
- directDictionaryEncodingArray = CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
implicitColumnArray = CarbonUtil.getImplicitColumnArray(queryDimensions);
complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
parentToChildColumnsMap.clear();
queryDimensionToComplexParentOrdinal.clear();
for (int i = 0; i < queryDimensions.length; i++) {
- if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) || queryDimensions[i]
- .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
actualIndexInSurrogateKey[index++] =
Arrays.binarySearch(primitive, queryDimensions[i].getDimension().getOrdinal());
}
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index 719d322..9f18894 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
@@ -97,13 +98,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
- } else if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ } else if (queryDimensions[i].getDimension().getDataType() != DataTypes.DATE) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
noDictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
- } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ } else if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
dictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 522aaf1..fa0b2a9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -90,7 +90,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
for (int i = 0; i < queryDimensions.length; i++) {
// fill default value in case the dimension does not exist in the current block
if (!dimensionInfo.getDimensionExists()[i]) {
- if (dictionaryEncodingArray[i] || directDictionaryEncodingArray[i]) {
+ if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
row[order[i]] = dimensionInfo.getDefaultValues()[i];
dictionaryColumnIndex++;
} else if (queryDimensions[i].getDimension().getDataType() == DataTypes.STRING) {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
index 71b1013..f055ad8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
@@ -72,7 +72,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
int dimCounterInCurrentBlock = 0;
for (int i = 0; i < queryDimensions.length; i++) {
- if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (queryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
if (executionInfo.getDimensionInfo().getDimensionExists()[i]) {
// get the dictionary key ordinal as column cardinality in segment properties
// will only be for dictionary encoded columns
@@ -86,27 +86,16 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
} else {
// partitioner index will be 1 every column will be in columnar format
updatedDimensionPartitioner.add(1);
- // for direct dictionary 4 bytes need to be allocated else 1
- if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- updatedColumnCardinality.add(Integer.MAX_VALUE);
- } else {
- // cardinality will be 2 will user has provided a default value
- byte[] defaultValue = queryDimensions[i].getDimension().getDefaultValue();
- if (null != defaultValue) {
- updatedColumnCardinality
- .add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
- } else {
- updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
- }
- }
+ // for direct dictionary 4 bytes need to be allocated
+ updatedColumnCardinality.add(Integer.MAX_VALUE);
}
}
}
if (!updatedColumnCardinality.isEmpty()) {
int[] latestColumnCardinality = ArrayUtils.toPrimitive(
- updatedColumnCardinality.toArray(new Integer[updatedColumnCardinality.size()]));
+ updatedColumnCardinality.toArray(new Integer[0]));
int[] latestColumnPartitioner = ArrayUtils.toPrimitive(
- updatedDimensionPartitioner.toArray(new Integer[updatedDimensionPartitioner.size()]));
+ updatedDimensionPartitioner.toArray(new Integer[0]));
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(latestColumnCardinality, latestColumnPartitioner);
restructuredKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
@@ -186,8 +175,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
int existingColumnKeyArrayIndex = 0;
int newKeyArrayIndex = 0;
for (int i = 0; i < dimensionInfo.getDimensionExists().length; i++) {
- if (CarbonUtil.hasEncoding(actualQueryDimensions[i].getDimension().getEncoder(),
- Encoding.DICTIONARY)) {
+ if (actualQueryDimensions[i].getDimension().getDataType() == DataTypes.DATE) {
// if dimension exists then add the key array value else add the default value
if (dimensionInfo.getDimensionExists()[i] && null != keyArray && 0 != keyArray.length) {
keyArrayWithNewAddedColumns[newKeyArrayIndex++] =
@@ -225,7 +213,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
int existingColumnValueIndex = 0;
int newKeyArrayIndex = 0;
for (int i = 0; i < dimensionInfo.getDimensionExists().length; i++) {
- if (!actualQueryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)
+ if (actualQueryDimensions[i].getDimension().getDataType() != DataTypes.DATE
&& !actualQueryDimensions[i].getDimension().hasEncoding(Encoding.IMPLICIT)) {
// if dimension exists then add the byte array value else add the default value
if (dimensionInfo.getDimensionExists()[i]) {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index 9b7da25..0274627 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
@@ -140,14 +139,10 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
int queryOrder = executionInfo.getActualQueryDimensions()[i].getOrdinal();
CarbonDimension dimension =
executionInfo.getActualQueryDimensions()[i].getDimension();
- if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimension.getDataType() == DataTypes.DATE) {
// fill direct dictionary column data
fillDirectDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder],
dimensionInfo.getDefaultValues()[i]);
- } else if (dimension.hasEncoding(Encoding.DICTIONARY)) {
- // fill dictionary column data
- fillDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder],
- dimensionInfo.getDefaultValues()[i]);
} else {
// fill no dictionary data
fillNoDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder],
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index adfce1f..a8789f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -47,8 +47,6 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
private boolean isDirectDictionary;
- private DirectDictionaryGenerator directDictGenForDate;
-
public PrimitiveQueryType(String name, String parentName, int blockIndex, DataType dataType,
int keySize, boolean isDirectDictionary) {
super(name, parentName, blockIndex);
@@ -57,8 +55,6 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
this.name = name;
this.parentName = parentName;
this.isDirectDictionary = isDirectDictionary;
- this.directDictGenForDate =
- DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.DATE);
}
@Override
@@ -167,7 +163,9 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
if (value.length == 0) {
actualData = null;
} else {
- actualData = this.directDictGenForDate.getValueFromSurrogate(
+ DirectDictionaryGenerator directDictGenForDate =
+ DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.DATE);
+ actualData = directDictGenForDate.getValueFromSurrogate(
ByteUtil.toXorInt(value, 0, CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
} else {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index dd14b04..6d591de 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -47,7 +47,7 @@ import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -641,8 +641,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
while (counter < queryDimension.size()) {
if (queryDimension.get(counter).getDimension().getNumberOfChild() > 0) {
counter += queryDimension.get(counter).getDimension().getNumberOfChild();
- } else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
- Encoding.DICTIONARY)) {
+ } else if (queryDimension.get(counter).getDimension().getDataType() != DataTypes.DATE) {
counter++;
} else {
fixedLengthDimensionOrdinal.add(blockMetadataInfo.getDimensionOrdinalToChunkMapping()
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index b3d3383..b3db954 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -354,8 +354,7 @@ public class QueryUtil {
List<Integer> dictionaryDimensionChunkIndex,
List<Integer> noDictionaryDimensionChunkIndex) {
for (ProjectionDimension queryDimension : projectDimensions) {
- if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
- && queryDimension.getDimension().getNumberOfChild() == 0) {
+ if (queryDimension.getDimension().getDataType() == DataTypes.DATE) {
dictionaryDimensionChunkIndex
.add(columnOrdinalToChunkIndexMapping.get(queryDimension.getDimension().getOrdinal()));
} else if (
@@ -534,11 +533,11 @@ public class QueryUtil {
private static void getChildDimensionOrdinal(CarbonDimension queryDimensions,
Set<Integer> filterDimensionsOrdinal) {
for (int j = 0; j < queryDimensions.getNumberOfChild(); j++) {
- List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
- if (queryDimensions.getListOfChildDimensions().get(j).getNumberOfChild() > 0) {
+ CarbonDimension dimension = queryDimensions.getListOfChildDimensions().get(j);
+ if (dimension.getNumberOfChild() > 0) {
getChildDimensionOrdinal(queryDimensions.getListOfChildDimensions().get(j),
filterDimensionsOrdinal);
- } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ } else if (dimension.getDataType() != DataTypes.DATE) {
filterDimensionsOrdinal.add(queryDimensions.getListOfChildDimensions().get(j).getOrdinal());
}
}
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index afadaeb..b573e0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.commons.lang3.ArrayUtils;
@@ -137,7 +136,7 @@ public class RestructureUtil {
// set the flag to say whether a new dictionary column or no dictionary column
// has been added. This will be useful after restructure for compaction scenarios where
// newly added columns data need to be filled
- if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (queryDimension.getDimension().getDataType() == DataTypes.DATE) {
dimensionInfo.setDictionaryColumnAdded(true);
newDictionaryColumnCount++;
} else {
@@ -235,15 +234,10 @@ public class RestructureUtil {
public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) {
byte[] defaultValue = queryDimension.getDefaultValue();
Object defaultValueToBeConsidered = null;
- if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) {
+ if (queryDimension.getDataType() == DataTypes.DATE) {
// direct dictionary case
- if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
- defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),
- queryDimension.getDefaultValue());
- } else {
- // dictionary case
- defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue);
- }
+ defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),
+ queryDimension.getDefaultValue());
} else {
// no dictionary
defaultValueToBeConsidered =
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 8a1105c..c607028 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -225,30 +225,24 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
- // getting new dim index.
- if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY) || currentCondExpression
- .isAlreadyResolved()) {
- // In case of Range Column Dictionary Include we do not need to resolve the range
- // expression as it is already resolved and has the surrogates in the filter value
- if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
- && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
- FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
- || FilterUtil
- .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
- return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier);
- }
- if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
- || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
- || currentCondExpression.getFilterExpressionType()
- == ExpressionType.GREATERTHAN_EQUALTO
- || currentCondExpression.getFilterExpressionType()
- == ExpressionType.LESSTHAN_EQUALTO) {
- return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier);
- }
+ // In case of Range Column Dictionary Include we do not need to resolve the range
+ // expression as it is already resolved and has the surrogates in the filter value
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.GREATERTHAN_EQUALTO
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.LESSTHAN_EQUALTO) {
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
}
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
@@ -282,28 +276,23 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, true);
}
- if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
- && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
- FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
- || FilterUtil
- .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
- return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier);
- }
- if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
- || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
- || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
- || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
-
- return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier);
- }
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+ if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
}
+
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
}
break;
@@ -314,10 +303,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
column = condExpression.getColumnList().get(0).getCarbonColumn();
if (condExpression.isSingleColumn() && !column.isComplex()) {
condExpression = (ConditionalExpression) expression;
- if ((condExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY))
- || (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) {
+ if (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
return new ConditionalFilterResolverImpl(expression, true, true,
condExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 0e01f2a..22ac5ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -788,7 +788,7 @@ public final class FilterUtil {
public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo,
CarbonDimension carbonDimension, SegmentProperties segmentProperties, boolean isExclude,
boolean isDictRange) {
- if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
+ if (carbonDimension.getDataType() != DataTypes.DATE) {
return columnFilterInfo.getNoDictionaryFilterValuesList()
.toArray((new byte[columnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
}
@@ -870,7 +870,7 @@ public final class FilterUtil {
dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
// step 1
for (Map.Entry<CarbonDimension, List<ColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ if (entry.getKey().getDataType() != DataTypes.DATE) {
List<ColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
if (null == listOfDimColFilterInfo) {
continue;
@@ -933,7 +933,7 @@ public final class FilterUtil {
dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
// step 1
for (Map.Entry<CarbonDimension, List<ColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ if (entry.getKey().getDataType() != DataTypes.DATE) {
List<ColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
if (null == listOfDimColFilterInfo) {
continue;
@@ -984,7 +984,7 @@ public final class FilterUtil {
SegmentProperties segmentProperties, long[] startKey, List<long[]> startKeyList) {
for (Map.Entry<CarbonDimension, List<ColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
List<ColumnFilterInfo> values = entry.getValue();
- if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ if (null == values || entry.getKey().getDataType() != DataTypes.DATE) {
continue;
}
boolean isExcludePresent = false;
@@ -1037,11 +1037,9 @@ public final class FilterUtil {
List<CarbonDimension> listOfCarbonDimPartOfKeyGen =
new ArrayList<CarbonDimension>(carbonDimensions.size());
for (CarbonDimension carbonDim : carbonDimensions) {
- if (CarbonUtil.hasEncoding(carbonDim.getEncoder(), Encoding.DICTIONARY) || CarbonUtil
- .hasEncoding(carbonDim.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+ if (carbonDim.getDataType() == DataTypes.DATE) {
listOfCarbonDimPartOfKeyGen.add(carbonDim);
}
-
}
return listOfCarbonDimPartOfKeyGen;
}
@@ -1051,7 +1049,7 @@ public final class FilterUtil {
SegmentProperties segmentProperties, long[] endKey, List<long[]> endKeyList) {
for (Map.Entry<CarbonDimension, List<ColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
List<ColumnFilterInfo> values = entry.getValue();
- if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ if (null == values || entry.getKey().getDataType() != DataTypes.DATE) {
continue;
}
boolean isExcludeFilterPresent = false;
@@ -1717,8 +1715,8 @@ public final class FilterUtil {
public static int compareValues(byte[] filterValue, byte[] minMaxBytes,
CarbonDimension carbonDimension, boolean isMin) {
DataType dataType = carbonDimension.getDataType();
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !carbonDimension
- .hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dataType != DataTypes.DATE) {
Object value =
DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(minMaxBytes, dataType);
// filter value should be in range of max and min value i.e
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index e8e6cc8..41a8f85 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.filter.FilterExecutorUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
@@ -179,7 +179,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
// for no dictionary measure column comparison can be done
// on the original data as like measure column
if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
- && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ && dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.DATE) {
scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex],
dimensionRawColumnChunk.getMinValues()[columnIndex],
dimColumnExecuterInfo.getFilterKeys(),
@@ -500,9 +500,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
chunkIndex = dimColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil
- .isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
- && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType()) &&
+ dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.DATE) {
isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues,
dimColumnEvaluatorInfo.getDimension().getDataType());
} else {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 4031e77..f18e007 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
@@ -642,16 +641,15 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter {
}
} else {
byte[] defaultValue = null;
- if (dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimColEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) {
defaultValue =
FilterUtil.getDefaultNullValue(dimColEvaluatorInfo.getDimension(), segmentProperties);
- } else {
- if (dimColEvaluatorInfo.getDimension().getDataType() == DataTypes.STRING) {
- defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
- } else if (!dimensionColumnPage.isAdaptiveEncoded()) {
- defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
- }
+ } else if (dimColEvaluatorInfo.getDimension().getDataType() == DataTypes.STRING) {
+ defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
+ } else if (!dimensionColumnPage.isAdaptiveEncoded()) {
+ defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
+
// evaluate result for lower range value first and then perform and operation in the
// upper range value in order to compute the final result
bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[0],
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
index c507122..452e07f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -56,7 +55,7 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
ColumnFilterInfo filterValues = dimColumnEvaluatorInfo.getFilterValues();
CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
byte[] defaultValue = dimension.getDefaultValue();
- if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
+ if (dimension.getDataType() != DataTypes.DATE) {
// for no dictionary cases
// 3 cases: is NUll, is Not Null and filter on default value of newly added column
if (null == defaultValue && dimension.getDataType() == DataTypes.STRING) {
@@ -79,7 +78,7 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
// 3 cases: is NUll, is Not Null and filter on default value of newly added column
int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
if (null != defaultValue) {
- if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimension.getDataType() == DataTypes.DATE) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dimension.getDataType());
if (directDictionaryGenerator != null) {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index d26c58b..1eda5c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -42,7 +41,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -396,7 +394,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
DimensionColumnPage columnDataChunk =
blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]
.decodeColumnPage(pageIndex);
- if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) &&
+ if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.DATE &&
(columnDataChunk instanceof VariableLengthDimensionColumnPage ||
columnDataChunk instanceof ColumnPageWrapper)) {
@@ -494,38 +492,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @return
*/
private Object getDimensionDefaultValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
- Object dimensionDefaultValue = null;
CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
- if (dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- byte[] defaultValue = dimension.getDefaultValue();
- if (null != defaultValue) {
- dimensionDefaultValue =
- new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- }
- } else {
- dimensionDefaultValue = RestructureUtil.validateAndGetDefaultValue(dimension);
- }
- return dimensionDefaultValue;
- }
-
- /**
- * method will read the actual data from the direct dictionary generator
- * by passing direct dictionary value.
- *
- * @param dimColumnEvaluatorInfo
- * @param dictionaryValue
- * @return
- */
- private Object getFilterActualValueFromDirectDictionaryValue(
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) {
- if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.DATE) {
- return dateDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
- } else if (dimColumnEvaluatorInfo.getDimension().getDataType() == DataTypes.TIMESTAMP) {
- return timestampDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
- } else {
- throw new RuntimeException("Invalid data type for dierct dictionary");
- }
+ return RestructureUtil.validateAndGetDefaultValue(dimension);
}
@Override
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 45ee863..8f5d8c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -135,8 +134,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
isScanRequired = isScanRequired(maxValue, filterRangeValues, dataType);
} else {
isScanRequired =
@@ -386,8 +385,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
scanRequired =
isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
} else {
@@ -454,16 +453,14 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
- } else if (dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ } else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
defaultValue = FilterUtil
.getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties);
} else if (!dimensionColumnPage.isAdaptiveEncoded()) {
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
- if (dimensionColumnPage.isNoDicitionaryColumn() || (
- dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
- && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) {
+ if (dimensionColumnPage.isNoDicitionaryColumn() ||
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
@@ -593,7 +590,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
@Override
public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
super.readColumnChunks(rawBlockletColumnChunks);
}
int chunkIndex = dimensionChunkIndex[0];
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index b986b30..0510516 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -133,8 +132,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
isScanRequired = isScanRequired(maxValue, filterRangeValues, dataType);
} else {
isScanRequired =
@@ -329,8 +328,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
scanRequired =
isScanRequired(rawColumnChunk.getMaxValues()[columnIndex], this.filterRangeValues,
dataType);
@@ -457,16 +456,14 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
- } else if (dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ } else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
defaultValue = FilterUtil
.getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties);
} else if (!dimensionColumnPage.isAdaptiveEncoded()) {
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
- if (dimensionColumnPage.isNoDicitionaryColumn() || (
- dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
- && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) {
+ if (dimensionColumnPage.isNoDicitionaryColumn() ||
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
@@ -581,7 +578,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
@Override
public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
super.readColumnChunks(rawBlockletColumnChunks);
}
int chunkIndex = dimensionChunkIndex[0];
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 4305da0..f68a29a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -133,8 +132,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
isScanRequired = isScanRequired(minValue, filterRangeValues, dataType);
} else {
isScanRequired =
@@ -310,8 +309,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
scanRequired =
isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
} else {
@@ -427,7 +426,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
- if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
defaultValue = FilterUtil
.getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties);
} else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.STRING) {
@@ -444,9 +443,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
}
- if (dimensionColumnPage.isNoDicitionaryColumn() || (
- dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
- && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) {
+ if (dimensionColumnPage.isNoDicitionaryColumn() ||
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
@@ -601,7 +599,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)
throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
super.readColumnChunks(rawBlockletColumnChunks);
}
int chunkIndex = dimensionChunkIndex[0];
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
index 37566f8..a25a452 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -133,8 +132,8 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
isScanRequired = isScanRequired(minValue, filterRangeValues, dataType);
} else {
isScanRequired =
@@ -306,8 +305,8 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
// for no dictionary measure column comparison can be done
// on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (DataTypeUtil.isPrimitiveColumn(dataType) &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
scanRequired =
isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
} else {
@@ -422,7 +421,7 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
- if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
defaultValue = FilterUtil
.getDefaultNullValue(dimColEvaluatorInfoList.get(0).getDimension(), segmentProperties);
} else if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.STRING) {
@@ -439,9 +438,8 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.STRING) {
defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
}
- if (dimensionColumnPage.isNoDicitionaryColumn() || (
- dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
- && dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY))) {
+ if (dimensionColumnPage.isNoDicitionaryColumn() ||
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
@@ -609,7 +607,7 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
@Override
public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
super.readColumnChunks(rawBlockletColumnChunks);
}
int chunkIndex = dimensionChunkIndex[0];
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 2cf08d8..deee498 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -23,7 +23,6 @@ import java.util.SortedMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
@@ -86,11 +85,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
// column expression.
// we need to check if the other expression contains column
// expression or not in depth.
- if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
- || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) && ((
- (null != columnExpression.getDimension()) && (columnExpression.getDimension()
- .hasEncoding(Encoding.DICTIONARY) && !columnExpression.getDimension()
- .hasEncoding(Encoding.DIRECT_DICTIONARY))))) {
+ if (FilterUtil.checkIfExpressionContainsColumn(rightExp)) {
isExpressionResolve = true;
} else {
//Visitor pattern is been used in this scenario inorder to populate the
@@ -150,10 +145,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
metadata.setColumnExpression(columnList.get(0));
metadata.setExpression(exp);
metadata.setIncludeFilter(isIncludeFilter);
- if ((null != columnList.get(0).getDimension()) &&
- (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) ||
- columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY))
- || (exp instanceof RangeExpression)) {
+ if ((null != columnList.get(0).getDimension()) || (exp instanceof RangeExpression)) {
dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0), exp),
metadata);
@@ -253,31 +245,13 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
case NOT_IN:
return FilterExecuterType.EXCLUDE;
case RANGE:
- if (isColDictionary()) {
- return FilterExecuterType.INCLUDE;
- } else {
- return FilterExecuterType.RANGE;
- }
+ return FilterExecuterType.RANGE;
default:
return FilterExecuterType.INCLUDE;
}
}
- private boolean isColDictionary() {
- RangeExpression condExp = (RangeExpression) exp;
- List<ColumnExpression> columnList = condExp.getColumnList();
- if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- if (columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return false;
- } else {
- return true;
- }
- } else {
- return false;
- }
- }
-
/**
* This method will return the filter values which is present in the range level
* conditional expressions.
@@ -286,13 +260,13 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
*/
public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
- if (null != dimColResolvedFilterInfo.getFilterValues() && !dimColResolvedFilterInfo
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (null != dimColResolvedFilterInfo.getFilterValues() &&
+ dimColResolvedFilterInfo.getDimension().getDataType() != DataTypes.DATE) {
List<byte[]> noDictFilterValuesList =
dimColResolvedFilterInfo.getFilterValues().getNoDictionaryFilterValuesList();
return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
- } else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo
- .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ } else if (null != dimColResolvedFilterInfo.getFilterValues() &&
+ dimColResolvedFilterInfo.getDimension().getDataType() == DataTypes.DATE) {
return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(),
this.dimColResolvedFilterInfo.getDimension(), segmentProperties, false, false);
}
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index bf08a45..e630c03 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -29,7 +29,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
@@ -76,15 +75,15 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
*/
public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
- if (dimColEvaluatorInfoList.size() > 0 && null != dimColEvaluatorInfoList.get(0)
- .getFilterValues() && !dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DICTIONARY)) {
+ if (dimColEvaluatorInfoList.size() > 0 &&
+ null != dimColEvaluatorInfoList.get(0).getFilterValues() &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() != DataTypes.DATE) {
List<byte[]> noDictFilterValuesList =
dimColEvaluatorInfoList.get(0).getFilterValues().getNoDictionaryFilterValuesList();
return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
- } else if (dimColEvaluatorInfoList.size() > 0 && null != dimColEvaluatorInfoList.get(0)
- .getFilterValues() && dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ } else if (dimColEvaluatorInfoList.size() > 0 &&
+ null != dimColEvaluatorInfoList.get(0).getFilterValues() &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
CarbonDimension dimensionFromCurrentBlock = segmentProperties
.getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension());
if (null != dimensionFromCurrentBlock) {
@@ -93,9 +92,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
} else {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(), false);
}
- } else if (dimColEvaluatorInfoList.size() > 0 && null != dimColEvaluatorInfoList.get(0)
- .getFilterValues() && dimColEvaluatorInfoList.get(0).getDimension()
- .hasEncoding(Encoding.DICTIONARY)) {
+ } else if (dimColEvaluatorInfoList.size() > 0 &&
+ null != dimColEvaluatorInfoList.get(0).getFilterValues() &&
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType() == DataTypes.DATE) {
CarbonDimension dimensionFromCurrentBlock = segmentProperties
.getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension());
if (null != dimensionFromCurrentBlock) {
@@ -255,19 +254,12 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
dimColumnEvaluatorInfo.setRowIndex(index++);
dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (columnExpression.getDimension().getDataType() == DataTypes.DATE) {
if (!isIncludeFilter) {
filterInfo.setExcludeFilterList(getDirectSurrogateValues(columnExpression));
} else {
filterInfo.setFilterList(getDirectSurrogateValues(columnExpression));
}
- } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)
- && !columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- if (!isIncludeFilter) {
- filterInfo.setExcludeFilterList(getSurrogateValues());
- } else {
- filterInfo.setFilterList(getSurrogateValues());
- }
} else {
filterInfo.setFilterListForNoDictionaryCols(getNoDictionaryRangeValues());
}
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
index 8bad320..3a11f2a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -34,21 +35,21 @@ public class FilterInfoTypeVisitorFactory {
public static ResolvedFilterInfoVisitorIntf getResolvedFilterInfoVisitor(
ColumnExpression columnExpression, Expression exp) {
if (exp instanceof RangeExpression) {
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (columnExpression.getDimension().getDataType() == DataTypes.DATE) {
return new RangeDirectDictionaryVisitor();
} else if (columnExpression.getDimension().hasEncoding(Encoding.IMPLICIT)) {
return new ImplicitColumnVisitor();
- } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ } else if (columnExpression.getDimension().getDataType() != DataTypes.DATE) {
return new RangeNoDictionaryTypeVisitor();
}
}
else {
if (null != columnExpression.getDimension()) {
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (columnExpression.getDimension().getDataType() == DataTypes.DATE) {
return new CustomTypeDictionaryVisitor();
} else if (columnExpression.getDimension().hasEncoding(Encoding.IMPLICIT)) {
return new ImplicitColumnVisitor();
- } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ } else if (columnExpression.getDimension().getDataType() != DataTypes.DATE) {
return new NoDictionaryTypeVisitor();
}
} else if (columnExpression.getMeasure().isMeasure()) {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
index 4a0af29..696993d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
@@ -26,7 +26,7 @@ import java.util.Objects;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapFilter;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -63,7 +63,7 @@ public class QueryModelBuilder {
CarbonDimension dimension = table.getDimensionByName(projectionColumnName);
if (dimension != null) {
CarbonDimension complexParentDimension = dimension.getComplexParentDimension();
- if (null != complexParentDimension && dimension.hasEncoding(Encoding.DICTIONARY)) {
+ if (null != complexParentDimension && dimension.getDataType() == DataTypes.DATE) {
if (!isAlreadyExists(complexParentDimension, projection.getDimensions())) {
projection.addDimension(complexParentDimension, i);
i++;
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ColumnDriftRawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ColumnDriftRawResultIterator.java
index 3b54b6d..6b3f16c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ColumnDriftRawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ColumnDriftRawResultIterator.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
@@ -67,7 +67,7 @@ public class ColumnDriftRawResultIterator extends RawResultIterator {
new ArrayList<>(destinationSegProperties.getDimensions().size());
for (CarbonDimension dimension : destinationSegProperties.getDimensions()) {
if (dimension.getNumberOfChild() == 0) {
- if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
+ if (dimension.getDataType() != DataTypes.DATE) {
noDictDims.add(dimension);
}
}
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d6607dc..a01827a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -850,24 +850,6 @@ public final class CarbonUtil {
return false;
}
- public static boolean[] getDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
- boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
- for (int i = 0; i < queryDimensions.length; i++) {
- dictionaryEncodingArray[i] =
- queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY);
- }
- return dictionaryEncodingArray;
- }
-
- public static boolean[] getDirectDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
- boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
- for (int i = 0; i < queryDimensions.length; i++) {
- dictionaryEncodingArray[i] =
- queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY);
- }
- return dictionaryEncodingArray;
- }
-
public static boolean[] getImplicitColumnArray(ProjectionDimension[] queryDimensions) {
boolean[] implicitColumnArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
@@ -1028,7 +1010,7 @@ public final class CarbonUtil {
if (null != childs && childs.size() > 0) {
break;
}
- if (hasEncoding(carbonDimension.getEncoder(), Encoding.DICTIONARY)) {
+ if (carbonDimension.getDataType() == DataTypes.DATE) {
isDictionaryDimensions.add(true);
} else {
isDictionaryDimensions.add(false);
@@ -2916,9 +2898,9 @@ public final class CarbonUtil {
continue;
}
// column should be no dictionary string datatype column or varchar datatype column
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING)
- || column.getDataType().equals(DataTypes.VARCHAR)) && !column
- .hasEncoding(Encoding.DICTIONARY)) {
+ if (column.isDimensionColumn() &&
+ (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR))) {
column.setLocalDictColumn(true);
}
// if local dictionary exclude columns is defined, then set for all no dictionary string
@@ -2950,9 +2932,9 @@ public final class CarbonUtil {
}
//if column is primitive string or varchar and no dictionary column,then set local
// dictionary if not specified as local dictionary exclude
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING)
- || column.getDataType().equals(DataTypes.VARCHAR)) && !column
- .hasEncoding(Encoding.DICTIONARY)) {
+ if (column.isDimensionColumn() &&
+ (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR))) {
if (!Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName())) {
column.setLocalDictColumn(true);
}
@@ -2977,11 +2959,11 @@ public final class CarbonUtil {
} else {
continue;
}
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) ||
- column.getDataType().equals(DataTypes.VARCHAR)) &&
- !column.hasEncoding(Encoding.DICTIONARY)
- && localDictIncludeColumns.toLowerCase()
- .contains(column.getColumnName().toLowerCase())) {
+ if (column.isDimensionColumn() &&
+ (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR)) &&
+ localDictIncludeColumns.toLowerCase()
+ .contains(column.getColumnName().toLowerCase())) {
for (String dictColumn : listOfDictionaryIncludeColumns) {
if (dictColumn.trim().equalsIgnoreCase(column.getColumnName())) {
column.setLocalDictColumn(true);
@@ -3009,9 +2991,9 @@ public final class CarbonUtil {
dimensionOrdinal++;
setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild());
} else {
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) ||
- column.getDataType().equals(DataTypes.VARCHAR)) &&
- !column.hasEncoding(Encoding.DICTIONARY)) {
+ if (column.isDimensionColumn() &&
+ (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR))) {
column.setLocalDictColumn(true);
}
}
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 58137a0..7c5dc1d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -892,7 +891,7 @@ public final class DataTypeUtil {
return String.valueOf(value)
.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
} else if (dataType == DataTypes.TIMESTAMP) {
- if (columnSchema.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (columnSchema.getDataType() == DataTypes.DATE) {
DirectDictionaryGenerator directDictionaryGenerator1 = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(columnSchema.getDataType());
int value1 = directDictionaryGenerator1.generateDirectSurrogateKey(data);
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index baf4149..4dd4316 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -493,58 +493,6 @@ public class CarbonUtilTest {
assertTrue(!DataTypes.DATE.isComplexType());
}
- @Test public void testToGetDictionaryEncodingArray() {
- ColumnSchema column1Schema = new ColumnSchema();
- ColumnSchema column2Schema = new ColumnSchema();
- column1Schema.setColumnName("Column1");
- List<Encoding> encoding = new ArrayList<>();
- encoding.add(Encoding.DICTIONARY);
- column1Schema.setEncodingList(encoding);
- ProjectionDimension
- column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
-
- column2Schema.setColumnName("Column2");
- List<Encoding> encoding2 = new ArrayList<>();
- encoding2.add(Encoding.DELTA);
- column2Schema.setEncodingList(encoding2);
- ProjectionDimension
- column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
-
- ProjectionDimension[] queryDimensions = { column1, column2 };
-
- boolean[] dictionaryEncoding = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
- boolean[] expectedDictionaryEncoding = { true, false };
- for (int i = 0; i < dictionaryEncoding.length; i++) {
- assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
- }
- }
-
- @Test public void testToGetDirectDictionaryEncodingArray() {
- ColumnSchema column1Schema = new ColumnSchema();
- ColumnSchema column2Schema = new ColumnSchema();
- column1Schema.setColumnName("Column1");
- List<Encoding> encoding = new ArrayList<>();
- encoding.add(Encoding.DIRECT_DICTIONARY);
- column1Schema.setEncodingList(encoding);
- ProjectionDimension
- column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
-
- column2Schema.setColumnName("Column2");
- List<Encoding> encoding2 = new ArrayList<>();
- encoding2.add(Encoding.DELTA);
- column2Schema.setEncodingList(encoding2);
- ProjectionDimension
- column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
-
- ProjectionDimension[] queryDimensions = { column1, column2 };
-
- boolean[] dictionaryEncoding = CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
- boolean[] expectedDictionaryEncoding = { true, false };
- for (int i = 0; i < dictionaryEncoding.length; i++) {
- assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
- }
- }
-
@Test public void testToGetComplexDataTypeArray() {
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
@@ -755,8 +703,10 @@ public class CarbonUtilTest {
ColumnSchema column2Schema = new ColumnSchema();
ColumnSchema column3Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
+ column1Schema.setDataType(DataTypes.DATE);
column1Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
column2Schema.setColumnName("Column2");
+ column2Schema.setDataType(DataTypes.DATE);
column2Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
column3Schema.setColumnName("Column3");
column3Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.INVERTED_INDEX));
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
index 17e00f7..289f43d 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
@@ -31,7 +31,6 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -135,8 +134,7 @@ public abstract class AbstractBloomDataMapWriter extends DataMapWriter {
}
indexValue = CarbonUtil.getValueAsBytes(indexColumns.get(indexColIdx).getDataType(), value);
} else {
- if (indexColumns.get(indexColIdx).hasEncoding(Encoding.DICTIONARY)
- || indexColumns.get(indexColIdx).hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (indexColumns.get(indexColIdx).getDataType() == DataTypes.DATE) {
indexValue = convertDictionaryValue(indexColIdx, value);
} else {
indexValue = convertNonDictionaryValue(indexColIdx, value);
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index 6d18e47..6917205 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -37,7 +37,6 @@ import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
@@ -306,8 +305,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
convertedValue = BooleanConvert.boolean2Byte((Boolean)convertedValue);
}
internalFilterValue = CarbonUtil.getValueAsBytes(carbonColumn.getDataType(), convertedValue);
- } else if (carbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY) ||
- carbonColumn.hasEncoding(Encoding.DICTIONARY)) {
+ } else if (carbonColumn.getDataType() == DataTypes.DATE) {
// for dictionary/date columns, convert the surrogate key to bytes
internalFilterValue = CarbonUtil.getValueAsBytes(DataTypes.INT, convertedValue);
} else {
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomDataMapWriter.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomDataMapWriter.java
index 6fed0ce..d507409 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomDataMapWriter.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomDataMapWriter.java
@@ -60,7 +60,7 @@ public class BloomDataMapWriter extends AbstractBloomDataMapWriter {
this.indexCol2MdkIdx = new HashMap<>();
int idx = 0;
for (final CarbonDimension dimension : segmentProperties.getDimensions()) {
- if (!dimension.isGlobalDictionaryEncoding() && !dimension.isDirectDictionaryEncoding()) {
+ if (dimension.getDataType() != DataTypes.DATE) {
continue;
}
boolean isExistInIndex = CollectionUtils.exists(indexColumns, new Predicate() {
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 247e856..fe4542a 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -343,7 +342,7 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
throw new MalformedDataMapCommandException(String.format(
"Only String column is supported, column '%s' is %s type. ",
column.getColName(), column.getDataType()));
- } else if (column.getEncoder().contains(Encoding.DICTIONARY)) {
+ } else if (column.getDataType() == DataTypes.DATE) {
throw new MalformedDataMapCommandException(String.format(
"Dictionary column is not supported, column '%s' is dictionary column",
column.getColName()));
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/stream/StreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/stream/StreamRecordReader.java
index ac4e48e..bf4c7d4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/stream/StreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/stream/StreamRecordReader.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -152,7 +151,7 @@ public class StreamRecordReader extends RecordReader<Void, Object> {
isNoDictColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(storageColumns);
directDictionaryGenerators = new DirectDictionaryGenerator[storageColumns.length];
for (int i = 0; i < storageColumns.length; i++) {
- if (storageColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (storageColumns[i].getDataType() == DataTypes.DATE) {
directDictionaryGenerators[i] = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(storageColumns[i].getDataType());
}
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index fd7dde7..7ac2ca5 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.StructField;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
@@ -142,18 +141,16 @@ class CarbondataPageSource implements ConnectorPageSource {
fields = new StructField[queryDimension.size() + queryMeasures.size()];
for (int i = 0; i < queryDimension.size(); i++) {
ProjectionDimension dim = queryDimension.get(i);
- if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dim.getDimension().isComplex()) {
+ fields[dim.getOrdinal()] =
+ new StructField(dim.getColumnName(), dim.getDimension().getDataType());
+ } else if (dim.getDimension().getDataType() == DataTypes.DATE) {
DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dim.getDimension().getDataType());
fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), generator.getReturnType());
- } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getOrdinal()] =
- new StructField(dim.getColumnName(), dim.getDimension().getDataType());
- } else if (dim.getDimension().isComplex()) {
+ } else {
fields[dim.getOrdinal()] =
new StructField(dim.getColumnName(), dim.getDimension().getDataType());
- } else {
- fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), DataTypes.INT);
}
}
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
index c98225e..b3438d8 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.StructField;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
@@ -176,18 +175,16 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
for (int i = 0; i < queryDimension.size(); i++) {
ProjectionDimension dim = queryDimension.get(i);
- if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dim.getDimension().isComplex()) {
+ fields[dim.getOrdinal()] =
+ new StructField(dim.getColumnName(), dim.getDimension().getDataType());
+ } else if (dim.getDimension().getDataType() == DataTypes.DATE) {
DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dim.getDimension().getDataType());
fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), generator.getReturnType());
- } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getOrdinal()] =
- new StructField(dim.getColumnName(), dim.getDimension().getDataType());
- } else if (dim.getDimension().isComplex()) {
+ } else {
fields[dim.getOrdinal()] =
new StructField(dim.getColumnName(), dim.getDimension().getDataType());
- } else {
- fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), DataTypes.INT);
}
}
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonDropCacheCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonDropCacheCommand.scala
index fc9d805..974fc22 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonDropCacheCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonDropCacheCommand.scala
@@ -28,6 +28,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.cache.CacheProvider
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.datatype.DataTypes
class TestCarbonDropCacheCommand extends QueryTest with BeforeAndAfterAll {
@@ -64,7 +65,7 @@ class TestCarbonDropCacheCommand extends QueryTest with BeforeAndAfterAll {
val tableIdentifier = new TableIdentifier(tableName, Some(dbName))
val carbonTable = CarbonEnv.getCarbonTable(tableIdentifier)(sqlContext.sparkSession)
val tablePath = carbonTable.getTablePath + CarbonCommonConstants.FILE_SEPARATOR
- val dictIds = carbonTable.getAllDimensions.asScala.filter(_.isGlobalDictionaryEncoding)
+ val dictIds = carbonTable.getAllDimensions.asScala.filter(_.getDataType == DataTypes.DATE)
.map(_.getColumnId).toArray
// Check if table index entries are dropped
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index ec9f40a..1d167b8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -373,7 +373,7 @@ object DataLoadProcessBuilderOnSpark {
private def createOrderingForColumn(column: CarbonColumn): Ordering[Object] = {
if (column.isDimension) {
val dimension = column.asInstanceOf[CarbonDimension]
- if (dimension.isGlobalDictionaryEncoding || dimension.isDirectDictionaryEncoding) {
+ if (dimension.getDataType == DataTypes.DATE) {
new PrimtiveOrdering(DataTypes.INT)
} else {
if (DataTypeUtil.isPrimitiveColumn(column.getDataType)) {
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 71e91cc..76a6f0d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -693,8 +693,7 @@ class CarbonMergerRDD[K, V](
private def createOrderingForColumn(column: CarbonColumn): Ordering[Object] = {
if (column.isDimension) {
val dimension = column.asInstanceOf[CarbonDimension]
- if ((dimension.isGlobalDictionaryEncoding || dimension.isDirectDictionaryEncoding) &&
- dimension.getDataType != DataTypes.TIMESTAMP) {
+ if (dimension.getDataType == DataTypes.DATE) {
new PrimtiveOrdering(DataTypes.INT)
} else {
if (DataTypeUtil.isPrimitiveColumn(column.getDataType)) {
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index c8fa80d..1b4425d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -183,7 +183,7 @@ object CarbonScalaUtil {
value: String,
column: ColumnSchema): String = {
try {
- if (column.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (column.getDataType.equals(CarbonDataTypes.DATE)) {
if (column.getDataType.equals(CarbonDataTypes.TIMESTAMP)) {
return DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
column.getDataType,
diff --git a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 91d00b3..d96d9d5 100644
--- a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -218,12 +218,12 @@ public class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
this.isNoDictStringField = new boolean[queryDimension.size() + queryMeasures.size()];
for (int i = 0; i < queryDimension.size(); i++) {
ProjectionDimension dim = queryDimension.get(i);
- if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dim.getDimension().getDataType() == DataTypes.DATE) {
DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dim.getDimension().getDataType());
fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
- } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ } else {
if (dim.getDimension().getDataType() == DataTypes.STRING
|| dim.getDimension().getDataType() == DataTypes.VARCHAR || dim.getDimension()
.getColumnSchema().isLocalDictColumn()) {
@@ -232,13 +232,6 @@ public class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
null);
- } else if (dim.getDimension().isComplex()) {
- fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
- CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
- null);
- } else {
- fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
- CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(DataTypes.INT), true, null);
}
}
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index 8079fa0..415a667 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -227,7 +227,7 @@ class RawBytesReadSupport(segmentProperties: SegmentProperties, indexColumns: Ar
indexColumns.foreach { col =>
if (col.isDimension) {
val dim = carbonTable.getDimensionByName(col.getColName)
- if (!dim.isGlobalDictionaryEncoding && !dim.isDirectDictionaryEncoding) {
+ if (dim.getDataType != DataTypes.DATE) {
indexCol2IdxInNoDictArray =
indexCol2IdxInNoDictArray + (col.getColName -> indexCol2IdxInNoDictArray.size)
} else {
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
index f90c303..6495a43 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
@@ -80,11 +80,6 @@ case class CarbonAddLoadCommand(
throw new MalformedCarbonCommandException("Unsupported operation on non transactional table")
}
- if (carbonTable.getTableInfo.getFactTable.getListOfColumns.asScala.exists(
- c => c.hasEncoding(Encoding.DICTIONARY) && !c.hasEncoding(Encoding.DIRECT_DICTIONARY))) {
- throw new MalformedCarbonCommandException(
- "Unsupported operation on global dictionary columns table")
- }
if (carbonTable.isChildTableForMV) {
throw new MalformedCarbonCommandException("Unsupported operation on MV table")
}
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index f31a8cb..7697719 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -678,7 +678,8 @@ case class CarbonLoadDataCommand(
// Update attribute datatypes in case of dictionary columns, in case of dictionary columns
// datatype is always int
val column = table.getColumnByName(attr.name)
- val updatedDataType = if (column.hasEncoding(Encoding.DICTIONARY)) {
+ val updatedDataType = if (column.getDataType ==
+ org.apache.carbondata.core.metadata.datatype.DataTypes.DATE) {
IntegerType
} else {
attr.dataType match {
@@ -852,7 +853,8 @@ case class CarbonLoadDataCommand(
val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
val metastoreSchema = StructType(catalogTable.schema.fields.map{f =>
val column = table.getColumnByName(f.name)
- val updatedDataType = if (column.hasEncoding(Encoding.DICTIONARY)) {
+ val updatedDataType = if (column.getDataType ==
+ org.apache.carbondata.core.metadata.datatype.DataTypes.DATE) {
IntegerType
} else {
f.dataType match {
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index 2cd4603..254e766 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -30,6 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.features.TableOperation
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.events.{AlterTableDropColumnPostEvent, AlterTableDropColumnPreEvent, OperationContext, OperationListenerBus}
@@ -97,7 +98,7 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
// column should not be already deleted and should exist in the table
if (!tableColumn.isInvisible && column.equalsIgnoreCase(tableColumn.getColName)) {
if (tableColumn.isDimension) {
- if (tableColumn.hasEncoding(Encoding.DICTIONARY)) {
+ if (tableColumn.getDataType == DataTypes.DATE) {
dictionaryColumns ++= Seq(tableColumn.getColumnSchema)
}
}
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index dc0187c..5916bfe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -96,18 +96,6 @@ case class CarbonCreateTableCommand(
val partitionString =
if (partitionInfo != null &&
partitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
- // Restrict dictionary encoding on partition columns.
- // TODO Need to decide whether it is required
- val dictionaryOnPartitionColumn =
- partitionInfo.getColumnSchemaList.asScala.exists{p =>
- p.hasEncoding(Encoding.DICTIONARY) && !p.hasEncoding(Encoding.DIRECT_DICTIONARY)
- }
- if (dictionaryOnPartitionColumn) {
- throwMetadataException(
- dbName,
- tableName,
- s"Dictionary include cannot be applied on partition columns")
- }
s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
_.getColumnName.toLowerCase).mkString(",")})"
} else {
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
index 18a7de1..4d8f0f9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
@@ -43,9 +43,7 @@ import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOp
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore
-import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.datatype.DataTypes
-import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, DataTypeConverter, DataTypeConverterImpl, DataTypeUtil, ObjectSerializationUtil, OutputFilesInfoHolder, ThreadLocalSessionInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -54,7 +52,7 @@ import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWrit
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable
import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.processing.util.CarbonBadRecordUtil
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil}
class SparkCarbonTableFormat
extends FileFormat
@@ -589,7 +587,7 @@ object CarbonOutputWriter {
model.getCarbonDataLoadSchema.getCarbonTable.getTableInfo.getFactTable.getPartitionInfo
.getColumnSchemaList.asScala.zipWithIndex.map { case (col, index) =>
- val dataType = if (col.hasEncoding(Encoding.DICTIONARY)) {
+ val dataType = if (col.getDataType.equals(DataTypes.DATE)) {
DataTypes.INT
} else if (col.getDataType.equals(DataTypes.TIMESTAMP) ||
col.getDataType.equals(DataTypes.DATE)) {
@@ -600,7 +598,7 @@ object CarbonOutputWriter {
if (staticPartition != null && staticPartition.get(col.getColumnName.toLowerCase)) {
val converetedVal =
CarbonScalaUtil.convertStaticPartitions(partitionData(index), col)
- if (col.hasEncoding(Encoding.DICTIONARY)) {
+ if (col.getDataType.equals(DataTypes.DATE)) {
converetedVal.toInt.asInstanceOf[AnyRef]
} else {
DataTypeUtil.getDataBasedOnDataType(
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java
index 5f0907e..1b49e8b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java
@@ -62,9 +62,9 @@ public class ArrayDataType implements GenericDataType<ArrayObject> {
private int outputArrayIndex;
/**
- * Dictionary column
+ * True if this is for dictionary column
*/
- private boolean isDictionaryColumn;
+ private boolean isDictionary;
/**
* current data counter
@@ -99,14 +99,14 @@ public class ArrayDataType implements GenericDataType<ArrayObject> {
* @param name
* @param parentName
* @param columnId
- * @param isDictionaryColumn
+ * @param isDictionary
*/
public ArrayDataType(String name, String parentName, String columnId,
- Boolean isDictionaryColumn) {
+ Boolean isDictionary) {
this.name = name;
this.parentName = parentName;
this.columnId = columnId;
- this.isDictionaryColumn = isDictionaryColumn;
+ this.isDictionary = isDictionary;
}
/*
@@ -175,7 +175,7 @@ public class ArrayDataType implements GenericDataType<ArrayObject> {
@Override
public boolean getIsColumnDictionary() {
- return isDictionaryColumn;
+ return isDictionary;
}
@Override
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java
index 51b8b9a..588afbf 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java
@@ -58,9 +58,9 @@ public class StructDataType implements GenericDataType<StructObject> {
private int outputArrayIndex;
/**
- * Dictionary column
+ * True if this is for dictionary column
*/
- private boolean isDictionaryColumn;
+ private boolean isDictionary;
/**
* data counter
@@ -95,14 +95,14 @@ public class StructDataType implements GenericDataType<StructObject> {
* @param name
* @param parentName
* @param columnId
- * @param isDictionaryColumn
+ * @param isDictionary
*/
public StructDataType(String name, String parentName, String columnId,
- Boolean isDictionaryColumn) {
+ Boolean isDictionary) {
this.name = name;
this.parentName = parentName;
this.columnId = columnId;
- this.isDictionaryColumn = isDictionaryColumn;
+ this.isDictionary = isDictionary;
}
/*
@@ -177,7 +177,7 @@ public class StructDataType implements GenericDataType<StructObject> {
@Override
public boolean getIsColumnDictionary() {
- return isDictionaryColumn;
+ return isDictionary;
}
@Override
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index f8aaebd..0627229 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.SortColumnRangeInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -128,12 +128,12 @@ public class CarbonDataLoadConfiguration {
if (column.isDimension()) {
dimensionCount++;
if (column.isComplex()) {
- if (!dataField.hasDictionaryEncoding()) {
+ if (!dataField.isDateDataType()) {
complexNonDictionaryColumnCount++;
} else {
complexDictionaryColumnCount++;
}
- } else if (!dataField.hasDictionaryEncoding()) {
+ } else if (!dataField.isDateDataType()) {
noDictionaryCount++;
}
}
@@ -283,7 +283,7 @@ public class CarbonDataLoadConfiguration {
int noDicCount = 0;
for (int i = 0; i < dataFields.length; i++) {
if (dataFields[i].getColumn().isDimension() && (
- !(dataFields[i].getColumn().hasEncoding(Encoding.DICTIONARY)) || dataFields[i].getColumn()
+ dataFields[i].getColumn().getDataType() != DataTypes.DATE || dataFields[i].getColumn()
.isComplex())) {
noDicOrCompIndexes.add(i);
noDicCount++;
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
index 4018f75..7a9f0c3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.processing.loading;
import java.io.Serializable;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
/**
@@ -39,8 +39,8 @@ public class DataField implements Serializable {
private boolean useActualData;
- public boolean hasDictionaryEncoding() {
- return column.hasEncoding(Encoding.DICTIONARY);
+ public boolean isDateDataType() {
+ return column.getDataType() == DataTypes.DATE;
}
public CarbonColumn getColumn() {
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index ad668ab..6c3ca6e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -328,7 +328,7 @@ public final class DataLoadProcessBuilder {
columnExist = true;
sortColIndex[j] = i;
- isSortColNoDict[j] = !outFields[i].hasDictionaryEncoding();
+ isSortColNoDict[j] = !outFields[i].isDateDataType();
j++;
}
}
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 68c505a..b9c2038 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -76,7 +75,7 @@ public class FieldEncoderFactory {
if (dataField.getColumn().isIndexColumn()) {
return new IndexFieldConverterImpl(dataField, nullFormat, index, isEmptyBadRecord,
configuration);
- } else if (dataField.getColumn().hasEncoding(Encoding.DIRECT_DICTIONARY) &&
+ } else if (dataField.getColumn().getDataType() == DataTypes.DATE &&
!dataField.getColumn().isComplex()) {
return new DirectDictionaryFieldConverterImpl(dataField, nullFormat, index,
isEmptyBadRecord);
@@ -146,7 +145,7 @@ public class FieldEncoderFactory {
// Create array parser with complex delimiter
ArrayDataType arrayDataType =
new ArrayDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId(),
- carbonColumn.hasEncoding(Encoding.DICTIONARY));
+ carbonColumn.getDataType() == DataTypes.DATE);
for (CarbonDimension dimension : listOfChildDimensions) {
arrayDataType.addChildren(
createComplexType(dimension, carbonColumn.getColName(), nullFormat, binaryDecoder));
@@ -158,7 +157,7 @@ public class FieldEncoderFactory {
// Create struct parser with complex delimiter
StructDataType structDataType =
new StructDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId(),
- carbonColumn.hasEncoding(Encoding.DICTIONARY));
+ carbonColumn.getDataType() == DataTypes.DATE);
for (CarbonDimension dimension : dimensions) {
structDataType.addChildren(
createComplexType(dimension, carbonColumn.getColName(), nullFormat, binaryDecoder));
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
index 6d9ef39..fb1a62a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -97,7 +96,7 @@ public class InputProcessorStepWithNoConverterImpl extends AbstractDataLoadProce
dataTypes = new DataType[configuration.getDataFields().length];
for (int i = 0; i < dataTypes.length; i++) {
- if (configuration.getDataFields()[i].getColumn().hasEncoding(Encoding.DICTIONARY)) {
+ if (configuration.getDataFields()[i].getColumn().getDataType() == DataTypes.DATE) {
dataTypes[i] = DataTypes.INT;
} else {
dataTypes[i] = configuration.getDataFields()[i].getColumn().getDataType();
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 98122e1..3bdc9dc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -38,7 +38,6 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -419,14 +418,8 @@ public class CarbonCompactionUtil {
*/
private static int getDimensionDefaultCardinality(CarbonDimension dimension) {
int cardinality = 0;
- if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (dimension.getDataType() == DataTypes.DATE) {
cardinality = Integer.MAX_VALUE;
- } else if (dimension.hasEncoding(Encoding.DICTIONARY)) {
- if (null != dimension.getDefaultValue()) {
- cardinality = CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1;
- } else {
- cardinality = CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY;
- }
} else {
cardinality = -1;
}
@@ -502,7 +495,7 @@ public class CarbonCompactionUtil {
} else {
exp2 = new LessThanEqualToExpression(new ColumnExpression(colName, dataType),
new LiteralExpression(maxVal, dataType));
- if (rangeColumn.hasEncoding(Encoding.DICTIONARY)) {
+ if (rangeColumn.getDataType() == DataTypes.DATE) {
exp2.setAlreadyResolved(true);
}
finalExpr = new OrExpression(exp1, exp2);
@@ -511,7 +504,7 @@ public class CarbonCompactionUtil {
// Last task
finalExpr = new GreaterThanExpression(new ColumnExpression(colName, dataType),
new LiteralExpression(minVal, dataType));
- if (rangeColumn.hasEncoding(Encoding.DICTIONARY)) {
+ if (rangeColumn.getDataType() == DataTypes.DATE) {
finalExpr.setAlreadyResolved(true);
}
} else {
@@ -520,7 +513,7 @@ public class CarbonCompactionUtil {
new LiteralExpression(minVal, dataType));
exp2 = new LessThanEqualToExpression(new ColumnExpression(colName, dataType),
new LiteralExpression(maxVal, dataType));
- if (rangeColumn.hasEncoding(Encoding.DICTIONARY)) {
+ if (rangeColumn.getDataType() == DataTypes.DATE) {
exp2.setAlreadyResolved(true);
exp1.setAlreadyResolved(true);
}
@@ -538,7 +531,7 @@ public class CarbonCompactionUtil {
int idx = -1;
DataType dataType = rangeCol.getDataType();
Object[] minMaxVals = new Object[2];
- boolean isDictEncode = rangeCol.hasEncoding(Encoding.DICTIONARY);
+ boolean isDictEncode = rangeCol.getDataType() == DataTypes.DATE;
try {
for (CarbonInputSplit split : carbonInputSplits) {
DataFileFooter dataFileFooter = null;
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 166b4ec..5144269 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -31,7 +31,6 @@ import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -276,7 +275,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
Object[] preparedRow = new Object[dimensions.size() + measureCount];
for (int i = 0; i < dimensions.size(); i++) {
CarbonDimension dims = dimensions.get(i);
- if (dims.hasEncoding(Encoding.DICTIONARY)) {
+ if (dims.getDataType() == DataTypes.DATE) {
// dictionary
preparedRow[i] = row[i];
} else {
@@ -325,7 +324,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
for (int i = 0; i < dimensions.size(); i++) {
CarbonDimension dims = dimensions.get(i);
- if (dims.hasEncoding(Encoding.DICTIONARY) && !dims.isComplex()) {
+ if (dims.getDataType() == DataTypes.DATE && !dims.isComplex()) {
// dictionary
preparedRow[i] = dictionaryValues[dictionaryIndex++];
} else if (!dims.isComplex()) {
@@ -443,7 +442,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
if (dimension.isSortColumn()) {
sortColumnMapping[i] = true;
}
- if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ if (dimension.getDataType() == DataTypes.DATE) {
i++;
continue;
}
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
index 6a5dbdc..9f540a1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.processing.sort.DummyRowUpdater;
import org.apache.carbondata.processing.sort.SchemaBasedRowUpdater;
@@ -125,7 +125,7 @@ public class TableFieldStat implements Serializable {
List<CarbonDimension> updatedDimensions = updateDimensionsBasedOnSortColumns(allDimensions);
for (int i = 0; i < updatedDimensions.size(); i++) {
CarbonDimension carbonDimension = updatedDimensions.get(i);
- if (carbonDimension.hasEncoding(Encoding.DICTIONARY) && !carbonDimension.isComplex()) {
+ if (carbonDimension.getDataType() == DataTypes.DATE && !carbonDimension.isComplex()) {
if (carbonDimension.isSortColumn()) {
dictSortDimIdx[tmpDictSortCnt++] = i;
} else {
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index e2c7c8b..a4a52d5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGener
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -257,7 +257,7 @@ public class CarbonFactDataHandlerModel {
}
List<DataType> noDictDataTypesList = new ArrayList<>();
for (DataField dataField : configuration.getDataFields()) {
- if (!dataField.hasDictionaryEncoding() && dataField.getColumn().isDimension()) {
+ if (!dataField.isDateDataType() && dataField.getColumn().isDimension()) {
noDictDataTypesList.add(dataField.getColumn().getDataType());
}
}
@@ -337,7 +337,7 @@ public class CarbonFactDataHandlerModel {
int noDicAndComp = 0;
List<DataType> noDictDataTypesList = new ArrayList<>();
for (CarbonDimension dim : allDimensions) {
- if (!dim.hasEncoding(Encoding.DICTIONARY)) {
+ if (dim.getDataType() != DataTypes.DATE) {
noDicAndComplexColumns[noDicAndComp++] =
new CarbonColumn(dim.getColumnSchema(), dim.getOrdinal(), dim.getSchemaOrdinal());
noDictDataTypesList.add(dim.getDataType());
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 6258af1..3cadb45 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -106,7 +106,7 @@ public class TablePage {
TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
ColumnType columnType = tableSpec.getDimensionSpec(i).getColumnType();
ColumnPage page;
- if (ColumnType.DIRECT_DICTIONARY == columnType) {
+ if (spec.getSchemaDataType() == DataTypes.DATE) {
page = ColumnPage.newPage(
new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize);
page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY));
@@ -423,7 +423,7 @@ public class TablePage {
int numDimensions = spec.getNumDimensions();
for (int i = 0; i < numDimensions; i++) {
ColumnType type = spec.getDimensionSpec(i).getColumnType();
- if (type == ColumnType.DIRECT_DICTIONARY) {
+ if (spec.getDimensionSpec(i).getSchemaDataType() == DataTypes.DATE) {
page = dictDimensionPages[++dictDimensionIndex];
} else if (type == ColumnType.PLAIN_VALUE) {
page = noDictDimensionPages[++noDictDimensionIndex];
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 7221b3d..072329c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
@@ -375,8 +376,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
for (int i = 0; i < wrapperColumnSchemaList.size(); i++) {
columnSchemaList
.add(schemaConverter.fromWrapperToExternalColumnSchema(wrapperColumnSchemaList.get(i)));
- if (CarbonUtil.hasEncoding(wrapperColumnSchemaList.get(i).getEncodingList(),
- org.apache.carbondata.core.metadata.encoder.Encoding.DICTIONARY)) {
+ if (wrapperColumnSchemaList.get(i).getDataType() == DataTypes.DATE) {
cardinality.add(dictionaryColumnCardinality[counter]);
counter++;
} else if (!wrapperColumnSchemaList.get(i).isDimensionColumn()) {
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 5383607..55f177a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -39,7 +39,6 @@ import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.DatabaseLocationProvider;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -179,7 +178,7 @@ public final class CarbonDataProcessorUtil {
break;
}
- if (!field.hasDictionaryEncoding() && field.getColumn().isDimension()) {
+ if (!field.isDateDataType() && field.getColumn().isDimension()) {
noDictionaryMapping.add(true);
} else if (field.getColumn().isDimension()) {
noDictionaryMapping.add(false);
@@ -216,7 +215,7 @@ public final class CarbonDataProcessorUtil {
if (column.isComplex()) {
break;
}
- if (!column.hasEncoding(Encoding.DICTIONARY) && column.isDimension()) {
+ if (column.getDataType() != DataTypes.DATE && column.isDimension()) {
noDictionaryMapping.add(true);
} else if (column.isDimension()) {
noDictionaryMapping.add(false);
@@ -238,11 +237,11 @@ public final class CarbonDataProcessorUtil {
}
private static String isDictionaryType(CarbonDimension dimension) {
- Boolean isDictionary = true;
- if (!(dimension.hasEncoding(Encoding.DICTIONARY))) {
+ boolean isDictionary = true;
+ if (dimension.getDataType() != DataTypes.DATE) {
isDictionary = false;
}
- return isDictionary.toString();
+ return String.valueOf(isDictionary);
}
/**
@@ -371,7 +370,7 @@ public final class CarbonDataProcessorUtil {
List<CarbonDimension> dimensions = carbonTable.getVisibleDimensions();
List<DataType> type = new ArrayList<>();
for (int i = 0; i < dimensions.size(); i++) {
- if (dimensions.get(i).isSortColumn() && !dimensions.get(i).hasEncoding(Encoding.DICTIONARY)) {
+ if (dimensions.get(i).isSortColumn() && dimensions.get(i).getDataType() != DataTypes.DATE) {
type.add(dimensions.get(i).getDataType());
}
}
@@ -389,7 +388,7 @@ public final class CarbonDataProcessorUtil {
List<Boolean> noDicSortColMap = new ArrayList<>();
for (int i = 0; i < dimensions.size(); i++) {
if (dimensions.get(i).isSortColumn()) {
- if (!dimensions.get(i).hasEncoding(Encoding.DICTIONARY)) {
+ if (dimensions.get(i).getDataType() != DataTypes.DATE) {
noDicSortColMap.add(true);
} else {
noDicSortColMap.add(false);
@@ -417,7 +416,7 @@ public final class CarbonDataProcessorUtil {
List<Integer> noDicSortColMap = new ArrayList<>();
int counter = 0;
for (CarbonDimension dimension : dimensions) {
- if (dimension.hasEncoding(Encoding.DICTIONARY)) {
+ if (dimension.getDataType() == DataTypes.DATE) {
continue;
}
if (dimension.isSortColumn() && DataTypeUtil.isPrimitiveColumn(dimension.getDataType())) {
@@ -444,7 +443,7 @@ public final class CarbonDataProcessorUtil {
List<DataType> noDictSortType = new ArrayList<>();
List<DataType> noDictNoSortType = new ArrayList<>();
for (int i = 0; i < dimensions.size(); i++) {
- if (!dimensions.get(i).hasEncoding(Encoding.DICTIONARY)) {
+ if (dimensions.get(i).getDataType() != DataTypes.DATE) {
if (dimensions.get(i).isSortColumn()) {
noDictSortType.add(dimensions.get(i).getDataType());
} else {
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
index 15aecca..cb9f1d6 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataFile.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
import org.apache.carbondata.core.reader.CarbonHeaderReader;
@@ -447,7 +446,7 @@ class DataFile {
*/
private double computePercentage(byte[] data, byte[] min, byte[] max, ColumnSchema column) {
if (column.getDataType() == DataTypes.STRING || column.getDataType() == DataTypes.BOOLEAN
- || column.hasEncoding(Encoding.DICTIONARY) || column.getDataType().isComplexType()) {
+ || column.getDataType() == DataTypes.DATE || column.getDataType().isComplexType()) {
// for string, we do not calculate
return 0;
} else if (DataTypes.isDecimal(column.getDataType())) {
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
index 908ce02..7639510 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/DataSummary.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonHeaderReader;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
@@ -312,7 +311,7 @@ class DataSummary implements Command {
maxPercent = "NA";
// for complex types min max can be given as NA and for varchar where min max is not
// written, can give NA
- if (blocklet.getColumnChunk().column.hasEncoding(Encoding.DICTIONARY) ||
+ if (blocklet.getColumnChunk().column.getDataType() == DataTypes.DATE ||
blocklet.getColumnChunk().column.isComplexColumn() ||
!blocklet.getColumnChunk().isMinMaxPresent) {
min = "NA";
@@ -324,7 +323,7 @@ class DataSummary implements Command {
} else {
// for column has global dictionary and for complex columns,min and max percentage can be
// NA
- if (blocklet.getColumnChunk().column.hasEncoding(Encoding.DICTIONARY) ||
+ if (blocklet.getColumnChunk().column.getDataType() == DataTypes.DATE ||
blocklet.getColumnChunk().column.isComplexColumn() ||
blocklet.getColumnChunk().column.getDataType().isComplexType()) {
minPercent = "NA";
@@ -337,7 +336,7 @@ class DataSummary implements Command {
}
DataFile.ColumnChunk columnChunk = blocklet.columnChunk;
// need to consider dictionary and complex columns
- if (columnChunk.column.hasEncoding(Encoding.DICTIONARY) ||
+ if (columnChunk.column.getDataType() == DataTypes.DATE ||
blocklet.getColumnChunk().column.isComplexColumn() ||
blocklet.getColumnChunk().column.getDataType().isComplexType()) {
min = "NA";