You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:35 UTC
[48/50] [abbrv] incubator-carbondata git commit: Merge
remote-tracking branch 'carbon_master/master' into apache/master
Merge remote-tracking branch 'carbon_master/master' into apache/master
Conflicts:
README.md
core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
core/src/main/java/org/carbondata/core/util/CarbonUtil.java
core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
core/src/main/java/org/carbondata/query/aggregator/impl/CustomAggregatorHelper.java
core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
core/src/main/java/org/carbondata/query/carbon/processor/impl/AggregateQueryBlockProcessor.java
core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java
core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java
core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
core/src/main/java/org/carbondata/scan/expression/Expression.java
core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/ListExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
core/src/main/java/org/carbondata/scan/expression/exception/FilterUnsupportedException.java
core/src/main/java/org/carbondata/scan/expression/logical/AndExpression.java
core/src/main/java/org/carbondata/scan/expression/logical/NotExpression.java
core/src/main/java/org/carbondata/scan/expression/logical/OrExpression.java
core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
core/src/main/java/org/carbondata/scan/filter/executer/ColGroupFilterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
core/src/main/java/org/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/FilterResolverIntf.java
core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase1.scala
integration-testcases/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase2.scala
integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonSQLDialect.scala
integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
integration/spark/src/test/scala/org/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
processing/src/main/java/org/carbondata/processing/store/CarbonFactDataHandlerModel.java
processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupDataHolder.java
processing/src/main/java/org/carbondata/processing/store/colgroup/ColGroupMinMax.java
processing/src/main/java/org/carbondata/processing/store/writer/AbstractFactDataWriter.java
processing/src/test/java/org/carbondata/processing/store/colgroup/ColGroupMinMaxTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f722186
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f722186
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f722186
Branch: refs/heads/master
Commit: 7f72218602926ca5f6629825df7b8f70ab1c6aa6
Parents: e9bbf75 d44c9e7
Author: ravipesala <ra...@gmail.com>
Authored: Thu Jun 30 17:41:47 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jun 30 17:41:47 2016 +0530
----------------------------------------------------------------------
common/pom.xml | 8 +-
.../common/logging/impl/StandardLogService.java | 15 +-
conf/carbon.properties.template | 98 +
.../common/ext/ColumnUniqueIdGenerator.java | 41 +
.../common/ext/DictionaryFactory.java | 119 +
.../org/carbondata/common/ext/PathFactory.java | 48 +
.../common/factory/CarbonCommonFactory.java | 54 +
.../dictionary/AbstractDictionaryCache.java | 43 +-
.../cache/dictionary/ColumnDictionaryInfo.java | 5 +-
.../cache/dictionary/DictionaryCacheLoader.java | 7 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 28 +-
.../DictionaryColumnUniqueIdentifier.java | 9 +-
.../dictionary/ForwardDictionaryCache.java | 11 +-
.../dictionary/ReverseDictionaryCache.java | 11 +-
.../core/carbon/ColumnIdentifier.java | 109 +
.../core/carbon/datastore/BlockIndexStore.java | 184 +-
.../carbon/datastore/SegmentTaskIndexStore.java | 21 +-
.../datastore/block/SegmentProperties.java | 94 +-
.../carbon/datastore/block/TableBlockInfo.java | 18 +-
.../impl/ColumnGroupDimensionDataChunk.java | 7 +-
.../ThriftWrapperSchemaConverterImpl.java | 4 +
.../carbon/metadata/index/BlockIndexInfo.java | 92 +
.../metadata/schema/table/CarbonTable.java | 2 +-
.../schema/table/column/CarbonColumn.java | 23 +
.../schema/table/column/ColumnSchema.java | 49 +
.../core/carbon/path/CarbonTablePath.java | 120 +-
.../core/constants/CarbonCommonConstants.java | 285 +-
.../store/columnar/ColumnGroupModel.java | 21 -
.../store/filesystem/HDFSCarbonFile.java | 3 +-
.../store/filesystem/ViewFSCarbonFile.java | 6 +-
.../core/datastorage/util/StoreFactory.java | 18 +-
.../DirectDictionaryGenerator.java | 13 +
.../TimeStampDirectDictionaryGenerator.java | 46 +-
.../core/load/LoadMetadataDetails.java | 43 +
.../CarbonDictionaryMetadataReaderImpl.java | 17 +-
.../core/reader/CarbonDictionaryReaderImpl.java | 28 +-
.../core/reader/CarbonIndexFileReader.java | 95 +
.../carbondata/core/reader/ThriftReader.java | 47 +-
.../CarbonDictionarySortIndexReaderImpl.java | 21 +-
.../core/service/ColumnUniqueIdService.java | 34 +
.../core/service/DictionaryService.java | 92 +
.../carbondata/core/service/PathService.java | 38 +
.../core/util/CarbonMetadataUtil.java | 114 +-
.../carbondata/core/util/CarbonProperties.java | 401 +--
.../org/carbondata/core/util/CarbonUtil.java | 245 +-
.../core/util/DataFileFooterConverter.java | 65 +-
.../core/writer/CarbonDictionaryWriterImpl.java | 45 +-
.../core/writer/CarbonIndexFileWriter.java | 64 +
.../core/writer/HierarchyValueWriterForCSV.java | 2 +-
.../CarbonDictionarySortIndexWriterImpl.java | 21 +-
.../scan/executor/infos/KeyStructureInfo.java | 18 -
.../scan/executor/util/QueryUtil.java | 105 +-
.../scan/expression/ColumnExpression.java | 3 +-
.../carbondata/scan/expression/Expression.java | 4 +-
.../scan/expression/ExpressionResult.java | 104 +-
.../expression/arithmetic/AddExpression.java | 4 +-
.../expression/arithmetic/DivideExpression.java | 4 +-
.../arithmetic/MultiplyExpression.java | 4 +-
.../arithmetic/SubstractExpression.java | 4 +-
.../conditional/EqualToExpression.java | 5 +-
.../GreaterThanEqualToExpression.java | 4 +-
.../conditional/GreaterThanExpression.java | 5 +-
.../expression/conditional/InExpression.java | 6 +-
.../conditional/LessThanEqualToExpression.java | 5 +-
.../conditional/LessThanExpression.java | 4 +-
.../expression/conditional/ListExpression.java | 7 +-
.../conditional/NotEqualsExpression.java | 4 +-
.../expression/conditional/NotInExpression.java | 6 +-
.../exception/FilterIllegalMemberException.java | 98 +
.../exception/FilterUnsupportedException.java | 1 -
.../scan/expression/logical/AndExpression.java | 6 +-
.../scan/expression/logical/NotExpression.java | 4 +-
.../scan/expression/logical/OrExpression.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 36 +-
.../org/carbondata/scan/filter/FilterUtil.java | 163 +-
.../executer/ColGroupFilterExecuterImpl.java | 190 --
.../ExcludeColGroupFilterExecuterImpl.java | 137 +
.../executer/ExcludeFilterExecuterImpl.java | 49 +-
.../IncludeColGroupFilterExecuterImpl.java | 209 ++
.../executer/IncludeFilterExecuterImpl.java | 8 +-
.../executer/RowLevelFilterExecuterImpl.java | 116 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 148 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 139 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 187 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 190 +-
.../RowLevelRangeTypeExecuterFacory.java | 13 +-
.../filter/resolver/AndFilterResolverImpl.java | 5 +-
.../resolver/ConditionalFilterResolverImpl.java | 19 +-
.../filter/resolver/FilterResolverIntf.java | 5 +-
.../resolver/LogicalFilterResolverImpl.java | 3 +-
.../resolver/RestructureFilterResolverImpl.java | 5 +-
.../RowLevelRangeFilterResolverImpl.java | 77 +-
.../visitor/CustomTypeDictionaryVisitor.java | 16 +-
.../visitor/DictionaryColumnVisitor.java | 11 +-
.../visitor/NoDictionaryTypeVisitor.java | 12 +-
.../scan/result/iterator/RawResultIterator.java | 2 +
.../org/carbondata/scan/util/DataTypeUtil.java | 2 +-
.../dictionary/AbstractDictionaryCacheTest.java | 7 +-
.../dictionary/ForwardDictionaryCacheTest.java | 4 +-
.../dictionary/ReverseDictionaryCacheTest.java | 4 +-
.../datastore/SegmentTaskIndexStoreTest.java | 143 -
...CarbonDictionarySortIndexReaderImplTest.java | 6 +-
.../writer/CarbonDictionaryWriterImplTest.java | 9 +-
.../core/writer/CarbonFooterWriterTest.java | 58 +-
...CarbonDictionarySortIndexWriterImplTest.java | 13 +-
dev/molap.properties.template | 94 -
dev/molap.properties_spark | 90 -
.../examples/GenerateDictionaryExample.scala | 4 +-
format/src/main/thrift/carbondataindex.thrift | 45 +
format/src/main/thrift/schema.thrift | 13 +-
.../org/carbondata/hadoop/CarbonPathFilter.java | 4 +-
.../AbstractDictionaryDecodedReadSupport.java | 4 +-
.../hadoop/test/util/StoreCreator.java | 11 +-
.../FACT_UNITED_DATA_INFO_sample_cube.csv | 20 -
.../FACT_UNITED_DATA_INFO_sample_table.csv | 20 +
.../src/test/resources/restructure_cube.csv | 101 -
.../src/test/resources/restructure_table.csv | 101 +
.../allqueries/AllDataTypesTestCase1.scala | 82 +-
.../allqueries/AllDataTypesTestCase2.scala | 116 +-
.../allqueries/AllDataTypesTestCase3.scala | 211 +-
.../allqueries/AllDataTypesTestCase4.scala | 2576 +++++-------------
.../allqueries/AllDataTypesTestCase5.scala | 165 +-
.../allqueries/AllDataTypesTestCase6.scala | 2377 ++++------------
.../spark/merger/RowResultMerger.java | 19 +-
.../carbondata/spark/load/CarbonLoadModel.java | 2 +
.../carbondata/spark/load/CarbonLoaderUtil.java | 189 +-
.../spark/load/DeleteLoadFolders.java | 5 +-
.../spark/merger/CarbonDataMergerUtil.java | 401 +--
.../spark/sql/CarbonDatasourceRelation.scala | 22 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 7 +-
.../org/apache/spark/sql/CarbonSqlParser.scala | 374 +--
.../execution/command/carbonTableSchema.scala | 180 +-
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 85 +-
.../spark/sql/hive/CarbonStrategies.scala | 4 +-
.../spark/CarbonColumnValidator.scala | 36 +
.../carbondata/spark/CarbonSparkFactory.scala | 63 +
.../spark/DictionaryDetailHelper.scala | 66 +
.../spark/rdd/CarbonDataLoadRDD.scala | 4 +
.../spark/rdd/CarbonDataRDDFactory.scala | 240 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 7 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 13 +-
.../org/carbondata/spark/rdd/Compactor.scala | 19 +-
.../org/carbondata/spark/util/CommonUtil.scala | 79 +-
.../spark/util/GlobalDictionaryUtil.scala | 85 +-
.../spark/load/CarbonLoaderUtilTest.java | 4 +-
.../validation/FileFooterValidator.java | 6 +-
integration/spark/src/test/resources/data2.csv | 6 +-
.../src/test/resources/data2_DiffTimeFormat.csv | 4 +
.../resources/datawithcomplexspecialchar.csv | 151 +
.../test/resources/datawithescapecharacter.csv | 22 +
.../src/test/resources/datawithnullmeasure.csv | 5 +
.../test/resources/datawithspecialcharacter.csv | 38 +
.../spark/src/test/resources/filter/emp2.csv | 9 +
.../src/test/resources/filter/emp2allnull.csv | 9 +
.../src/test/resources/filter/emp2nonull.csv | 8 +
.../test/resources/nullandnonparsableValue.csv | 3 +
.../apache/spark/sql/TestCarbonSqlParser.scala | 37 +-
.../dataload/TestLoadDataWithBlankLine.scala | 18 +-
.../TestLoadDataWithEmptyArrayColumns.scala | 12 +-
.../TestLoadDataWithMaxMinInteger.scala | 6 +-
.../dataload/TestLoadDataWithNullMeasures.scala | 15 +-
.../AllDataTypesTestCaseAggregate.scala | 44 +-
.../aggquery/IntegerDataTypeTestCase.scala | 7 +-
.../AllDataTypesTestCaseAggregate.scala | 8 +-
.../testsuite/bigdecimal/TestBigDecimal.scala | 10 +
.../bigdecimal/TestNullAndEmptyFields.scala | 110 +
.../createtable/TestCreateTableSyntax.scala | 32 +-
.../DataCompactionCardinalityBoundryTest.scala | 12 +-
.../datacompaction/DataCompactionLockTest.scala | 111 +
.../datacompaction/DataCompactionTest.scala | 12 +-
.../dataload/TestLoadDataWithHiveSyntax.scala | 108 +-
...ataWithMalformedCarbonCommandException.scala | 4 +-
.../dataretention/DataRetentionTestCase.scala | 37 +
.../detailquery/AllDataTypesTestCase.scala | 6 +-
.../ColumnGroupDataTypesTestCase.scala | 51 +-
.../ColumnPropertyValidationTestCase.scala | 28 +
.../HighCardinalityDataTypesTestCase.scala | 66 +-
.../detailquery/IntegerDataTypeTestCase.scala | 6 +-
...estampDataTypeDirectDictionaryTestCase.scala | 12 +-
...TypeDirectDictionaryWithNoDictTestCase.scala | 4 +-
.../filterexpr/AllDataTypesTestCaseFilter.scala | 6 +-
.../filterexpr/FilterProcessorTestCase.scala | 131 +-
.../GrtLtFilterProcessorTestCase.scala | 176 ++
.../filterexpr/IntegerDataTypeTestCase.scala | 6 +-
.../NullMeasureValueTestCaseFilter.scala | 39 +
.../HadoopFSRelationTestCase.scala | 16 +-
.../joinquery/AllDataTypesTestCaseJoin.scala | 6 +-
.../joinquery/IntegerDataTypeTestCase.scala | 6 +-
.../NullMeasureValueTestCaseAggregate.scala | 2 +-
.../sortexpr/AllDataTypesTestCaseSort.scala | 6 +-
.../sortexpr/IntegerDataTypeTestCase.scala | 13 +-
.../util/GlobalDictionaryUtilTestCase.scala | 51 +-
.../carbonaggregatesurrogategenerator/icon.png | Bin 9058 -> 0 bytes
.../plugin.xml | 1 -
.../steps/carbonautoagggraphgenerator/icon.png | Bin 9058 -> 0 bytes
.../carbonautoagggraphgenerator/plugin.xml | 1 -
.../steps/carbonautoaggslicemerger/icon.png | Bin 9058 -> 0 bytes
.../steps/carbonautoaggslicemerger/plugin.xml | 1 -
.../plugins/steps/carboncsvbasedseqgen/icon.png | Bin 24093 -> 0 bytes
.../steps/carboncsvbasedseqgen/plugin.xml | 1 -
.../plugins/steps/carboncsvreader/icon.png | Bin 24093 -> 0 bytes
.../plugins/steps/carboncsvreader/plugin.xml | 1 -
.../plugins/steps/carboncsvreaderstrep/icon.png | Bin 9058 -> 0 bytes
.../steps/carboncsvreaderstrep/plugin.xml | 1 -
.../steps/carbondatawriter/Binary-icon.png | Bin 14573 -> 0 bytes
.../plugins/steps/carbondatawriter/plugin.xml | 1 -
.../plugins/steps/carbonfactreader/icon.png | Bin 9058 -> 0 bytes
.../plugins/steps/carbonfactreader/plugin.xml | 1 -
.../plugins/steps/carbongroupby/icon.png | Bin 9058 -> 0 bytes
.../plugins/steps/carbongroupby/plugin.xml | 1 -
.../steps/carboninmemoryfactreader/icon.png | Bin 9058 -> 0 bytes
.../steps/carboninmemoryfactreader/plugin.xml | 1 -
.../.kettle/plugins/steps/carbonseqgen/icon.png | Bin 24093 -> 0 bytes
.../plugins/steps/carbonseqgen/plugin.xml | 1 -
.../plugins/steps/carbonslicemerger/icon.png | Bin 9058 -> 0 bytes
.../plugins/steps/carbonslicemerger/plugin.xml | 1 -
.../steps/carbonsortkeyandgroupby/icon.png | Bin 9058 -> 0 bytes
.../steps/carbonsortkeyandgroupby/plugin.xml | 1 -
.../.kettle/plugins/steps/mdkeygenstep/icon.png | Bin 9058 -> 0 bytes
.../plugins/steps/mdkeygenstep/plugin.xml | 1 -
.../.kettle/plugins/steps/sortkeystep/icon.png | Bin 9058 -> 0 bytes
.../plugins/steps/sortkeystep/plugin.xml | 1 -
.../org/carbondata/lcm/locks/LocalFileLock.java | 4 +-
.../org/carbondata/lcm/locks/LockUsage.java | 3 +-
.../lcm/status/SegmentStatusManager.java | 111 +-
.../api/dataloader/DataLoadModel.java | 9 +
.../processing/csvload/DataGraphExecuter.java | 8 +
.../csvreaderstep/BlockDataHandler.java | 83 +-
.../processing/csvreaderstep/CsvInput.java | 4 +-
.../processing/csvreaderstep/CsvInputData.java | 2 +
.../processing/csvreaderstep/CsvInputMeta.java | 21 +-
.../dataprocessor/DataProcessTaskStatus.java | 10 +
.../dataprocessor/IDataProcessStatus.java | 2 +
.../graphgenerator/GraphGenerator.java | 6 +
.../configuration/GraphConfigurationInfo.java | 10 +
.../processing/mdkeygen/MDKeyGenStep.java | 40 +-
.../merger/step/CarbonSliceMergerStep.java | 3 +-
.../processing/schema/metadata/ColumnsInfo.java | 10 +
.../store/CarbonFactDataHandlerColumnar.java | 111 +-
.../store/CarbonFactDataHandlerModel.java | 63 +-
.../store/SingleThreadFinalSortFilesMerger.java | 6 +-
.../store/colgroup/ColGroupDataHolder.java | 4 +-
.../store/colgroup/ColGroupMinMax.java | 30 +-
.../store/writer/AbstractFactDataWriter.java | 147 +-
...actDataWriterImplForIntIndexAndAggBlock.java | 6 +-
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 36 +
.../csvbased/CarbonCSVBasedSeqGenStep.java | 19 +-
.../FileStoreSurrogateKeyGenForCSV.java | 12 +-
.../processing/util/CarbonSchemaParser.java | 44 +
.../store/colgroup/ColGroupMinMaxTest.java | 385 +--
250 files changed, 8562 insertions(+), 7434 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
index c177feb,f6f369a..bfa5cdc
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/block/SegmentProperties.java
@@@ -32,6 -34,6 +34,7 @@@ import org.carbondata.core.carbon.metad
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
import org.carbondata.core.constants.CarbonCommonConstants;
++import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
index 532c6e9,0000000..26b2519
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
@@@ -1,137 -1,0 +1,116 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.core.datastorage.store.columnar;
+
+public class ColumnGroupModel {
+
+ /**
- * cardinality all dimension
- */
- private int[] colGroupCardinality;
-
- /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
- * @return cardinality of column groups
- */
- public int[] getColumnGroupCardinality() {
- return colGroupCardinality;
- }
-
- /**
- * set columngroup cardinality
- *
- * @param columnGroupCardinality
- */
- public void setColumnGroupCardinality(int[] columnGroupCardinality) {
- this.colGroupCardinality = columnGroupCardinality;
- }
-
- /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
index 8fc5aac,63f8157..e8efa21
--- a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
@@@ -19,35 -19,38 +19,19 @@@
package org.carbondata.core.datastorage.util;
- import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.FileHolder;
-import org.carbondata.core.datastorage.store.NodeKeyStore;
import org.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStore;
-import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataFileStore;
import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
-
-import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataFileStore;
import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore;
- import org.carbondata.core.util.CarbonProperties;
-import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarFileKeyStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarFileKeyStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyFileStore;
-import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyFileStore;
-import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyInMemoryStore;
public final class StoreFactory {
/**
- * Double array data store.
- * key type.
-- */
- private static final String COMPRESSED_DOUBLE_ARRAY = "COMPRESSED_DOUBLE_ARRAY";
- private static StoreType keyType;
-- /**
* value type.
*/
private static StoreType valueType;
static {
- String valuetype = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.VALUESTORE_TYPE,
- CarbonCommonConstants.VALUESTORE_TYPE_DEFAULT_VAL);
-
- // set value type
- if (COMPRESSED_DOUBLE_ARRAY.equals(valuetype)) {
- valueType = StoreType.COMPRESSED_DOUBLE_ARRAY;
- } else {
- valueType = StoreType.HEAVY_VALUE_COMPRESSION;
- }
- keyType = StoreType.COMPRESSED_SINGLE_ARRAY;
+ valueType = StoreType.HEAVY_VALUE_COMPRESSION;
}
private StoreFactory() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 771e5f5,97baf3d..0320621
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@@ -39,20 -39,7 +39,14 @@@ import static org.carbondata.core.keyge
*/
public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator {
+ private TimeStampDirectDictionaryGenerator() {
+
+ }
+
+ public static TimeStampDirectDictionaryGenerator instance =
+ new TimeStampDirectDictionaryGenerator();
+
/**
- * Logger instance
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
-
- /**
* The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis.
*/
public static final long granularityFactor;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/util/CarbonUtil.java
index 31d41b3,b19f8d3..2368139
--- a/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonUtil.java
@@@ -47,9 -40,14 +40,13 @@@ import org.carbondata.core.carbon.metad
import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.carbon.metadata.encoder.Encoding;
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+ import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
+ import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+ import org.carbondata.core.carbon.path.CarbonStorePath;
+ import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.FileHolder;
+import org.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
import org.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
import org.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
index 6641c18,0000000..52a139c
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
@@@ -1,418 -1,0 +1,463 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.core.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
++import org.carbondata.core.carbon.datastore.block.TableBlockInfo;
+import org.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
+import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
+import org.carbondata.core.carbon.metadata.blocklet.SegmentInfo;
+import org.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta;
+import org.carbondata.core.carbon.metadata.blocklet.compressor.CompressionCodec;
+import org.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
+import org.carbondata.core.carbon.metadata.blocklet.index.BlockletBTreeIndex;
+import org.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
+import org.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex;
+import org.carbondata.core.carbon.metadata.blocklet.sort.SortState;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.carbondata.core.datastorage.store.FileHolder;
+import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.metadata.ValueEncoderMeta;
+import org.carbondata.core.reader.CarbonFooterReader;
++import org.carbondata.core.reader.CarbonIndexFileReader;
++import org.carbondata.format.BlockIndex;
+import org.carbondata.format.FileFooter;
+
+/**
+ * Below class will be used to convert the thrift object of data file
+ * meta data to wrapper object
+ */
- class DataFileFooterConverter {
++public class DataFileFooterConverter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DataFileFooterConverter.class.getName());
+
+ /**
++ * Below method will be used to get the index info from index file
++ *
++ * @param filePath file path of the index file
++ * @param tableBlockInfoList table block index
++ * @return list of index info
++ * @throws IOException problem while reading the index file
++ */
++ public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList)
++ throws IOException {
++ CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
++ // open the reader
++ indexReader.openThriftReader(filePath);
++ // get the index header
++ org.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
++ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
++ List<org.carbondata.format.ColumnSchema> table_columns = readIndexHeader.getTable_columns();
++ for (int i = 0; i < table_columns.size(); i++) {
++ columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
++ }
++ // get the segment info
++ SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
++ BlockletIndex blockletIndex = null;
++ int counter = 0;
++ DataFileFooter dataFileFooter = null;
++ List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
++ // read the block info from file
++ while (indexReader.hasNext()) {
++ BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
++ blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
++ dataFileFooter = new DataFileFooter();
++ dataFileFooter.setBlockletIndex(blockletIndex);
++ dataFileFooter.setColumnInTable(columnSchemaList);
++ dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
++ dataFileFooter.setTableBlockInfo(tableBlockInfoList.get(counter++));
++ dataFileFooter.setSegmentInfo(segmentInfo);
++ dataFileFooters.add(dataFileFooter);
++ }
++ return dataFileFooters;
++ }
++
++ /**
+ * Below method will be used to convert thrift file meta to wrapper file meta
+ */
+ public DataFileFooter readDataFileFooter(String filePath, long blockOffset, long blockLength)
+ throws IOException {
+ DataFileFooter dataFileFooter = new DataFileFooter();
+ FileHolder fileReader = null;
+ try {
+ long completeBlockLength = blockOffset + blockLength;
+ long footerPointer = completeBlockLength - 8;
+ fileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath));
+ long actualFooterOffset = fileReader.readLong(filePath, footerPointer);
+ CarbonFooterReader reader = new CarbonFooterReader(filePath, actualFooterOffset);
+ FileFooter footer = reader.readFooter();
+ dataFileFooter.setVersionId(footer.getVersion());
+ dataFileFooter.setNumberOfRows(footer.getNum_rows());
+ dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ List<org.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
+ for (int i = 0; i < table_columns.size(); i++) {
+ columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
+ }
+ dataFileFooter.setColumnInTable(columnSchemaList);
+
+ List<org.carbondata.format.BlockletIndex> leaf_node_indices_Thrift =
+ footer.getBlocklet_index_list();
+ List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
+ for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
+ BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
+ blockletIndexList.add(blockletIndex);
+ }
+
+ List<org.carbondata.format.BlockletInfo> leaf_node_infos_Thrift =
+ footer.getBlocklet_info_list();
+ List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
+ for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
+ BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
+ blockletInfo.setBlockletIndex(blockletIndexList.get(i));
+ blockletInfoList.add(blockletInfo);
+ }
+ dataFileFooter.setBlockletList(blockletInfoList);
+ dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
+ } finally {
+ if (null != fileReader) {
+ fileReader.finish();
+ }
+ }
+ return dataFileFooter;
+ }
+
+ /**
+ * Below method will be used to get blocklet index for data file meta
+ *
+ * @param blockletIndexList
+ * @return blocklet index
+ */
+ private BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
+ BlockletIndex blockletIndex = new BlockletIndex();
+ BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
+ blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
+ blockletBTreeIndex
+ .setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
+ blockletIndex.setBtreeIndex(blockletBTreeIndex);
+ byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
+ byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
+ byte[][] minValue = null;
+ byte[][] maxValue = null;
+ for (int i = 1; i < blockletIndexList.size(); i++) {
+ minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
+ maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
+ for (int j = 0; j < maxValue.length; j++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
+ currentMinValue[j] = minValue[j].clone();
+ }
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
+ currentMaxValue[j] = maxValue[j].clone();
+ }
+ }
+ }
+
+ BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
+ minMax.setMaxValues(currentMaxValue);
+ minMax.setMinValues(currentMinValue);
+ blockletIndex.setMinMaxIndex(minMax);
+ return blockletIndex;
+ }
+
+ private ColumnSchema thriftColumnSchmeaToWrapperColumnSchema(
+ org.carbondata.format.ColumnSchema externalColumnSchema) {
+ ColumnSchema wrapperColumnSchema = new ColumnSchema();
+ wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
+ wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
+ wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
+ wrapperColumnSchema
+ .setDataType(thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type));
+ wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
+ List<Encoding> encoders = new ArrayList<Encoding>();
+ for (org.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
+ encoders.add(fromExternalToWrapperEncoding(encoder));
+ }
+ wrapperColumnSchema.setEncodingList(encoders);
+ wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
+ wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
+ wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
+ wrapperColumnSchema.setScale(externalColumnSchema.getScale());
+ wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
+ wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function());
+ return wrapperColumnSchema;
+ }
+
+ /**
+ * Below method is to convert the blocklet info of the thrift to wrapper
+ * blocklet info
+ *
+ * @param blockletInfoThrift blocklet info of the thrift
+ * @return blocklet info wrapper
+ */
+ private BlockletInfo getBlockletInfo(org.carbondata.format.BlockletInfo blockletInfoThrift) {
+ BlockletInfo blockletInfo = new BlockletInfo();
+ List<DataChunk> dimensionColumnChunk = new ArrayList<DataChunk>();
+ List<DataChunk> measureChunk = new ArrayList<DataChunk>();
+ Iterator<org.carbondata.format.DataChunk> column_data_chunksIterator =
+ blockletInfoThrift.getColumn_data_chunksIterator();
- while (column_data_chunksIterator.hasNext()) {
- org.carbondata.format.DataChunk next = column_data_chunksIterator.next();
- if (next.isRowMajor()) {
- dimensionColumnChunk.add(getDataChunk(next, false));
- } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) {
- measureChunk.add(getDataChunk(next, true));
- } else {
-
- dimensionColumnChunk.add(getDataChunk(next, false));
++ if (null != column_data_chunksIterator) {
++ while (column_data_chunksIterator.hasNext()) {
++ org.carbondata.format.DataChunk next = column_data_chunksIterator.next();
++ if (next.isRowMajor()) {
++ dimensionColumnChunk.add(getDataChunk(next, false));
++ } else if (next.getEncoders().contains(org.carbondata.format.Encoding.DELTA)) {
++ measureChunk.add(getDataChunk(next, true));
++ } else {
++ dimensionColumnChunk.add(getDataChunk(next, false));
++ }
+ }
+ }
+ blockletInfo.setDimensionColumnChunk(dimensionColumnChunk);
+ blockletInfo.setMeasureColumnChunk(measureChunk);
+ blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
+ return blockletInfo;
+ }
+
+ /**
+ * Below method is convert the thrift encoding to wrapper encoding
+ *
+ * @param encoderThrift thrift encoding
+ * @return wrapper encoding
+ */
+ private Encoding fromExternalToWrapperEncoding(org.carbondata.format.Encoding encoderThrift) {
+ switch (encoderThrift) {
+ case DICTIONARY:
+ return Encoding.DICTIONARY;
+ case DELTA:
+ return Encoding.DELTA;
+ case RLE:
+ return Encoding.RLE;
+ case INVERTED_INDEX:
+ return Encoding.INVERTED_INDEX;
+ case BIT_PACKED:
+ return Encoding.BIT_PACKED;
+ case DIRECT_DICTIONARY:
+ return Encoding.DIRECT_DICTIONARY;
+ default:
+ return Encoding.DICTIONARY;
+ }
+ }
+
+ /**
+ * Below method will be used to convert the thrift compression to wrapper
+ * compression codec
+ *
+ * @param compressionCodecThrift
+ * @return wrapper compression codec
+ */
+ private CompressionCodec getCompressionCodec(
+ org.carbondata.format.CompressionCodec compressionCodecThrift) {
+ switch (compressionCodecThrift) {
+ case SNAPPY:
+ return CompressionCodec.SNAPPY;
+ default:
+ return CompressionCodec.SNAPPY;
+ }
+ }
+
+ /**
+ * Below method will be used to convert thrift segment object to wrapper
+ * segment object
+ *
+ * @param segmentInfo thrift segment info object
+ * @return wrapper segment info object
+ */
+ private SegmentInfo getSegmentInfo(org.carbondata.format.SegmentInfo segmentInfo) {
+ SegmentInfo info = new SegmentInfo();
+ int[] cardinality = new int[segmentInfo.getColumn_cardinalities().size()];
+ for (int i = 0; i < cardinality.length; i++) {
+ cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
+ }
+ info.setColumnCardinality(cardinality);
+ info.setNumberOfColumns(segmentInfo.getNum_cols());
+ return info;
+ }
+
+ /**
+ * Below method will be used to convert the blocklet index of thrift to
+ * wrapper
+ *
+ * @param blockletIndexThrift
+ * @return blocklet index wrapper
+ */
+ private BlockletIndex getBlockletIndex(org.carbondata.format.BlockletIndex blockletIndexThrift) {
+ org.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index();
+ org.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index();
+ return new BlockletIndex(
+ new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()),
+ new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
+ }
+
+ /**
+ * Below method will be used to convert the thrift compression meta to
+ * wrapper chunk compression meta
+ *
+ * @param chunkCompressionMetaThrift
+ * @return chunkCompressionMetaWrapper
+ */
+ private ChunkCompressorMeta getChunkCompressionMeta(
+ org.carbondata.format.ChunkCompressionMeta chunkCompressionMetaThrift) {
+ ChunkCompressorMeta compressorMeta = new ChunkCompressorMeta();
+ compressorMeta
+ .setCompressor(getCompressionCodec(chunkCompressionMetaThrift.getCompression_codec()));
+ compressorMeta.setCompressedSize(chunkCompressionMetaThrift.getTotal_compressed_size());
+ compressorMeta.setUncompressedSize(chunkCompressionMetaThrift.getTotal_uncompressed_size());
+ return compressorMeta;
+ }
+
+ /**
+ * Below method will be used to convert the thrift data type to wrapper data
+ * type
+ *
+ * @param dataTypeThrift
+ * @return dataType wrapper
+ */
+ private DataType thriftDataTyopeToWrapperDataType(org.carbondata.format.DataType dataTypeThrift) {
+ switch (dataTypeThrift) {
+ case STRING:
+ return DataType.STRING;
+ case SHORT:
+ return DataType.SHORT;
+ case INT:
+ return DataType.INT;
+ case LONG:
+ return DataType.LONG;
+ case DOUBLE:
+ return DataType.DOUBLE;
+ case DECIMAL:
+ return DataType.DECIMAL;
+ case TIMESTAMP:
+ return DataType.TIMESTAMP;
+ case ARRAY:
+ return DataType.ARRAY;
+ case STRUCT:
+ return DataType.STRUCT;
+ default:
+ return DataType.STRING;
+ }
+ }
+
+ /**
+ * Below method will be used to convert the thrift presence meta to wrapper
+ * presence meta
+ *
+ * @param presentMetadataThrift
+ * @return wrapper presence meta
+ */
+ private PresenceMeta getPresenceMeta(org.carbondata.format.PresenceMeta presentMetadataThrift) {
+ PresenceMeta presenceMeta = new PresenceMeta();
+ presenceMeta.setRepresentNullValues(presentMetadataThrift.isRepresents_presence());
+ presenceMeta.setBitSet(BitSet.valueOf(presentMetadataThrift.getPresent_bit_stream()));
+ return presenceMeta;
+ }
+
+ /**
+ * Below method will be used to convert the thrift object to wrapper object
+ *
+ * @param sortStateThrift
+ * @return wrapper sort state object
+ */
+ private SortState getSortState(org.carbondata.format.SortState sortStateThrift) {
+ if (sortStateThrift == org.carbondata.format.SortState.SORT_EXPLICIT) {
+ return SortState.SORT_EXPLICT;
+ } else if (sortStateThrift == org.carbondata.format.SortState.SORT_NATIVE) {
+ return SortState.SORT_NATIVE;
+ } else {
+ return SortState.SORT_NONE;
+ }
+ }
+
+ /**
+ * Below method will be used to convert the thrift data chunk to wrapper
+ * data chunk
+ *
+ * @param datachunkThrift
+ * @return wrapper data chunk
+ */
+ private DataChunk getDataChunk(org.carbondata.format.DataChunk datachunkThrift,
+ boolean isPresenceMetaPresent) {
+ DataChunk dataChunk = new DataChunk();
+ dataChunk.setColumnUniqueIdList(datachunkThrift.getColumn_ids());
+ dataChunk.setDataPageLength(datachunkThrift.getData_page_length());
+ dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());
+ if (isPresenceMetaPresent) {
+ dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence()));
+ }
+ dataChunk.setRlePageLength(datachunkThrift.getRle_page_length());
+ dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset());
+ dataChunk.setRowMajor(datachunkThrift.isRowMajor());
+ dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());
+ dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());
+ dataChunk.setSortState(getSortState(datachunkThrift.getSort_state()));
+ dataChunk.setChunkCompressionMeta(getChunkCompressionMeta(datachunkThrift.getChunk_meta()));
+ List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());
+ for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {
+ encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
+ }
+ dataChunk.setEncoderList(encodingList);
+ if (encodingList.contains(Encoding.DELTA)) {
+ List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();
+ List<ValueEncoderMeta> encodeMetaList =
+ new ArrayList<ValueEncoderMeta>(thriftEncoderMeta.size());
+ for (int i = 0; i < thriftEncoderMeta.size(); i++) {
+ encodeMetaList.add(deserializeEncoderMeta(thriftEncoderMeta.get(i).array()));
+ }
+ dataChunk.setValueEncoderMeta(encodeMetaList);
+ }
+ return dataChunk;
+ }
+
+ /**
+ * Below method will be used to convert the encode metadata to
+ * ValueEncoderMeta object
+ *
+ * @param encoderMeta
+ * @return ValueEncoderMeta object
+ */
+ private ValueEncoderMeta deserializeEncoderMeta(byte[] encoderMeta) {
+ // TODO : should remove the unnecessary fields.
+ ByteArrayInputStream aos = null;
+ ObjectInputStream objStream = null;
+ ValueEncoderMeta meta = null;
+ try {
+ aos = new ByteArrayInputStream(encoderMeta);
+ objStream = new ObjectInputStream(aos);
+ meta = (ValueEncoderMeta) objStream.readObject();
+ } catch (ClassNotFoundException e) {
+ LOGGER.error(e);
+ } catch (IOException e) {
+ CarbonUtil.closeStreams(objStream);
+ }
+ return meta;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f722186/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
index 51874b4,0000000..e19a3e9
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ b/core/src/main/java/org/carbondata/scan/executor/infos/KeyStructureInfo.java
@@@ -1,125 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.executor.infos;
+
+import org.carbondata.core.keygenerator.KeyGenerator;
+
+/**
+ * Below class will store the structure of the key
+ * used during query execution
+ */
+public class KeyStructureInfo {
+
+ /**
+ * it's actually a latest key generator
+ * last table block as this key generator will be used to
+ * to update the mdkey of the older slice with the new slice
+ */
+ private KeyGenerator keyGenerator;
+
+ /**
+ * mask bytes ranges for the query
+ */
+ private int[] maskByteRanges;
+
+ /**
+ * masked bytes of the query
+ */
+ private int[] maskedBytes;
+
+ /**
+ * max key for query execution
+ */
+ private byte[] maxKey;
+
+ /**
- * mdkey start index of block
- */
- private int blockMdKeyStartOffset;
-
- /**
+ * @return the keyGenerator
+ */
+ public KeyGenerator getKeyGenerator() {
+ return keyGenerator;
+ }
+
+ /**
+ * @param keyGenerator the keyGenerator to set
+ */
+ public void setKeyGenerator(KeyGenerator keyGenerator) {
+ this.keyGenerator = keyGenerator;
+ }
+
+ /**
+ * @return the maskByteRanges
+ */
+ public int[] getMaskByteRanges() {
+ return maskByteRanges;
+ }
+
+ /**
+ * @param maskByteRanges the maskByteRanges to set
+ */
+ public void setMaskByteRanges(int[] maskByteRanges) {
+ this.maskByteRanges = maskByteRanges;
+ }
+
+ /**
+ * @return the maskedBytes
+ */
+ public int[] getMaskedBytes() {
+ return maskedBytes;
+ }
+
+ /**
+ * @param maskedBytes the maskedBytes to set
+ */
+ public void setMaskedBytes(int[] maskedBytes) {
+ this.maskedBytes = maskedBytes;
+ }
+
+ /**
+ * @return the maxKey
+ */
+ public byte[] getMaxKey() {
+ return maxKey;
+ }
+
+ /**
+ * @param maxKey the maxKey to set
+ */
+ public void setMaxKey(byte[] maxKey) {
+ this.maxKey = maxKey;
+ }
+
- /**
- * @param startOffset
- */
- public void setBlockMdKeyStartOffset(int startOffset) {
- this.blockMdKeyStartOffset = startOffset;
- }
-
- /**
- * @return
- */
- public int getBlockMdKeyStartOffset() {
- return this.blockMdKeyStartOffset;
- }
+}