You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/10 03:46:42 UTC
[6/6] carbondata git commit: [CARBONDATA-1015] Extract interface in
data load write step. This closes #971
[CARBONDATA-1015] Extract interface in data load write step. This closes #971
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3df5ef81
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3df5ef81
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3df5ef81
Branch: refs/heads/master
Commit: 3df5ef81e4d2253b984847291acc3933306a36b9
Parents: f53ab4b dc83b2a
Author: ravipesala <ra...@gmail.com>
Authored: Sat Jun 10 09:16:01 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Jun 10 09:16:01 2017 +0530
----------------------------------------------------------------------
.../core/compression/BigIntCompressor.java | 34 +-
.../core/compression/DoubleCompressor.java | 22 +-
.../core/compression/ValueCompressor.java | 75 +---
.../core/datastore/DimensionType.java | 35 ++
.../carbondata/core/datastore/TableSpec.java | 220 ++++++++++
.../core/datastore/block/SegmentProperties.java | 6 +-
.../AbstractMeasureChunkReaderV2V3Format.java | 2 +
...CompressedMeasureChunkFileBasedReaderV2.java | 29 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 33 +-
...ndexerStorageForNoInvertedIndexForShort.java | 4 +-
.../compression/MeasureMetaDataModel.java | 2 +-
.../compression/WriterCompressModel.java | 243 -----------
.../dataholder/CarbonWriteDataHolder.java | 131 ------
.../exception/CarbonDataWriterException.java | 79 ++++
.../core/datastore/page/ColumnPage.java | 196 ++++++++-
.../core/datastore/page/ComplexColumnPage.java | 8 +-
.../datastore/page/FixLengthColumnPage.java | 159 -------
.../datastore/page/VarLengthColumnPage.java | 42 --
.../page/statistics/ColumnPageStatistics.java | 172 ++++++++
.../page/statistics/MeasurePageStatsVO.java | 103 +++++
.../page/statistics/PageStatistics.java | 132 ------
.../core/datastore/row/CarbonRow.java | 73 ++++
.../core/datastore/row/WriteStepRowUtil.java | 86 ++++
.../core/metadata/BlockletInfoColumnar.java | 20 +-
.../core/metadata/CarbonMetadata.java | 6 +-
.../core/metadata/datatype/DataType.java | 5 +-
.../schema/table/column/CarbonDimension.java | 23 +-
.../table/column/CarbonImplicitDimension.java | 9 +-
.../schema/table/column/ColumnSchema.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 4 +-
.../core/scan/executor/util/QueryUtil.java | 22 +-
.../scan/executor/util/RestructureUtil.java | 4 +-
.../util/AbstractDataFileFooterConverter.java | 2 +-
.../apache/carbondata/core/util/ByteUtil.java | 18 +
.../core/util/CarbonMetadataUtil.java | 113 ++---
.../apache/carbondata/core/util/CarbonUtil.java | 43 +-
.../apache/carbondata/core/util/NodeHolder.java | 61 ++-
.../core/util/ValueCompressionUtil.java | 56 +--
...ressedDimensionChunkFileBasedReaderTest.java | 132 ------
...mpressedMeasureChunkFileBasedReaderTest.java | 90 ----
.../core/metadata/CarbonMetadataTest.java | 2 +-
.../core/util/CarbonMetadataUtilTest.java | 29 +-
.../carbondata/core/util/CarbonUtilTest.java | 5 +-
.../core/util/ValueCompressionUtilTest.java | 134 ------
.../core/writer/CarbonFooterWriterTest.java | 30 +-
docs/useful-tips-on-carbondata.md | 3 +-
.../util/ExternalColumnDictionaryTestCase.scala | 11 +-
.../vectorreader/AddColumnTestCases.scala | 1 +
.../core/datastore/GenericDataType.java | 145 +++++++
.../columnar/ColGroupBlockStorage.java | 103 +++++
.../processing/datatypes/ArrayDataType.java | 1 +
.../processing/datatypes/GenericDataType.java | 145 -------
.../processing/datatypes/PrimitiveDataType.java | 1 +
.../processing/datatypes/StructDataType.java | 1 +
.../merger/CompactionResultSortProcessor.java | 4 +-
.../merger/RowResultMergerProcessor.java | 6 +-
.../newflow/AbstractDataLoadProcessorStep.java | 2 +-
.../newflow/CarbonDataLoadConfiguration.java | 12 +
.../newflow/DataLoadProcessBuilder.java | 3 +
.../newflow/converter/FieldConverter.java | 2 +-
.../newflow/converter/RowConverter.java | 2 +-
.../impl/ComplexFieldConverterImpl.java | 4 +-
.../impl/DictionaryFieldConverterImpl.java | 6 +-
.../DirectDictionaryFieldConverterImpl.java | 10 +-
.../converter/impl/FieldEncoderFactory.java | 2 +-
.../impl/MeasureFieldConverterImpl.java | 4 +-
.../impl/NonDictionaryFieldConverterImpl.java | 7 +-
.../converter/impl/RowConverterImpl.java | 2 +-
.../processing/newflow/row/CarbonRow.java | 73 ----
.../processing/newflow/row/CarbonRowBatch.java | 1 +
.../processing/newflow/row/CarbonSortBatch.java | 1 +
.../newflow/row/WriteStepRowUtil.java | 86 ----
.../sort/impl/ParallelReadMergeSorterImpl.java | 8 +-
...arallelReadMergeSorterWithBucketingImpl.java | 10 +-
.../UnsafeBatchParallelReadMergeSorterImpl.java | 4 +-
.../impl/UnsafeParallelReadMergeSorterImpl.java | 10 +-
...arallelReadMergeSorterWithBucketingImpl.java | 11 +-
.../newflow/sort/unsafe/IntPointerBuffer.java | 1 -
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 2 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 2 +-
.../CarbonRowDataWriterProcessorStepImpl.java | 6 +-
.../steps/DataConverterProcessorStepImpl.java | 2 +-
...ConverterProcessorWithBucketingStepImpl.java | 2 +-
.../steps/DataWriterBatchProcessorStepImpl.java | 2 +-
.../steps/DataWriterProcessorStepImpl.java | 4 +-
.../newflow/steps/DummyClassForTest.java | 2 +-
.../newflow/steps/InputProcessorStepImpl.java | 2 +-
.../newflow/steps/SortProcessorStepImpl.java | 2 +-
.../sortandgroupby/sortdata/RowComparator.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 429 ++-----------------
.../store/CarbonFactDataHandlerModel.java | 32 +-
.../processing/store/CarbonFactHandler.java | 4 +-
.../processing/store/DefaultEncoder.java | 265 ++++++++++++
.../store/SingleThreadFinalSortFilesMerger.java | 2 +-
.../carbondata/processing/store/TablePage.java | 58 ++-
.../processing/store/TablePageKey.java | 139 ++++++
.../processing/store/TablePageStatistics.java | 142 ++++++
.../store/colgroup/ColGroupBlockStorage.java | 101 -----
.../store/writer/AbstractFactDataWriter.java | 57 +--
.../store/writer/CarbonDataWriterVo.java | 30 +-
.../store/writer/CarbonFactDataWriter.java | 26 +-
.../processing/store/writer/Encoder.java | 38 ++
.../exception/CarbonDataWriterException.java | 79 ----
.../writer/v1/CarbonFactDataWriterImplV1.java | 67 ++-
.../writer/v2/CarbonFactDataWriterImplV2.java | 18 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 135 +++---
.../util/CarbonDataProcessorUtil.java | 2 +-
.../processing/util/NonDictionaryUtil.java | 2 +-
108 files changed, 2385 insertions(+), 2649 deletions(-)
----------------------------------------------------------------------