You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/11/10 14:49:27 UTC
[5/5] incubator-carbondata git commit: [CARBONDATA-2] Data load
integration of all steps for removing kettle This closes #263
[CARBONDATA-2] Data load integration of all steps for removing kettle This closes #263
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e3085397
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e3085397
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e3085397
Branch: refs/heads/master
Commit: e30853975a986ff15ec5c0823876a5da2f613eef
Parents: ae9d88b 496cde4
Author: jackylk <ja...@huawei.com>
Authored: Thu Nov 10 22:40:26 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Nov 10 22:40:26 2016 +0800
----------------------------------------------------------------------
.../core/carbon/CarbonTableIdentifier.java | 8 +
.../store/dataholder/CarbonWriteDataHolder.java | 42 ++
.../TimeStampDirectDictionaryGenerator.java | 7 +-
.../carbondata/core/load/BlockDetails.java | 31 +-
.../carbondata/hadoop/csv/CSVInputFormat.java | 54 ++
.../recorditerator/RecordReaderIterator.java | 68 ++
.../hadoop/test/util/StoreCreator.java | 4 +-
.../sql/common/util/CarbonHiveContext.scala | 3 +-
integration/spark/pom.xml | 1 +
.../spark/merger/RowResultMerger.java | 2 +-
.../carbondata/spark/load/CarbonLoadModel.java | 645 ------------------
.../carbondata/spark/load/CarbonLoaderUtil.java | 1 +
.../spark/load/DeleteLoadFolders.java | 1 +
.../spark/merger/CarbonDataMergerUtil.java | 2 +-
.../carbondata/spark/util/LoadMetadataUtil.java | 2 +-
.../spark/CarbonDataFrameWriter.scala | 3 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 5 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 58 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 3 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 272 ++++++++
.../spark/util/GlobalDictionaryUtil.scala | 2 +-
.../apache/spark/mapred/SparkMapRedUtil.scala | 32 +
.../org/apache/spark/sql/CarbonSqlParser.scala | 3 +-
.../execution/command/carbonTableSchema.scala | 59 +-
.../spark/sql/hive/CarbonStrategies.scala | 4 +-
.../org/apache/spark/util/SplitUtils.scala | 2 +-
.../TestLoadDataWithNotProperInputFile.scala | 2 +-
.../spark/util/AllDictionaryTestCase.scala | 4 +-
.../AutoHighCardinalityIdentifyTestCase.scala | 3 +-
.../util/ExternalColumnDictionaryTestCase.scala | 3 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 3 +-
.../util/GlobalDictionaryUtilTestCase.scala | 4 +-
.../sql/common/util/CarbonHiveContext.scala | 3 +-
pom.xml | 7 +
.../processing/csvload/DataGraphExecuter.java | 45 +-
.../processing/csvload/GraphExecutionUtil.java | 61 --
.../processing/datatypes/ArrayDataType.java | 27 +-
.../processing/datatypes/GenericDataType.java | 18 +-
.../processing/datatypes/PrimitiveDataType.java | 73 ++-
.../processing/datatypes/StructDataType.java | 37 +-
.../processing/mdkeygen/MDKeyGenStep.java | 2 +-
.../processing/model/CarbonLoadModel.java | 647 +++++++++++++++++++
.../processing/newflow/DataField.java | 17 +-
.../processing/newflow/DataLoadExecutor.java | 72 +++
.../newflow/DataLoadProcessBuilder.java | 167 +++++
.../constants/DataLoadProcessorConstants.java | 8 +
.../newflow/converter/BadRecordLogHolder.java | 46 ++
.../newflow/converter/FieldConverter.java | 4 +-
.../newflow/converter/RowConverter.java | 4 +
.../AbstractDictionaryFieldConverterImpl.java | 4 +-
.../impl/ComplexFieldConverterImpl.java | 36 +-
.../impl/DictionaryFieldConverterImpl.java | 31 +-
.../DirectDictionaryFieldConverterImpl.java | 51 +-
.../converter/impl/FieldEncoderFactory.java | 86 ++-
.../impl/MeasureFieldConverterImpl.java | 83 +++
.../impl/NonDictionaryFieldConverterImpl.java | 20 +-
.../converter/impl/RowConverterImpl.java | 50 +-
.../newflow/dictionary/DirectDictionary.java | 59 ++
.../dictionary/PreCreatedDictionary.java | 8 +-
.../exception/BadRecordFoundException.java | 67 ++
.../newflow/parser/CarbonParserFactory.java | 25 +-
.../newflow/parser/GenericParser.java | 4 +-
.../newflow/parser/impl/ArrayParserImpl.java | 33 +-
.../parser/impl/PrimitiveParserImpl.java | 2 +-
.../newflow/parser/impl/RowParserImpl.java | 68 +-
.../newflow/parser/impl/StructParserImpl.java | 27 +-
.../processing/newflow/row/CarbonRow.java | 18 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 28 +-
.../sort/impl/SortPreparatorIterator.java | 147 -----
.../steps/DataConverterProcessorStepImpl.java | 106 ++-
.../steps/DataWriterProcessorStepImpl.java | 216 +++++++
.../newflow/steps/DummyClassForTest.java | 84 +++
.../newflow/steps/InputProcessorStepImpl.java | 47 +-
.../newflow/steps/SortProcessorStepImpl.java | 1 +
.../writer/DataWriterProcessorStepImpl.java | 222 -------
.../sortdata/IntermediateFileMerger.java | 90 ++-
.../sortdata/NewRowComparator.java | 73 +++
.../sortdata/NewRowComparatorForNormalDims.java | 61 ++
.../sortandgroupby/sortdata/SortDataRows.java | 112 +++-
.../sortandgroupby/sortdata/SortParameters.java | 44 +-
.../sortdata/SortTempFileChunkHolder.java | 119 +++-
.../store/CarbonFactDataHandlerColumnar.java | 309 ++++++++-
.../store/CarbonFactDataHandlerModel.java | 40 +-
.../store/SingleThreadFinalSortFilesMerger.java | 8 +-
.../csvbased/BadRecordsLogger.java | 249 +++++++
.../csvbased/BadRecordslogger.java | 235 -------
.../csvbased/CarbonCSVBasedSeqGenStep.java | 57 +-
.../util/CarbonDataProcessorUtil.java | 226 ++++++-
.../processing/util/RemoveDictionaryUtil.java | 9 +
.../carbondata/test/util/StoreCreator.java | 3 +-
91 files changed, 4027 insertions(+), 1705 deletions(-)
----------------------------------------------------------------------