[carbondata] branch feature/DistributedIndexServer updated (9051ab7 -> 7f05e69)

[ku...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kunalkapoor pushed a change to branch feature/DistributedIndexServer
in repository https://gitbox.apache.org/repos/asf/carbondata.git.


    from 9051ab7  [CARBONDATA-3298]Removed Log Message for Already Deleted Segments
     add 718be37  [CARBONDATA-3305] Support show metacache command to list the cache sizes for all tables
     add 3f98c51  [CARBONDATA-3305] Added DDL to drop cache for a table
     add e443a94  [CARBONDATA-3281] Add validation for the size of the LRU cache
     add a5fc19d  [CARBONDATA-3301]Fix inserting null values to Array<date> columns in carbon file format data load
     add 182eeb0  [LOG] Optimize the logs of CarbonProperties
     add f5e4793  [CARBONDATA-3307] Fix Performance Issue in No Sort
     add 6840a18  [CARBONDATA-3297] Fix that the IndexoutOfBoundsException when creating table and dropping table are at the same time
     add dda9c4d  [CARBONDATA-3300] Fixed ClassNotFoundException when using UDF in spark-shell
     add 1825861  [DOC] Update the doc of "Show DataMap"
     add c55a5d0  [CARBONDATA-3304] Distinguish the thread names created by thread pool of CarbonThreadFactory
     add 89c3873  [CARBONDATA-3313] count(*) is not invalidating the invalid segments cache
     add 8328cc7  [CARBONDATA-3315] Fix for Range Filter failing with two between clauses as children of OR expression
     add 5fbc644  [CARBONDATA-3314] Fix for Index Cache Size in SHOW METACACHE DDL
     add 1c8e9b6  [CARBONDATA-3317] Fix NPE when execute 'show segments' command for stream table
     add cdcea33  [CARBONDATA-3311] support presto 0.217 #3142
     add dda3e7d  [TestCase][HOTFIX] Added drop database in beforeEach to avoid exception
     add 66982f3  [CARBONDATA-3302] [Spark-Integration] code cleaning related to CarbonCreateTable command
     add 2980803  [CARBONDATA-3318] Added PreAgg & Bloom Event-Listener for ShowCacheCommmand
     add 6fd81af  [CARBONDATA-3293] Prune datamaps improvement for count(*)
     add e61b97e  [CARBONDATA-3321] Improved Single/Concurrent query Performance
     add 9854f20  [DOC] Fix the spell mistake of enable.unsafe.in.query.processing
     add 5c52876  [CARBONDATA-3322] [CARBONDATA-3323] Added check for invalid tables in ShowCacheCommand & Standard output on ShowCacheCommand on table
     add e5a97d0  [CARBONDATA-3320]fix number of partitions issue in describe formatted and drop partition issue
     add dc3d5ab  [CARBONDATA-3329] Fixed deadlock issue during failed query
     add 91c6758  [CARBONDATA-3328]Fixed performance issue with merge small files distribution
     add 726bb7e  [HOTFIX][DOC] Optimize quick-start-guide.md and dml-of-carbondata.md
     add 176d6a7  [CARBONDATA-3319][TestCase]Added condition to check if datamap exist or not before caching
     add 35e63b1  [CARBONDATA-3330] Fix Invalid Exception while clearing datamap from SDK carbon reader
     add 4a7e3bd  [CARBONDATA-3333]Fixed No Sort Store Size issue and Compatibility issue after alter added column done in 1.1 and load in 1.5
     add 46006ab  [Document] update doc about presto version support details #3163
     add 36e6840  [CARBONDATA-3332] Blocked concurrent compaction and update/delete
     add 38c81e2  [CARBONDATA-3335]Fixed load and compaction failure after alter done in older version
     add 0b93ba3  [HOTFIX]Fixed data map loading issue when number of segments are high
     add 7f05e69  [HOTFIX]fixed loading issue for legacy store

No new revisions were added by this update.

Summary of changes:
 .../carbondata/core/cache/CacheProvider.java       |   4 +
 .../carbondata/core/cache/CarbonLRUCache.java      |  59 +++-
 .../core/constants/CarbonCommonConstants.java      |   5 +
 .../constants/CarbonCommonConstantsInternal.java   |   2 +
 .../core/datamap/DataMapStoreManager.java          |  25 +-
 .../core/datamap/DistributableDataMapFormat.java   |   1 +
 .../apache/carbondata/core/datamap/Segment.java    |  24 +-
 .../carbondata/core/datamap/TableDataMap.java      | 102 ++++--
 .../carbondata/core/datamap/dev/DataMap.java       |  15 +
 .../datamap/dev/cgdatamap/CoarseGrainDataMap.java  |  14 +
 .../datamap/dev/fgdatamap/FineGrainDataMap.java    |  13 +
 .../carbondata/core/datastore/TableSpec.java       |  86 ++++-
 .../block/SegmentPropertiesAndSchemaHolder.java    |  62 ++--
 .../UnsafeAbstractDimensionDataChunkStore.java     |  10 +-
 .../core/datastore/impl/FileFactory.java           |  43 ++-
 .../core/indexstore/ExtendedBlocklet.java          |  97 ++++--
 .../core/indexstore/SegmentPropertiesFetcher.java  |   3 +
 .../TableBlockIndexUniqueIdentifier.java           |   5 +-
 .../core/indexstore/UnsafeMemoryDMStore.java       | 161 ++++++---
 .../indexstore/blockletindex/BlockDataMap.java     | 149 ++++++---
 .../indexstore/blockletindex/BlockletDataMap.java  |  28 +-
 .../blockletindex/BlockletDataMapFactory.java      |   9 +-
 .../blockletindex/BlockletDataMapRowIndexes.java   |  14 +-
 .../carbondata/core/indexstore/row/DataMapRow.java |  12 +-
 .../core/indexstore/row/UnsafeDataMapRow.java      | 217 ++-----------
 .../core/indexstore/schema/CarbonRowSchema.java    |   8 +
 .../core/indexstore/schema/SchemaGenerator.java    |  72 ++++
 .../apache/carbondata/core/locks/LockUsage.java    |   1 +
 .../core/memory/UnsafeMemoryManager.java           |   3 +-
 .../carbondata/core/metadata/SegmentFileStore.java |  21 +-
 .../core/metadata/blocklet/BlockletInfo.java       |   4 +-
 .../carbondata/core/mutate/CarbonUpdateUtil.java   |  10 +-
 .../scan/expression/RangeExpressionEvaluator.java  |  30 +-
 .../carbondata/core/scan/model/QueryModel.java     |  30 --
 .../carbondata/core/util/CarbonProperties.java     | 132 ++++----
 .../apache/carbondata/core/util/CarbonUtil.java    |   4 +-
 .../core/util/ObjectSerializationUtil.java         |   3 +-
 .../apache/carbondata/hadoop/CarbonInputSplit.java | 361 +++++++++++++++------
 .../hadoop/internal/ObjectArrayWritable.java       |   0
 .../carbondata/hadoop/internal/index/Block.java    |   0
 .../carbondata/core/cache/CarbonLRUCacheTest.java  |   7 +
 .../datamap/bloom/BloomCacheKeyValue.java          |   2 +-
 .../bloom/BloomCoarseGrainDataMapFactory.java      |   9 +-
 docs/datamap/datamap-management.md                 |   1 +
 docs/ddl-of-carbondata.md                          |  43 +++
 docs/dml-of-carbondata.md                          |  24 +-
 docs/presto-guide.md                               |  33 +-
 docs/quick-start-guide.md                          |   6 +-
 docs/usecases.md                                   |   4 +-
 .../carbondata/hadoop/CarbonMultiBlockSplit.java   |  23 +-
 .../carbondata/hadoop/CarbonRecordReader.java      |   4 +-
 .../hadoop/api/CarbonFileInputFormat.java          |   6 +-
 .../carbondata/hadoop/api/CarbonInputFormat.java   |  55 +---
 .../hadoop/api/CarbonTableInputFormat.java         | 117 +++----
 .../hadoop/api/CarbonTableOutputFormat.java        |   3 +-
 .../hadoop/util/CarbonVectorizedRecordReader.java  |   2 +-
 integration/presto/pom.xml                         |   2 +-
 .../presto/CarbondataConnectorFactory.java         |   7 +-
 .../apache/carbondata/presto/CarbondataModule.java |   8 +-
 .../carbondata/presto/CarbondataSplitManager.java  |   2 +-
 .../presto/impl/CarbonLocalInputSplit.java         |   4 +-
 .../cluster/sdv/generated/QueriesBVATestCase.scala |   4 +-
 ...teTableUsingSparkCarbonFileFormatTestCase.scala |   7 +-
 .../datasource/SparkCarbonDataSourceTestCase.scala |  19 +-
 .../sql/common/util/DataSourceTestUtil.scala}      |  88 ++---
 ...ryWithColumnMetCacheAndCacheLevelProperty.scala |   5 +-
 .../dblocation/DBLocationCarbonTableTestCase.scala |  26 +-
 .../detailquery/RangeFilterTestCase.scala          |  38 +++
 .../TestAllDataTypeForPartitionTable.scala         |   4 +-
 .../partition/TestDDLForPartitionTable.scala       |  13 +-
 ...StandardPartitionWithPreaggregateTestCase.scala |  22 ++
 .../sql/commands/TestCarbonDropCacheCommand.scala  | 200 ++++++++++++
 .../sql/commands/TestCarbonShowCacheCommand.scala  | 233 +++++++++++++
 .../org/apache/carbondata/spark/util/Util.java     |   3 +-
 .../org/apache/carbondata/api/CarbonStore.scala    |  10 +-
 .../{CleanFilesEvents.scala => CacheEvents.scala}  |  25 +-
 .../org/apache/carbondata/events/Events.scala      |  14 +
 .../carbondata/spark/rdd/CarbonMergerRDD.scala     |   4 +-
 .../carbondata/spark/rdd/CarbonScanRDD.scala       |  13 +-
 .../apache/carbondata/spark/util/CommonUtil.scala  |  39 ++-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala    |   1 +
 .../datasources/SparkCarbonFileFormat.scala        |   6 +-
 .../datasource/SparkCarbonDataSourceTest.scala     |  11 +
 .../spark/rdd/CarbonDataRDDFactory.scala           |  48 +--
 .../org/apache/spark/sql/CarbonCountStar.scala     |   2 +-
 .../spark/sql/CarbonDatasourceHadoopRelation.scala |   6 +-
 .../scala/org/apache/spark/sql/CarbonEnv.scala     |   5 +
 .../sql/execution/command/cache/CacheUtil.scala    | 114 +++++++
 .../command/cache/CarbonDropCacheCommand.scala     |  66 ++++
 .../command/cache/CarbonShowCacheCommand.scala     | 225 +++++++++++++
 .../command/cache/DropCacheEventListeners.scala    | 121 +++++++
 .../command/cache/ShowCacheEventListeners.scala    | 126 +++++++
 .../CarbonAlterTableCompactionCommand.scala        |  55 ++--
 .../mutation/CarbonProjectForDeleteCommand.scala   |  18 +-
 .../mutation/CarbonProjectForUpdateCommand.scala   |  90 ++---
 .../command/mutation/DeleteExecution.scala         |   2 +-
 .../command/table/CarbonCreateTableCommand.scala   |   3 +-
 .../table/CarbonDescribeFormattedCommand.scala     |  19 +-
 .../spark/sql/hive/CarbonFileMetastore.scala       |  37 ++-
 .../spark/sql/parser/CarbonSpark2SqlParser.scala   |  18 +-
 .../BloomCoarseGrainDataMapFunctionSuite.scala     |   2 +-
 .../register/TestRegisterCarbonTable.scala         |  24 +-
 .../processing/datatypes/PrimitiveDataType.java    |   7 +-
 .../loading/TableProcessingOperations.java         |   3 +-
 .../loading/converter/impl/RowConverterImpl.java   |   2 +-
 .../loading/row/IntermediateSortTempRow.java       |   8 +
 .../loading/sort/SortStepRowHandler.java           |   7 +-
 .../sort/impl/ParallelReadMergeSorterImpl.java     |   3 +-
 .../impl/UnsafeParallelReadMergeSorterImpl.java    |   3 +-
 .../loading/sort/unsafe/UnsafeCarbonRowPage.java   |   8 +-
 .../loading/sort/unsafe/UnsafeSortDataRows.java    |   9 +-
 .../holder/UnsafeSortTempFileChunkHolder.java      |  16 +-
 .../merger/UnsafeIntermediateFileMerger.java       |   2 +-
 .../unsafe/merger/UnsafeIntermediateMerger.java    |   5 +-
 .../CarbonRowDataWriterProcessorStepImpl.java      |   2 +-
 .../loading/steps/DataWriterProcessorStepImpl.java |   2 +-
 .../loading/steps/InputProcessorStepImpl.java      |   2 +-
 .../DummyRowUpdater.java}                          |  25 +-
 .../processing/sort/SchemaBasedRowUpdater.java     |  91 ++++++
 .../SortTempRowUpdater.java}                       |  27 +-
 .../processing/sort/sortdata/SortDataRows.java     |   3 +-
 .../sort/sortdata/SortIntermediateFileMerger.java  |   3 +-
 .../processing/sort/sortdata/SortParameters.java   |  54 +++
 .../sort/sortdata/SortTempFileChunkHolder.java     |  18 +-
 .../processing/sort/sortdata/TableFieldStat.java   |  16 +
 .../store/writer/AbstractFactDataWriter.java       |   6 +-
 .../carbondata/sdk/file/CarbonReaderTest.java      |  38 +--
 127 files changed, 3185 insertions(+), 1142 deletions(-)
 rename {hadoop => core}/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java (56%)
 rename {hadoop => core}/src/main/java/org/apache/carbondata/hadoop/internal/ObjectArrayWritable.java (100%)
 rename {hadoop => core}/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java (100%)
 copy integration/{spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/TestUtil.scala => spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/DataSourceTestUtil.scala} (67%)
 create mode 100644 integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonDropCacheCommand.scala
 create mode 100644 integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/TestCarbonShowCacheCommand.scala
 copy integration/spark-common/src/main/scala/org/apache/carbondata/events/{CleanFilesEvents.scala => CacheEvents.scala} (72%)
 create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/CacheUtil.scala
 create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonDropCacheCommand.scala
 create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala
 create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/DropCacheEventListeners.scala
 create mode 100644 integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/ShowCacheEventListeners.scala
 copy processing/src/main/java/org/apache/carbondata/processing/{loading/sort/unsafe/holder/SortTempChunkHolder.java => sort/DummyRowUpdater.java} (54%)
 create mode 100644 processing/src/main/java/org/apache/carbondata/processing/sort/SchemaBasedRowUpdater.java
 copy processing/src/main/java/org/apache/carbondata/processing/{loading/sort/unsafe/holder/SortTempChunkHolder.java => sort/SortTempRowUpdater.java} (53%)