You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2021/07/02 10:33:35 UTC
[incubator-pinot] branch threshold_updater_patch updated (1047e1d
-> 78a8b86)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a change to branch threshold_updater_patch
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 1047e1d Remove unused import
add b826f2f Add LZ4 Compression Codec (#6804) (#7035)
add f369da1 Change controller API to allow use specify column filter when query segment metadata (#7061)
add 92868e6 Revert "Upgrade t-digest to latest release (#7029)" (#7069)
add 18051eb Clear legacy configs when converting to new TableConfig. (#7071)
add 849b4c1 Make BitmapDocIdSetOperator a top level operator (#7068)
add f773208 Add Query Options for GroupBy In-Segment Trim (#7052)
add ce419d2 Allow reloading consuming segment by default (#7078)
add 9e26c24 Add support for range index rule recommendation(#7034) (#7063)
add 282edba Adding support for config files in compat tests (#7074)
add bd792a3 Misc fixes for json data type (#7057)
add 80fda68 Startable class from config (#7075)
add f4e5518 Add partial upsert config and mergers (#6899)
add 01b6c22 Support data preprocessing for AVRO and ORC formats (#7062)
add 709abb0 Adding Additional JMX Metrics to Prometheus config (#7079)
add c856c6c Support json path expressions in query. (#6998)
add 064a492 Make TableConfigTunerRegistry configurable to scan packages. (#7089)
add ac8f098 Fixing docker build scripts (#7094)
add 1561414 Fixing docker build scripts by providing JDK_VERSION as parameter (#7095)
add 0c813bd Optimize BaseSingleSegmentConversionExecutor by skipping task if CRC not match (#7086)
add c950926 Added TaskMetricsEmitted periodic controller job (#7091)
add 43a67b8 Fix the deleted segments directory not exist warning (#7097)
add 8283d39 Enhance PinotHelixResourceManager.updateInstance() to preserve custom fields (#7098)
add 3503074 Add system info on instance config for servers. (#7093)
add 3976eac Move KafkaJsonDecoder code to pinot-json (#7021)
add 47f1d93 Add `PinotHelixResourceManager` to `TableConfigTuner.init()` method. (#7101)
add bbdd615 Added support for additional mvn options (#7103)
add 0981271 Add minion metrics of task queueing time and task numbers (#7099)
add 2475089 Fix maven options (#7105)
add 47a75e5 Remove parallel builds in compat scripts (#7108)
add ea3566f Fix the memory issue for selection query with large limit (#7112)
add f38609b Clean up SegmentMetadata and ColumnMetadata (#7104)
add f809e50 SegmentProcessorFramework Enhancement (#7092)
add 0fb9209 Implement Append merger for partial upsert (#7087)
add 0255757 Fix build (#7115)
add 423ac35 Validate the length of preprocessing operation from custom config (#7119)
add 70308f8 Fixing pinot k8s helm for jdk11 (#7118)
add 84c6a6e Add integration test for Kinesis realtime connector (#7016)
add d4d0b93 Cancel running Kinesis consumer tasks when timeout occurs (#7109)
add f9f58c7 Allow updating controller and broker helix hostname (#7064)
add 78a8b86 Merge branch 'master' into threshold_updater_patch
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 2 +-
compatibility-verifier/checkoutAndBuild.sh | 6 +-
compatibility-verifier/compCheck.sh | 76 +-
.../config/BrokerConfig.properties | 14 +-
.../config/ControllerConfig.properties | 16 +-
.../config/ServerConfig.properties | 17 +-
docker/images/pinot/Dockerfile | 3 +-
.../pinot/etc/jmx_prometheus_javaagent/README.md | 22 +
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 72 +-
.../helm/pinot/templates/controller/configmap.yaml | 2 +-
.../helm/pinot/templates/minion/configmap.yaml | 2 +-
kubernetes/helm/pinot/values.yaml | 10 +-
...ixBrokerStarter.java => BaseBrokerStarter.java} | 156 ++--
.../broker/broker/helix/HelixBrokerStarter.java | 391 +--------
.../requesthandler/BaseBrokerRequestHandler.java | 10 +-
.../broker/HelixBrokerStarterHostnamePortTest.java | 101 +++
.../broker/broker/HelixBrokerStarterTest.java | 3 +-
pinot-common/pom.xml | 4 +
.../pinot/common/metrics/ControllerGauge.java | 6 +
.../restlet/resources/SystemResourceInfo.java | 100 +++
.../pinot/common/utils/config/InstanceUtils.java | 64 +-
.../common/utils/config/TableConfigUtils.java | 48 +-
.../pinot/common/utils/helix/HelixHelper.java | 70 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 58 ++
.../common/utils/config/InstanceUtilsTest.java | 146 ++++
.../common/utils/config/TableConfigUtilsTest.java | 8 +
.../pinot/common/utils/helix/HelixHelperTest.java | 78 ++
...llerStarter.java => BaseControllerStarter.java} | 163 ++--
.../apache/pinot/controller/ControllerConf.java | 66 +-
.../apache/pinot/controller/ControllerStarter.java | 664 +---------------
.../api/resources/PinotQueryResource.java | 10 +-
.../api/resources/PinotSegmentRestletResource.java | 11 +-
.../api/resources/PinotTableRestletResource.java | 3 +-
.../api/resources/ServerTableSizeReader.java | 2 +-
.../api/resources/TableConfigsRestletResource.java | 3 +-
.../api/resources/TableDebugResource.java | 2 +-
.../helix/core/PinotHelixResourceManager.java | 24 +-
.../helix/core/SegmentDeletionManager.java | 10 +-
.../core/minion/PinotHelixTaskResourceManager.java | 118 ++-
.../helix/core/minion/PinotTaskManager.java | 3 +-
.../helix/core/minion/TaskMetricsEmitter.java | 90 +++
.../realtime/PinotLLCRealtimeSegmentManager.java | 16 +-
.../helix/core/util/ZKMetadataUtils.java | 13 +-
.../controller/recommender/io/InputManager.java | 6 +
.../recommender/rules/RulesToExecute.java | 14 +
.../NoDictionaryOnHeapDictionaryJointRule.java | 3 +-
.../recommender/rules/impl/RangeIndexRule.java | 116 +++
.../rules/io/params/RangeIndexRuleParams.java | 40 +-
.../rules/io/params/RecommenderConstants.java | 5 +
.../tuner/NoOpTableTableConfigTuner.java | 7 +-
.../controller}/tuner/RealTimeAutoIndexTuner.java | 7 +-
.../pinot/controller}/tuner/TableConfigTuner.java | 16 +-
.../tuner/TableConfigTunerRegistry.java | 47 +-
.../controller/tuner/TableConfigTunerUtils.java | 24 +-
.../org/apache/pinot/controller}/tuner/Tuner.java | 2 +-
.../controller/util/CompletionServiceHelper.java | 25 +-
.../util/ConsumingSegmentInfoReader.java | 2 +-
.../util/ServerSegmentMetadataReader.java | 37 +-
.../pinot/controller/util/TableMetadataReader.java | 15 +-
.../pinot/controller/ControllerStarterTest.java | 84 ++
.../pinot/controller/ControllerTestUtils.java | 13 +-
.../api/PinotInstanceRestletResourceTest.java | 17 +-
.../controller/api/PinotSegmentsMetadataTest.java | 6 +-
...ControllerPeriodicTaskStarterStatelessTest.java | 12 +-
.../pinot/controller/helix/ControllerTest.java | 14 +-
.../helix/PinotControllerModeStatelessTest.java | 27 +-
.../helix/core/minion/PinotTaskManagerTest.java | 3 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 7 +-
.../controller/recommender/TestConfigEngine.java | 14 +
.../tuner/RealTimeAutoIndexTunerTest.java | 9 +-
.../pinot/controller}/tuner/TunerRegistryTest.java | 9 +-
.../controller/utils/SegmentMetadataMockUtils.java | 6 +-
.../recommenderInput/RangeIndexInput.json | 140 ++++
.../manager/realtime/RealtimeTableDataManager.java | 46 +-
.../pinot/core/minion/MergeRollupConverter.java | 186 -----
.../pinot/core/minion/RawIndexConverter.java | 13 +-
.../core/minion/rollup/RollupRecordAggregator.java | 11 +-
.../core/operator/BitmapDocIdSetOperator.java | 74 ++
.../ExpressionScanDocIdIterator.java | 81 +-
.../query/AggregationGroupByOrderByOperator.java | 27 +-
.../operator/query/SelectionOrderByOperator.java | 39 +-
.../JsonExtractScalarTransformFunction.java | 304 ++++---
.../org/apache/pinot/core/plan/FilterPlanNode.java | 12 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 2 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 20 +-
.../statement/JsonStatementOptimizer.java | 587 ++++++++++++++
.../optimizer/statement/StatementOptimizer.java | 21 +-
.../query/pruner/SelectionQuerySegmentPruner.java | 2 +-
.../query/selection/SelectionOperatorUtils.java | 9 +-
.../MaxValueAggregator.java | 2 +-
.../MinValueAggregator.java | 2 +-
.../SumValueAggregator.java | 2 +-
.../{collector => aggregator}/ValueAggregator.java | 2 +-
.../ValueAggregatorFactory.java | 26 +-
.../segment/processing/collector/Collector.java | 56 --
.../processing/collector/CollectorConfig.java | 109 ---
.../processing/collector/CollectorFactory.java | 61 --
.../processing/collector/ConcatCollector.java | 153 ----
.../processing/collector/GenericRowSorter.java | 95 ---
.../processing/collector/RollupCollector.java | 174 ----
.../MergeType.java} | 21 +-
.../processing/framework/SegmentConfig.java | 26 +-
.../processing/framework/SegmentMapperConfig.java | 79 --
.../framework/SegmentProcessorConfig.java | 62 +-
.../framework/SegmentProcessorFramework.java | 94 ++-
.../processing/framework/SegmentReducer.java | 129 ---
.../processing/framework/SegmentReducerConfig.java | 61 --
.../genericrow/GenericRowDeserializer.java | 221 +++--
.../genericrow/GenericRowFileManager.java | 28 +-
.../genericrow/GenericRowFileReader.java | 34 +-
.../genericrow/GenericRowFileRecordReader.java | 127 +++
.../genericrow/GenericRowSerializer.java | 2 +
.../{framework => mapper}/SegmentMapper.java | 37 +-
.../segment/processing/reducer/ConcatReducer.java | 23 +-
.../segment/processing/reducer/DedupReducer.java | 89 +++
.../ValueAggregator.java => reducer/Reducer.java} | 15 +-
.../segment/processing/reducer/ReducerFactory.java | 48 ++
.../segment/processing/reducer/RollupReducer.java | 172 ++++
.../processing/utils/SegmentProcessingUtils.java | 84 --
.../processing/utils/SegmentProcessorUtils.java | 98 +++
.../processing/utils/SortOrderComparator.java | 72 --
.../org/apache/pinot/core/util/QueryOptions.java | 28 +
.../realtime/LLRealtimeSegmentDataManagerTest.java | 12 +-
...adataAndDictionaryAggregationPlanMakerTest.java | 4 +-
.../groupby/GroupByInSegmentTrimTest.java | 14 +
.../statement/JsonStatementOptimizerTest.java | 204 +++++
.../pruner/SelectionQuerySegmentPrunerTest.java | 6 +-
.../processing/framework/CollectorTest.java | 306 -------
.../processing/framework/GenericRowSorterTest.java | 77 --
.../segment/processing/framework/ReducerTest.java | 565 +++++++++++++
.../processing/framework/SegmentMapperTest.java | 166 ++--
.../framework/SegmentProcessingFrameworkTest.java | 551 -------------
.../framework/SegmentProcessorFrameworkTest.java | 885 +++++++++++++++++++++
.../processing/framework/SegmentReducerTest.java | 205 -----
.../processing/genericrow/GenericRowSerDeTest.java | 40 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 39 +
.../apache/pinot/queries/JsonPathQueriesTest.java | 302 +++++++
.../NoDictionaryCompressionQueriesTest.java | 73 +-
pinot-integration-tests/pom.xml | 14 +
.../pinot/compat/tests/ClusterDescriptor.java | 64 +-
.../pinot/compat/tests/CompatibilityOpsRunner.java | 4 +
.../org/apache/pinot/compat/tests/QueryOp.java | 2 +-
.../org/apache/pinot/compat/tests/SegmentOp.java | 13 +-
.../org/apache/pinot/compat/tests/StreamOp.java | 7 +-
.../org/apache/pinot/compat/tests/TableOp.java | 9 +-
.../pinot/integration/tests/ClusterTest.java | 22 +-
...vertToRawIndexMinionClusterIntegrationTest.java | 6 +-
...tlyOnceKafkaRealtimeClusterIntegrationTest.java | 3 +-
.../tests/HybridClusterIntegrationTest.java | 12 +-
.../tests/LLCRealtimeClusterIntegrationTest.java | 8 +-
...rDownloadLLCRealtimeClusterIntegrationTest.java | 5 +-
.../tests/RealtimeKinesisIntegrationTest.java | 464 +++++++++++
.../tests/ServerStarterIntegrationTest.java | 6 +-
.../tests/SimpleMinionClusterIntegrationTest.java | 36 +-
.../UpsertTableSegmentUploadIntegrationTest.java | 3 +-
.../ControllerLeaderLocatorIntegrationTest.java | 15 +-
.../kinesis/airlineStats_data_reduced.json | 200 +++++
.../kinesis/airlineStats_data_reduced.schema | 32 +
.../{MinionStarter.java => BaseMinionStarter.java} | 75 +-
.../org/apache/pinot/minion/MinionStarter.java | 257 +-----
.../apache/pinot/minion/metrics/MinionGauge.java | 1 +
.../pinot/minion/metrics/MinionQueryPhase.java | 3 +-
.../minion/taskfactory/TaskFactoryRegistry.java | 40 +-
.../BenchmarkNoDictionaryIntegerCompression.java | 94 ++-
.../perf/BenchmarkNoDictionaryLongCompression.java | 88 +-
.../BenchmarkNoDictionaryStringCompression.java | 92 ++-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 2 +-
.../v0_deprecated/pinot-hadoop/pom.xml | 4 +
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 382 +++------
.../pinot/hadoop/job/InternalConfigConstants.java | 13 +-
.../job/mappers/AvroDataPreprocessingMapper.java | 85 ++
.../job/mappers/OrcDataPreprocessingMapper.java | 87 ++
.../job/mappers/SegmentPreprocessingMapper.java | 53 +-
.../AvroDataPreprocessingPartitioner.java | 77 ++
.../OrcDataPreprocessingPartitioner.java | 83 ++
.../preprocess/AvroDataPreprocessingHelper.java | 155 ++++
.../job/preprocess/DataPreprocessingHelper.java | 228 ++++++
.../preprocess/DataPreprocessingHelperFactory.java | 55 ++
.../job/preprocess/OrcDataPreprocessingHelper.java | 231 ++++++
...ucer.java => AvroDataPreprocessingReducer.java} | 45 +-
...ducer.java => OrcDataPreprocessingReducer.java} | 57 +-
.../hadoop/utils/preprocess/DataFileUtils.java | 62 ++
.../utils/preprocess/DataPreprocessingUtils.java | 102 +++
.../pinot/hadoop/utils/preprocess/HadoopUtils.java | 29 +-
.../pinot/hadoop/utils/preprocess/OrcUtils.java | 88 ++
.../hadoop/utils/preprocess/TextComparator.java | 41 +-
.../ingestion/jobs/SegmentPreprocessingJob.java | 26 +-
.../inputformat/json/JSONMessageDecoder.java} | 6 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 8 +
.../plugin/minion/tasks/BaseTaskExecutor.java | 17 +
.../merge_rollup/MergeRollupTaskExecutor.java | 63 +-
.../tasks/merge_rollup/MergeRollupTaskUtils.java | 40 +-
.../RealtimeToOfflineSegmentsTaskExecutor.java | 60 +-
.../merge_rollup/MergeRollupTaskUtilsTest.java | 16 +-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 2 +-
.../SegmentGenerationAndPushTaskGeneratorTest.java | 3 +-
.../pinot-kafka-base/pom.xml | 1 -
.../stream/kafka/KafkaJSONMessageDecoder.java | 58 +-
.../pinot/plugin/stream/kinesis/KinesisConfig.java | 6 +
.../stream/kinesis/KinesisConnectionHandler.java | 27 +-
.../plugin/stream/kinesis/KinesisConsumer.java | 28 +-
.../plugin/stream/kinesis/KinesisConsumerTest.java | 5 +-
.../indexsegment/immutable/EmptyIndexSegment.java | 10 +-
.../immutable/ImmutableSegmentImpl.java | 20 +-
.../immutable/ImmutableSegmentLoader.java | 4 +-
.../indexsegment/mutable/IntermediateSegment.java | 4 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 97 +--
.../io/compression/ChunkCompressorFactory.java | 6 +
.../local/io/compression/LZ4Compressor.java | 50 ++
.../local/io/compression/LZ4Decompressor.java | 51 ++
.../realtime/impl/geospatial/MutableH3Index.java | 2 +-
.../invertedindex/RealtimeInvertedIndexReader.java | 2 +-
.../impl/nullvalue/MutableNullValueVector.java | 2 +-
.../index/column/BaseVirtualColumnProvider.java | 12 +-
.../DefaultNullValueVirtualColumnProvider.java | 8 +-
.../index/column/PhysicalColumnIndexContainer.java | 2 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../segment/index/datasource/EmptyDataSource.java | 2 +-
.../index/datasource/ImmutableDataSource.java | 2 +-
.../local/segment/index/loader/LoaderUtils.java | 5 +-
.../loader/bloomfilter/BloomFilterHandler.java | 4 +-
.../ColumnMinMaxValueGenerator.java | 2 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 23 +-
.../defaultcolumn/DefaultColumnHandlerFactory.java | 2 +-
.../index/loader/invertedindex/H3IndexHandler.java | 4 +-
.../loader/invertedindex/InvertedIndexHandler.java | 8 +-
.../loader/invertedindex/JsonIndexHandler.java | 7 +-
.../invertedindex/LuceneFSTIndexHandler.java | 8 +-
.../loader/invertedindex/RangeIndexHandler.java | 8 +-
.../loader/invertedindex/TextIndexHandler.java | 11 +-
.../index/readers/ValidDocIndexReaderImpl.java | 2 +-
.../local/segment/store/FilePerIndexDirectory.java | 36 +-
.../segment/store/SegmentLocalFSDirectory.java | 8 +-
.../virtualcolumn/DocIdVirtualColumnProvider.java | 9 +-
.../virtualcolumn/VirtualColumnProvider.java | 2 +-
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../startree/v2/builder/MultipleTreesBuilder.java | 5 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 6 +-
.../startree/v2/store/StarTreeLoaderUtils.java | 2 +-
.../segment/local/upsert/PartialUpsertHandler.java | 142 ++++
.../upsert/PartitionUpsertMetadataManager.java | 161 ++--
.../pinot/segment/local/upsert/RecordLocation.java | 18 +-
.../local/upsert/TableUpsertMetadataManager.java | 10 +-
.../segment/local/upsert/merger/AppendMerger.java | 45 ++
.../local/upsert/merger/IncrementMerger.java | 48 ++
.../local/upsert/merger/OverwriteMerger.java | 18 +-
.../local/upsert/merger/PartialUpsertMerger.java | 17 +-
.../upsert/merger/PartialUpsertMergerFactory.java | 47 ++
.../segment/local/upsert/merger/UnionMerger.java | 51 ++
.../segment/local/utils/TableConfigUtils.java | 62 +-
.../mutable/MutableSegmentImplUpsertTest.java | 4 +-
.../local/segment/creator/IntArraysTest.java | 6 +-
.../local/segment/index/ColumnMetadataTest.java | 59 +-
.../segment/index/SegmentMetadataImplTest.java | 6 +-
.../DefaultNullValueVirtualColumnProviderTest.java | 208 +++--
.../SegmentV1V2ToV3FormatConverterTest.java | 4 +-
.../SegmentGenerationWithBytesTypeTest.java | 6 +-
.../index/creator/SegmentPartitionTest.java | 10 +-
.../forward/FixedByteChunkSVForwardIndexTest.java | 10 +
.../forward/VarByteChunkSVForwardIndexTest.java | 13 +
.../local/segment/index/loader/LoaderTest.java | 38 +-
.../index/loader/SegmentPreProcessorTest.java | 194 ++---
.../store/ColumnIndexDirectoryTestHelper.java | 54 +-
.../store/SingleFileIndexDirectoryTest.java | 2 +-
.../v2/builder/StarTreeV2BuilderConfigTest.java | 2 +-
.../local/upsert/PartialUpsertHandlerTest.java | 81 ++
.../upsert/PartitionUpsertMetadataManagerTest.java | 94 ++-
.../upsert/merger/PartialUpsertMergerTest.java | 59 ++
.../segment/local/utils}/TableConfigUtilsTest.java | 190 +++--
.../apache/pinot/segment/spi/ColumnMetadata.java | 90 +++
.../org/apache/pinot/segment/spi/IndexSegment.java | 4 +-
.../apache/pinot/segment/spi/SegmentMetadata.java | 59 +-
.../org/apache/pinot/segment/spi/V1Constants.java | 18 +-
.../spi/compression/ChunkCompressionType.java | 2 +-
.../spi/creator/SegmentGeneratorConfig.java | 27 +-
.../creator/name/SimpleSegmentNameGenerator.java | 10 +-
.../spi/index}/ThreadSafeMutableRoaringBitmap.java | 2 +-
.../segment/spi/index/metadata/ColumnMetadata.java | 623 ---------------
.../spi/index/metadata/ColumnMetadataImpl.java | 390 +++++++++
.../spi/index/metadata/SegmentMetadataImpl.java | 284 ++-----
.../api/resources/SegmentMetadataFetcher.java | 6 +-
.../pinot/server/api/resources/TablesResource.java | 23 +-
...ixServerStarter.java => BaseServerStarter.java} | 182 ++---
.../helix/DefaultHelixStarterServerConfig.java | 2 -
.../server/starter/helix/HelixServerStarter.java | 666 +---------------
.../pinot/server/api/TablesResourceTest.java | 48 +-
.../apache/pinot/spi/config/table/FieldConfig.java | 2 +-
.../pinot/spi/config/table/UpsertConfig.java | 28 +-
.../java/org/apache/pinot/spi/data/FieldSpec.java | 7 +
.../java/org/apache/pinot/spi/data/Schema.java | 14 +-
.../apache/pinot/spi/data/readers/GenericRow.java | 10 +-
.../org/apache/pinot/spi/plugin/PluginManager.java | 4 +-
.../pinot/spi/services/ServiceStartable.java | 3 +
.../apache/pinot/spi/utils/CommonConstants.java | 23 +-
.../pinot/spi/config/table/UpsertConfigTest.java | 18 +-
pinot-tools/pom.xml | 11 +
.../pinot/tools/PartialUpsertQuickStart.java | 134 ++++
.../command/SegmentProcessorFrameworkCommand.java | 5 +-
.../anonymizer/ArrayBasedGlobalDictionaries.java | 2 +-
.../pinot/tools/anonymizer/GlobalDictionaries.java | 2 +-
.../anonymizer/MapBasedGlobalDictionaries.java | 2 +-
.../anonymizer/PinotDataAndQueryAnonymizer.java | 2 +-
.../pinot/tools/perf/PerfBenchmarkDriver.java | 28 +-
.../apache/pinot/tools/scan/query/Projection.java | 2 +-
.../tools/scan/query/SegmentQueryProcessor.java | 3 +-
.../processor/SegmentProcessorFrameworkSpec.java | 23 +-
.../pinot/tools/service/PinotServiceManager.java | 63 +-
.../batch/githubEvents/githubEvents_schema.json | 9 +-
.../stream/meetupRsvp/json_meetupRsvp_schema.json | 12 +-
...t_partial_meetupRsvp_realtime_table_config.json | 55 ++
.../upsert_partial_meetupRsvp_schema.json | 67 ++
pom.xml | 13 +-
312 files changed, 11422 insertions(+), 7992 deletions(-)
copy kubernetes/helm/pinot/templates/minion/configmap.yaml => compatibility-verifier/sample-test-suite/config/BrokerConfig.properties (72%)
copy kubernetes/helm/pinot/templates/minion/configmap.yaml => compatibility-verifier/sample-test-suite/config/ControllerConfig.properties (72%)
copy kubernetes/helm/pinot/templates/minion/configmap.yaml => compatibility-verifier/sample-test-suite/config/ServerConfig.properties (72%)
create mode 100644 docker/images/pinot/etc/jmx_prometheus_javaagent/README.md
copy pinot-broker/src/main/java/org/apache/pinot/broker/broker/helix/{HelixBrokerStarter.java => BaseBrokerStarter.java} (79%)
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/broker/HelixBrokerStarterHostnamePortTest.java
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SystemResourceInfo.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/config/InstanceUtilsTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/helix/HelixHelperTest.java
copy pinot-controller/src/main/java/org/apache/pinot/controller/{ControllerStarter.java => BaseControllerStarter.java} (88%)
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/TaskMetricsEmitter.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/RangeIndexRule.java
copy pinot-minion/src/main/java/org/apache/pinot/minion/metrics/MinionGauge.java => pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RangeIndexRuleParams.java (52%)
rename {pinot-common/src/main/java/org/apache/pinot/common/config => pinot-controller/src/main/java/org/apache/pinot/controller}/tuner/NoOpTableTableConfigTuner.java (83%)
rename {pinot-common/src/main/java/org/apache/pinot/common/config => pinot-controller/src/main/java/org/apache/pinot/controller}/tuner/RealTimeAutoIndexTuner.java (88%)
copy {pinot-spi/src/main/java/org/apache/pinot/spi/config/table => pinot-controller/src/main/java/org/apache/pinot/controller}/tuner/TableConfigTuner.java (63%)
rename {pinot-common/src/main/java/org/apache/pinot/common/config => pinot-controller/src/main/java/org/apache/pinot/controller}/tuner/TableConfigTunerRegistry.java (65%)
copy pinot-spi/src/main/java/org/apache/pinot/spi/config/table/tuner/TableConfigTuner.java => pinot-controller/src/main/java/org/apache/pinot/controller/tuner/TableConfigTunerUtils.java (58%)
rename {pinot-spi/src/main/java/org/apache/pinot/spi/config/table => pinot-controller/src/main/java/org/apache/pinot/controller}/tuner/Tuner.java (96%)
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterTest.java
rename {pinot-common/src/test/java/org/apache/pinot/common/config => pinot-controller/src/test/java/org/apache/pinot/controller}/tuner/RealTimeAutoIndexTunerTest.java (91%)
rename {pinot-common/src/test/java/org/apache/pinot/common/config => pinot-controller/src/test/java/org/apache/pinot/controller}/tuner/TunerRegistryTest.java (86%)
create mode 100644 pinot-controller/src/test/resources/recommenderInput/RangeIndexInput.json
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/minion/MergeRollupConverter.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/BitmapDocIdSetOperator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizer.java
rename pinot-spi/src/main/java/org/apache/pinot/spi/config/table/tuner/TableConfigTuner.java => pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StatementOptimizer.java (65%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector => aggregator}/MaxValueAggregator.java (96%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector => aggregator}/MinValueAggregator.java (96%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector => aggregator}/SumValueAggregator.java (96%)
copy pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector => aggregator}/ValueAggregator.java (94%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector => aggregator}/ValueAggregatorFactory.java (68%)
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/Collector.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/CollectorConfig.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/CollectorFactory.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/GenericRowSorter.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/RollupCollector.java
copy pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector/ValueAggregator.java => framework/MergeType.java} (68%)
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentMapperConfig.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentReducer.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentReducerConfig.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{framework => mapper}/SegmentMapper.java (86%)
copy pinot-minion/src/main/java/org/apache/pinot/minion/metrics/MinionQueryPhase.java => pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/ConcatReducer.java (64%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/DedupReducer.java
copy pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector/ValueAggregator.java => reducer/Reducer.java} (67%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/ReducerFactory.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/RollupReducer.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SegmentProcessingUtils.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SegmentProcessorUtils.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SortOrderComparator.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/JsonStatementOptimizerTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/CollectorTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/GenericRowSorterTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/ReducerTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentReducerTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/JsonPathQueriesTest.java
create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeKinesisIntegrationTest.java
create mode 100644 pinot-integration-tests/src/test/resources/kinesis/airlineStats_data_reduced.json
create mode 100644 pinot-integration-tests/src/test/resources/kinesis/airlineStats_data_reduced.schema
copy pinot-minion/src/main/java/org/apache/pinot/minion/{MinionStarter.java => BaseMinionStarter.java} (81%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/AvroDataPreprocessingMapper.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/OrcDataPreprocessingMapper.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/AvroDataPreprocessingPartitioner.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/partitioners/OrcDataPreprocessingPartitioner.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/preprocess/AvroDataPreprocessingHelper.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/preprocess/DataPreprocessingHelper.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/preprocess/DataPreprocessingHelperFactory.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/preprocess/OrcDataPreprocessingHelper.java
copy pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/reducers/{SegmentPreprocessingReducer.java => AvroDataPreprocessingReducer.java} (63%)
rename pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/reducers/{SegmentPreprocessingReducer.java => OrcDataPreprocessingReducer.java} (54%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/preprocess/DataFileUtils.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/preprocess/DataPreprocessingUtils.java
copy pinot-spi/src/test/java/org/apache/pinot/spi/config/table/UpsertConfigTest.java => pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/preprocess/HadoopUtils.java (59%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/preprocess/OrcUtils.java
copy pinot-minion/src/main/java/org/apache/pinot/minion/metrics/MinionGauge.java => pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/preprocess/TextComparator.java (53%)
copy pinot-plugins/{pinot-stream-ingestion/pinot-kafka-base/src/main/java/org/apache/pinot/plugin/stream/kafka/KafkaJSONMessageDecoder.java => pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONMessageDecoder.java} (93%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/compression/LZ4Compressor.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/compression/LZ4Decompressor.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/PartialUpsertHandler.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/AppendMerger.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/IncrementMerger.java
copy pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ValueAggregator.java => pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/OverwriteMerger.java (68%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ValueAggregator.java => pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/PartialUpsertMerger.java (66%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/PartialUpsertMergerFactory.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/UnionMerger.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/PartialUpsertHandlerTest.java
rename {pinot-core/src/test/java/org/apache/pinot/core => pinot-segment-local/src/test/java/org/apache/pinot/segment/local}/upsert/PartitionUpsertMetadataManagerTest.java (76%)
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/merger/PartialUpsertMergerTest.java
rename {pinot-core/src/test/java/org/apache/pinot/core/util => pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils}/TableConfigUtilsTest.java (90%)
create mode 100644 pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
rename {pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl => pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index}/ThreadSafeMutableRoaringBitmap.java (96%)
delete mode 100644 pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadata.java
create mode 100644 pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
copy pinot-server/src/main/java/org/apache/pinot/server/starter/helix/{HelixServerStarter.java => BaseServerStarter.java} (82%)
create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java
create mode 100644 pinot-tools/src/main/resources/examples/stream/meetupRsvp/upsert_partial_meetupRsvp_realtime_table_config.json
create mode 100644 pinot-tools/src/main/resources/examples/stream/meetupRsvp/upsert_partial_meetupRsvp_schema.json
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org