You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2022/04/25 19:36:57 UTC
[pinot] branch helix-1.0 updated (fce772482a -> 0f3698c4ab)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch helix-1.0
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard fce772482a Upgrade Helix to 1.0.2
add aadd0cddcf make ConfigUtils testable without illegal reflective access (#8530)
add fe7247fb96 create task queue as needed for adhoc task (#8540)
add d503d505b2 Only discover public methods annotated with @ScalarFunction (#8544)
add 05a7f28b73 don't snapshot environment in ConfigUtils (#8548)
add 734d203a50 Fix BigDecimal ser/de on negative scale (#8553)
add 87fb007f7d add scalar function for cast so it can be calculated at compile time (#8535)
add 6fb7e521ef Add tracing for operator planning (#8523)
add ce0a08a49d Add connection based FailureDetector (#8491)
add 2704d88732 Make execute adhoc task call async to prevent timeout (#8555)
add acfc618d0c This dependency should be packaged in the jar itself and not provided scope (#8561)
add f48a22a141 Make spark dependency provided (#8559)
add 82f7aefe3d Ensure all records are pushed in Pulsar Consumer Test (#8554)
add 829ede946c Bump up azure dep versions to resolve conflict with netty (#8552)
add 399cb97f30 Add brokerURL and better handle TimeoutException in QueryRunner (#8539)
add bc363d8247 endpoints to get and delete minion task metadata (#8551)
add 6eff17754a Enable range index on raw columns and push counts down to range index when v2 is in use (#8513)
add 5687f27214 Simplify the parameters for DistinctCountSmartHLLAggregationFunction (#8566)
add b025f437eb Add PercentileSmartTDigestAggregationFunction (#8565)
add a08f8c637a Use MultiThreadedHttpConnectionManager in SegmentStatusChecker (#8568)
add 727d0c9cf1 range index shouldn’t be loaded when the column is sorted (#8571)
add 7e060fdbdd Improve StarTree traversal performance (#8570)
add d2cfa0737a Invoke checkpoint before closing partition group consumer (#8564)
add fe1423162e Move controllerUrlUtils to SPI (#8573)
add 4f788b2995 Fix the deadlock in ClusterChangeMediator (#8572)
add 2fba4b0cb7 Add query option to use more replica groups (#8550)
add a954d85d29 Adding missing prometheus metrics to controller.yml (#8576)
add d15847828a Package kafka 0.9 shaded jar to pinot-distribution (#8569)
add f200ec5156 Added multi column partitioning for offline table (#8255)
add 5eddb19d24 Merge ControllerSegmentUrlBuilder into ControllerRequestURLBuilder (#8575)
add 8b5314dd97 Add minion health check endpoint (#8574)
add d6001a258a Minimize data movement between instances in pools (#8483)
add 0f3698c4ab Upgrade Helix to 1.0.2
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fce772482a)
\
N -- N -- N refs/heads/helix-1.0 (0f3698c4ab)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
config/checkstyle.xml | 1 +
docker/images/pinot/Dockerfile | 2 +-
.../jmx_prometheus_javaagent/configs/broker.yml | 3 +
.../configs/controller.yml | 17 +-
.../etc/jmx_prometheus_javaagent/configs/pinot.yml | 3 +
.../broker/broker/helix/ClusterChangeMediator.java | 112 ++-
...BaseExponentialBackoffRetryFailureDetector.java | 190 +++++
.../failuredetector/ConnectionFailureDetector.java | 27 +-
.../broker/failuredetector/FailureDetector.java | 105 +++
.../failuredetector/FailureDetectorFactory.java | 73 ++
.../failuredetector/NoOpFailureDetector.java | 55 +-
.../requesthandler/BaseBrokerRequestHandler.java | 13 +
.../requesthandler/GrpcBrokerRequestHandler.java | 1 +
.../SingleConnectionBrokerRequestHandler.java | 77 +-
.../pinot/broker/routing/BrokerRoutingManager.java | 179 +++-
.../instanceselector/BalancedInstanceSelector.java | 2 +-
.../instanceselector/BaseInstanceSelector.java | 8 +-
.../routing/instanceselector/InstanceSelector.java | 2 +-
.../ReplicaGroupInstanceSelector.java | 19 +-
...ava => MultiPartitionColumnsSegmentPruner.java} | 140 ++--
.../segmentpruner/SegmentPrunerFactory.java | 31 +-
...ava => SinglePartitionColumnSegmentPruner.java} | 10 +-
.../broker/helix/ClusterChangeMediatorTest.java | 89 ++
.../ConnectionFailureDetectorTest.java | 155 ++++
...elixExternalViewBasedQueryQuotaManagerTest.java | 2 +-
.../broker/requesthandler/QueryValidationTest.java | 7 +
.../instanceselector/InstanceSelectorTest.java | 243 ++++++
.../routing/segmentpruner/SegmentPrunerTest.java | 143 +++-
pinot-clients/pom.xml | 1 -
.../assignment/InstanceAssignmentConfigUtils.java | 5 +-
.../apache/pinot/common/function/FunctionInfo.java | 1 -
.../pinot/common/function/FunctionInvoker.java | 1 -
.../pinot/common/function/FunctionRegistry.java | 4 +
.../scalar/DataTypeConversionFunctions.java | 34 +
.../apache/pinot/common/http/MultiGetRequest.java | 8 +-
.../pinot/common/metadata/ZKMetadataProvider.java | 2 +-
.../metadata/instance/InstanceZKMetadata.java | 2 +-
.../apache/pinot/common/metrics/BrokerGauge.java | 3 +-
.../common/minion/MinionRequestURLBuilder.java | 2 +-
.../common/utils/FileUploadDownloadClient.java | 1 +
.../scalar/DataTypeConversionFunctionsTest.java | 63 ++
.../apache/pinot/common/utils/StringUtilTest.java | 1 +
.../common/utils/config/TableConfigSerDeTest.java | 2 +-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 30 +-
.../java/org/apache/pinot/compat/SegmentOp.java | 2 +-
.../java/org/apache/pinot/compat/StreamOp.java | 2 +-
.../main/java/org/apache/pinot/compat/TableOp.java | 2 +-
.../pinot/connector/flink/FlinkQuickStart.java | 2 +-
.../spark/connector/PinotServerDataFetcher.scala | 2 +-
.../presto/PinotScatterGatherQueryClient.java | 5 +-
.../pinot/controller/BaseControllerStarter.java | 7 +-
.../apache/pinot/controller/ControllerConf.java | 2 +-
...Exception.java => NoTaskMetadataException.java} | 6 +-
.../PinotInstanceAssignmentRestletResource.java | 31 +-
.../PinotSegmentUploadDownloadRestletResource.java | 2 +-
.../api/resources/PinotTableRestletResource.java | 2 +-
.../api/resources/PinotTaskRestletResource.java | 48 +-
.../controller/helix/ControllerRequestClient.java | 3 +-
.../controller/helix/SegmentStatusChecker.java | 7 +-
.../helix/core/PinotHelixResourceManager.java | 2 +-
.../helix/core/PinotTableIdealStateBuilder.java | 2 +-
.../instance/InstanceAssignmentDriver.java | 6 +-
.../InstanceReplicaGroupPartitionSelector.java | 256 +++++-
.../segment/OfflineSegmentAssignment.java | 2 +-
.../segment/RealtimeSegmentAssignment.java | 2 +-
.../core/minion/PinotHelixTaskResourceManager.java | 30 +-
.../helix/core/minion/PinotTaskManager.java | 5 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 2 +-
.../helix/core/rebalance/TableRebalancer.java | 8 +-
.../helix/core/relocation/SegmentRelocator.java | 2 +-
.../controller/util/AutoAddInvertedIndex.java | 2 +-
.../pinot/controller/ControllerTestUtils.java | 18 +-
.../pinot/controller/api/AccessControlTest.java | 5 +-
...PinotInstanceAssignmentRestletResourceTest.java | 4 +-
.../api/PinotTableRestletResourceTest.java | 2 +-
.../PinotDummyExtraRestletResourceTest.java | 2 +-
.../pinot/controller/helix/ControllerTest.java | 1 +
.../instance/InstanceAssignmentTest.java | 556 ++++++++++++-
...fflineNonReplicaGroupSegmentAssignmentTest.java | 2 +-
...NonReplicaGroupTieredSegmentAssignmentTest.java | 2 +-
.../OfflineReplicaGroupSegmentAssignmentTest.java | 2 +-
...altimeNonReplicaGroupSegmentAssignmentTest.java | 2 +-
...NonReplicaGroupTieredSegmentAssignmentTest.java | 2 +-
.../RealtimeReplicaGroupSegmentAssignmentTest.java | 2 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 2 +-
.../TableRebalancerClusterStatelessTest.java | 3 +-
.../realtime/LLRealtimeSegmentDataManager.java | 27 +-
.../realtime/PeerSchemeSplitSegmentCommitter.java | 2 +-
.../manager/realtime/PinotFSSegmentUploader.java | 2 +-
.../apache/pinot/core/operator/BaseOperator.java | 2 +-
.../filter/RangeIndexBasedFilterOperator.java | 305 +++++--
.../predicate/RangePredicateEvaluatorFactory.java | 8 +-
.../query/NonScanBasedAggregationOperator.java | 4 +-
.../apache/pinot/core/plan/CombinePlanNode.java | 16 +-
.../function/AggregationFunctionFactory.java | 3 +
.../DistinctCountSmartHLLAggregationFunction.java | 79 +-
.../PercentileSmartTDigestAggregationFunction.java | 346 ++++++++
.../segment/processing/mapper/SegmentMapper.java | 2 +-
.../startree/operator/StarTreeFilterOperator.java | 4 +-
.../pinot/core/transport/AsyncQueryResponse.java | 71 +-
.../pinot/core/transport/DataTableHandler.java | 3 +-
.../apache/pinot/core/transport/QueryResponse.java | 76 ++
.../apache/pinot/core/transport/QueryRouter.java | 24 +-
.../pinot/core/transport/ServerChannels.java | 42 +-
.../pinot/core/transport/ServerInstance.java | 52 +-
.../pinot/core/transport/ServerResponse.java | 2 +
.../core/transport/ServerRoutingInstance.java | 37 +-
.../pinot/core/util/PeerServerSegmentFinder.java | 2 +-
.../apache/pinot/core/util/QueryOptionsUtils.java | 5 +
.../function/InbuiltFunctionEvaluatorTest.java | 4 +-
.../realtime/PinotFSSegmentUploaderTest.java | 2 +-
.../function/CastTransformFunctionTest.java | 23 +-
.../query/reduce/StreamingReduceServiceTest.java | 6 +-
.../pinot/core/transport/QueryRoutingTest.java | 14 +-
.../core/util/PeerServerSegmentFinderTest.java | 2 +-
...tLeastQueriesTest.java => CastQueriesTest.java} | 72 +-
.../pinot/queries/DistinctCountQueriesTest.java | 25 +-
.../pinot/queries/FastFilteredCountTest.java | 19 +-
.../queries/PercentileTDigestMVQueriesTest.java | 7 +
.../queries/PercentileTDigestQueriesTest.java | 89 +-
.../org/apache/pinot/queries/RangeQueriesTest.java | 201 +++++
pinot-distribution/pinot-assembly.xml | 9 +-
.../tests/BaseClusterIntegrationTest.java | 7 +-
.../tests/ClusterIntegrationTestUtils.java | 2 +-
.../tests/HybridClusterIntegrationTest.java | 26 +-
.../MergeRollupMinionClusterIntegrationTest.java | 22 +-
.../MultiNodesOfflineClusterIntegrationTest.java | 63 ++
.../tests/RealtimeKinesisIntegrationTest.java | 2 +-
...fflineSegmentsMinionClusterIntegrationTest.java | 44 +-
...mentGenerationMinionClusterIntegrationTest.java | 139 ++++
.../integration/tests/TlsIntegrationTest.java | 6 +-
.../org/apache/pinot/minion/BaseMinionStarter.java | 23 +-
.../pinot/minion/MinionAdminApiApplication.java | 4 +-
.../minion}/api/resources/HealthCheckResource.java | 15 +-
.../org/apache/pinot/perf/BenchmarkQueries.java | 12 +-
.../pinot-batch-ingestion-spark/pom.xml | 1 +
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 4 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 28 +-
.../mergerollup/MergeRollupTaskGenerator.java | 41 +-
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 14 +
.../mergerollup/MergeRollupTaskGeneratorTest.java | 56 +-
.../pinot-stream-ingestion/pinot-kafka-0.9/pom.xml | 86 ++
.../pinot-kafka-2.0/README.md | 2 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 154 +---
.../plugin/stream/pulsar/PulsarConsumerTest.java | 128 ++-
.../stream/pulsar/PulsarStandaloneCluster.java | 132 ---
.../src/test/resources/standalone.properties | 904 ---------------------
pinot-plugins/pom.xml | 4 +
.../indexsegment/mutable/IntermediateSegment.java | 23 +-
.../recordtransformer/SanitizationTransformer.java | 2 +-
.../index/column/PhysicalColumnIndexContainer.java | 16 +-
.../index/readers/BitSlicedRangeIndexReader.java | 39 +-
.../index/readers/RangeIndexReaderImpl.java | 41 +
.../local/startree/OffHeapStarTreeNode.java | 58 +-
.../mutable/IntermediateSegmentTest.java | 32 +-
.../index/creator/BitSlicedIndexCreatorTest.java | 18 +-
.../segment/index/creator/RawIndexCreatorTest.java | 2 +-
.../pinot/segment/spi/AggregationFunctionType.java | 1 +
.../segment/spi/index/reader/RangeIndexReader.java | 46 ++
.../pinot/server/starter/ServerInstance.java | 11 +-
.../apache/pinot/server/api/BaseResourceTest.java | 2 +-
.../org/apache/pinot/spi/config/ConfigUtils.java | 22 +-
.../SegmentsValidationAndRetentionConfig.java | 9 +
.../InstanceReplicaGroupPartitionConfig.java | 10 +-
.../apache/pinot/spi/utils/BigDecimalUtils.java | 4 +-
.../apache/pinot/spi/utils/CommonConstants.java | 36 +-
.../pinot/spi/utils}/RebalanceConfigConstants.java | 2 +-
.../org/apache/pinot/spi}/utils/StringUtil.java | 3 +-
.../builder}/ControllerRequestURLBuilder.java | 49 +-
.../apache/pinot/spi/config/ConfigUtilsTest.java | 41 +-
.../pinot/spi/utils/BigDecimalUtilsTest.java | 10 +-
.../apache/pinot/tools/PinotTableRebalancer.java | 2 +-
.../pinot/tools/admin/command/AddTableCommand.java | 2 +-
.../tools/admin/command/AddTenantCommand.java | 2 +-
.../tools/admin/command/ImportDataCommand.java | 2 +-
.../pinot/tools/perf/PerfBenchmarkDriver.java | 4 +-
.../pinot/tools/perf/PerfBenchmarkDriverConf.java | 9 +
.../org/apache/pinot/tools/perf/QueryRunner.java | 115 +--
.../pinot/tools/utils/KafkaStarterUtils.java | 12 +-
pom.xml | 24 -
180 files changed, 4778 insertions(+), 2293 deletions(-)
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/failuredetector/BaseExponentialBackoffRetryFailureDetector.java
copy pinot-core/src/main/java/org/apache/pinot/core/query/reduce/StreamingReducer.java => pinot-broker/src/main/java/org/apache/pinot/broker/failuredetector/ConnectionFailureDetector.java (52%)
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/failuredetector/FailureDetector.java
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/failuredetector/FailureDetectorFactory.java
copy pinot-core/src/main/java/org/apache/pinot/core/query/utils/idset/EmptyIdSet.java => pinot-broker/src/main/java/org/apache/pinot/broker/failuredetector/NoOpFailureDetector.java (51%)
copy pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/{PartitionSegmentPruner.java => MultiPartitionColumnsSegmentPruner.java} (59%)
rename pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/{PartitionSegmentPruner.java => SinglePartitionColumnSegmentPruner.java} (95%)
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/broker/helix/ClusterChangeMediatorTest.java
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/failuredetector/ConnectionFailureDetectorTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/function/scalar/DataTypeConversionFunctionsTest.java
copy pinot-controller/src/main/java/org/apache/pinot/controller/api/exception/{UnknownTaskTypeException.java => NoTaskMetadataException.java} (83%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/PercentileSmartTDigestAggregationFunction.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/transport/QueryResponse.java
copy pinot-core/src/test/java/org/apache/pinot/queries/{SumGreatestLeastQueriesTest.java => CastQueriesTest.java} (73%)
create mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/RangeQueriesTest.java
create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentGenerationMinionClusterIntegrationTest.java
copy {pinot-server/src/main/java/org/apache/pinot/server => pinot-minion/src/main/java/org/apache/pinot/minion}/api/resources/HealthCheckResource.java (82%)
delete mode 100644 pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarStandaloneCluster.java
delete mode 100644 pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/resources/standalone.properties
rename {pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance => pinot-spi/src/main/java/org/apache/pinot/spi/utils}/RebalanceConfigConstants.java (98%)
rename {pinot-common/src/main/java/org/apache/pinot/common => pinot-spi/src/main/java/org/apache/pinot/spi}/utils/StringUtil.java (95%)
rename {pinot-controller/src/main/java/org/apache/pinot/controller/helix => pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder}/ControllerRequestURLBuilder.java (93%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org