You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2022/07/11 20:37:27 UTC
[pinot] branch helix-1.0 updated (04aeeee2f2 -> f6b42691ad)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch helix-1.0
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit 04aeeee2f2 Upgrade Helix to 1.0.4
add ad32a2a5ec adding in AggregateNode and related contents. (#8946)
add db452c5df1 Part 2 - Add support for multi-value noDict column types for group-by and filter type queries (#8993)
add bc27ad99ec Emit metrics if there's no consuming segment for a partition (#8877)
add d86d4abc63 Add data schema to stage nodes (#8985)
add d4a64e3853 add project/transform operator to multistage engine (#8967)
add eda488f5d7 adding MultiStageEngineQuickStart (#8980)
add 792ff66eaf Fix allSegmentLoaded check (#9010)
add 3ff4d13af7 Retry validateJobs for a while to avoid flaky errors (#8999)
add de16a0a35d specify how many segments were pruned by each server segment pruner (#8884)
add 04cc62b363 Adding constructor override for KinesisDataProducer (#8975)
add e2ba55bed5 Null check for partitionGroupSmallestOffset and metric for failure (#9020)
add bd251e4dba Remove realtime segments from metadata upon destroy (#9021)
add ebf8a103c0 adding explicit error on record reader (#9024)
add 1bcc032d0b Enable key value byte stitching in PulsarMessageBatch (#8897)
add c1dea5951f [8835] Fix for CSV files surrounding space (#9028)
add 8d0afe3fbb Upgrade apache commons lib version (#9013)
add bb0505f056 Adding auth zk basic quickstart (#9034)
add 7b3e01e5d3 Change Helm chart to log info to console by default (#9015)
add 524dd0a3f2 Remove double counting on pinot-server (#9001)
add 5625beab41 add api to check segment storage tier (#8914)
add 892896b494 Fix thread safety issue with java client (#8971)
add 63d53d57e5 Allow broker list to be passed to JDBC (#9037)
add f6b42691ad Upgrade Helix to 1.0.4
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (04aeeee2f2)
\
N -- N -- N refs/heads/helix-1.0 (f6b42691ad)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../images/pinot/etc/conf/pinot-broker-log4j2.xml | 2 +-
.../pinot/etc/conf/pinot-controller-log4j2.xml | 2 +-
.../images/pinot/etc/conf/pinot-server-log4j2.xml | 2 +-
.../configs/controller.yml | 7 +
kubernetes/helm/pinot/values.yaml | 20 +-
.../org/apache/pinot/client/ConnectionFactory.java | 6 +-
.../org/apache/pinot/client/PinotConnection.java | 17 +-
.../java/org/apache/pinot/client/PinotDriver.java | 6 +-
.../org/apache/pinot/client/utils/Constants.java | 2 +-
.../org/apache/pinot/client/utils/DriverUtils.java | 4 +-
.../pinot/common/metrics/ControllerGauge.java | 11 +-
.../pinot/common/metrics/ControllerMeter.java | 1 +
.../apache/pinot/common/metrics/ServerMeter.java | 6 +-
.../pinot/common/response/BrokerResponse.java | 42 +
.../response/broker/BrokerResponseNative.java | 39 +
.../{TableSizeInfo.java => TableTierInfo.java} | 32 +-
.../org/apache/pinot/common/utils/DataTable.java | 3 +
.../pinot/common/utils/request/RequestUtils.java | 2 +-
pinot-common/src/main/proto/plan.proto | 4 +-
.../api/resources/PinotSegmentRestletResource.java | 60 ++
.../controller/helix/SegmentStatusChecker.java | 29 +-
.../core/periodictask/ControllerPeriodicTask.java | 2 +
.../realtime/MissingConsumingSegmentFinder.java | 229 +++++
.../realtime/PinotLLCRealtimeSegmentManager.java | 20 +-
.../ServerTableTierReader.java} | 49 +-
.../pinot/controller/util/TableTierReader.java | 129 +++
.../src/main/resources/app/pages/Query.tsx | 21 +
.../pinot/controller/api/TableTierReaderTest.java | 311 +++++++
.../controller/helix/SegmentStatusCheckerTest.java | 24 +-
.../core/minion/PinotTaskManagerStatelessTest.java | 23 +-
.../MissingConsumingSegmentFinderTest.java | 306 +++++++
.../dociditerators/MVScanDocIdIterator.java | 138 ++-
.../NoDictionarySingleColumnGroupKeyGenerator.java | 111 ++-
.../core/query/config/SegmentPrunerConfig.java | 2 +-
.../query/executor/ServerQueryExecutorV1Impl.java | 17 +-
.../pinot/core/query/pruner/SegmentPruner.java | 2 +
.../core/query/pruner/SegmentPrunerProvider.java | 7 +-
.../core/query/pruner/SegmentPrunerService.java | 76 +-
.../core/query/pruner/SegmentPrunerStatistics.java | 35 +-
.../pinot/core/query/reduce/BaseReduceService.java | 23 +-
.../core/query/scheduler/PriorityScheduler.java | 2 -
.../pinot/core/query/scheduler/QueryScheduler.java | 30 +-
.../query/pruner/SegmentPrunerServiceTest.java | 117 +++
...Test.java => BaseMultiValueRawQueriesTest.java} | 16 +-
...egmentAggregationMultiValueRawQueriesTest.java} | 2 +-
...rSegmentSelectionMultiValueRawQueriesTest.java} | 2 +-
...egmentAggregationMultiValueRawQueriesTest.java} | 54 +-
...terSegmentGroupByMultiValueRawQueriesTest.java} | 21 +-
.../pinot/queries/MultiValueRawQueriesTest.java | 939 ++++++++++++++++++++-
pinot-distribution/pom.xml | 4 -
.../tests/BaseClusterIntegrationTest.java | 21 +
.../integration/tests/DedupIntegrationTest.java | 153 ++++
.../tests/MultiStageEngineIntegrationTest.java | 10 +-
.../tests/OfflineClusterIntegrationTest.java | 37 +
.../src/test/resources/dedupIngestionTestData.avro | Bin 0 -> 294 bytes
.../test/resources/dedupIngestionTestData.tar.gz | Bin 0 -> 321 bytes
...schema.json => dedupIngestionTestSchema.schema} | 17 +-
.../plugin/inputformat/csv/CSVRecordReader.java | 2 +-
.../inputformat/csv/CSVRecordExtractorTest.java | 53 +-
.../stream/kafka20/server/KafkaDataProducer.java | 6 +-
.../pinot/plugin/stream/kinesis/KinesisConfig.java | 17 +
.../stream/kinesis/server/KinesisDataProducer.java | 65 +-
.../pinot/plugin/stream/pulsar/PulsarConfig.java | 14 +-
.../plugin/stream/pulsar/PulsarMessageBatch.java | 47 +-
.../pulsar/PulsarPartitionLevelConsumer.java | 14 +-
.../stream/pulsar/PulsarMessageBatchTest.java | 199 +++++
pinot-plugins/pom.xml | 4 -
.../query/parser/CalciteRexExpressionParser.java | 34 +-
.../query/planner/hints/PinotRelationalHints.java | 2 +
.../query/planner/logical/RelToStageConverter.java | 66 +-
.../pinot/query/planner/logical/RexExpression.java | 17 +-
.../pinot/query/planner/logical/StagePlanner.java | 18 +-
.../partitioning/FieldSelectionKeySelector.java | 2 +-
.../query/planner/stage/AbstractStageNode.java | 20 +-
.../stage/{ProjectNode.java => AggregateNode.java} | 34 +-
.../pinot/query/planner/stage/FilterNode.java | 5 +-
.../apache/pinot/query/planner/stage/JoinNode.java | 5 +-
.../query/planner/stage/MailboxReceiveNode.java | 5 +-
.../pinot/query/planner/stage/MailboxSendNode.java | 9 +-
.../pinot/query/planner/stage/ProjectNode.java | 5 +-
.../pinot/query/planner/stage/StageNode.java | 7 +-
.../query/planner/stage/StageNodeSerDeUtils.java | 19 +
.../pinot/query/planner/stage/TableScanNode.java | 5 +-
.../PinotAggregateExchangeNodeInsertRule.java | 181 ++++
...e.java => PinotJoinExchangeNodeInsertRule.java} | 10 +-
.../pinot/query/rules/PinotQueryRuleSets.java | 3 +-
.../pinot/query/QueryEnvironmentTestBase.java | 10 +-
.../pinot/query/planner/stage/SerDeUtilsTest.java | 3 +
.../runtime/executor/WorkerQueryExecutor.java | 12 +-
.../query/runtime/operator/AggregateOperator.java | 214 +++++
.../query/runtime/operator/OperatorUtils.java | 67 ++
.../query/runtime/operator/TransformOperator.java | 185 ++++
.../query/runtime/utils/ServerRequestUtils.java | 12 +-
.../apache/pinot/query/service/QueryConfig.java | 4 +-
.../apache/pinot/query/QueryServerEnclosure.java | 2 +-
.../pinot/query/runtime/QueryRunnerTest.java | 39 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 31 +-
.../local/utils/tablestate/TableStateUtils.java | 69 +-
.../MultiValueFixedByteRawIndexCreatorTest.java | 58 +-
.../server/api/resources/TableTierResource.java | 156 ++++
.../server/starter/helix/BaseServerStarter.java | 5 +-
.../pinot/server/api/TableTierResourceTest.java | 77 ++
pinot-tools/pom.xml | 31 +
...hQuickstart.java => AuthZkBasicQuickstart.java} | 25 +-
...kStart.java => MultistageEngineQuickStart.java} | 66 +-
.../tools/admin/command/PostQueryCommand.java | 18 +-
.../tools/admin/command/QuickstartRunner.java | 15 +-
.../tools/admin/command/StartBrokerCommand.java | 17 +-
.../tools/admin/command/StartServerCommand.java | 39 +-
.../admin/command/StartServiceManagerCommand.java | 7 +-
.../apache/pinot/tools/utils/PinotConfigUtils.java | 12 +-
pinot-tools/src/main/resources/conf/log4j2.xml | 2 +-
.../baseballStats_offline_table_config.json | 2 +-
.../baseballStats_offline_table_config.json | 2 +-
pom.xml | 2 +-
115 files changed, 4875 insertions(+), 422 deletions(-)
copy pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/{TableSizeInfo.java => TableTierInfo.java} (57%)
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/MissingConsumingSegmentFinder.java
copy pinot-controller/src/main/java/org/apache/pinot/controller/{api/resources/ServerTableSizeReader.java => util/ServerTableTierReader.java} (57%)
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/util/TableTierReader.java
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/api/TableTierReaderTest.java
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/MissingConsumingSegmentFinderTest.java
copy pinot-plugins/pinot-stream-ingestion/pinot-kafka-base/src/main/java/org/apache/pinot/plugin/stream/kafka/MessageAndOffset.java => pinot-core/src/main/java/org/apache/pinot/core/query/pruner/SegmentPrunerStatistics.java (58%)
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/query/pruner/SegmentPrunerServiceTest.java
copy pinot-core/src/test/java/org/apache/pinot/queries/{BaseMultiValueQueriesTest.java => BaseMultiValueRawQueriesTest.java} (92%)
copy pinot-core/src/test/java/org/apache/pinot/queries/{InnerSegmentAggregationMultiValueQueriesTest.java => InnerSegmentAggregationMultiValueRawQueriesTest.java} (98%)
copy pinot-core/src/test/java/org/apache/pinot/queries/{InnerSegmentSelectionMultiValueQueriesTest.java => InnerSegmentSelectionMultiValueRawQueriesTest.java} (99%)
copy pinot-core/src/test/java/org/apache/pinot/queries/{InterSegmentAggregationMultiValueQueriesTest.java => InterSegmentAggregationMultiValueRawQueriesTest.java} (94%)
copy pinot-core/src/test/java/org/apache/pinot/queries/{InterSegmentGroupByMultiValueQueriesTest.java => InterSegmentGroupByMultiValueRawQueriesTest.java} (89%)
create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/DedupIntegrationTest.java
create mode 100644 pinot-integration-tests/src/test/resources/dedupIngestionTestData.avro
create mode 100644 pinot-integration-tests/src/test/resources/dedupIngestionTestData.tar.gz
copy pinot-integration-tests/src/test/resources/{chaos-monkey-schema.json => dedupIngestionTestSchema.schema} (60%)
create mode 100644 pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageBatchTest.java
copy pinot-query-planner/src/main/java/org/apache/pinot/query/planner/stage/{ProjectNode.java => AggregateNode.java} (53%)
create mode 100644 pinot-query-planner/src/main/java/org/apache/pinot/query/rules/PinotAggregateExchangeNodeInsertRule.java
rename pinot-query-planner/src/main/java/org/apache/pinot/query/rules/{PinotExchangeNodeInsertRule.java => PinotJoinExchangeNodeInsertRule.java} (91%)
create mode 100644 pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/AggregateOperator.java
create mode 100644 pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/OperatorUtils.java
create mode 100644 pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/TransformOperator.java
create mode 100644 pinot-server/src/main/java/org/apache/pinot/server/api/resources/TableTierResource.java
create mode 100644 pinot-server/src/test/java/org/apache/pinot/server/api/TableTierResourceTest.java
copy pinot-tools/src/main/java/org/apache/pinot/tools/{AuthQuickstart.java => AuthZkBasicQuickstart.java} (62%)
copy pinot-tools/src/main/java/org/apache/pinot/tools/{JoinQuickStart.java => MultistageEngineQuickStart.java} (64%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org