You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/07/14 08:00:53 UTC
[incubator-pinot] branch kafka_2.0 updated (c01cc2e -> a163b4d)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch kafka_2.0
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard c01cc2e fixing the bytes type conversion and adding consumer test
discard d9e0316 Adding support for Kafka 2.0
discard 3f682d0 WIP: adding kafka 2 stream provider
add 732a7b9 Add more key/value pairs into LOG2M_TO_SIZE_IN_BYTES in HllSizeUtils (#4398)
add 49d8fa7 #4317 Feature/variable length bytes offline dictionary for indexing bytes and string dicts. (#4321)
add 02f6181 CompletionConfig for realtime tables (#4367)
add ae1ecef [TE] add a thread pool to run preview tasks (#4405)
add de62d8e [TE] Make email template and subject pluggable under email alert scheme (#4409)
add 7df1a43 [TE] Add Merger after metric grouper; other minor clean up (#4410)
add e71ac1b [TE] Do not authorize when auth is disabled (#4414)
add 0b8af63 Fix codecov link from old url to new (#4412)
add 07e2c72 Move tests for API resources to a different package (#4415)
add 52f69d1 [TE] Added support for Vertica as a data source (#4404)
add a7c419a Adding Calcite SQL Parser and make another entry point to query Pinot (#4387)
add e041ec5 [TE] frontend - harleyjj/detection-health - UI for model performance (#4413)
add 6bec451 [TE] Inject the port from config into Commons mail SSL Smtp port (#4420)
add dac9ae2 prompt on fail schema (#4395)
add b6fe4e6 [TE] detection health ui adjustments (#4421)
add bca6756 make kafka version number controlled by config number (#4396)
add 0993974 [TE] Fix exception handling - Propagate and display the error message/exception on frontind (#4419)
add aadcd36 In ClusterChangeMediator, stop enqueue/process changes if already stopped (#4422)
add 52f9b08 Add pinot community inviter (#4424)
add 2f691fc [TE] detection health - coverage fix (#4428)
add 079b86d Config for overriding initial rows threshold in segment size auto tuning (#4376)
add 50aaaf9 [TE] added ThirdEye configuration documents (#4429)
add f671072 Add URIUtils class to handle URI/URL encoding/decoding (#4426)
add 626c43a Misc fix for controller tests (#4431)
add 220df69 WIP: adding kafka 2 stream provider
add cad9466 Adding support for Kafka 2.0
add ee87138 fixing the bytes type conversion and adding consumer test
add a163b4d address comments
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c01cc2e)
\
N -- N -- N refs/heads/kafka_2.0 (a163b4d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
README.md | 4 +-
.../java/org/apache/pinot/client/Connection.java | 44 +-
.../client/JsonAsyncHttpPinotClientTransport.java | 30 +-
.../apache/pinot/client/PinotClientTransport.java | 6 +
.../{PinotClientTransport.java => Request.java} | 34 +-
.../apache/pinot/client/PreparedStatementTest.java | 40 +-
.../apache/pinot/client/ResultSetGroupTest.java | 38 +-
.../broker/broker/helix/ClusterChangeMediator.java | 12 +-
.../requesthandler/BaseBrokerRequestHandler.java | 28 +-
...enNestedPredicatesFilterQueryTreeOptimizer.java | 3 +-
.../requesthandler/PinotQueryParserFactory.java | 49 +++
.../broker/requesthandler/PinotQueryRequest.java | 31 +-
.../request/PqlAndCalciteSqlCompatibilityTest.java | 123 ++++++
pinot-common/pom.xml | 4 +
.../pinot/common/config/CompletionConfig.java | 63 +++
.../apache/pinot/common/config/IndexingConfig.java | 20 +-
.../SegmentsValidationAndRetentionConfig.java | 30 +-
.../apache/pinot/common/utils/CommonConstants.java | 9 +
.../common/utils/FileUploadDownloadClient.java | 16 +-
.../org/apache/pinot/common/utils/URIUtils.java | 88 ++++
.../pinot/common/utils/request/RequestUtils.java | 55 ++-
.../org/apache/pinot/filesystem/LocalPinotFS.java | 91 ++--
.../pinot/{pql => }/parsers/AbstractCompiler.java | 7 +-
.../utils/BrokerRequestComparisonUtils.java | 235 +++++++++++
.../parsers/PinotQuery2BrokerRequestConverter.java | 5 +-
.../org/apache/pinot/pql/parsers/Pql2Compiler.java | 63 +--
.../parsers/pql2/ast/BetweenPredicateAstNode.java | 2 +-
.../pql2/ast/ComparisonPredicateAstNode.java | 20 +-
.../pql/parsers/pql2/ast/InPredicateAstNode.java | 2 +-
.../pinot/pql/parsers/pql2/ast/OrderByAstNode.java | 2 +-
.../pql/parsers/pql2/ast/OutputColumnAstNode.java | 2 +-
.../pql/parsers/pql2/ast/PredicateListAstNode.java | 2 +-
.../pql2/ast/RegexpLikePredicateAstNode.java | 2 +-
.../parsers/CalciteSqlCompiler.java} | 18 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 273 ++++++++++++
.../parsers/SqlCompilationException.java} | 19 +-
.../apache/pinot/startree/hll/HllSizeUtils.java | 14 +-
.../pinot/common/config/IndexingConfigTest.java | 3 +
.../pinot/common/config/TableConfigTest.java | 28 ++
.../apache/pinot/common/utils/URIUtilsTest.java | 68 +++
.../request/BrokerRequestSerializationTest.java | 10 +-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 465 +++++++++++++++++++++
pinot-common/src/test/resources/pql_queries.list | 5 +
.../{pql_queries.list => sql_queries.list} | 247 +++++------
pinot-connectors/pinot-connector-kafka-0.9/pom.xml | 12 +-
.../pinot-connector-kafka-2.0/README.md | 3 +-
pinot-connectors/pinot-connector-kafka-2.0/pom.xml | 19 +-
.../impl/kafka2/KafkaAvroMessageDecoder.java | 290 -------------
...sumerFactory.java => KafkaConsumerFactory.java} | 10 +-
...Config.java => KafkaHighLevelStreamConfig.java} | 18 +-
.../impl/kafka2/KafkaJSONMessageDecoder.java | 63 ---
...ka2MessageBatch.java => KafkaMessageBatch.java} | 4 +-
...a => KafkaPartitionLevelConnectionHandler.java} | 10 +-
...sumer.java => KafkaPartitionLevelConsumer.java} | 8 +-
...g.java => KafkaPartitionLevelStreamConfig.java} | 34 +-
...KafkaPartitionLevelStreamMetadataProvider.java} | 6 +-
...rties.java => KafkaStreamConfigProperties.java} | 2 +-
...Consumer.java => KafkaStreamLevelConsumer.java} | 14 +-
...r.java => KafkaStreamLevelConsumerManager.java} | 46 +-
.../kafka2/KafkaPartitionLevelConsumerTest.java | 44 +-
.../KafkaPartitionLevelStreamConfigTest.java | 42 +-
.../README.md | 5 +-
.../pom.xml | 38 +-
.../impl/kafka/KafkaAvroMessageDecoder.java | 0
.../impl/kafka/KafkaJSONMessageDecoder.java | 0
.../realtime/impl/kafka}/MessageAndOffset.java | 0
pinot-connectors/pom.xml | 1 +
.../apache/pinot/controller/ControllerConf.java | 46 +-
.../api/resources/FileUploadPathProvider.java | 5 +-
.../resources/LLCSegmentCompletionHandlers.java | 25 +-
.../api/resources/PinotControllerHealthCheck.java | 4 +-
.../api/resources/PinotSegmentRestletResource.java | 20 +-
.../PinotSegmentUploadRestletResource.java | 61 +--
.../resources/PinotTableConfigRestletResource.java | 2 +
.../api/resources/PinotTableRestletResource.java | 10 +-
.../pinot/controller/api/resources/TableViews.java | 4 +-
.../helix/ControllerRequestURLBuilder.java | 21 +-
.../helix/core/PinotHelixResourceManager.java | 34 +-
.../helix/core/SegmentDeletionManager.java | 34 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 34 +-
.../segment/FlushThresholdUpdateManager.java | 11 +-
.../SegmentSizeBasedFlushThresholdUpdater.java | 14 +-
.../controller/validation/StorageQuotaChecker.java | 32 +-
.../src/main/resources/static/js/init.js | 2 +
.../api/{resources => }/AccessControlTest.java | 10 +-
.../FileUploadPathProviderTest.java | 3 +-
.../api/{resources => }/PinotFileUploadTest.java | 2 +-
.../PinotInstanceRestletResourceTest.java | 2 +-
.../PinotSchemaRestletResourceTest.java | 2 +-
.../PinotSegmentRestletResourceTest.java | 2 +-
.../PinotTableRestletResourceTest.java | 2 +-
.../PinotTenantRestletResourceTest.java | 2 +-
.../SegmentCompletionProtocolDeserTest.java | 2 +-
.../SegmentCompletionUtilsTest.java | 2 +-
.../{resources => }/ServerTableSizeReaderTest.java | 3 +-
.../api/{resources => }/TableSizeReaderTest.java | 2 +-
.../api/{resources => }/TableViewsTest.java | 3 +-
.../helix/core/realtime/SegmentCompletionTest.java | 7 +-
.../segment/FlushThresholdUpdaterTest.java | 28 +-
.../realtime/HLRealtimeSegmentDataManager.java | 5 +
.../realtime/LLRealtimeSegmentDataManager.java | 91 ++--
.../generator/SegmentGeneratorConfig.java | 13 +
.../org/apache/pinot/core/io/util/ValueReader.java | 3 +-
.../io/util/VarLengthBytesValueReaderWriter.java | 240 +++++++++++
.../converter/RealtimeSegmentConverter.java | 11 +-
.../core/realtime/impl/RealtimeSegmentConfig.java | 20 +-
.../pinot/core/realtime/stream/StreamConfig.java | 55 ++-
.../realtime/stream/StreamConfigProperties.java | 8 +-
.../segment/creator/ColumnIndexCreationInfo.java | 10 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 3 +-
.../creator/impl/SegmentDictionaryCreator.java | 72 +++-
.../impl/SegmentIndexCreationDriverImpl.java | 6 +-
.../segment/index/loader/IndexLoadingConfig.java | 11 +
.../defaultcolumn/BaseDefaultColumnHandler.java | 8 +-
.../index/readers/ImmutableDictionaryReader.java | 18 +-
.../segment/store/SegmentLocalFSDirectory.java | 2 +-
.../MutableSegmentImplAggregateMetricsTest.java | 1 +
.../mutable/MutableSegmentImplTest.java | 3 +-
.../mutable/MutableSegmentImplTestUtils.java | 4 +-
.../util/VarLengthBytesValueReaderWriterTest.java | 118 ++++++
.../pinot/core/startree/hll/HllFieldSizeTest.java | 2 +-
pinot-integration-tests/pom.xml | 3 +-
.../tests/BaseClusterIntegrationTest.java | 18 +-
.../tests/BaseClusterIntegrationTestSet.java | 85 ++++
.../tests/ClusterIntegrationTestUtils.java | 41 +-
.../pinot/integration/tests/ClusterTest.java | 11 +-
.../DeleteAPIHybridClusterIntegrationTest.java | 24 +-
...mentBuildPushOfflineClusterIntegrationTest.java | 7 +
.../tests/HybridClusterIntegrationTest.java | 7 +
...ridClusterIntegrationTestCommandLineRunner.java | 4 +-
.../MultiNodesOfflineClusterIntegrationTest.java | 7 +
.../tests/OfflineClusterIntegrationTest.java | 20 +-
.../tests/PinotURIUploadIntegrationTest.java | 8 +-
.../tests/RealtimeClusterIntegrationTest.java | 13 +
.../tests/SegmentCompletionIntegrationTests.java | 7 +
pinot-perf/pom.xml | 3 +-
.../pinot/perf/BenchmarkDictionaryCreation.java | 12 +
.../pinot/perf/StringDictionaryPerfTest.java | 86 +++-
.../api/{resources => }/BaseResourceTest.java | 2 +-
.../api/{resources => }/TableSizeResourceTest.java | 2 +-
.../api/{resources => }/TablesResourceTest.java | 2 +-
pinot-tools/pom.xml | 3 +-
.../tools/query/comparison/ClusterStarter.java | 4 +-
.../realtime/provisioning/MemoryEstimator.java | 18 +-
pom.xml | 91 ++++
thirdeye/README.md | 3 +
thirdeye/docs/configuration.rst | 210 ++++++++++
thirdeye/docs/index.rst | 2 +-
thirdeye/docs/production.rst | 27 --
thirdeye/docs/quick_start.rst | 8 +-
.../app/pods/components/alert-details/component.js | 2 +
.../app/pods/components/alert-details/template.hbs | 80 +++-
.../pods/components/detection-health/component.js | 197 +++++++++
.../pods/components/detection-health/template.hbs | 84 ++++
.../app/pods/components/stats-cards/component.js | 42 +-
.../app/pods/components/stats-cards/template.hbs | 61 ++-
.../app/pods/components/yaml-editor/template.hbs | 2 +-
.../app/pods/manage/explore/route.js | 35 +-
.../app/pods/manage/explore/template.hbs | 1 +
.../app/pods/manage/yaml/route.js | 44 +-
.../app/pods/manage/yaml/template.hbs | 3 +-
.../app/pods/self-serve/create-alert/template.hbs | 2 +-
.../app/styles/shared/_styles.scss | 118 ++++++
.../dashboard/DetectionPreviewConfiguration.java | 50 +++
.../dashboard/ThirdEyeDashboardApplication.java | 2 +-
.../dashboard/ThirdEyeDashboardConfiguration.java | 9 +
.../datasource/sql/SqlResponseCacheLoader.java | 37 ++
.../pinot/thirdeye/datasource/sql/SqlUtils.java | 3 +
.../thirdeye/detection/DetectionPipeline.java | 35 +-
.../detection/DetectionPipelineLoader.java | 10 +-
.../detection/alert/DetectionAlertTaskFactory.java | 13 +-
.../alert/scheme/DetectionEmailAlerter.java | 62 ++-
.../components/TriggerConditionGrouper.java | 2 +-
.../thirdeye/detection/health/DetectionHealth.java | 4 +-
.../detection/health/DetectionTaskStatus.java | 14 +
.../validators/DetectionConfigValidator.java | 23 +-
.../thirdeye/detection/wrapper/GrouperWrapper.java | 2 +-
.../thirdeye/detection/yaml/YamlResource.java | 63 ++-
.../yaml/translator/DetectionConfigTranslator.java | 13 +-
.../translator/SubscriptionConfigTranslator.java | 9 +-
.../thirdeye/detection/alert/SendAlertTest.java | 2 +-
.../thirdeye/detection/yaml/YamlResourceTest.java | 6 +-
.../detection/yaml/alertconfig/alert-config-5.yaml | 3 +
.../compositePipelineTranslatorTestResult-1.json | 91 ++--
184 files changed, 4408 insertions(+), 1558 deletions(-)
copy pinot-api/src/main/java/org/apache/pinot/client/{PinotClientTransport.java => Request.java} (61%)
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/PinotQueryParserFactory.java
copy pinot-common/src/main/java/org/apache/pinot/pql/parsers/AbstractCompiler.java => pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/PinotQueryRequest.java (60%)
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/request/PqlAndCalciteSqlCompatibilityTest.java
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/CompletionConfig.java
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/utils/URIUtils.java
copy pinot-common/src/main/java/org/apache/pinot/{pql => }/parsers/AbstractCompiler.java (81%)
create mode 100644 pinot-common/src/main/java/org/apache/pinot/parsers/utils/BrokerRequestComparisonUtils.java
copy pinot-common/src/main/java/org/apache/pinot/{pql/parsers/AbstractCompiler.java => sql/parsers/CalciteSqlCompiler.java} (61%)
create mode 100644 pinot-common/src/main/java/org/apache/pinot/sql/parsers/CalciteSqlParser.java
rename pinot-common/src/main/java/org/apache/pinot/{pql/parsers/AbstractCompiler.java => sql/parsers/SqlCompilationException.java} (69%)
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/URIUtilsTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java
copy pinot-common/src/test/resources/{pql_queries.list => sql_queries.list} (87%)
delete mode 100644 pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/KafkaAvroMessageDecoder.java
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2ConsumerFactory.java => KafkaConsumerFactory.java} (79%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2HighLevelStreamConfig.java => KafkaHighLevelStreamConfig.java} (85%)
delete mode 100644 pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/KafkaJSONMessageDecoder.java
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2MessageBatch.java => KafkaMessageBatch.java} (92%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2PartitionLevelConnectionHandler.java => KafkaPartitionLevelConnectionHandler.java} (86%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2PartitionLevelPartitionLevelConsumer.java => KafkaPartitionLevelConsumer.java} (87%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2PartitionLevelStreamConfig.java => KafkaPartitionLevelStreamConfig.java} (77%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2PartitionLevelStreamMetadataProvider.java => KafkaPartitionLevelStreamMetadataProvider.java} (86%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2StreamConfigProperties.java => KafkaStreamConfigProperties.java} (98%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2StreamLevelConsumer.java => KafkaStreamLevelConsumer.java} (90%)
rename pinot-connectors/pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2/{Kafka2ConsumerManager.java => KafkaStreamLevelConsumerManager.java} (82%)
copy pinot-connectors/{pinot-connector-kafka-2.0 => pinot-connector-kafka-common}/README.md (61%)
copy pinot-connectors/{pinot-connector-kafka-0.9 => pinot-connector-kafka-common}/pom.xml (56%)
rename pinot-connectors/{pinot-connector-kafka-0.9 => pinot-connector-kafka-common}/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaAvroMessageDecoder.java (100%)
rename pinot-connectors/{pinot-connector-kafka-0.9 => pinot-connector-kafka-common}/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaJSONMessageDecoder.java (100%)
rename pinot-connectors/{pinot-connector-kafka-2.0/src/main/java/org/apache/pinot/core/realtime/impl/kafka2 => pinot-connector-kafka-common/src/main/java/org/apache/pinot/core/realtime/impl/kafka}/MessageAndOffset.java (100%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/AccessControlTest.java (87%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/FileUploadPathProviderTest.java (95%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotFileUploadTest.java (98%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotInstanceRestletResourceTest.java (99%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotSchemaRestletResourceTest.java (98%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotSegmentRestletResourceTest.java (98%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotTableRestletResourceTest.java (99%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/PinotTenantRestletResourceTest.java (98%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/SegmentCompletionProtocolDeserTest.java (99%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/SegmentCompletionUtilsTest.java (96%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/ServerTableSizeReaderTest.java (98%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/TableSizeReaderTest.java (99%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{resources => }/TableViewsTest.java (98%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriterTest.java
rename pinot-server/src/test/java/org/apache/pinot/server/api/{resources => }/BaseResourceTest.java (99%)
rename pinot-server/src/test/java/org/apache/pinot/server/api/{resources => }/TableSizeResourceTest.java (98%)
rename pinot-server/src/test/java/org/apache/pinot/server/api/{resources => }/TablesResourceTest.java (99%)
create mode 100644 thirdeye/docs/configuration.rst
delete mode 100644 thirdeye/docs/production.rst
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/detection-health/component.js
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/detection-health/template.hbs
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/DetectionPreviewConfiguration.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org