You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2021/03/12 23:28:50 UTC
[incubator-pinot] branch basic-auth-controller updated (90b699b ->
d2bb5c7)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch basic-auth-controller
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 90b699b basic auth integration test
discard 78663c2 integration test fix
discard 1b130c7 manual review fixes
discard 2907987 dedicated auth quickstart
discard 559a19d config overrides in starter commands
discard b162807 force quickstart broker auth
discard 369648c make minion task auth augmentation optional
discard 9b100fb auth workflow info
discard eddf171 rebase fix
discard e26ee1d refactor FileUploadDownloadClient
discard 527212e transitioning more calls for auth support
discard 31a8cdb tokens everywhere
discard e720aeb progress
add 3b70930 Kinesis implementation Part 1: Rename partitionId to partitionGroupId (#6655)
add 31bdc2b Don't throw exception in json_extract_scalar when default value is available. (#6651)
add 59d02c6 complete compatibility regression testing (#6650)
add b72917c Remove unused shaded jars (#6666)
add f386ede Skip loading columns not in the schema (#6649)
add 74e4037 SumPrecision: support all data types and star-tree (#6668)
add 4fac51c Add realtime provisioning rule to recommendation engine (#6625)
add 2bdc92e Add pinot-yammer to pinot-tools pom (#6675)
add c982721 Add an option to fail segment creation job when getting empty files (#6664)
add 36d6db6 Fix the percentile star-tree tests (#6673)
add 02e901b progress
add 6c5ce4f tokens everywhere
add a198d7a transitioning more calls for auth support
add afd9f6d refactor FileUploadDownloadClient
add f6f67c6 rebase fix
add 94a23e8 auth workflow info
add 96b9cba make minion task auth augmentation optional
add 4d719a3 force quickstart broker auth
add 88b47a2 config overrides in starter commands
add fd7ac78 dedicated auth quickstart
add f88a71c manual review fixes
add f54a414 integration test fix
add 3f65e7d basic auth integration test
add d2bb5c7 review fixes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (90b699b)
\
N -- N -- N refs/heads/basic-auth-controller (d2bb5c7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
{pinot-perf => compatibility-verifier}/README.md | 27 +--
compatibility-verifier/checkoutAndBuild.sh | 79 +++++++
compatibility-verifier/compCheck.sh | 150 +++++--------
pinot-broker/pom.xml | 23 --
.../org/apache/pinot/broker/api/AccessControl.java | 22 +-
.../requesthandler/BaseBrokerRequestHandler.java | 2 +-
.../segmentselector/RealtimeSegmentSelector.java | 2 +-
.../scalar/DataTypeConversionFunctions.java | 58 +----
.../apache/pinot/common/utils/CommonConstants.java | 13 ++
.../common/utils/FileUploadDownloadClient.java | 232 ++++++++++++++++++--
.../apache/pinot/common/utils/LLCSegmentName.java | 24 +-
.../org/apache/pinot/common/utils/SegmentName.java | 4 +-
.../pinot/common/utils/SegmentNameBuilderTest.java | 20 +-
pinot-controller/pom.xml | 41 +---
.../pinot/controller/api/access/AccessControl.java | 6 +-
.../api/resources/PinotControllerAuthResource.java | 15 ++
.../segment/RealtimeSegmentAssignment.java | 23 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 106 ++++-----
.../SegmentSizeBasedFlushThresholdUpdater.java | 4 +-
.../recommender}/data/generator/AvroWriter.java | 2 +-
.../data/generator/BytesGenerator.java | 2 +-
.../recommender}/data/generator/DataGenerator.java | 2 +-
.../data/generator/DataGeneratorSpec.java | 2 +-
.../recommender}/data/generator/Generator.java | 2 +-
.../data/generator/GeneratorFactory.java | 2 +-
.../data/generator/MultiValueGeneratorHelper.java | 2 +-
.../data/generator/NumberGenerator.java | 80 ++-----
.../data/generator/PatternMixtureGenerator.java | 2 +-
.../data/generator/PatternSeasonalGenerator.java | 2 +-
.../data/generator/PatternSequenceGenerator.java | 2 +-
.../data/generator/PatternSpikeGenerator.java | 2 +-
.../data/generator/PatternStringGenerator.java | 2 +-
.../recommender}/data/generator/PatternType.java | 2 +-
.../data/generator/RangeDoubleGenerator.java | 2 +-
.../data/generator/RangeFloatGenerator.java | 2 +-
.../data/generator/RangeIntGenerator.java | 2 +-
.../data/generator/RangeLongGenerator.java | 2 +-
.../data/generator/SchemaAnnotation.java | 2 +-
.../data/generator/StringGenerator.java | 12 +-
.../recommender}/data/generator/TimeGenerator.java | 2 +-
.../controller/recommender/io/ConfigManager.java | 18 ++
.../controller/recommender/io/InputManager.java | 15 ++
.../realtime/provisioning/MemoryEstimator.java | 67 +++---
.../recommender/rules/RulesToExecute.java | 16 +-
.../rules/impl/RealtimeProvisioningRule.java | 187 ++++++++++++++++
.../recommender/rules/io/configs/IndexConfig.java | 4 +
.../rules/io/configs/PartitionConfig.java | 12 +
.../io/params/RealtimeProvisioningRuleParams.java | 105 +++++++++
.../rules/io/params/RecommenderConstants.java | 8 +
.../PinotLLCRealtimeSegmentManagerTest.java | 79 +++----
.../controller/recommender/TestConfigEngine.java | 195 +++++++++-------
.../generator/MultiValueGeneratorHelperTest.java | 2 +-
.../data/generator/NumberGeneratorTest.java | 86 ++++++++
.../data/generator/TimeGeneratorTest.java | 2 +-
.../realtime/provisioning/MemoryEstimatorTest.java | 10 +-
.../memory_estimation/schema-with-metadata.json | 0
.../schema-with-metadata__dateTimeFieldSpec.json | 0
.../resources/memory_estimation/table-config.json | 0
...exInput.json => RealtimeProvisioningInput.json} | 53 +++--
.../apache/pinot/core/common/ObjectSerDeUtils.java | 8 +-
...gator.java => SumPrecisionValueAggregator.java} | 59 +++--
.../data/aggregator/ValueAggregatorFactory.java | 4 +
.../realtime/LLRealtimeSegmentDataManager.java | 46 ++--
.../manager/realtime/RealtimeTableDataManager.java | 17 +-
.../generator/SegmentGeneratorConfig.java | 9 +
.../immutable/ImmutableSegmentLoader.java | 19 +-
.../JsonExtractScalarTransformFunction.java | 25 ++-
.../function/SumPrecisionAggregationFunction.java | 229 +++++++++++++------
.../segment/creator/ColumnIndexCreationInfo.java | 4 -
.../core/segment/creator/ColumnStatistics.java | 4 +
.../creator/impl/SegmentColumnarIndexCreator.java | 3 -
.../impl/SegmentIndexCreationDriverImpl.java | 20 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 9 +-
.../index/metadata/SegmentMetadataImpl.java | 40 ++--
.../realtime/LLRealtimeSegmentDataManagerTest.java | 20 +-
.../SegmentGenerationWithBytesTypeTest.java | 2 +-
.../startree/v2/PercentileEstStarTreeV2Test.java | 9 +-
.../v2/PercentileTDigestStarTreeV2Test.java | 8 +-
.../PreAggregatedPercentileEstStarTreeV2Test.java | 11 +-
...eAggregatedPercentileTDigestStarTreeV2Test.java | 10 +-
...V2Test.java => SumPrecisionStarTreeV2Test.java} | 17 +-
.../pinot/queries/JsonMatchPredicateTest.java | 31 +++
.../pinot/queries/SumPrecisionQueriesTest.java | 244 +++++++++++++++++++++
.../apache/pinot/queries/SumWithPrecisionTest.java | 221 -------------------
.../org/apache/pinot/compat/tests/SegmentOp.java | 8 +-
.../pinot/compat/tests/SqlResultComparator.java | 35 +--
.../tests/BasicAuthClusterIntegrationTest.java | 5 +-
.../tests/OfflineClusterIntegrationTest.java | 142 +++++++-----
...PartitionLLCRealtimeClusterIntegrationTest.java | 22 +-
.../compat-tests/configs/feature-test-1.json | 4 +-
.../compat-tests/data/FeatureTest1-data-00.csv | 25 +--
.../compat-tests/data/FeatureTest1-data-01.csv | 17 +-
.../compat-tests/data/recordReaderConfig.json | 2 +-
...p-publish.yaml => kafka-stream-produce-op.yaml} | 12 +-
.../compat-tests/kafka-topic-create-op.yaml | 16 +-
.../compat-tests/offline-table-create-op.yaml | 17 +-
.../compat-tests/offline-table-delete-op.yaml | 17 +-
.../queries/feature-test-1-sql.queries | 17 +-
.../src/test/resources/compat-tests/query-op.yaml | 28 +--
.../query-results/feature-test-1-rest-sql.results | 1 -
.../compat-tests/realtime-table-create-op.yaml | 17 +-
.../compat-tests/realtime-table-delete-op.yaml | 17 +-
...ream-op-publish.yaml => segment-create-op.yaml} | 24 +-
.../resources/compat-tests/segment-delete-op.yaml | 17 +-
.../resources/compat-tests/table-segment-op.yaml | 45 ----
pinot-minion/pom.xml | 23 --
.../batch/common/SegmentGenerationTaskRunner.java | 3 +-
.../batch/hadoop/HadoopSegmentCreationMapper.java | 1 +
.../HadoopSegmentGenerationJobRunnerTest.java | 34 +--
.../spark/SparkSegmentGenerationJobRunner.java | 1 +
.../standalone/SegmentGenerationJobRunner.java | 1 +
.../hadoop/job/mappers/SegmentCreationMapper.java | 3 +
.../RealtimeToOfflineSegmentsTaskGenerator.java | 11 +-
.../SegmentGenerationAndPushTaskExecutor.java | 3 +
pinot-server/pom.xml | 24 --
.../spi/ingestion/batch/BatchConfigProperties.java | 1 +
.../batch/spec/SegmentGenerationJobSpec.java | 13 ++
.../batch/spec/SegmentGenerationTaskSpec.java | 11 +-
.../apache/pinot/spi/utils/BigDecimalUtils.java | 61 ++++++
.../BigDecimalUtilsTest.java} | 28 +--
pinot-tools/pom.xml | 5 +
.../tools/admin/command/CreateSegmentCommand.java | 16 +-
.../tools/admin/command/GenerateDataCommand.java | 6 +-
.../command/RealtimeProvisioningHelperCommand.java | 37 +++-
124 files changed, 2271 insertions(+), 1408 deletions(-)
copy {pinot-perf => compatibility-verifier}/README.md (53%)
create mode 100644 compatibility-verifier/checkoutAndBuild.sh
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/AvroWriter.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/BytesGenerator.java (95%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/DataGenerator.java (99%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/DataGeneratorSpec.java (98%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/Generator.java (93%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/GeneratorFactory.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/MultiValueGeneratorHelper.java (96%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/NumberGenerator.java (54%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternMixtureGenerator.java (98%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternSeasonalGenerator.java (98%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternSequenceGenerator.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternSpikeGenerator.java (98%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternStringGenerator.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/PatternType.java (94%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/RangeDoubleGenerator.java (95%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/RangeFloatGenerator.java (95%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/RangeIntGenerator.java (95%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/RangeLongGenerator.java (95%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/SchemaAnnotation.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/StringGenerator.java (90%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/data/generator/TimeGenerator.java (97%)
rename {pinot-tools/src/main/java/org/apache/pinot/tools => pinot-controller/src/main/java/org/apache/pinot/controller/recommender}/realtime/provisioning/MemoryEstimator.java (92%)
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/RealtimeProvisioningRule.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/io/params/RealtimeProvisioningRuleParams.java
rename {pinot-tools/src/test/java/org/apache/pinot/tools => pinot-controller/src/test/java/org/apache/pinot/controller/recommender}/data/generator/MultiValueGeneratorHelperTest.java (96%)
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/recommender/data/generator/NumberGeneratorTest.java
rename {pinot-tools/src/test/java/org/apache/pinot/tools => pinot-controller/src/test/java/org/apache/pinot/controller/recommender}/data/generator/TimeGeneratorTest.java (96%)
rename {pinot-tools/src/test/java/org/apache/pinot/tools => pinot-controller/src/test/java/org/apache/pinot/controller/recommender}/realtime/provisioning/MemoryEstimatorTest.java (92%)
rename {pinot-tools => pinot-controller}/src/test/resources/memory_estimation/schema-with-metadata.json (100%)
rename {pinot-tools => pinot-controller}/src/test/resources/memory_estimation/schema-with-metadata__dateTimeFieldSpec.json (100%)
rename {pinot-tools => pinot-controller}/src/test/resources/memory_estimation/table-config.json (100%)
copy pinot-controller/src/test/resources/recommenderInput/{SortedInvertedIndexInput.json => RealtimeProvisioningInput.json} (76%)
copy pinot-core/src/main/java/org/apache/pinot/core/data/aggregator/{DistinctCountBitmapValueAggregator.java => SumPrecisionValueAggregator.java} (50%)
copy pinot-core/src/test/java/org/apache/pinot/core/startree/v2/{MaxStarTreeV2Test.java => SumPrecisionStarTreeV2Test.java} (71%)
create mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/SumPrecisionQueriesTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/SumWithPrecisionTest.java
copy pinot-integration-tests/src/test/resources/compat-tests/{stream-op-publish.yaml => kafka-stream-produce-op.yaml} (70%)
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/kafka-topic-create-op.yaml (70%)
mode change 100755 => 100644
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/offline-table-create-op.yaml (68%)
mode change 100755 => 100644
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/offline-table-delete-op.yaml (68%)
mode change 100755 => 100644
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/realtime-table-create-op.yaml (67%)
mode change 100755 => 100644
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/realtime-table-delete-op.yaml (67%)
mode change 100755 => 100644
rename pinot-integration-tests/src/test/resources/compat-tests/{stream-op-publish.yaml => segment-create-op.yaml} (59%)
copy .github/workflows/scripts/.pinot_test.sh => pinot-integration-tests/src/test/resources/compat-tests/segment-delete-op.yaml (73%)
mode change 100755 => 100644
delete mode 100644 pinot-integration-tests/src/test/resources/compat-tests/table-segment-op.yaml
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/utils/BigDecimalUtils.java
copy pinot-spi/src/test/java/org/apache/pinot/spi/{config/table/UpsertConfigTest.java => utils/BigDecimalUtilsTest.java} (52%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org