You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2021/10/20 16:42:34 UTC
[pinot] branch threshold_updater_patch updated (8dfa4a2 -> 00ed50a)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a change to branch threshold_updater_patch
in repository https://gitbox.apache.org/repos/asf/pinot.git.
from 8dfa4a2 edit comment to reflect new partition id calculation
add 9ef1f49 Add timezone as argument in toDateTime (#7552)
add 6c01ce2 Add IN function (#7542)
add 95b0eda map bitmaps through a bounded window to avoid excessive disk pressure… (#7535)
add 9a8ba67 [Refactor] Remove the index access within the PlanNode constructor (#7550)
add 7fe59d5 Return 0 in the response metadata when thread cpu time measurement is disabled (#7544)
add 493ed2c MergeRollupTaskGenerator enhancement: enable parallel buckets scheduling (#7481)
add 01cb4cd Make utility functions in `SegmentAssignmentUtils` public. (#7561)
add 1dda74a Return non zero from main with exception (#7482)
add 4ba8061 Adjust tuner api (#7553)
add bd39dd8 [Auto Reformat] Reformat TableRebalancer related classes (#7571)
add 8150322 Adding config for metrics library (#7551)
add ea31137 Refactor toDateTime with time zone (#7558)
add 5eee297 fix: Sorting tables by number of segments is broken (#7564)
add 0ecfc11 Add geo type conversion scalar functions (#7573)
add e681916 Enhance TableRebalancer to offload the segments from most loaded instances first (#7574)
add 9f0562e Adding generic util method for persisting task metadata in a ZNode (#7575)
add c9e2785 Improve get tenant API to differentiate offline and realtime tenants (#7548)
add ac49978 Need to make abstract methods public since its going to be overriden and called from different packages (#7579)
add 5106a92 Adding new generic method in ClusterInfoAccessor for getting ZNRecord (#7580)
add 0421197 Introduce Native Text Indices (Core Functionality) (#7405)
add 3ef5a37 Change FetchContext to have list of index per colum to fetch (#7587)
add 9d2a9f3 Fix mergeRollupTaskGenerator metrics and logs. (#7588)
add b8d3c2e allow rle compression of bitmaps for smaller file sizes (#7582)
add 85e0d9e add BOOLEAN_ARRAY and TIMESTAMP_ARRAY types (#7581)
add 00ed50a Merge branch 'master' into threshold_updater_patch
No new revisions were added by this update.
Summary of changes:
.github/workflows/scripts/.pinot_quickstart.sh | 3 +-
config/suppressions.xml | 1 +
.../common/function/DateTimePatternHandler.java | 18 +-
.../common/function/TransformFunctionType.java | 1 +
.../common/function/scalar/DateTimeFunctions.java | 8 +
.../pinot/common/metrics/PinotMetricUtils.java | 40 +-
.../pinot/common/minion/BaseTaskMetadata.java | 59 ++
.../common/minion/MergeRollupTaskMetadata.java | 17 +-
.../common/minion/MinionTaskMetadataUtils.java | 30 +-
.../RealtimeToOfflineSegmentsTaskMetadata.java | 17 +-
.../org/apache/pinot/common/utils/DataSchema.java | 184 ++--
.../apache/pinot/common/utils/PinotDataType.java | 74 +-
.../common/utils/config/TableConfigUtils.java | 17 +-
.../pinot/common/utils/helix/HelixHelper.java | 27 +-
.../pinot/common/metrics/PinotMetricUtilsTest.java | 13 +
.../common/minion/MinionTaskMetadataUtilsTest.java | 88 ++
.../apache/pinot/common/utils/DataSchemaTest.java | 38 +-
.../pinot/common/utils/PinotDataTypeTest.java | 45 +-
.../common/utils/config/TableConfigSerDeTest.java | 7 +-
.../api/resources/PinotTableRestletResource.java | 2 +-
.../api/resources/PinotTenantRestletResource.java | 33 +-
.../api/resources/TableConfigsRestletResource.java | 2 +-
.../helix/core/PinotHelixResourceManager.java | 5 +
.../assignment/segment/SegmentAssignmentUtils.java | 14 +-
.../helix/core/minion/ClusterInfoAccessor.java | 27 +-
.../helix/core/rebalance/TableRebalancer.java | 241 +++--
.../tuner/NoOpTableTableConfigTuner.java | 7 +-
.../controller/tuner/RealTimeAutoIndexTuner.java | 18 +-
.../pinot/controller/tuner/TableConfigTuner.java | 16 +-
.../controller/tuner/TableConfigTunerUtils.java | 34 +-
.../src/main/resources/app/components/Table.tsx | 12 +-
.../api/TableConfigsRestletResourceTest.java | 4 +-
.../TableRebalancerClusterStatelessTest.java | 18 +-
.../helix/core/rebalance/TableRebalancerTest.java | 428 +++++----
.../tuner/RealTimeAutoIndexTunerTest.java | 7 +-
.../pinot/controller/tuner/TunerRegistryTest.java | 7 +-
.../apache/pinot/core/common/MinionConstants.java | 1 +
.../function/ConstructFromTextFunction.java | 9 +-
.../function/ConstructFromWKBFunction.java | 9 +-
.../transform/function/ScalarFunctions.java | 52 +-
.../transform/function/StAsBinaryFunction.java | 13 +-
.../transform/function/StAsTextFunction.java | 16 +-
.../transform/function/StGeogFromTextFunction.java | 6 +-
.../transform/function/StGeogFromWKBFunction.java | 6 +-
.../transform/function/StGeomFromTextFunction.java | 10 +-
.../transform/function/StGeomFromWKBFunction.java | 10 +-
.../transform/function/StPolygonFunction.java | 6 +-
.../core/operator/InstanceResponseOperator.java | 57 +-
.../query/DictionaryBasedAggregationOperator.java | 20 +-
.../transform/function/InTransformFunction.java | 381 ++++++++
.../function/TransformFunctionFactory.java | 2 +
.../plan/AggregationGroupByOrderByPlanNode.java | 80 +-
.../core/plan/AggregationGroupByPlanNode.java | 69 +-
.../pinot/core/plan/AggregationPlanNode.java | 112 ++-
.../apache/pinot/core/plan/CombinePlanNode.java | 4 +-
.../plan/DictionaryBasedAggregationPlanNode.java | 62 --
.../core/plan/DictionaryBasedDistinctPlanNode.java | 67 --
.../apache/pinot/core/plan/DistinctPlanNode.java | 46 +-
.../apache/pinot/core/plan/DocIdSetPlanNode.java | 12 +-
.../plan/MetadataBasedAggregationPlanNode.java | 65 --
.../apache/pinot/core/plan/ProjectionPlanNode.java | 18 +-
.../apache/pinot/core/plan/SelectionPlanNode.java | 46 +-
.../core/plan/StreamingSelectionPlanNode.java | 15 +-
.../apache/pinot/core/plan/TransformPlanNode.java | 34 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 99 +-
.../function/AggregationFunctionUtils.java | 10 -
.../core/query/request/context/ThreadTimer.java | 4 +
.../core/data/function/DateTimeFunctionsTest.java | 7 +
.../function/InTransformFunctionTest.java | 194 ++++
...adataAndDictionaryAggregationPlanMakerTest.java | 83 +-
...erSegmentResultTableSingleValueQueriesTest.java | 20 +
.../apache/pinot/queries/StUnionQueriesTest.java | 42 +-
.../MergeRollupMinionClusterIntegrationTest.java | 15 +-
.../tests/OfflineClusterIntegrationTest.java | 2 +-
...fflineSegmentsMinionClusterIntegrationTest.java | 5 +-
.../org/apache/pinot/minion/BaseMinionStarter.java | 2 +-
.../java/org/apache/pinot/minion/MinionConf.java | 4 +
.../executor/MinionTaskZkMetadataManager.java | 2 +-
.../java/org/apache/pinot/perf/BenchmarkFST.java | 100 ++
.../pinot/perf/BenchmarkGeospatialSerde.java | 4 +-
...BenchmarkOffheapBitmapInvertedIndexCreator.java | 7 +-
.../java/org/apache/pinot/perf/BytesCounter.java | 38 +-
.../mergerollup/MergeRollupTaskGenerator.java | 294 +++---
.../tasks/mergerollup/MergeRollupTaskUtils.java | 3 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 6 +-
.../mergerollup/MergeRollupTaskGeneratorTest.java | 201 +++-
.../mergerollup/MergeRollupTaskUtilsTest.java | 4 +-
...RealtimeToOfflineSegmentsTaskGeneratorTest.java | 38 +-
.../impl/inv/BitmapInvertedIndexWriter.java | 54 +-
.../inv/OffHeapBitmapInvertedIndexCreator.java | 2 +-
.../impl/inv/OnHeapBitmapInvertedIndexCreator.java | 2 +-
.../impl/inv/json/BaseJsonIndexCreator.java | 2 +-
.../pinot/segment/local/utils/GeometryUtils.java | 10 +
.../utils/nativefst/ByteSequenceIterator.java | 180 ++++
.../local/utils/nativefst/ConstantArcSizeFST.java | 157 +++
.../pinot/segment/local/utils/nativefst/FST.java | 438 +++++++++
.../segment/local/utils/nativefst/FSTFlags.java | 57 +-
.../segment/local/utils/nativefst/FSTHeader.java | 80 ++
.../local/utils/nativefst/FSTTraversal.java | 127 +++
.../local/utils/nativefst/ImmutableFST.java | 396 ++++++++
.../segment/local/utils/nativefst/MatchResult.java | 90 ++
.../utils/nativefst/NativeFSTIndexCreator.java | 82 ++
.../utils/nativefst/NativeFSTIndexReader.java | 75 ++
.../local/utils/nativefst/StateVisitor.java | 27 +-
.../local/utils/nativefst/automaton/Automaton.java | 658 +++++++++++++
.../nativefst/automaton/AutomatonMatcher.java | 264 +++++
.../nativefst/automaton/AutomatonProvider.java | 27 +-
.../utils/nativefst/automaton/BasicAutomata.java | 250 +++++
.../utils/nativefst/automaton/BasicOperations.java | 647 +++++++++++++
.../nativefst/automaton/CharacterRunAutomaton.java | 66 ++
.../automaton/MinimizationOperations.java | 710 ++++++++++++++
.../local/utils/nativefst/automaton/README.md | 84 ++
.../local/utils/nativefst/automaton/RegExp.java | 771 +++++++++++++++
.../utils/nativefst/automaton/RunAutomaton.java | 311 ++++++
.../nativefst/automaton/SpecialOperations.java | 360 +++++++
.../local/utils/nativefst/automaton/State.java | 173 ++++
.../local/utils/nativefst/automaton/StatePair.java | 68 ++
.../nativefst/automaton/StringUnionOperations.java | 372 ++++++++
.../utils/nativefst/automaton/Transition.java | 149 +++
.../nativefst/automaton/TransitionComparator.java | 79 ++
.../local/utils/nativefst/builder/FSTBuilder.java | 566 +++++++++++
.../local/utils/nativefst/builder/FSTInfo.java | 143 +++
.../utils/nativefst/builder/FSTSerializer.java | 57 ++
.../utils/nativefst/builder/FSTSerializerImpl.java | 370 ++++++++
.../local/utils/nativefst/builder/FSTUtils.java | 180 ++++
.../local/utils/nativefst/utils/RegexpMatcher.java | 169 ++++
.../creator/inv/BitmapInvertedIndexWriterTest.java | 144 +++
.../local/utils/nativefst/FSTBuilderTest.java | 99 ++
.../utils/nativefst/FSTRegexpWithWeirdTest.java | 85 ++
.../local/utils/nativefst/FSTSanityTest.java | 84 ++
.../local/utils/nativefst/FSTTestUtils.java | 139 +++
.../local/utils/nativefst/FSTTraversalTest.java | 324 +++++++
.../nativefst/ImmutableFSTDeserializedTest.java | 50 +
.../nativefst/ImmutableFSTSerializerTest.java | 21 +-
.../local/utils/nativefst/ImmutableFSTTest.java | 147 +++
.../segment/local/utils/nativefst/MinMax.java | 24 +-
.../local/utils/nativefst/SerializerTestBase.java | 195 ++++
.../src/test/resources/data/abc-numbers.native.fst | Bin 0 -> 29 bytes
.../src/test/resources/data/abc.native.fst | Bin 0 -> 24 bytes
.../src/test/resources/data/en_tst.dict | Bin 0 -> 1070678 bytes
.../src/test/resources/data/serfst.txt | Bin 0 -> 29951 bytes
.../src/test/resources/data/words.txt | 1004 ++++++++++++++++++++
.../org/apache/pinot/segment/spi/FetchContext.java | 36 +-
.../org/apache/pinot/segment/spi/V1Constants.java | 1 +
.../apache/pinot/spi/config/table/TableConfig.java | 19 +-
.../apache/pinot/spi/utils/CommonConstants.java | 5 +
.../spi/utils/builder/TableConfigBuilder.java | 8 +-
pinot-tools/pom.xml | 1 +
.../pinot/tools/admin/PinotAdministrator.java | 3 +
.../command/LaunchDataIngestionJobCommand.java | 28 -
150 files changed, 13096 insertions(+), 1548 deletions(-)
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/minion/BaseTaskMetadata.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/minion/MinionTaskMetadataUtilsTest.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/plan/DictionaryBasedAggregationPlanNode.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/plan/DictionaryBasedDistinctPlanNode.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/plan/MetadataBasedAggregationPlanNode.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InTransformFunctionTest.java
create mode 100644 pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkFST.java
copy pinot-core/src/main/java/org/apache/pinot/core/geospatial/transform/function/StGeomFromWKBFunction.java => pinot-perf/src/main/java/org/apache/pinot/perf/BytesCounter.java (59%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ByteSequenceIterator.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ConstantArcSizeFST.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FST.java
copy pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/FetchContext.java => pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FSTFlags.java (51%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FSTHeader.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/FSTTraversal.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFST.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/MatchResult.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexCreator.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/NativeFSTIndexReader.java
copy pinot-core/src/main/java/org/apache/pinot/core/geospatial/transform/function/StGeomFromWKBFunction.java => pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/StateVisitor.java (60%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/Automaton.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/AutomatonMatcher.java
copy pinot-core/src/main/java/org/apache/pinot/core/geospatial/transform/function/StGeomFromWKBFunction.java => pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/AutomatonProvider.java (60%)
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/BasicAutomata.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/BasicOperations.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/CharacterRunAutomaton.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/MinimizationOperations.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/README.md
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/RegExp.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/RunAutomaton.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/SpecialOperations.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/State.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/StatePair.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/StringUnionOperations.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/Transition.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/automaton/TransitionComparator.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/builder/FSTBuilder.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/builder/FSTInfo.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/builder/FSTSerializer.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/builder/FSTSerializerImpl.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/builder/FSTUtils.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/nativefst/utils/RegexpMatcher.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/inv/BitmapInvertedIndexWriterTest.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTBuilderTest.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTRegexpWithWeirdTest.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTSanityTest.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTTestUtils.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/FSTTraversalTest.java
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTDeserializedTest.java
copy pinot-core/src/main/java/org/apache/pinot/core/geospatial/transform/function/StGeomFromWKBFunction.java => pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTSerializerTest.java (60%)
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/ImmutableFSTTest.java
copy pinot-core/src/main/java/org/apache/pinot/core/geospatial/transform/function/StGeomFromWKBFunction.java => pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/MinMax.java (60%)
create mode 100644 pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/nativefst/SerializerTestBase.java
create mode 100644 pinot-segment-local/src/test/resources/data/abc-numbers.native.fst
create mode 100644 pinot-segment-local/src/test/resources/data/abc.native.fst
create mode 100644 pinot-segment-local/src/test/resources/data/en_tst.dict
create mode 100644 pinot-segment-local/src/test/resources/data/serfst.txt
create mode 100644 pinot-segment-local/src/test/resources/data/words.txt
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org