You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2024/02/01 06:56:38 UTC
(pinot) branch stop-all-segments-in-releaseAndRemoveAllSegments updated (2abd892fe1 -> 4f05fd71ae)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch stop-all-segments-in-releaseAndRemoveAllSegments
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 2abd892fe1 Release all segments of a table in releaseAndRemoveAllSegments method
add 7b69d094be Refactoring the upsert compaction related code (#12275)
add ced6bc282e Use higher fetch timeout for Kinesis (#12214)
add f1fec060a6 Support runtime reload for TLS resources (#12277)
add eeaf1f0811 [feature] allow dim table config to detect/disallow duplicate PK (#12290)
add 35faeb6712 Revert allowing tag override with upserts (#12311)
add 63e91ef95f Support table suffix in ZkBasicAuthAccessControlFactory (#12310)
add 91ffcc759f Reduce Disk Footprint for Segment Processor Framework to Avoid Out of Disk Issues (#12220)
add f43664dd13 Support server level consumption throttle (#12292)
add 9c1bb02dec Misc fixes for upsert metadata manager (#12319)
add 76d0eb25db Reduce Heap Usage of OnHeapStringDictionary (#12223)
add 7978d29031 Mark distribution as multi-release (#12300)
add 5a382f2e7d Shared aggregations in StarTree (#12164)
add 2acf8ea354 correct errmsg format for Preconditions.check... (#12327)
add 23dbb08e8d Make server resource classes configurable (#12324)
add ed6761a982 Fix "rewind()" for CompactedPinotSegmentRecordReader (#12329)
add 4823802886 Wire soft upsert delete for compaction task (#12330)
add 6cc1915140 Add HttpHeaders in broker event listener requestContext (#12258)
add e1d20d75d5 Making taskManager resources protected for derived classes to override in their setUp() method. (#12335)
add 1a82ba6fc6 Add CPU metrics for minion purge task (#12337)
add d5014786db Add first implementation of clpMatch that doesn't explicitly use indexes. (#12291)
add 3ab28348fb auto reload TLS resources when loading from local files (#12325)
add 8988b755d6 Remove segments with empty download url in UpsertCompactionTask (#12320)
add dd8be2a229 add TextMatchFilterOptimizer to maximally push down `text_match` filters to Lucene (#12339)
add 17db0fd17b add ScalingThreadPoolExecutor and use for realtime Lucene thread pool (#12274)
add 4f05fd71ae Release all segments of a table in releaseAndRemoveAllSegments method
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (2abd892fe1)
\
N -- N -- N refs/heads/stop-all-segments-in-releaseAndRemoveAllSegments (4f05fd71ae)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 2 +-
.../broker/ZkBasicAuthAccessControlFactory.java | 3 +-
.../requesthandler/BaseBrokerRequestHandler.java | 10 +-
pinot-common/pom.xml | 8 +
.../common/function/TransformFunctionType.java | 3 +
.../apache/pinot/common/http/MultiHttpRequest.java | 51 +-
.../pinot/common/metrics/AbstractMetrics.java | 6 +
.../apache/pinot/common/metrics/MinionTimer.java | 2 +-
...leSizeInfo.java => ValidDocIdMetadataInfo.java} | 42 +-
.../common/tier/FixedTierSegmentSelector.java | 2 +-
.../common/utils/ScalingThreadPoolExecutor.java | 128 +++++
.../org/apache/pinot/common/utils/TlsUtils.java | 258 +++++++--
.../pinot/common/utils/grpc/GrpcQueryClient.java | 24 +-
.../sql/parsers/rewriter/CLPDecodeRewriter.java | 177 ------
.../pinot/sql/parsers/rewriter/ClpRewriter.java | 633 +++++++++++++++++++++
.../function/FunctionDefinitionRegistryTest.java | 6 +-
.../pinot/common/http/MultiHttpRequestTest.java | 106 +++-
.../pinot/common/utils/FALFInternerTest.java | 168 ++++++
.../utils/ScalingThreadPoolExecutorTest.java | 69 +++
.../apache/pinot/common/utils/TlsUtilsTest.java | 286 ++++++++++
.../parsers/rewriter/CLPDecodeRewriterTest.java | 65 ---
.../sql/parsers/rewriter/ClpRewriterTest.java | 282 +++++++++
.../src/test/resources/tls/keystore-updated.p12 | Bin 0 -> 2581 bytes
pinot-common/src/test/resources/tls/keystore.p12 | Bin 0 -> 2581 bytes
.../src/test/resources/tls/truststore-updated.p12 | Bin 0 -> 1186 bytes
pinot-common/src/test/resources/tls/truststore.p12 | Bin 0 -> 1186 bytes
.../access/ZkBasicAuthAccessControlFactory.java | 4 +-
.../api/resources/PinotRunningQueryResource.java | 2 +-
.../api/resources/PinotTableRestletResource.java | 37 ++
.../helix/core/PinotHelixResourceManager.java | 2 +-
.../helix/core/rebalance/RebalanceChecker.java | 2 +-
.../helix/core/relocation/SegmentRelocator.java | 2 +-
.../controller/util/CompletionServiceHelper.java | 52 +-
.../util/ServerSegmentMetadataReader.java | 161 +++++-
.../pinot/controller/util/TableMetadataReader.java | 30 +-
.../apache/pinot/core/common/MinionConstants.java | 1 +
.../core/data/manager/BaseTableDataManager.java | 3 +-
.../manager/offline/DimensionTableDataManager.java | 9 +-
.../realtime/RealtimeConsumptionRateManager.java | 39 +-
.../realtime/RealtimeSegmentDataManager.java | 10 +-
.../manager/realtime/RealtimeTableDataManager.java | 4 +-
.../ClpEncodedVarsMatchTransformFunction.java | 150 +++++
.../function/TransformFunctionFactory.java | 1 +
.../function/AggregationFunctionUtils.java | 17 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 8 +-
.../optimizer/filter/TextMatchFilterOptimizer.java | 186 ++++++
.../processing/framework/SegmentConfig.java | 30 +-
.../framework/SegmentProcessorFramework.java | 139 +++--
.../genericrow/AdaptiveConstraintsWriter.java | 17 +-
.../genericrow/AdaptiveSizeBasedWriter.java | 34 +-
.../segment/processing/genericrow/FileWriter.java | 16 +-
.../genericrow/GenericRowFileWriter.java | 11 +-
.../segment/processing/mapper/SegmentMapper.java | 70 ++-
.../apache/pinot/core/startree/StarTreeUtils.java | 2 +-
.../executor/StarTreeAggregationExecutor.java | 2 +-
.../startree/executor/StarTreeGroupByExecutor.java | 2 +-
.../pinot/core/transport/grpc/GrpcQueryServer.java | 12 +-
.../apache/pinot/core/util/ListenerConfigUtil.java | 2 +-
.../offline/DimensionTableDataManagerTest.java | 75 ++-
.../RealtimeConsumptionRateManagerTest.java | 49 +-
...ionTest.java => ClpTransformFunctionsTest.java} | 111 +++-
.../core/query/optimizer/QueryOptimizerTest.java | 55 ++
.../core/realtime/stream/StreamConfigTest.java | 27 +
.../framework/SegmentProcessorFrameworkTest.java | 205 ++++++-
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 23 +-
...java => DistinctCountRawHLLStarTreeV2Test.java} | 8 +-
.../src/test/resources/data/dimBaseballTeams.avro | Bin 2214 -> 0 bytes
.../src/test/resources/data/dimBaseballTeams.avsc | 8 -
.../src/test/resources/data/dimBaseballTeams.csv | 1 +
.../resources/data/dimBaseballTeams_config.json | 0
pinot-distribution/pom.xml | 5 +
...nsumptionRateLimiterClusterIntegrationTest.java | 298 ++++++++++
...sertCompactionMinionClusterIntegrationTest.java | 221 -------
.../tests/UpsertTableIntegrationTest.java | 289 ++++++++--
.../src/test/resources/gameScores_large_csv.tar.gz | Bin 0 -> 12810 bytes
.../test/resources/upsert_compaction_test.tar.gz | Bin 9405 -> 0 bytes
.../batch/hadoop/HadoopSegmentCreationMapper.java | 2 +-
.../inputformat/clplog/CLPLogRecordExtractor.java | 8 +-
.../clplog/CLPLogRecordExtractorTest.java | 14 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 4 +
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 41 ++
.../minion/tasks/purge/PurgeTaskExecutor.java | 19 +-
.../minion/tasks/purge/PurgeTaskGenerator.java | 2 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 6 +-
.../SegmentGenerationAndPushTaskGenerator.java | 2 +-
.../UpsertCompactionTaskExecutor.java | 141 +----
.../UpsertCompactionTaskGenerator.java | 120 ++--
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 6 +-
.../minion/tasks/purge/PurgeTaskExecutorTest.java | 3 +-
.../UpsertCompactionTaskExecutorTest.java | 7 +-
.../UpsertCompactionTaskGeneratorTest.java | 86 +--
.../query/planner/plannode/AggregateNode.java | 2 +-
.../DistinctCountHLLPlusValueAggregator.java | 2 +-
.../RealtimeLuceneTextIndexSearcherPool.java | 6 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 8 +-
.../index/dictionary/DictionaryIndexType.java | 55 +-
.../index/dictionary/DictionaryInternerHolder.java | 58 ++
.../index/readers/OnHeapStringDictionary.java | 17 +-
.../readers/CompactedPinotSegmentRecordReader.java | 111 ++++
.../local/segment/store/StarTreeIndexReader.java | 29 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 12 +-
.../startree/v2/store/StarTreeIndexMapUtils.java | 25 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 56 +-
...oncurrentMapPartitionUpsertMetadataManager.java | 43 +-
.../segment/local/utils/TableConfigUtils.java | 20 +-
.../index/dictionary/DictionaryIndexTypeTest.java | 70 ++-
.../index/readers/ImmutableDictionaryTest.java | 2 +-
.../ImmutableDictionaryTypeConversionTest.java | 25 +-
... => CompactedPinotSegmentRecordReaderTest.java} | 80 ++-
.../v2/builder/StarTreeV2BuilderConfigTest.java | 57 +-
.../segment/local/utils/TableConfigUtilsTest.java | 10 +-
.../spi/creator/SegmentGeneratorConfig.java | 2 +-
.../segment/spi/index/DictionaryIndexConfig.java | 37 +-
.../startree/AggregationFunctionColumnPair.java | 42 ++
.../spi/index/startree/AggregationSpec.java | 8 +
.../spi/index/startree/StarTreeV2Metadata.java | 14 +-
.../AggregationFunctionColumnPairTest.java | 59 ++
.../spi/index/startree/StarTreeV2MetadataTest.java | 130 +++++
.../pinot/server/api/AdminApiApplication.java | 10 +-
.../pinot/server/api/resources/TablesResource.java | 88 ++-
.../server/starter/helix/BaseServerStarter.java | 5 +-
.../starter/helix/HelixInstanceDataManager.java | 15 +-
.../pinot/server/api/TablesResourceTest.java | 22 +
.../spi/config/table/DimensionTableConfig.java | 11 +-
.../config/table/{IndexConfig.java => Intern.java} | 50 +-
.../apache/pinot/spi/data/DateTimeFormatSpec.java | 2 +-
.../spi/data/readers/RecordReaderFileConfig.java | 51 +-
.../PinotBrokerQueryEventListenerFactory.java | 34 ++
.../org/apache/pinot/spi/stream/StreamConfig.java | 7 +-
.../pinot/spi/trace/DefaultRequestContext.java | 11 +
.../org/apache/pinot/spi/trace/RequestContext.java | 4 +
.../apache/pinot/spi/utils/CommonConstants.java | 8 +
.../org/apache/pinot/spi/utils/FALFInterner.java | 148 +++++
.../pinot/spi/config/table/IndexingConfigTest.java | 7 +-
.../org/apache/pinot/tools/utils/JarUtils.java | 47 +-
pom.xml | 7 +
136 files changed, 5344 insertions(+), 1387 deletions(-)
copy pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/{TableSizeInfo.java => ValidDocIdMetadataInfo.java} (55%)
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/utils/ScalingThreadPoolExecutor.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/CLPDecodeRewriter.java
create mode 100644 pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriter.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/FALFInternerTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/ScalingThreadPoolExecutorTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/common/utils/TlsUtilsTest.java
delete mode 100644 pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/CLPDecodeRewriterTest.java
create mode 100644 pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/ClpRewriterTest.java
create mode 100644 pinot-common/src/test/resources/tls/keystore-updated.p12
create mode 100644 pinot-common/src/test/resources/tls/keystore.p12
create mode 100644 pinot-common/src/test/resources/tls/truststore-updated.p12
create mode 100644 pinot-common/src/test/resources/tls/truststore.p12
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ClpEncodedVarsMatchTransformFunction.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/TextMatchFilterOptimizer.java
copy pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/InvertedIndexCreator.java => pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/AdaptiveConstraintsWriter.java (73%)
copy pinot-connectors/pinot-flink-connector/src/main/java/org/apache/pinot/connector/flink/common/MapGenericRowConverter.java => pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/AdaptiveSizeBasedWriter.java (55%)
copy pinot-controller/src/main/java/org/apache/pinot/controller/recommender/data/generator/Generator.java => pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/FileWriter.java (75%)
rename pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/{CLPDecodeTransformFunctionTest.java => ClpTransformFunctionsTest.java} (69%)
copy pinot-core/src/test/java/org/apache/pinot/core/startree/v2/{DistinctCountHLLStarTreeV2Test.java => DistinctCountRawHLLStarTreeV2Test.java} (86%)
delete mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avro
delete mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avsc
copy pinot-tools/src/main/resources/examples/batch/dimBaseballTeams/dimBaseballTeams_offline_table_config.json => pinot-core/src/test/resources/data/dimBaseballTeams_config.json (100%)
create mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeConsumptionRateLimiterClusterIntegrationTest.java
delete mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertCompactionMinionClusterIntegrationTest.java
create mode 100644 pinot-integration-tests/src/test/resources/gameScores_large_csv.tar.gz
delete mode 100644 pinot-integration-tests/src/test/resources/upsert_compaction_test.tar.gz
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryInternerHolder.java
create mode 100644 pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java
copy pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/{PinotSegmentRecordReaderTest.java => CompactedPinotSegmentRecordReaderTest.java} (62%)
create mode 100644 pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/startree/AggregationFunctionColumnPairTest.java
create mode 100644 pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/startree/StarTreeV2MetadataTest.java
copy pinot-spi/src/main/java/org/apache/pinot/spi/config/table/{IndexConfig.java => Intern.java} (56%)
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/utils/FALFInterner.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org