You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/07/14 05:38:52 UTC
(pinot) branch master updated: Add compression statistics tracking and table APIs (#18185)
This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9a93618392c Add compression statistics tracking and table APIs (#18185)
9a93618392c is described below
commit 9a93618392cf3290b5713b71f1f2c73e2383b3de
Author: John Solomon J <45...@users.noreply.github.com>
AuthorDate: Mon Jul 13 22:38:46 2026 -0700
Add compression statistics tracking and table APIs (#18185)
Track uncompressed serialized value sizes and raw forward-index codecs in segment metadata behind the default-off compressionStatsEnabled table flag.
Expose bounded, replica-aware compression summaries and optional per-column details through the table size and metadata APIs, with rolling-upgrade fallback, controller metrics, documentation, and focused coverage.
Co-authored-by: John Solomon J <js...@splunk.com>
---
.../ColumnCompressionStatsAccumulator.java | 174 +++
.../apache/pinot/common/http/MultiHttpRequest.java | 166 ++-
.../http/MultiHttpRequestBufferedResponse.java | 64 ++
.../pinot/common/metrics/ControllerGauge.java | 9 +
.../ColumnCompressionStatsContribution.java | 144 +++
.../resources/ColumnCompressionStatsInfo.java | 212 ++++
.../restlet/resources/CompressionStatsSummary.java | 103 ++
.../SegmentCompressionStatsContribution.java | 82 ++
.../common/restlet/resources/SegmentSizeInfo.java | 66 +-
.../resources/ServerCompressionStatsResponse.java | 59 +
.../restlet/resources/ServerTableMetadataInfo.java | 197 ++++
.../restlet/resources/TableMetadataInfo.java | 56 +-
.../common/restlet/resources/TableSizeInfo.java | 20 +-
.../ColumnCompressionStatsAccumulatorTest.java | 57 +
.../pinot/common/http/MultiHttpRequestTest.java | 96 ++
.../pinot/common/metrics/ControllerGaugeTest.java | 35 +-
.../resources/ColumnCompressionStatsInfoTest.java | 135 +++
.../resources/CompressionStatsSummaryTest.java | 82 ++
.../restlet/resources/SegmentSizeInfoTest.java | 101 ++
.../ServerTransportCompatibilityTest.java | 141 +++
.../TableMetadataInfoCompressionTest.java | 157 +++
.../api/resources/PinotTableRestletResource.java | 25 +-
.../api/resources/ServerTableSizeReader.java | 46 +-
.../pinot/controller/api/resources/TableSize.java | 4 +-
.../controller/helix/SegmentStatusChecker.java | 34 +-
.../controller/util/CompletionServiceHelper.java | 152 ++-
.../util/ServerCompressionStatsReader.java | 648 +++++++++++
.../util/ServerSegmentMetadataReader.java | 70 +-
.../pinot/controller/util/TableMetadataReader.java | 19 +-
.../pinot/controller/util/TableSizeReader.java | 242 ++++-
.../ServerTableSizeReaderCompressionStatsTest.java | 224 ++++
.../controller/api/ServerTableSizeReaderTest.java | 51 +-
.../api/TableMetadataReaderCompressionTest.java | 1128 ++++++++++++++++++++
.../api/TableSizeReaderCompressionStatsTest.java | 682 ++++++++++++
.../pinot/controller/api/TableSizeReaderTest.java | 3 +-
.../controller/helix/SegmentStatusCheckerTest.java | 75 +-
...essionStatsOfflineIngestionIntegrationTest.java | 402 +++++++
...ssionStatsRealtimeIngestionIntegrationTest.java | 231 ++++
.../writer/impl/BaseChunkForwardIndexWriter.java | 16 +
.../impl/FixedByteChunkForwardIndexWriter.java | 10 +
.../impl/VarByteChunkForwardIndexWriter.java | 8 +
.../impl/VarByteChunkForwardIndexWriterV4.java | 25 +
.../local/io/writer/impl/VarByteChunkWriter.java | 6 +
.../segment/creator/impl/BaseSegmentCreator.java | 46 +-
.../segment/creator/impl/ColumnIndexCreators.java | 12 +
.../creator/impl/SameValueForwardIndexCreator.java | 13 +
.../creator/impl/SegmentDictionaryCreator.java | 150 ++-
.../creator/impl/fwd/CLPForwardIndexCreatorV1.java | 23 +-
.../creator/impl/fwd/CLPForwardIndexCreatorV2.java | 35 +-
...ompressionStatsTrackingForwardIndexCreator.java | 41 +
.../fwd/MultiValueFixedByteRawIndexCreator.java | 20 +-
.../impl/fwd/MultiValueVarByteRawIndexCreator.java | 20 +-
.../fwd/SingleValueFixedByteRawIndexCreator.java | 20 +-
.../fwd/SingleValueVarByteRawIndexCreator.java | 20 +-
.../index/dictionary/DictionaryIndexType.java | 2 +-
.../index/forward/CompressionStatsMetadata.java | 100 ++
.../index/forward/ForwardIndexCreatorFactory.java | 59 +-
.../segment/index/loader/ForwardIndexHandler.java | 325 +++++-
...IndexAndDictionaryBasedForwardIndexCreator.java | 56 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 44 +-
.../SegmentGeneratorConfigPropagationTest.java | 70 ++
.../impl/SameValueForwardIndexCreatorTest.java | 39 +
.../index/creator/CLPForwardIndexCreatorTest.java | 64 +-
.../creator/CLPForwardIndexCreatorV2Test.java | 45 +
.../creator/CompressionStatsCornerCaseTest.java | 294 +++++
.../CompressionStatsSegmentCreationTest.java | 281 +++++
...orwardIndexWriterUncompressedValueSizeTest.java | 448 ++++++++
.../MultiValueFixedByteRawIndexCreatorTest.java | 30 +
.../MultiValueVarByteRawIndexCreatorTest.java | 31 +
...DictionaryCreatorUncompressedValueSizeTest.java | 440 ++++++++
.../forward/CompressionStatsMetadataTest.java | 59 +
.../forward/ForwardIndexCreatorFactoryTest.java | 35 +
.../forward/VarByteChunkSVForwardIndexTest.java | 32 +
.../ForwardIndexHandlerCompressionStatsTest.java | 607 +++++++++++
.../index/loader/ForwardIndexHandlerTest.java | 57 +-
.../index/loader/SegmentPreProcessorTest.java | 43 +-
.../apache/pinot/segment/spi/ColumnMetadata.java | 21 +
.../org/apache/pinot/segment/spi/V1Constants.java | 8 +
.../segment/spi/creator/IndexCreationContext.java | 27 +
.../spi/creator/SegmentGeneratorConfig.java | 12 +
.../spi/index/creator/ForwardIndexCreator.java | 13 +
.../spi/index/metadata/ColumnMetadataImpl.java | 134 ++-
.../spi/index/metadata/ColumnMetadataImplTest.java | 85 ++
.../spi/utils/SegmentMetadataUtilsTest.java | 80 ++
.../resources/SegmentCompressionStatsReader.java | 168 +++
.../server/api/resources/TableSizeResource.java | 79 +-
.../pinot/server/api/resources/TablesResource.java | 112 +-
.../apache/pinot/server/api/BaseResourceTest.java | 7 +
.../pinot/server/api/TableSizeResourceTest.java | 90 +-
.../pinot/server/api/TablesResourceTest.java | 221 ++++
.../SegmentCompressionStatsReaderTest.java | 143 +++
.../pinot/spi/config/table/IndexingConfig.java | 17 +
.../java/org/apache/pinot/spi/utils/Utf8Utils.java | 25 +
.../spi/utils/builder/TableConfigBuilder.java | 8 +
.../table/IndexingConfigCompressionFlagTest.java | 70 ++
.../org/apache/pinot/spi/utils/Utf8UtilsTest.java} | 31 +-
.../pinot/sql/ddl/compile/PropertyMapping.java | 4 +
.../pinot/sql/ddl/reverse/PropertyExtractor.java | 3 +
.../pinot/sql/ddl/roundtrip/RoundTripTest.java | 1 +
.../resources/examples/compression-stats/README.md | 104 ++
.../compressionStats_offline_table_config.json | 26 +
101 files changed, 10964 insertions(+), 343 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulator.java b/pinot-common/src/main/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulator.java
new file mode 100644
index 00000000000..21102c84af2
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulator.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.compression;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+
+
+/// Mutable, non-thread-safe accumulator for one column's compression statistics.
+///
+/// The accumulator owns encoding and chunk-type merging, unavailable-value handling, index names, and breakdown entries
+/// so server and controller aggregation paths use the same rules.
+public final class ColumnCompressionStatsAccumulator {
+ private long _uncompressedValueSizeInBytes;
+ private long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private int _numSegments;
+ private final Set<String> _observedIndexes = new LinkedHashSet<>();
+ private final Map<EncodingKey, EncodingAccumulator> _encodingAccumulators = new LinkedHashMap<>();
+
+ /// Adds one or more independent contributions for one encoding and chunk-compression type.
+ public void add(long uncompressedValueSizeInBytes, long forwardIndexAndDictionaryStorageSizeInBytes,
+ EncodingType encoding, @Nullable ChunkCompressionType chunkCompressionType,
+ @Nullable Collection<String> observedIndexes, int numSegments) {
+ if (uncompressedValueSizeInBytes < 0 || forwardIndexAndDictionaryStorageSizeInBytes < 0 || numSegments <= 0) {
+ return;
+ }
+ addAggregate(uncompressedValueSizeInBytes, forwardIndexAndDictionaryStorageSizeInBytes, observedIndexes,
+ numSegments);
+ _encodingAccumulators.computeIfAbsent(new EncodingKey(encoding, chunkCompressionType),
+ ignored -> new EncodingAccumulator())
+ .add(uncompressedValueSizeInBytes, forwardIndexAndDictionaryStorageSizeInBytes, numSegments);
+ }
+
+ /// Adds an already-aggregated contribution.
+ public void add(ColumnCompressionStatsContribution contribution) {
+ if (contribution.getUncompressedValueSizeInBytes() < 0
+ || contribution.getForwardIndexAndDictionaryStorageSizeInBytes() < 0
+ || contribution.getNumSegments() <= 0) {
+ return;
+ }
+ addAggregate(contribution.getUncompressedValueSizeInBytes(),
+ contribution.getForwardIndexAndDictionaryStorageSizeInBytes(), contribution.getObservedIndexes(),
+ contribution.getNumSegments());
+ for (ColumnCompressionStatsContribution.EncodingContribution entry : contribution.getEncodingBreakdown()) {
+ EncodingKey key = new EncodingKey(entry.getEncoding(), entry.getChunkCompressionType());
+ _encodingAccumulators.computeIfAbsent(key, ignored -> new EncodingAccumulator())
+ .add(entry.getUncompressedValueSizeInBytes(), entry.getForwardIndexAndDictionaryStorageSizeInBytes(),
+ entry.getNumSegments());
+ }
+ }
+
+ /// Builds the public DTO without replica normalization.
+ public ColumnCompressionStatsInfo toColumnCompressionStatsInfo(String column) {
+ double ratio = _forwardIndexAndDictionaryStorageSizeInBytes > 0
+ ? (double) _uncompressedValueSizeInBytes / _forwardIndexAndDictionaryStorageSizeInBytes : 0;
+ List<String> observedIndexes = _observedIndexes.isEmpty() ? null : List.copyOf(_observedIndexes);
+ List<ColumnCompressionStatsInfo.EncodingBreakdownEntry> breakdown = new ArrayList<>();
+ for (Map.Entry<EncodingKey, EncodingAccumulator> entry : _encodingAccumulators.entrySet()) {
+ breakdown.add(entry.getValue().toEntry(entry.getKey()));
+ }
+ breakdown.sort(Comparator.comparing((ColumnCompressionStatsInfo.EncodingBreakdownEntry entry) ->
+ entry.getEncoding().name())
+ .thenComparing(entry -> entry.getChunkCompressionType() != null ? entry.getChunkCompressionType().name() : ""));
+ return ColumnCompressionStatsInfo.builder(column)
+ .withUncompressedValueSizeInBytes(_uncompressedValueSizeInBytes)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(_forwardIndexAndDictionaryStorageSizeInBytes)
+ .withCompressionRatio(ratio)
+ .withObservedIndexes(observedIndexes)
+ .withEncodingBreakdown(breakdown)
+ .withNumSegments(_numSegments)
+ .build();
+ }
+
+ /// Builds a server-local contribution without replica normalization.
+ public ColumnCompressionStatsContribution toColumnCompressionStatsContribution(String column) {
+ List<String> observedIndexes = _observedIndexes.isEmpty() ? null : List.copyOf(_observedIndexes);
+ List<ColumnCompressionStatsContribution.EncodingContribution> breakdown = new ArrayList<>();
+ for (Map.Entry<EncodingKey, EncodingAccumulator> entry : _encodingAccumulators.entrySet()) {
+ EncodingKey key = entry.getKey();
+ EncodingAccumulator value = entry.getValue();
+ breakdown.add(new ColumnCompressionStatsContribution.EncodingContribution(key._encoding,
+ key._chunkCompressionType, value._numSegments, value._uncompressedValueSizeInBytes,
+ value._forwardIndexAndDictionaryStorageSizeInBytes));
+ }
+ breakdown.sort(Comparator.comparing((ColumnCompressionStatsContribution.EncodingContribution entry) ->
+ entry.getEncoding().name())
+ .thenComparing(entry -> entry.getChunkCompressionType() != null ? entry.getChunkCompressionType().name() : ""));
+ return new ColumnCompressionStatsContribution(column, _uncompressedValueSizeInBytes,
+ _forwardIndexAndDictionaryStorageSizeInBytes, observedIndexes, breakdown, _numSegments);
+ }
+
+ private void addAggregate(long uncompressedValueSizeInBytes, long forwardIndexAndDictionaryStorageSizeInBytes,
+ @Nullable Collection<String> observedIndexes, int numSegments) {
+ _uncompressedValueSizeInBytes += uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes += forwardIndexAndDictionaryStorageSizeInBytes;
+ _numSegments += numSegments;
+ if (observedIndexes != null) {
+ _observedIndexes.addAll(observedIndexes);
+ }
+ }
+
+ private static final class EncodingKey {
+ private final EncodingType _encoding;
+ private final ChunkCompressionType _chunkCompressionType;
+
+ private EncodingKey(EncodingType encoding, @Nullable ChunkCompressionType chunkCompressionType) {
+ _encoding = encoding;
+ _chunkCompressionType = chunkCompressionType;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (!(object instanceof EncodingKey)) {
+ return false;
+ }
+ EncodingKey that = (EncodingKey) object;
+ return _encoding == that._encoding && _chunkCompressionType == that._chunkCompressionType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(_encoding, _chunkCompressionType);
+ }
+ }
+
+ private static final class EncodingAccumulator {
+ private long _uncompressedValueSizeInBytes;
+ private long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private int _numSegments;
+
+ private void add(long uncompressedValueSizeInBytes, long forwardIndexAndDictionaryStorageSizeInBytes,
+ int numSegments) {
+ _uncompressedValueSizeInBytes += uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes += forwardIndexAndDictionaryStorageSizeInBytes;
+ _numSegments += numSegments;
+ }
+
+ private ColumnCompressionStatsInfo.EncodingBreakdownEntry toEntry(EncodingKey key) {
+ return new ColumnCompressionStatsInfo.EncodingBreakdownEntry(key._encoding, key._chunkCompressionType,
+ _numSegments, _uncompressedValueSizeInBytes, _forwardIndexAndDictionaryStorageSizeInBytes);
+ }
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequest.java b/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequest.java
index a1f829e6715..f02f5984700 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequest.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequest.java
@@ -21,11 +21,13 @@ package org.apache.pinot.common.http;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletionService;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import javax.annotation.Nullable;
@@ -80,6 +82,25 @@ public class MultiHttpRequest {
return execute(urlsAndRequestBodies, requestHeaders, timeoutMs, "GET", HttpGet::new);
}
+ /// Executes cancellable GET requests that buffer each response body before releasing its connection.
+ public BufferedRequestExecution executeGetBuffered(List<String> urls, @Nullable Map<String, String> requestHeaders,
+ int timeoutMs) {
+ BufferedRequestExecution execution = createBufferedExecution(timeoutMs);
+ try {
+ for (String url : urls) {
+ HttpGet request = new HttpGet(url);
+ if (requestHeaders != null) {
+ requestHeaders.forEach(request::setHeader);
+ }
+ execution.submit(request);
+ }
+ return execution;
+ } catch (RuntimeException | Error e) {
+ execution.close();
+ throw e;
+ }
+ }
+
/**
* POST urls in parallel using the executor service.
* @param urlsAndRequestBodies absolute URLs to POST
@@ -118,34 +139,141 @@ public class MultiHttpRequest {
CompletionService<MultiHttpRequestResponse> completionService = new ExecutorCompletionService<>(_executor);
CloseableHttpClient client = httpClientBuilder.build();
for (Pair<String, String> pair : urlsAndRequestBodies) {
+ String url = pair.getLeft();
+ String body = pair.getRight();
+ HttpUriRequestBase httpMethod = httpRequestBaseSupplier.apply(url);
+ // If the http method is POST, set the request body
+ if (httpMethod instanceof HttpPost) {
+ ((HttpPost) httpMethod).setEntity(new StringEntity(body));
+ }
+ if (requestHeaders != null) {
+ requestHeaders.forEach(httpMethod::setHeader);
+ }
completionService.submit(() -> {
- String url = pair.getLeft();
- String body = pair.getRight();
- HttpUriRequestBase httpMethod = httpRequestBaseSupplier.apply(url);
- // If the http method is POST, set the request body
- if (httpMethod instanceof HttpPost) {
- ((HttpPost) httpMethod).setEntity(new StringEntity(body));
- }
- if (requestHeaders != null) {
- requestHeaders.forEach(((HttpUriRequestBase) httpMethod)::setHeader);
- }
CloseableHttpResponse response = null;
+ boolean responseHandedOff = false;
try {
response = client.execute(httpMethod);
httpMethod.setAbsoluteRequestUri(true);
- return new MultiHttpRequestResponse(URI.create(httpMethod.getRequestUri()), response);
- } catch (IOException ex) {
- if (response != null) {
- String error = EntityUtils.toString(response.getEntity());
- LOGGER.warn("Caught '{}' while executing: {} on URL: {}", error, httpMethodName, url);
- } else {
- // Log only exception type and message instead of the whole stack trace
- LOGGER.warn("Caught '{}' while executing: {} on URL: {}", ex, httpMethodName, url);
+ MultiHttpRequestResponse result =
+ new MultiHttpRequestResponse(URI.create(httpMethod.getRequestUri()), response);
+ responseHandedOff = true;
+ return result;
+ } finally {
+ if (response != null && !responseHandedOff) {
+ response.close();
}
- throw ex;
}
});
}
return completionService;
}
+
+ /// Creates a cancellable execution that buffers each response body before releasing its connection. Callers may
+ /// submit requests with per-request configuration and must close the execution when collection finishes.
+ public BufferedRequestExecution createBufferedExecution(int timeoutMs) {
+ Timeout timeout = Timeout.of(timeoutMs, TimeUnit.MILLISECONDS);
+ RequestConfig defaultRequestConfig =
+ RequestConfig.custom().setConnectionRequestTimeout(timeout).setResponseTimeout(timeout).build();
+ CloseableHttpClient client = HttpClients.custom().setConnectionManager(_connectionManager)
+ .setConnectionManagerShared(true).setDefaultRequestConfig(defaultRequestConfig).build();
+ return new BufferedRequestExecution(new ExecutorCompletionService<>(_executor), client);
+ }
+
+ private static MultiHttpRequestBufferedResponse executeBufferedRequest(CloseableHttpClient client,
+ HttpUriRequestBase httpMethod)
+ throws Exception {
+ try (CloseableHttpResponse response = client.execute(httpMethod)) {
+ httpMethod.setAbsoluteRequestUri(true);
+ URI uri = URI.create(httpMethod.getRequestUri());
+ return new MultiHttpRequestBufferedResponse(uri, response.getCode(), response.getReasonPhrase(),
+ EntityUtils.toString(response.getEntity()));
+ } catch (Exception e) {
+ throw new IOException("Failed to execute request on URL: " + httpMethod.getRequestUri() + ": "
+ + e.getMessage(), e);
+ }
+ }
+
+ /// Mutable execution state for buffered requests. A single collector thread must poll it and close it when done.
+ /// Closing cancels every unfinished request, including when collection exits early.
+ public static final class BufferedRequestExecution implements AutoCloseable {
+ private final CompletionService<MultiHttpRequestBufferedResponse> _completionService;
+ private final CloseableHttpClient _client;
+ private final Map<Future<MultiHttpRequestBufferedResponse>, HttpUriRequestBase> _pendingRequests = new HashMap<>();
+ private boolean _closed;
+
+ private BufferedRequestExecution(CompletionService<MultiHttpRequestBufferedResponse> completionService,
+ CloseableHttpClient client) {
+ _completionService = completionService;
+ _client = client;
+ }
+
+ /// Submits one request and returns its future. The request is cancelled automatically when this execution closes.
+ public Future<MultiHttpRequestBufferedResponse> submit(HttpUriRequestBase request) {
+ if (_closed) {
+ throw new IllegalStateException("Buffered request execution is closed");
+ }
+ try {
+ Future<MultiHttpRequestBufferedResponse> future =
+ _completionService.submit(() -> executeBufferedRequest(_client, request));
+ _pendingRequests.put(future, request);
+ return future;
+ } catch (RuntimeException | Error e) {
+ request.cancel();
+ throw e;
+ }
+ }
+
+ /// Waits for one completed request. The request's own HTTP timeout remains the only timeout applied.
+ public Future<MultiHttpRequestBufferedResponse> take()
+ throws InterruptedException {
+ if (_closed) {
+ throw new IllegalStateException("Buffered request execution is closed");
+ }
+ Future<MultiHttpRequestBufferedResponse> completedFuture = _completionService.take();
+ _pendingRequests.remove(completedFuture);
+ return completedFuture;
+ }
+
+ /// Waits up to the supplied timeout for one completed request.
+ @Nullable
+ public Future<MultiHttpRequestBufferedResponse> poll(long timeout, TimeUnit unit)
+ throws InterruptedException {
+ if (_closed) {
+ throw new IllegalStateException("Buffered request execution is closed");
+ }
+ Future<MultiHttpRequestBufferedResponse> completedFuture = _completionService.poll(timeout, unit);
+ if (completedFuture != null) {
+ _pendingRequests.remove(completedFuture);
+ }
+ return completedFuture;
+ }
+
+ /// Cancels one pending request. Calling this for a request that already completed is harmless.
+ public void cancel(Future<MultiHttpRequestBufferedResponse> future) {
+ HttpUriRequestBase request = _pendingRequests.remove(future);
+ if (request != null) {
+ request.cancel();
+ }
+ future.cancel(true);
+ }
+
+ /// Aborts unfinished HTTP requests and cancels their executor tasks. This method is idempotent.
+ @Override
+ public void close() {
+ if (_closed) {
+ return;
+ }
+ _closed = true;
+ for (Map.Entry<Future<MultiHttpRequestBufferedResponse>, HttpUriRequestBase> entry
+ : new ArrayList<>(_pendingRequests.entrySet())) {
+ cancel(entry.getKey());
+ }
+ try {
+ _client.close();
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close buffered HTTP client: {}", e.getMessage());
+ }
+ }
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequestBufferedResponse.java b/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequestBufferedResponse.java
new file mode 100644
index 00000000000..239aeeb54ce
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/http/MultiHttpRequestBufferedResponse.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.http;
+
+import java.net.URI;
+import javax.annotation.Nullable;
+
+
+/// Immutable, thread-safe HTTP response whose entity has been fully buffered and whose connection has been released.
+public final class MultiHttpRequestBufferedResponse {
+ private final URI _uri;
+ private final int _statusCode;
+ @Nullable
+ private final String _reasonPhrase;
+ @Nullable
+ private final String _responseBody;
+
+ /// Creates a response whose body has already been consumed and whose HTTP connection has been released.
+ public MultiHttpRequestBufferedResponse(URI uri, int statusCode, @Nullable String reasonPhrase,
+ @Nullable String responseBody) {
+ _uri = uri;
+ _statusCode = statusCode;
+ _reasonPhrase = reasonPhrase;
+ _responseBody = responseBody;
+ }
+
+ /// Returns the request URI.
+ public URI getURI() {
+ return _uri;
+ }
+
+ /// Returns the HTTP status code.
+ public int getStatusCode() {
+ return _statusCode;
+ }
+
+ /// Returns the optional HTTP reason phrase.
+ @Nullable
+ public String getReasonPhrase() {
+ return _reasonPhrase;
+ }
+
+ /// Returns the buffered response body, or `null` when the response had no entity.
+ @Nullable
+ public String getResponseBody() {
+ return _responseBody;
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index 3660d3524e2..44291ba29ce 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -116,6 +116,15 @@ public enum ControllerGauge implements AbstractMetrics.Gauge {
// Percentage of segments we failed to get size for
TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT("TableStorageEstMissingSegmentPercent", false),
+ // Uncompressed-value to forward-index-and-dictionary storage ratio scaled by 100 (e.g., 4.5x becomes 450).
+ TABLE_COMPRESSION_STATS_RATIO_PERCENT("percent", false),
+
+ // Uncompressed column-value size represented by compression statistics, per replica
+ TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA("bytes", false),
+
+ // Forward-index and dictionary size represented by compression statistics, per replica
+ TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA("bytes", false),
+
// Number of scheduled Cron jobs
CRON_SCHEDULER_JOB_SCHEDULED("cronSchedulerJobScheduled", false),
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsContribution.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsContribution.java
new file mode 100644
index 00000000000..cce98a1ab5f
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsContribution.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+
+
+/// Server-local compression statistics for one column.
+///
+/// This transport type deliberately omits a compression ratio because server values are physical contributions.
+/// Controllers de-duplicate replicas before building [ColumnCompressionStatsInfo]. Instances are immutable and
+/// thread-safe.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ColumnCompressionStatsContribution {
+ private final String _column;
+ private final long _uncompressedValueSizeInBytes;
+ private final long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private final List<String> _observedIndexes;
+ private final List<EncodingContribution> _encodingBreakdown;
+ private final int _numSegments;
+
+ @JsonCreator
+ public ColumnCompressionStatsContribution(
+ @JsonProperty("column") String column,
+ @JsonProperty("uncompressedValueSizeInBytes") long uncompressedValueSizeInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizeInBytes") long forwardIndexAndDictionaryStorageSizeInBytes,
+ @JsonProperty("observedIndexes") @Nullable List<String> observedIndexes,
+ @JsonProperty("encodingBreakdown") @Nullable List<EncodingContribution> encodingBreakdown,
+ @JsonProperty("numSegments") int numSegments) {
+ _column = column;
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ _observedIndexes = observedIndexes != null ? List.copyOf(observedIndexes) : null;
+ _encodingBreakdown = encodingBreakdown != null ? List.copyOf(encodingBreakdown) : List.of();
+ _numSegments = numSegments;
+ }
+
+ /// Returns the column name.
+ public String getColumn() {
+ return _column;
+ }
+
+ /// Returns represented uncompressed serialized column-value bytes.
+ public long getUncompressedValueSizeInBytes() {
+ return _uncompressedValueSizeInBytes;
+ }
+
+ /// Returns represented forward-index and dictionary bytes on disk.
+ public long getForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+
+ /// Returns observed index names, or null when index details were not requested.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public List<String> getObservedIndexes() {
+ return _observedIndexes;
+ }
+
+ /// Returns one entry for each represented encoding and chunk-compression type.
+ public List<EncodingContribution> getEncodingBreakdown() {
+ return _encodingBreakdown;
+ }
+
+ /// Returns the represented segment count.
+ public int getNumSegments() {
+ return _numSegments;
+ }
+
+ /// Server-local statistics for one encoding and chunk-compression type.
+ /// Instances are immutable and thread-safe.
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static final class EncodingContribution {
+ private final EncodingType _encoding;
+ private final ChunkCompressionType _chunkCompressionType;
+ private final int _numSegments;
+ private final long _uncompressedValueSizeInBytes;
+ private final long _forwardIndexAndDictionaryStorageSizeInBytes;
+
+ @JsonCreator
+ public EncodingContribution(
+ @JsonProperty("encoding") EncodingType encoding,
+ @JsonProperty("chunkCompressionType") @Nullable ChunkCompressionType chunkCompressionType,
+ @JsonProperty("numSegments") int numSegments,
+ @JsonProperty("uncompressedValueSizeInBytes") long uncompressedValueSizeInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizeInBytes") long forwardIndexAndDictionaryStorageSizeInBytes) {
+ _encoding = encoding;
+ _chunkCompressionType = chunkCompressionType;
+ _numSegments = numSegments;
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+
+ /// Returns `RAW` or `DICTIONARY`.
+ public EncodingType getEncoding() {
+ return _encoding;
+ }
+
+ /// Returns the raw forward-index chunk-compression type, or null for dictionary encoding.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public ChunkCompressionType getChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ /// Returns the represented segment count.
+ public int getNumSegments() {
+ return _numSegments;
+ }
+
+ /// Returns represented uncompressed serialized column-value bytes.
+ public long getUncompressedValueSizeInBytes() {
+ return _uncompressedValueSizeInBytes;
+ }
+
+ /// Returns represented forward-index and dictionary bytes.
+ public long getForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfo.java
new file mode 100644
index 00000000000..5ea44700c35
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfo.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+
+
+/// Compression statistics for one column across one or more segments.
+///
+/// Raw forward indexes contribute the uncompressed serialized value bytes presented to their chunk compressor and the
+/// forward-index file size. Dictionary-encoded forward indexes contribute uncompressed value bytes and the combined
+/// dictionary plus forward-index file size. Segments without persisted value-size metadata are excluded from both sizes
+/// and from `numSegments` so coverage remains explicit. [#getEncodingBreakdown()] separates forward-index encoding from
+/// raw-index chunk-compression type and lists every represented combination. Instances are immutable and thread-safe.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ColumnCompressionStatsInfo {
+ private final String _column;
+ private final long _uncompressedValueSizeInBytes;
+ private final long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private final double _compressionRatio;
+ private final List<String> _observedIndexes;
+ private final List<EncodingBreakdownEntry> _encodingBreakdown;
+ private final int _numSegments;
+
+ @JsonCreator
+ private ColumnCompressionStatsInfo(
+ @JsonProperty("column") String column,
+ @JsonProperty("uncompressedValueSizeInBytes") long uncompressedValueSizeInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizeInBytes") long forwardIndexAndDictionaryStorageSizeInBytes,
+ @JsonProperty("compressionRatio") double compressionRatio,
+ @JsonProperty("observedIndexes") @Nullable List<String> observedIndexes,
+ @JsonProperty("encodingBreakdown") @Nullable List<EncodingBreakdownEntry> encodingBreakdown,
+ @JsonProperty("numSegments") int numSegments) {
+ _column = column;
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ _compressionRatio = compressionRatio;
+ _observedIndexes = observedIndexes != null ? List.copyOf(observedIndexes) : null;
+ _encodingBreakdown = encodingBreakdown != null ? List.copyOf(encodingBreakdown) : List.of();
+ _numSegments = numSegments;
+ }
+
+ /// Returns a builder for one column aggregate.
+ public static Builder builder(String column) {
+ return new Builder(column);
+ }
+
+ /// Returns the column name.
+ public String getColumn() {
+ return _column;
+ }
+
+ /// Returns uncompressed serialized column-value bytes represented by this aggregate.
+ public long getUncompressedValueSizeInBytes() {
+ return _uncompressedValueSizeInBytes;
+ }
+
+ /// Returns tracked forward-index and dictionary bytes on disk.
+ public long getForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+
+ /// Returns `uncompressedValueSizeInBytes / forwardIndexAndDictionaryStorageSizeInBytes`, or `0` when the denominator
+ /// is zero.
+ public double getCompressionRatio() {
+ return _compressionRatio;
+ }
+
+ /// Returns all index names observed for the column, or null when index details were not requested.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public List<String> getObservedIndexes() {
+ return _observedIndexes;
+ }
+
+ /// Returns one entry for each represented encoding and chunk-compression type.
+ public List<EncodingBreakdownEntry> getEncodingBreakdown() {
+ return _encodingBreakdown;
+ }
+
+ /// Returns the number of segment contributions represented by this aggregate.
+ public int getNumSegments() {
+ return _numSegments;
+ }
+
+ /// Mutable, non-thread-safe builder for [ColumnCompressionStatsInfo].
+ public static final class Builder {
+ private final String _column;
+ private long _uncompressedValueSizeInBytes;
+ private long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private double _compressionRatio;
+ private List<String> _observedIndexes;
+ private List<EncodingBreakdownEntry> _encodingBreakdown = List.of();
+ private int _numSegments;
+
+ private Builder(String column) {
+ _column = column;
+ }
+
+ public Builder withUncompressedValueSizeInBytes(long uncompressedValueSizeInBytes) {
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ return this;
+ }
+
+ public Builder withForwardIndexAndDictionaryStorageSizeInBytes(
+ long forwardIndexAndDictionaryStorageSizeInBytes) {
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ return this;
+ }
+
+ public Builder withCompressionRatio(double compressionRatio) {
+ _compressionRatio = compressionRatio;
+ return this;
+ }
+
+ public Builder withObservedIndexes(@Nullable List<String> observedIndexes) {
+ _observedIndexes = observedIndexes;
+ return this;
+ }
+
+ public Builder withEncodingBreakdown(@Nullable List<EncodingBreakdownEntry> encodingBreakdown) {
+ _encodingBreakdown = encodingBreakdown;
+ return this;
+ }
+
+ public Builder withNumSegments(int numSegments) {
+ _numSegments = numSegments;
+ return this;
+ }
+
+ public ColumnCompressionStatsInfo build() {
+ return new ColumnCompressionStatsInfo(_column, _uncompressedValueSizeInBytes,
+ _forwardIndexAndDictionaryStorageSizeInBytes, _compressionRatio, _observedIndexes, _encodingBreakdown,
+ _numSegments);
+ }
+ }
+
+ /// Statistics for one forward-index encoding and chunk-compression type combination.
+ /// Instances are immutable and thread-safe.
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static final class EncodingBreakdownEntry {
+ private final EncodingType _encoding;
+ private final ChunkCompressionType _chunkCompressionType;
+ private final int _numSegments;
+ private final long _uncompressedValueSizeInBytes;
+ private final long _forwardIndexAndDictionaryStorageSizeInBytes;
+
+ @JsonCreator
+ public EncodingBreakdownEntry(
+ @JsonProperty("encoding") EncodingType encoding,
+ @JsonProperty("chunkCompressionType") @Nullable ChunkCompressionType chunkCompressionType,
+ @JsonProperty("numSegments") int numSegments,
+ @JsonProperty("uncompressedValueSizeInBytes") long uncompressedValueSizeInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizeInBytes") long forwardIndexAndDictionaryStorageSizeInBytes) {
+ _encoding = encoding;
+ _chunkCompressionType = chunkCompressionType;
+ _numSegments = numSegments;
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+
+ /// Returns `RAW` or `DICTIONARY`.
+ public EncodingType getEncoding() {
+ return _encoding;
+ }
+
+ /// Returns the raw forward-index chunk-compression type, or null for dictionary encoding.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public ChunkCompressionType getChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ /// Returns the represented segment count.
+ public int getNumSegments() {
+ return _numSegments;
+ }
+
+ /// Returns uncompressed serialized column-value bytes for this encoding and chunk-compression type.
+ public long getUncompressedValueSizeInBytes() {
+ return _uncompressedValueSizeInBytes;
+ }
+
+ /// Returns forward-index and dictionary bytes for this encoding and chunk-compression type.
+ public long getForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummary.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummary.java
new file mode 100644
index 00000000000..690f3b17f42
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummary.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/// Table-level compression statistics summary, aggregated across all servers for an offline or realtime table.
+/// Reported under the `compressionStats` key in table-size and aggregate-metadata responses.
+///
+/// Sizes are per replica; the controller selects one coherent contribution for each logical segment, so totals reflect
+/// one copy of the data rather than the physical replication factor.
+///
+/// `partialCoverage` is true when one or more segments lack stats (e.g. built before
+/// `compressionStatsEnabled` was set), meaning the ratio is computed from a subset of segments only.
+/// Instances are immutable and thread-safe.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class CompressionStatsSummary {
+ /// Sum of selected per-segment uncompressed column-value sizes.
+ private final long _uncompressedValueSizePerReplicaInBytes;
+
+ /// Sum of selected per-segment forward-index and dictionary sizes on disk.
+ private final long _forwardIndexAndDictionaryStorageSizePerReplicaInBytes;
+
+ /// Overall ratio of uncompressed value bytes to forward-index and dictionary storage bytes.
+ private final double _compressionRatio;
+
+ /// Number of segments that have compression stats (built with `compressionStatsEnabled=true`).
+ private final int _segmentsWithCompleteStats;
+
+ /// Total number of segments in the sub-table.
+ private final int _totalSegments;
+
+ /// True when `segmentsWithCompleteStats < totalSegments`, meaning the ratio covers only a subset of segments.
+ private final boolean _partialCoverage;
+
+ @JsonCreator
+ public CompressionStatsSummary(
+ @JsonProperty("uncompressedValueSizePerReplicaInBytes") long uncompressedValueSizePerReplicaInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizePerReplicaInBytes")
+ long forwardIndexAndDictionaryStorageSizePerReplicaInBytes,
+ @JsonProperty("compressionRatio") double compressionRatio,
+ @JsonProperty("segmentsWithCompleteStats") int segmentsWithCompleteStats,
+ @JsonProperty("totalSegments") int totalSegments,
+ @JsonProperty("partialCoverage") boolean partialCoverage) {
+ _uncompressedValueSizePerReplicaInBytes = uncompressedValueSizePerReplicaInBytes;
+ _forwardIndexAndDictionaryStorageSizePerReplicaInBytes = forwardIndexAndDictionaryStorageSizePerReplicaInBytes;
+ _compressionRatio = compressionRatio;
+ _segmentsWithCompleteStats = segmentsWithCompleteStats;
+ _totalSegments = totalSegments;
+ _partialCoverage = partialCoverage;
+ }
+
+ /// Returns uncompressed serialized column-value bytes for one logical copy of the covered segments.
+ public long getUncompressedValueSizePerReplicaInBytes() {
+ return _uncompressedValueSizePerReplicaInBytes;
+ }
+
+ /// Returns forward-index and dictionary bytes for one logical copy of the covered segments.
+ public long getForwardIndexAndDictionaryStorageSizePerReplicaInBytes() {
+ return _forwardIndexAndDictionaryStorageSizePerReplicaInBytes;
+ }
+
+ /// Returns `uncompressedValueSizePerReplicaInBytes / forwardIndexAndDictionaryStorageSizePerReplicaInBytes`, or `0`
+ /// when no bytes are covered.
+ public double getCompressionRatio() {
+ return _compressionRatio;
+ }
+
+ /// Returns the number of logical segments with complete statistics.
+ public int getSegmentsWithCompleteStats() {
+ return _segmentsWithCompleteStats;
+ }
+
+ /// Returns the total logical segment count used as the coverage denominator.
+ public int getTotalSegments() {
+ return _totalSegments;
+ }
+
+ /// Returns whether at least one logical segment is not covered.
+ @JsonProperty("partialCoverage")
+ public boolean isPartialCoverage() {
+ return _partialCoverage;
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentCompressionStatsContribution.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentCompressionStatsContribution.java
new file mode 100644
index 00000000000..83fa8bf8d36
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentCompressionStatsContribution.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+/// Compression-statistics contribution for one immutable segment on one server.
+///
+/// A contribution is complete only when every column with a forward index has usable metadata. Controllers select
+/// one complete replica per segment before aggregating table-level totals. Instances are immutable and thread-safe.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class SegmentCompressionStatsContribution {
+ private final String _segmentName;
+ private final boolean _complete;
+ private final long _uncompressedValueSizeInBytes;
+ private final long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private final Map<String, ColumnCompressionStatsContribution> _columnCompressionStats;
+
+ @JsonCreator
+ public SegmentCompressionStatsContribution(
+ @JsonProperty("segmentName") String segmentName,
+ @JsonProperty("complete") boolean complete,
+ @JsonProperty("uncompressedValueSizeInBytes") long uncompressedValueSizeInBytes,
+ @JsonProperty("forwardIndexAndDictionaryStorageSizeInBytes") long forwardIndexAndDictionaryStorageSizeInBytes,
+ @JsonProperty("columnCompressionStats") @Nullable
+ Map<String, ColumnCompressionStatsContribution> columnCompressionStats) {
+ _segmentName = segmentName;
+ _complete = complete;
+ _uncompressedValueSizeInBytes = uncompressedValueSizeInBytes;
+ _forwardIndexAndDictionaryStorageSizeInBytes = forwardIndexAndDictionaryStorageSizeInBytes;
+ _columnCompressionStats = columnCompressionStats != null ? Map.copyOf(columnCompressionStats) : null;
+ }
+
+ /// Returns the logical segment name.
+ public String getSegmentName() {
+ return _segmentName;
+ }
+
+ /// Returns whether every eligible forward-index column has usable statistics.
+ public boolean isComplete() {
+ return _complete;
+ }
+
+ /// Returns segment-level uncompressed serialized column-value bytes, or `-1` when this contribution is incomplete.
+ public long getUncompressedValueSizeInBytes() {
+ return _uncompressedValueSizeInBytes;
+ }
+
+ /// Returns segment-level forward-index and dictionary bytes, or `-1` when this contribution is incomplete.
+ public long getForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _forwardIndexAndDictionaryStorageSizeInBytes;
+ }
+
+ /// Returns independently available per-column contributions, or null when details were not requested.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public Map<String, ColumnCompressionStatsContribution> getColumnCompressionStats() {
+ return _columnCompressionStats;
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfo.java
index 6a9fdf59e0c..0fe01c65dea 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfo.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfo.java
@@ -19,20 +19,60 @@
package org.apache.pinot.common.restlet.resources;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+import javax.annotation.Nullable;
@JsonIgnoreProperties(ignoreUnknown = true)
public class SegmentSizeInfo {
private final String _segmentName;
private final long _diskSizeInBytes;
+ /// Segment-level uncompressed serialized column-value size covered by compression statistics.
+ @JsonProperty("compressionStatsUncompressedValueSizeInBytes")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private final Long _compressionStatsUncompressedValueSizeInBytes;
+ /// Segment-level forward-index and dictionary size covered by compression statistics.
+ @JsonProperty("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private final Long _compressionStatsForwardIndexAndDictionaryStorageSizeInBytes;
+ private final Map<String, ColumnCompressionStatsInfo> _columnCompressionStats;
+
+ public SegmentSizeInfo(String segmentName, long sizeBytes) {
+ this(segmentName, sizeBytes, (Long) null, null, null);
+ }
+
+ public SegmentSizeInfo(String segmentName, long sizeBytes, long compressionStatsUncompressedValueSizeInBytes,
+ long compressionStatsForwardIndexAndDictionaryStorageSizeInBytes) {
+ this(segmentName, sizeBytes, compressionStatsUncompressedValueSizeInBytes,
+ compressionStatsForwardIndexAndDictionaryStorageSizeInBytes, null);
+ }
+
+ public SegmentSizeInfo(String segmentName, long sizeBytes, long compressionStatsUncompressedValueSizeInBytes,
+ long compressionStatsForwardIndexAndDictionaryStorageSizeInBytes,
+ @Nullable Map<String, ColumnCompressionStatsInfo> columnCompressionStats) {
+ this(segmentName, sizeBytes, availableValue(compressionStatsUncompressedValueSizeInBytes),
+ availableValue(compressionStatsForwardIndexAndDictionaryStorageSizeInBytes), columnCompressionStats);
+ }
@JsonCreator
public SegmentSizeInfo(@JsonProperty("segmentName") String segmentName,
- @JsonProperty("diskSizeInBytes") long sizeBytes) {
+ @JsonProperty("diskSizeInBytes") long sizeBytes,
+ @JsonProperty("compressionStatsUncompressedValueSizeInBytes") @Nullable
+ Long compressionStatsUncompressedValueSizeInBytes,
+ @JsonProperty("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes") @Nullable
+ Long compressionStatsForwardIndexAndDictionaryStorageSizeInBytes,
+ @JsonProperty("columnCompressionStats") @Nullable Map<String, ColumnCompressionStatsInfo>
+ columnCompressionStats) {
_segmentName = segmentName;
_diskSizeInBytes = sizeBytes;
+ _compressionStatsUncompressedValueSizeInBytes = compressionStatsUncompressedValueSizeInBytes;
+ _compressionStatsForwardIndexAndDictionaryStorageSizeInBytes =
+ compressionStatsForwardIndexAndDictionaryStorageSizeInBytes;
+ _columnCompressionStats = columnCompressionStats != null ? Map.copyOf(columnCompressionStats) : null;
}
public String getSegmentName() {
@@ -43,6 +83,30 @@ public class SegmentSizeInfo {
return _diskSizeInBytes;
}
+ /// Returns segment-level uncompressed serialized column-value bytes, or `-1` when unavailable.
+ @JsonIgnore
+ public long getCompressionStatsUncompressedValueSizeInBytes() {
+ return _compressionStatsUncompressedValueSizeInBytes != null ? _compressionStatsUncompressedValueSizeInBytes : -1;
+ }
+
+ /// Returns segment-level forward-index and dictionary bytes, or `-1` when unavailable.
+ @JsonIgnore
+ public long getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes() {
+ return _compressionStatsForwardIndexAndDictionaryStorageSizeInBytes != null
+ ? _compressionStatsForwardIndexAndDictionaryStorageSizeInBytes : -1;
+ }
+
+ /// Returns public per-column compression statistics, or null when details were not requested.
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public Map<String, ColumnCompressionStatsInfo> getColumnCompressionStats() {
+ return _columnCompressionStats;
+ }
+
+ private static Long availableValue(long value) {
+ return value >= 0 ? value : null;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerCompressionStatsResponse.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerCompressionStatsResponse.java
new file mode 100644
index 00000000000..4cd3281f15f
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerCompressionStatsResponse.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import javax.annotation.Nullable;
+
+
+/// Immutable, thread-safe response for one batch of logical segment replicas on a server.
+///
+/// Per-segment contributions are retained so the controller can de-duplicate replicas with the same policy used by
+/// the table-size API and fall back to another replica during rolling upgrades.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ServerCompressionStatsResponse {
+ public static final int CURRENT_METADATA_VERSION = 1;
+ public static final int MAX_COLUMN_CONTRIBUTIONS_PER_RESPONSE = 10_000;
+
+ private final int _metadataVersion;
+ private final List<SegmentCompressionStatsContribution> _segmentCompressionStats;
+
+ /// Creates a response containing one contribution for each requested segment replica.
+ @JsonCreator
+ public ServerCompressionStatsResponse(
+ @JsonProperty("metadataVersion") @Nullable Integer metadataVersion,
+ @JsonProperty("segmentCompressionStats") @Nullable
+ List<SegmentCompressionStatsContribution> segmentCompressionStats) {
+ _metadataVersion = metadataVersion != null ? metadataVersion : 0;
+ _segmentCompressionStats = segmentCompressionStats != null ? List.copyOf(segmentCompressionStats) : List.of();
+ }
+
+ /// Returns the response protocol version.
+ public int getMetadataVersion() {
+ return _metadataVersion;
+ }
+
+ /// Returns the per-segment replica contributions in this response batch.
+ public List<SegmentCompressionStatsContribution> getSegmentCompressionStats() {
+ return _segmentCompressionStats;
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerTableMetadataInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerTableMetadataInfo.java
new file mode 100644
index 00000000000..f7a44558ed3
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ServerTableMetadataInfo.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Maps;
+import java.util.Collections;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+/// Server-to-controller transport for table metadata contributions.
+///
+/// Values in this server-to-controller transport DTO are server-local totals. Construction takes ownership of supplied
+/// maps and exposes unmodifiable views without copying request-local data. Instances are thread-safe after construction
+/// provided callers honor that ownership transfer.
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ServerTableMetadataInfo {
+ private final String _tableName;
+ private final long _diskSizeInBytes;
+ private final long _numSegments;
+ private final long _numRows;
+ private final Map<String, Double> _columnLengthMap;
+ private final Map<String, Double> _columnCardinalityMap;
+ private final Map<String, Double> _maxNumMultiValuesMap;
+ private final Map<String, Map<String, Double>> _columnIndexSizeMap;
+ private final Map<Integer, Map<String, Long>> _partitionToServerPrimaryKeyCountMap;
+
+ @JsonCreator
+ private ServerTableMetadataInfo(@JsonProperty("tableName") String tableName,
+ @JsonProperty("diskSizeInBytes") long sizeInBytes, @JsonProperty("numSegments") long numSegments,
+ @JsonProperty("numRows") long numRows,
+ @JsonProperty("columnLengthMap") @Nullable Map<String, Double> columnLengthMap,
+ @JsonProperty("columnCardinalityMap") @Nullable Map<String, Double> columnCardinalityMap,
+ @JsonProperty("maxNumMultiValuesMap") @Nullable Map<String, Double> maxNumMultiValuesMap,
+ @JsonProperty("columnIndexSizeMap") @Nullable Map<String, Map<String, Double>> columnIndexSizeMap,
+ @JsonProperty("upsertPartitionToServerPrimaryKeyCountMap")
+ @Nullable Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap) {
+ _tableName = tableName;
+ _diskSizeInBytes = sizeInBytes;
+ _numSegments = numSegments;
+ _numRows = numRows;
+ _columnLengthMap = unmodifiableView(columnLengthMap);
+ _columnCardinalityMap = unmodifiableView(columnCardinalityMap);
+ _maxNumMultiValuesMap = unmodifiableView(maxNumMultiValuesMap);
+ _columnIndexSizeMap = nestedUnmodifiableView(columnIndexSizeMap);
+ _partitionToServerPrimaryKeyCountMap = nestedUnmodifiableView(partitionToServerPrimaryKeyCountMap);
+ }
+
+ public static Builder builder(String tableName) {
+ return new Builder(tableName);
+ }
+
+ /// Returns the typed table name.
+ public String getTableName() {
+ return _tableName;
+ }
+
+ /// Returns server-local segment bytes.
+ public long getDiskSizeInBytes() {
+ return _diskSizeInBytes;
+ }
+
+ /// Returns the number of server-local segments represented.
+ public long getNumSegments() {
+ return _numSegments;
+ }
+
+ /// Returns the number of rows across server-local segments.
+ public long getNumRows() {
+ return _numRows;
+ }
+
+ /// Returns summed per-column maximum lengths.
+ public Map<String, Double> getColumnLengthMap() {
+ return _columnLengthMap;
+ }
+
+ /// Returns summed per-column cardinalities.
+ public Map<String, Double> getColumnCardinalityMap() {
+ return _columnCardinalityMap;
+ }
+
+ /// Returns summed per-column maximum multi-value counts.
+ public Map<String, Double> getMaxNumMultiValuesMap() {
+ return _maxNumMultiValuesMap;
+ }
+
+ /// Returns summed per-column index sizes.
+ public Map<String, Map<String, Double>> getColumnIndexSizeMap() {
+ return _columnIndexSizeMap;
+ }
+
+ /// Returns primary-key counts by partition and server.
+ @JsonProperty("upsertPartitionToServerPrimaryKeyCountMap")
+ public Map<Integer, Map<String, Long>> getPartitionToServerPrimaryKeyCountMap() {
+ return _partitionToServerPrimaryKeyCountMap;
+ }
+
+ private static <K, V> Map<K, V> unmodifiableView(@Nullable Map<K, V> source) {
+ if (source == null || source.isEmpty()) {
+ return Map.of();
+ }
+ return Collections.unmodifiableMap(source);
+ }
+
+ private static <K, N, V> Map<K, Map<N, V>> nestedUnmodifiableView(
+ @Nullable Map<K, Map<N, V>> source) {
+ if (source == null || source.isEmpty()) {
+ return Map.of();
+ }
+ return Collections.unmodifiableMap(
+ Maps.transformValues(source, value -> Collections.unmodifiableMap(value)));
+ }
+
+ /// Mutable, non-thread-safe builder for [ServerTableMetadataInfo].
+ public static final class Builder {
+ private final String _tableName;
+ private long _diskSizeInBytes;
+ private long _numSegments;
+ private long _numRows;
+ private Map<String, Double> _columnLengthMap = Map.of();
+ private Map<String, Double> _columnCardinalityMap = Map.of();
+ private Map<String, Double> _maxNumMultiValuesMap = Map.of();
+ private Map<String, Map<String, Double>> _columnIndexSizeMap = Map.of();
+ private Map<Integer, Map<String, Long>> _partitionToServerPrimaryKeyCountMap = Map.of();
+
+ private Builder(String tableName) {
+ _tableName = tableName;
+ }
+
+ public Builder withDiskSizeInBytes(long diskSizeInBytes) {
+ _diskSizeInBytes = diskSizeInBytes;
+ return this;
+ }
+
+ public Builder withNumSegments(long numSegments) {
+ _numSegments = numSegments;
+ return this;
+ }
+
+ public Builder withNumRows(long numRows) {
+ _numRows = numRows;
+ return this;
+ }
+
+ public Builder withColumnLengthMap(Map<String, Double> columnLengthMap) {
+ _columnLengthMap = columnLengthMap;
+ return this;
+ }
+
+ public Builder withColumnCardinalityMap(Map<String, Double> columnCardinalityMap) {
+ _columnCardinalityMap = columnCardinalityMap;
+ return this;
+ }
+
+ public Builder withMaxNumMultiValuesMap(Map<String, Double> maxNumMultiValuesMap) {
+ _maxNumMultiValuesMap = maxNumMultiValuesMap;
+ return this;
+ }
+
+ public Builder withColumnIndexSizeMap(Map<String, Map<String, Double>> columnIndexSizeMap) {
+ _columnIndexSizeMap = columnIndexSizeMap;
+ return this;
+ }
+
+ public Builder withPartitionToServerPrimaryKeyCountMap(
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap) {
+ _partitionToServerPrimaryKeyCountMap = partitionToServerPrimaryKeyCountMap;
+ return this;
+ }
+
+ public ServerTableMetadataInfo build() {
+ return new ServerTableMetadataInfo(_tableName, _diskSizeInBytes, _numSegments, _numRows, _columnLengthMap,
+ _columnCardinalityMap, _maxNumMultiValuesMap, _columnIndexSizeMap,
+ _partitionToServerPrimaryKeyCountMap);
+ }
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java
index 21468d7d426..7acabd313a2 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java
@@ -20,19 +20,17 @@ package org.apache.pinot.common.restlet.resources;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
-/**
- * Class to represent aggregated segment metadata for a table.
- * - When return by server API, columnLengthMap/columnCardinalityMap is the total columnLength/columnCardinality across
- * all segments on that server.
- * - When return by controller API, columnLengthMap/columnCardinalityMap is the average columnLength/columnCardinality
- * across all segments for the given table.
- * - For diskSizeInBytes/numSegments/numRows, both server API and controller API will return the total value across all
- * segments (if a segment has multiple replicas, only consider one replica).
- */
+/// Aggregated controller response for table segment metadata.
+///
+/// Column length and cardinality values are averages across logical segments. Disk size, segment count, and row count
+/// are table totals with replicas de-duplicated.
@JsonIgnoreProperties(ignoreUnknown = true)
public class TableMetadataInfo {
private final String _tableName;
@@ -46,6 +44,8 @@ public class TableMetadataInfo {
// JSON property name kept as "upsertPartitionToServerPrimaryKeyCountMap" to avoid silent data loss during rolling
// upgrades where servers and controllers may temporarily run different versions of this class.
private final Map<Integer, Map<String, Long>> _partitionToServerPrimaryKeyCountMap;
+ private final List<ColumnCompressionStatsInfo> _columnCompressionStats;
+ private final CompressionStatsSummary _compressionStats;
@JsonCreator
public TableMetadataInfo(@JsonProperty("tableName") String tableName,
@@ -55,7 +55,10 @@ public class TableMetadataInfo {
@JsonProperty("maxNumMultiValuesMap") Map<String, Double> maxNumMultiValuesMap,
@JsonProperty("columnIndexSizeMap") Map<String, Map<String, Double>> columnIndexSizeMap,
@JsonProperty("upsertPartitionToServerPrimaryKeyCountMap")
- Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap) {
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap,
+ @JsonProperty("columnCompressionStats") @Nullable
+ List<ColumnCompressionStatsInfo> columnCompressionStats,
+ @JsonProperty("compressionStats") @Nullable CompressionStatsSummary compressionStats) {
_tableName = tableName;
_diskSizeInBytes = sizeInBytes;
_numSegments = numSegments;
@@ -65,6 +68,27 @@ public class TableMetadataInfo {
_maxNumMultiValuesMap = maxNumMultiValuesMap;
_columnIndexSizeMap = columnIndexSizeMap;
_partitionToServerPrimaryKeyCountMap = partitionToServerPrimaryKeyCountMap;
+ _columnCompressionStats = columnCompressionStats != null ? List.copyOf(columnCompressionStats) : null;
+ _compressionStats = compressionStats;
+ }
+
+ /// Constructor for callers that provide column compression statistics but not the aggregate summary.
+ public TableMetadataInfo(String tableName, long sizeInBytes, long numSegments, long numRows,
+ Map<String, Double> columnLengthMap, Map<String, Double> columnCardinalityMap,
+ Map<String, Double> maxNumMultiValuesMap, Map<String, Map<String, Double>> columnIndexSizeMap,
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap,
+ @Nullable List<ColumnCompressionStatsInfo> columnCompressionStats) {
+ this(tableName, sizeInBytes, numSegments, numRows, columnLengthMap, columnCardinalityMap, maxNumMultiValuesMap,
+ columnIndexSizeMap, partitionToServerPrimaryKeyCountMap, columnCompressionStats, null);
+ }
+
+ /// Backwards-compatible constructor for callers that do not provide compression fields.
+ public TableMetadataInfo(String tableName, long sizeInBytes, long numSegments, long numRows,
+ Map<String, Double> columnLengthMap, Map<String, Double> columnCardinalityMap,
+ Map<String, Double> maxNumMultiValuesMap, Map<String, Map<String, Double>> columnIndexSizeMap,
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap) {
+ this(tableName, sizeInBytes, numSegments, numRows, columnLengthMap, columnCardinalityMap, maxNumMultiValuesMap,
+ columnIndexSizeMap, partitionToServerPrimaryKeyCountMap, null, null);
}
public String getTableName() {
@@ -103,4 +127,16 @@ public class TableMetadataInfo {
public Map<Integer, Map<String, Long>> getPartitionToServerPrimaryKeyCountMap() {
return _partitionToServerPrimaryKeyCountMap;
}
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public List<ColumnCompressionStatsInfo> getColumnCompressionStats() {
+ return _columnCompressionStats;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public CompressionStatsSummary getCompressionStats() {
+ return _compressionStats;
+ }
}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableSizeInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableSizeInfo.java
index b6621d9dbf2..4f6417a1cfa 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableSizeInfo.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableSizeInfo.java
@@ -20,22 +20,34 @@ package org.apache.pinot.common.restlet.resources;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
+import javax.annotation.Nullable;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TableSizeInfo {
+ /// Version of the server size contribution protocol emitted by this build.
+ public static final int CURRENT_METADATA_VERSION = 1;
+
private final String _tableName;
private final long _diskSizeInBytes;
private final List<SegmentSizeInfo> _segments;
+ private final int _metadataVersion;
+
+ public TableSizeInfo(String tableName, long sizeInBytes, List<SegmentSizeInfo> segments) {
+ this(tableName, sizeInBytes, segments, 0);
+ }
@JsonCreator
public TableSizeInfo(@JsonProperty("tableName") String tableName, @JsonProperty("diskSizeInBytes") long sizeInBytes,
- @JsonProperty("segments") List<SegmentSizeInfo> segments) {
+ @JsonProperty("segments") List<SegmentSizeInfo> segments,
+ @JsonProperty("metadataVersion") @Nullable Integer metadataVersion) {
_tableName = tableName;
_diskSizeInBytes = sizeInBytes;
_segments = segments;
+ _metadataVersion = metadataVersion != null ? metadataVersion : 0;
}
public String getTableName() {
@@ -49,4 +61,10 @@ public class TableSizeInfo {
public List<SegmentSizeInfo> getSegments() {
return _segments;
}
+
+ /// Returns the server size contribution protocol version, or `0` for a legacy response.
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public int getMetadataVersion() {
+ return _metadataVersion;
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulatorTest.java b/pinot-common/src/test/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulatorTest.java
new file mode 100644
index 00000000000..5b2ec467906
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/compression/ColumnCompressionStatsAccumulatorTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.compression;
+
+import java.util.List;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies column compression contribution aggregation and public DTO conversion.
+public class ColumnCompressionStatsAccumulatorTest {
+ @Test
+ public void testMixedEncodingAggregation() {
+ ColumnCompressionStatsAccumulator accumulator = new ColumnCompressionStatsAccumulator();
+ accumulator.add(9_000, 2_000, EncodingType.RAW, ChunkCompressionType.LZ4, List.of("forward_index"), 3);
+ accumulator.add(1_000, 1_500, EncodingType.DICTIONARY, null,
+ List.of("dictionary", "forward_index"), 2);
+
+ ColumnCompressionStatsInfo info = accumulator.toColumnCompressionStatsInfo("mixed");
+ assertEquals(info.getUncompressedValueSizeInBytes(), 10_000L);
+ assertEquals(info.getForwardIndexAndDictionaryStorageSizeInBytes(), 3_500L);
+ assertEquals(info.getNumSegments(), 5);
+ assertEquals(info.getEncodingBreakdown().size(), 2);
+ assertTrue(info.getObservedIndexes().contains("dictionary"));
+ }
+
+ @Test
+ public void testUnavailableContributionIsIgnoredButZeroIsRetained() {
+ ColumnCompressionStatsAccumulator accumulator = new ColumnCompressionStatsAccumulator();
+ accumulator.add(-1, 100, EncodingType.RAW, ChunkCompressionType.LZ4, null, 1);
+ assertEquals(accumulator.toColumnCompressionStatsInfo("empty").getNumSegments(), 0);
+
+ accumulator.add(0, 0, EncodingType.RAW, ChunkCompressionType.PASS_THROUGH, null, 1);
+ assertEquals(accumulator.toColumnCompressionStatsInfo("empty").getNumSegments(), 1);
+ }
+}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/http/MultiHttpRequestTest.java b/pinot-common/src/test/java/org/apache/pinot/common/http/MultiHttpRequestTest.java
index 840cb090025..5615ee4bac8 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/http/MultiHttpRequestTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/http/MultiHttpRequestTest.java
@@ -31,10 +31,19 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -204,6 +213,93 @@ public class MultiHttpRequestTest {
Assert.assertEquals(result.getTimeouts(), 1);
}
+ @Test
+ public void testBufferedExecutionSubmitsAndPollsRequests()
+ throws Exception {
+ String url = "http://localhost:" + (_portStart + 3) + URI_PATH;
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ try (MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(executor, connectionManager).createBufferedExecution(TIMEOUT_MS)) {
+ for (String body : List.of("b0", "b1", "b2")) {
+ HttpPost request = new HttpPost(url);
+ request.setEntity(new StringEntity(body));
+ execution.submit(request);
+ }
+ for (int i = 0; i < 3; i++) {
+ Future<MultiHttpRequestBufferedResponse> completed = execution.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ Assert.assertNotNull(completed);
+ Assert.assertEquals(completed.get().getResponseBody(), SUCCESS_MSG);
+ }
+ } finally {
+ executor.shutdownNow();
+ connectionManager.close();
+ }
+ }
+
+ @Test
+ public void testBufferedExecutionRejectsOperationsAfterClose()
+ throws Exception {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ try {
+ MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(executor, connectionManager).createBufferedExecution(TIMEOUT_MS);
+ execution.close();
+ Assert.expectThrows(IllegalStateException.class,
+ () -> execution.poll(1, TimeUnit.MILLISECONDS));
+ Assert.expectThrows(IllegalStateException.class, execution::take);
+ Assert.expectThrows(IllegalStateException.class,
+ () -> execution.submit(new HttpPost("http://localhost:" + (_portStart + 3) + URI_PATH)));
+ } finally {
+ executor.shutdownNow();
+ connectionManager.close();
+ }
+ }
+
+ @Test
+ public void testBufferedExecutionCancelsAcceptedRequestAfterSubmissionFailure()
+ throws Exception {
+ AtomicReference<Runnable> acceptedTask = new AtomicReference<>();
+ AtomicInteger handledRequests = new AtomicInteger();
+ String rollbackPath = "/rollback";
+ HttpServer server = _servers.get(3);
+ server.createContext(rollbackPath, exchange -> {
+ handledRequests.incrementAndGet();
+ exchange.sendResponseHeaders(SUCCESS_CODE, 0);
+ exchange.close();
+ });
+ Executor rejectingExecutor = task -> {
+ if (!acceptedTask.compareAndSet(null, task)) {
+ throw new RejectedExecutionException("reject second request");
+ }
+ };
+ PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ String url = "http://localhost:" + (_portStart + 3) + rollbackPath;
+ try {
+ MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(rejectingExecutor, connectionManager).createBufferedExecution(TIMEOUT_MS);
+ try {
+ HttpPost first = new HttpPost(url);
+ first.setEntity(new StringEntity("first"));
+ execution.submit(first);
+ HttpPost second = new HttpPost(url);
+ second.setEntity(new StringEntity("second"));
+ Assert.expectThrows(RejectedExecutionException.class, () -> execution.submit(second));
+ } finally {
+ execution.close();
+ }
+
+ // Running the accepted task after close must be a no-op because its Future was cancelled.
+ Assert.assertNotNull(acceptedTask.get());
+ acceptedTask.get().run();
+ Assert.assertEquals(handledRequests.get(), 0);
+ } finally {
+ server.removeContext(rollbackPath);
+ connectionManager.close();
+ }
+ }
+
private TestResult collectResult(CompletionService<MultiHttpRequestResponse> completionService, int size) {
int success = 0;
int errors = 0;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerGaugeTest.java
similarity index 56%
copy from pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
copy to pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerGaugeTest.java
index 80739ca63d5..9736c0bbb5c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/metrics/ControllerGaugeTest.java
@@ -16,30 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.segment.local.io.writer.impl;
+package org.apache.pinot.common.metrics;
-import java.io.Closeable;
-import java.math.BigDecimal;
+import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
-public interface VarByteChunkWriter extends Closeable {
- void putBigDecimal(BigDecimal value);
- void putString(String value);
-
- void putBytes(byte[] value);
-
- void putIntMV(int[] values);
-
- void putLongMV(long[] values);
-
- void putFloatMV(float[] values);
-
- void putDoubleMV(double[] values);
-
- void putBigDecimalMV(BigDecimal[] values);
-
- void putStringMV(String[] values);
-
- void putBytesMV(byte[][] values);
+/// Verifies controller compression gauge names and metadata.
+public class ControllerGaugeTest {
+ @Test
+ public void testCompressionGaugeUnits() {
+ assertEquals(ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT.getUnit(), "percent");
+ assertEquals(ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA.getUnit(), "bytes");
+ assertEquals(
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA.getUnit(),
+ "bytes");
+ }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfoTest.java b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfoTest.java
new file mode 100644
index 00000000000..1e1d077c182
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ColumnCompressionStatsInfoTest.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.List;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies JSON compatibility and immutability for [ColumnCompressionStatsInfo].
+public class ColumnCompressionStatsInfoTest {
+ @Test
+ public void testRawEncodingJsonRoundTrip()
+ throws Exception {
+ ColumnCompressionStatsInfo original = info("col1", 10_000, 2_500, 4.0,
+ List.of("forward_index", "range_index"),
+ List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, ChunkCompressionType.ZSTANDARD, 3, 10_000, 2_500)), 3);
+
+ String json = JsonUtils.objectToString(original);
+ ColumnCompressionStatsInfo deserialized = JsonUtils.stringToObject(json, ColumnCompressionStatsInfo.class);
+
+ assertEquals(deserialized.getColumn(), "col1");
+ assertEquals(deserialized.getUncompressedValueSizeInBytes(), 10_000L);
+ assertEquals(deserialized.getForwardIndexAndDictionaryStorageSizeInBytes(), 2_500L);
+ assertEquals(deserialized.getCompressionRatio(), 4.0, 1e-9);
+ assertEquals(deserialized.getObservedIndexes(), List.of("forward_index", "range_index"));
+ assertEquals(deserialized.getNumSegments(), 3);
+ assertEquals(deserialized.getEncodingBreakdown().size(), 1);
+ ColumnCompressionStatsInfo.EncodingBreakdownEntry entry = deserialized.getEncodingBreakdown().get(0);
+ assertEquals(entry.getEncoding(), EncodingType.RAW);
+ assertEquals(entry.getChunkCompressionType(), ChunkCompressionType.ZSTANDARD);
+ assertEquals(entry.getNumSegments(), 3);
+ }
+
+ @Test
+ public void testDictionaryEncodingHasNoChunkCompressionType()
+ throws Exception {
+ ColumnCompressionStatsInfo original = info("dictCol", 4_000, 1_000, 4.0,
+ List.of("dictionary", "forward_index"),
+ List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.DICTIONARY, null, 2, 4_000, 1_000)), 2);
+
+ JsonNode json = JsonUtils.stringToJsonNode(JsonUtils.objectToString(original));
+ JsonNode entry = json.get("encodingBreakdown").get(0);
+ assertEquals(entry.get("encoding").asText(), "DICTIONARY");
+ assertFalse(entry.has("chunkCompressionType"));
+
+ ColumnCompressionStatsInfo deserialized =
+ JsonUtils.stringToObject(json.toString(), ColumnCompressionStatsInfo.class);
+ assertNull(deserialized.getEncodingBreakdown().get(0).getChunkCompressionType());
+ }
+
+ @Test
+ public void testMixedEncodingRoundTrip()
+ throws Exception {
+ List<ColumnCompressionStatsInfo.EncodingBreakdownEntry> breakdown = List.of(
+ new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, ChunkCompressionType.LZ4, 3, 9_000, 2_000),
+ new ColumnCompressionStatsInfo.EncodingBreakdownEntry(EncodingType.DICTIONARY, null, 2, 1_000, 1_500));
+ ColumnCompressionStatsInfo original = info(
+ "mixedCol", 10_000, 3_500, 10_000d / 3_500, List.of("forward_index", "dictionary"), breakdown, 5);
+
+ ColumnCompressionStatsInfo deserialized =
+ JsonUtils.stringToObject(JsonUtils.objectToString(original), ColumnCompressionStatsInfo.class);
+
+ assertEquals(deserialized.getEncodingBreakdown().size(), 2);
+ assertEquals(deserialized.getEncodingBreakdown().get(0).getEncoding(), EncodingType.RAW);
+ assertEquals(deserialized.getEncodingBreakdown().get(1).getEncoding(), EncodingType.DICTIONARY);
+ assertEquals(deserialized.getNumSegments(), 5);
+ }
+
+ @Test
+ public void testNullIndexesAreOmitted()
+ throws Exception {
+ ColumnCompressionStatsInfo info = info("col", 0, 0, 0, null,
+ List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, ChunkCompressionType.PASS_THROUGH, 1, 0, 0)),
+ 1);
+
+ JsonNode json = JsonUtils.stringToJsonNode(JsonUtils.objectToString(info));
+ assertFalse(json.has("observedIndexes"));
+ assertTrue(json.has("encodingBreakdown"));
+ }
+
+ @Test
+ public void testUnknownAndLegacyFieldsAreIgnored()
+ throws Exception {
+ String json = "{\"column\":\"futureCol\",\"uncompressedValueSizeInBytes\":6000,"
+ + "\"forwardIndexAndDictionaryStorageSizeInBytes\":1200,\"compressionRatio\":5.0,\"codec\":\"LZ4\","
+ + "\"hasDictionary\":false,\"unknownField\":\"ignored\"}";
+
+ ColumnCompressionStatsInfo deserialized = JsonUtils.stringToObject(json, ColumnCompressionStatsInfo.class);
+ assertEquals(deserialized.getColumn(), "futureCol");
+ assertEquals(deserialized.getUncompressedValueSizeInBytes(), 6_000L);
+ assertTrue(deserialized.getEncodingBreakdown().isEmpty());
+ }
+
+ private static ColumnCompressionStatsInfo info(String column, long uncompressedValueSizeInBytes,
+ long forwardIndexAndDictionaryStorageSizeInBytes, double compressionRatio, List<String> observedIndexes,
+ List<ColumnCompressionStatsInfo.EncodingBreakdownEntry> encodingBreakdown, int numSegments) {
+ return ColumnCompressionStatsInfo.builder(column)
+ .withUncompressedValueSizeInBytes(uncompressedValueSizeInBytes)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(forwardIndexAndDictionaryStorageSizeInBytes)
+ .withCompressionRatio(compressionRatio)
+ .withObservedIndexes(observedIndexes)
+ .withEncodingBreakdown(encodingBreakdown)
+ .withNumSegments(numSegments)
+ .build();
+ }
+}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummaryTest.java b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummaryTest.java
new file mode 100644
index 00000000000..6211585ba0e
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/CompressionStatsSummaryTest.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Verifies JSON compatibility for [CompressionStatsSummary].
+public class CompressionStatsSummaryTest {
+
+ @Test
+ public void testGetters() {
+ CompressionStatsSummary summary = new CompressionStatsSummary(100000, 40000, 2.5, 8, 10, true);
+ assertEquals(summary.getUncompressedValueSizePerReplicaInBytes(), 100000);
+ assertEquals(summary.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 40000);
+ assertEquals(summary.getCompressionRatio(), 2.5, 0.001);
+ assertEquals(summary.getSegmentsWithCompleteStats(), 8);
+ assertEquals(summary.getTotalSegments(), 10);
+ assertTrue(summary.isPartialCoverage());
+ }
+
+ @Test
+ public void testFullCoverage() {
+ CompressionStatsSummary summary = new CompressionStatsSummary(50000, 25000, 2.0, 5, 5, false);
+ assertEquals(summary.getSegmentsWithCompleteStats(), 5);
+ assertEquals(summary.getTotalSegments(), 5);
+ assertFalse(summary.isPartialCoverage());
+ }
+
+ @Test
+ public void testJsonRoundTrip()
+ throws Exception {
+ CompressionStatsSummary original = new CompressionStatsSummary(200000, 80000, 2.5, 3, 4, true);
+ String json = JsonUtils.objectToString(original);
+
+ assertTrue(json.contains("uncompressedValueSizePerReplicaInBytes"));
+ assertTrue(json.contains("forwardIndexAndDictionaryStorageSizePerReplicaInBytes"));
+ assertTrue(json.contains("compressionRatio"));
+ assertTrue(json.contains("segmentsWithCompleteStats"));
+ assertTrue(json.contains("totalSegments"));
+ assertTrue(json.contains("partialCoverage"));
+
+ CompressionStatsSummary deserialized = JsonUtils.stringToObject(json, CompressionStatsSummary.class);
+ assertEquals(deserialized.getUncompressedValueSizePerReplicaInBytes(), 200000);
+ assertEquals(deserialized.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 80000);
+ assertEquals(deserialized.getCompressionRatio(), 2.5, 0.001);
+ assertEquals(deserialized.getSegmentsWithCompleteStats(), 3);
+ assertEquals(deserialized.getTotalSegments(), 4);
+ assertTrue(deserialized.isPartialCoverage());
+ }
+
+ @Test
+ public void testJsonIgnoresUnknownFields()
+ throws Exception {
+ String json = "{\"uncompressedValueSizePerReplicaInBytes\":1000,"
+ + "\"forwardIndexAndDictionaryStorageSizePerReplicaInBytes\":500,"
+ + "\"compressionRatio\":2.0,\"segmentsWithCompleteStats\":1,\"totalSegments\":1,\"partialCoverage\":false,"
+ + "\"unknownFutureField\":\"ignored\"}";
+ CompressionStatsSummary summary = JsonUtils.stringToObject(json, CompressionStatsSummary.class);
+ assertEquals(summary.getUncompressedValueSizePerReplicaInBytes(), 1000);
+ assertFalse(summary.isPartialCoverage());
+ }
+}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfoTest.java b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfoTest.java
new file mode 100644
index 00000000000..1de765b872a
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/SegmentSizeInfoTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies legacy and compression-aware JSON forms of [SegmentSizeInfo].
+public class SegmentSizeInfoTest {
+ @Test
+ public void testJsonRoundTripWithCompressionStats()
+ throws Exception {
+ ColumnCompressionStatsInfo columnInfo = ColumnCompressionStatsInfo.builder("col1")
+ .withUncompressedValueSizeInBytes(10_000)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(2_500)
+ .withCompressionRatio(4.0)
+ .withObservedIndexes(List.of("forward_index"))
+ .withEncodingBreakdown(List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, ChunkCompressionType.LZ4, 1, 10_000, 2_500)))
+ .withNumSegments(1)
+ .build();
+ SegmentSizeInfo original =
+ new SegmentSizeInfo("seg1", 50_000, 10_000, 2_500, Map.of("col1", columnInfo));
+
+ String serialized = JsonUtils.objectToString(original);
+ JsonNode json = JsonUtils.stringToJsonNode(serialized);
+ assertTrue(json.has("compressionStatsUncompressedValueSizeInBytes"));
+ assertTrue(json.has("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes"));
+ assertFalse(json.has("uncompressedValueSizeBytes"));
+ assertFalse(json.has("forwardIndexStorageSizeBytes"));
+ assertEquals(json.get("columnCompressionStats").get("col1").get("compressionRatio").asDouble(), 4.0);
+
+ SegmentSizeInfo deserialized = JsonUtils.stringToObject(serialized, SegmentSizeInfo.class);
+ assertEquals(deserialized.getCompressionStatsUncompressedValueSizeInBytes(), 10_000L);
+ assertEquals(deserialized.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), 2_500L);
+ assertNotNull(deserialized.getColumnCompressionStats());
+ }
+
+ @Test
+ public void testUnreleasedAliasesAreIgnored()
+ throws Exception {
+ String json = "{\"segmentName\":\"seg1\",\"diskSizeInBytes\":50000,"
+ + "\"uncompressedValueSizeBytes\":1200,\"forwardIndexStorageSizeBytes\":300}";
+ SegmentSizeInfo deserialized = JsonUtils.stringToObject(json, SegmentSizeInfo.class);
+ assertEquals(deserialized.getCompressionStatsUncompressedValueSizeInBytes(), -1L);
+ assertEquals(deserialized.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), -1L);
+ }
+
+ @Test
+ public void testLegacyTwoArgConstructorEmitsOnlyLegacyFields()
+ throws Exception {
+ SegmentSizeInfo info = new SegmentSizeInfo("seg1", 1_000);
+ JsonNode json = JsonUtils.stringToJsonNode(JsonUtils.objectToString(info));
+
+ assertEquals(json.size(), 2);
+ assertEquals(json.get("segmentName").asText(), "seg1");
+ assertEquals(json.get("diskSizeInBytes").asLong(), 1_000L);
+ assertEquals(info.getCompressionStatsUncompressedValueSizeInBytes(), -1L);
+ assertEquals(info.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), -1L);
+ assertNull(info.getColumnCompressionStats());
+ }
+
+ @Test
+ public void testZeroSizesRemainAvailable()
+ throws Exception {
+ SegmentSizeInfo original = new SegmentSizeInfo("empty", 0, 0, 0);
+ String serialized = JsonUtils.objectToString(original);
+ SegmentSizeInfo deserialized = JsonUtils.stringToObject(serialized, SegmentSizeInfo.class);
+
+ assertEquals(deserialized.getCompressionStatsUncompressedValueSizeInBytes(), 0L);
+ assertEquals(deserialized.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), 0L);
+ }
+}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ServerTransportCompatibilityTest.java b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ServerTransportCompatibilityTest.java
new file mode 100644
index 00000000000..c86a173e738
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/ServerTransportCompatibilityTest.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+
+/// Guards the server-first half of rolling upgrades with DTOs frozen to the pre-compression response shapes.
+public class ServerTransportCompatibilityTest {
+ @Test
+ public void testCurrentTableSizePayloadDeserializesWithLegacyDto()
+ throws Exception {
+ TableSizeInfo current = new TableSizeInfo("myTable_OFFLINE", 120L,
+ List.of(new SegmentSizeInfo("segment0", 120L, 1_000L, 100L, Map.of())),
+ TableSizeInfo.CURRENT_METADATA_VERSION);
+
+ LegacyTableSizeInfo legacy =
+ JsonUtils.stringToObject(JsonUtils.objectToString(current), LegacyTableSizeInfo.class);
+
+ assertEquals(legacy._tableName, "myTable_OFFLINE");
+ assertEquals(legacy._diskSizeInBytes, 120L);
+ assertEquals(legacy._segments.size(), 1);
+ assertEquals(legacy._segments.get(0)._segmentName, "segment0");
+ assertEquals(legacy._segments.get(0)._diskSizeInBytes, 120L);
+ }
+
+ @Test
+ public void testCurrentTableMetadataPayloadDeserializesWithLegacyDto()
+ throws Exception {
+ ServerTableMetadataInfo current = ServerTableMetadataInfo.builder("myTable_OFFLINE")
+ .withDiskSizeInBytes(120L)
+ .withNumSegments(2L)
+ .withNumRows(30L)
+ .withColumnLengthMap(Map.of("col", 4.0))
+ .withColumnCardinalityMap(Map.of("col", 5.0))
+ .withMaxNumMultiValuesMap(Map.of("col", 1.0))
+ .withColumnIndexSizeMap(Map.of("col", Map.of("forward_index", 12.0)))
+ .withPartitionToServerPrimaryKeyCountMap(Map.of(0, Map.of("server0", 10L)))
+ .build();
+
+ LegacyTableMetadataInfo legacy =
+ JsonUtils.stringToObject(JsonUtils.objectToString(current), LegacyTableMetadataInfo.class);
+
+ assertEquals(legacy._tableName, "myTable_OFFLINE");
+ assertEquals(legacy._diskSizeInBytes, 120L);
+ assertEquals(legacy._numSegments, 2L);
+ assertEquals(legacy._numRows, 30L);
+ assertEquals(legacy._columnLengthMap, Map.of("col", 4.0));
+ assertEquals(legacy._columnCardinalityMap, Map.of("col", 5.0));
+ assertEquals(legacy._maxNumMultiValuesMap, Map.of("col", 1.0));
+ assertEquals(legacy._columnIndexSizeMap, Map.of("col", Map.of("forward_index", 12.0)));
+ assertEquals(legacy._partitionToServerPrimaryKeyCountMap, Map.of(0, Map.of("server0", 10L)));
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ private static final class LegacyTableSizeInfo {
+ private final String _tableName;
+ private final long _diskSizeInBytes;
+ private final List<LegacySegmentSizeInfo> _segments;
+
+ @JsonCreator
+ private LegacyTableSizeInfo(@JsonProperty("tableName") String tableName,
+ @JsonProperty("diskSizeInBytes") long diskSizeInBytes,
+ @JsonProperty("segments") List<LegacySegmentSizeInfo> segments) {
+ _tableName = tableName;
+ _diskSizeInBytes = diskSizeInBytes;
+ _segments = segments;
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ private static final class LegacySegmentSizeInfo {
+ private final String _segmentName;
+ private final long _diskSizeInBytes;
+
+ @JsonCreator
+ private LegacySegmentSizeInfo(@JsonProperty("segmentName") String segmentName,
+ @JsonProperty("diskSizeInBytes") long diskSizeInBytes) {
+ _segmentName = segmentName;
+ _diskSizeInBytes = diskSizeInBytes;
+ }
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ private static final class LegacyTableMetadataInfo {
+ private final String _tableName;
+ private final long _diskSizeInBytes;
+ private final long _numSegments;
+ private final long _numRows;
+ private final Map<String, Double> _columnLengthMap;
+ private final Map<String, Double> _columnCardinalityMap;
+ private final Map<String, Double> _maxNumMultiValuesMap;
+ private final Map<String, Map<String, Double>> _columnIndexSizeMap;
+ private final Map<Integer, Map<String, Long>> _partitionToServerPrimaryKeyCountMap;
+
+ @JsonCreator
+ private LegacyTableMetadataInfo(@JsonProperty("tableName") String tableName,
+ @JsonProperty("diskSizeInBytes") long diskSizeInBytes, @JsonProperty("numSegments") long numSegments,
+ @JsonProperty("numRows") long numRows,
+ @JsonProperty("columnLengthMap") Map<String, Double> columnLengthMap,
+ @JsonProperty("columnCardinalityMap") Map<String, Double> columnCardinalityMap,
+ @JsonProperty("maxNumMultiValuesMap") Map<String, Double> maxNumMultiValuesMap,
+ @JsonProperty("columnIndexSizeMap") Map<String, Map<String, Double>> columnIndexSizeMap,
+ @JsonProperty("upsertPartitionToServerPrimaryKeyCountMap")
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap) {
+ _tableName = tableName;
+ _diskSizeInBytes = diskSizeInBytes;
+ _numSegments = numSegments;
+ _numRows = numRows;
+ _columnLengthMap = columnLengthMap;
+ _columnCardinalityMap = columnCardinalityMap;
+ _maxNumMultiValuesMap = maxNumMultiValuesMap;
+ _columnIndexSizeMap = columnIndexSizeMap;
+ _partitionToServerPrimaryKeyCountMap = partitionToServerPrimaryKeyCountMap;
+ }
+ }
+}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/TableMetadataInfoCompressionTest.java b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/TableMetadataInfoCompressionTest.java
new file mode 100644
index 00000000000..112d3502f66
--- /dev/null
+++ b/pinot-common/src/test/java/org/apache/pinot/common/restlet/resources/TableMetadataInfoCompressionTest.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.restlet.resources;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Tests the [TableMetadataInfo] response schema for compression statistics.
+///
+/// Validates that the server response includes `columnCompressionStats` when present and suppresses it when absent.
+public class TableMetadataInfoCompressionTest {
+
+ @Test
+ public void testSerializationWithCompressionStats()
+ throws Exception {
+ List<ColumnCompressionStatsInfo> colStats = new ArrayList<>();
+ colStats.add(rawStats("col_a", 10000, 2000, ChunkCompressionType.LZ4, List.of("forward_index")));
+ colStats.add(rawStats("col_b", 20000, 5000, ChunkCompressionType.ZSTANDARD,
+ List.of("forward_index", "inverted_index")));
+
+ TableMetadataInfo info = new TableMetadataInfo("testTable", 50000, 3, 1000,
+ Map.of("col_a", 4.0), Map.of("col_a", 50.0), Map.of(), Map.of(), Map.of(), colStats);
+
+ String json = JsonUtils.objectToString(info);
+ JsonNode node = JsonUtils.stringToJsonNode(json);
+
+ // columnCompressionStats should be present as an array
+ assertTrue(node.has("columnCompressionStats"));
+ JsonNode colStatsNode = node.get("columnCompressionStats");
+ assertTrue(colStatsNode.isArray(), "columnCompressionStats should be a JSON array");
+ assertEquals(colStatsNode.size(), 2);
+
+ // Validate col_a values (first element)
+ JsonNode colA = colStatsNode.get(0);
+ assertEquals(colA.get("column").asText(), "col_a");
+ assertEquals(colA.get("uncompressedValueSizeInBytes").asLong(), 10000);
+ assertEquals(colA.get("forwardIndexAndDictionaryStorageSizeInBytes").asLong(), 2000);
+ assertEquals(colA.get("compressionRatio").asDouble(), 5.0, 0.01);
+ assertEquals(colA.get("encodingBreakdown").get(0).get("encoding").asText(), "RAW");
+ assertEquals(colA.get("encodingBreakdown").get(0).get("chunkCompressionType").asText(), "LZ4");
+ assertFalse(colA.has("hasDictionary"));
+ assertFalse(colA.has("codec"));
+ assertTrue(colA.has("observedIndexes"));
+
+ // Validate col_b values (second element)
+ JsonNode colB = colStatsNode.get(1);
+ assertEquals(colB.get("column").asText(), "col_b");
+ assertEquals(colB.get("uncompressedValueSizeInBytes").asLong(), 20000);
+ assertEquals(colB.get("forwardIndexAndDictionaryStorageSizeInBytes").asLong(), 5000);
+ assertEquals(colB.get("compressionRatio").asDouble(), 4.0, 0.01);
+ assertEquals(colB.get("encodingBreakdown").get(0).get("chunkCompressionType").asText(), "ZSTANDARD");
+ assertFalse(colB.has("hasDictionary"));
+ assertEquals(colB.get("observedIndexes").size(), 2);
+ }
+
+ @Test
+ public void testSerializationWithoutCompressionStats()
+ throws Exception {
+ // Use backwards-compatible constructor (no compression stats)
+ TableMetadataInfo info = new TableMetadataInfo("testTable", 50000, 3, 1000,
+ Map.of("col_a", 4.0), Map.of("col_a", 50.0), Map.of(), Map.of(), Map.of());
+
+ String json = JsonUtils.objectToString(info);
+ JsonNode node = JsonUtils.stringToJsonNode(json);
+
+ // columnCompressionStats should be absent (suppressed by NON_NULL)
+ assertFalse(node.has("columnCompressionStats"),
+ "columnCompressionStats should be suppressed from JSON when null");
+ }
+
+ @Test
+ public void testDeserializationRoundTrip()
+ throws Exception {
+ List<ColumnCompressionStatsInfo> colStats = new ArrayList<>();
+ colStats.add(rawStats("metric_col", 50000, 8000, ChunkCompressionType.SNAPPY, List.of("forward_index")));
+
+ TableMetadataInfo original = new TableMetadataInfo("roundTripTable", 100000, 5, 5000,
+ Map.of("metric_col", 8.0), Map.of("metric_col", 100.0), Map.of(), Map.of(), Map.of(), colStats);
+
+ String json = JsonUtils.objectToString(original);
+ TableMetadataInfo deserialized = JsonUtils.stringToObject(json, TableMetadataInfo.class);
+
+ assertEquals(deserialized.getTableName(), "roundTripTable");
+ assertEquals(deserialized.getDiskSizeInBytes(), 100000);
+ assertNotNull(deserialized.getColumnCompressionStats());
+ assertEquals(deserialized.getColumnCompressionStats().size(), 1);
+
+ ColumnCompressionStatsInfo stats = deserialized.getColumnCompressionStats().get(0);
+ assertNotNull(stats);
+ assertEquals(stats.getColumn(), "metric_col");
+ assertEquals(stats.getUncompressedValueSizeInBytes(), 50000);
+ assertEquals(stats.getForwardIndexAndDictionaryStorageSizeInBytes(), 8000);
+ assertEquals(stats.getCompressionRatio(), 6.25, 0.01);
+ assertEquals(stats.getEncodingBreakdown().get(0).getChunkCompressionType(), ChunkCompressionType.SNAPPY);
+ assertNotNull(stats.getObservedIndexes());
+ }
+
+ @Test
+ public void testBackwardCompatDeserialization()
+ throws Exception {
+ // Simulate JSON from an old server that doesn't include columnCompressionStats
+ String oldJson = "{\"tableName\":\"oldTable\",\"diskSizeInBytes\":30000,"
+ + "\"numSegments\":2,\"numRows\":500,"
+ + "\"columnLengthMap\":{\"col\":4.0},"
+ + "\"columnCardinalityMap\":{\"col\":10.0},"
+ + "\"maxNumMultiValuesMap\":{},"
+ + "\"columnIndexSizeMap\":{},"
+ + "\"upsertPartitionToServerPrimaryKeyCountMap\":{}}";
+
+ TableMetadataInfo info = JsonUtils.stringToObject(oldJson, TableMetadataInfo.class);
+ assertNotNull(info);
+ assertEquals(info.getTableName(), "oldTable");
+ assertEquals(info.getDiskSizeInBytes(), 30000);
+ // columnCompressionStats should be null (not present in old JSON)
+ assertNull(info.getColumnCompressionStats());
+ }
+
+ private static ColumnCompressionStatsInfo rawStats(String column, long rawSize,
+ long forwardIndexAndDictionaryStorageSize, ChunkCompressionType chunkCompressionType,
+ List<String> observedIndexes) {
+ return ColumnCompressionStatsInfo.builder(column)
+ .withUncompressedValueSizeInBytes(rawSize)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(forwardIndexAndDictionaryStorageSize)
+ .withCompressionRatio(forwardIndexAndDictionaryStorageSize > 0
+ ? (double) rawSize / forwardIndexAndDictionaryStorageSize : 0)
+ .withObservedIndexes(observedIndexes)
+ .withEncodingBreakdown(List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, chunkCompressionType, 1, rawSize, forwardIndexAndDictionaryStorageSize)))
+ .withNumSegments(1)
+ .build();
+ }
+}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index d723fae45f5..035c36a1243 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -1256,6 +1256,8 @@ public class PinotTableRestletResource {
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
@ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr,
@ApiParam(value = "Columns name", allowMultiple = true) @QueryParam("columns") List<String> columns,
+ @ApiParam(value = "Include per-column compression stats in response (default false to avoid large responses)")
+ @DefaultValue("false") @QueryParam("includeColumnCompressionStats") boolean includeColumnCompressionStats,
@Context HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
LOGGER.info("Received a request to fetch aggregate metadata for a table {}", tableName);
@@ -1269,9 +1271,16 @@ public class PinotTableRestletResource {
TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType);
int numReplica = tableConfig == null ? 1 : tableConfig.getReplication();
+ // compressionStatsEnabled gates server-side collection; includeColumnCompressionStats controls the per-column
+ // response list.
+ boolean compressionStatsEnabled = tableConfig != null && tableConfig.getIndexingConfig() != null
+ && tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+
String segmentsMetadata;
try {
- JsonNode segmentsMetadataJson = getAggregateMetadataFromServer(tableNameWithType, columns, numReplica);
+ JsonNode segmentsMetadataJson =
+ getAggregateMetadataFromServer(tableNameWithType, columns, numReplica, compressionStatsEnabled,
+ includeColumnCompressionStats);
segmentsMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson);
} catch (InvalidConfigException e) {
throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.BAD_REQUEST);
@@ -1292,6 +1301,8 @@ public class PinotTableRestletResource {
@ApiParam(value = "Name of the table with type suffix", required = true) @PathParam("tableNameWithType")
String tableNameWithType,
@ApiParam(value = "Comma separated list of columns") @QueryParam("columns") @Nullable String columns,
+ @ApiParam(value = "Include per-column compression stats in response (default false to avoid large responses)")
+ @DefaultValue("false") @QueryParam("includeColumnCompressionStats") boolean includeColumnCompressionStats,
@Context HttpHeaders headers) {
tableNameWithType = DatabaseUtils.translateTableName(tableNameWithType, headers);
LOGGER.info("Received a request to fetch aggregate metadata for a table {}", tableNameWithType);
@@ -1321,9 +1332,13 @@ public class PinotTableRestletResource {
}
}
+ boolean compressionStatsEnabled = tableConfig != null && tableConfig.getIndexingConfig() != null
+ && tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+
try {
JsonNode segmentsMetadataJson =
- getAggregateMetadataFromServer(existingTableNameWithType, columnsList, numReplica);
+ getAggregateMetadataFromServer(existingTableNameWithType, columnsList, numReplica,
+ compressionStatsEnabled, includeColumnCompressionStats);
return JsonUtils.objectToPrettyString(segmentsMetadataJson);
} catch (InvalidConfigException e) {
throw new ControllerApplicationException(LOGGER, e.getMessage(), Response.Status.BAD_REQUEST);
@@ -1453,12 +1468,14 @@ public class PinotTableRestletResource {
* @param numReplica num or replica for the table
* @return aggregated metadata of the table segments
*/
- private JsonNode getAggregateMetadataFromServer(String tableNameWithType, List<String> columns, int numReplica)
+ private JsonNode getAggregateMetadataFromServer(String tableNameWithType, List<String> columns, int numReplica,
+ boolean compressionStatsEnabled, boolean includeColumnCompressionStats)
throws InvalidConfigException, IOException {
TableMetadataReader tableMetadataReader =
new TableMetadataReader(_executor, _connectionManager, _pinotHelixResourceManager);
return tableMetadataReader.getAggregateTableMetadata(tableNameWithType, columns, numReplica,
- _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
+ _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000, compressionStatsEnabled,
+ includeColumnCompressionStats);
}
@GET
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
index d90aa0d8ff0..2527cc942ec 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ServerTableSizeReader.java
@@ -49,16 +49,54 @@ public class ServerTableSizeReader {
_connectionManager = connectionManager;
}
+ /// Reads server segment sizes without compression statistics.
public Map<String, List<SegmentSizeInfo>> getSegmentSizeInfoFromServers(BiMap<String, String> serverEndPoints,
String tableNameWithType, int timeoutMs) {
+ return extractSegmentSizeInfo(getTableSizeInfoFromServers(serverEndPoints, tableNameWithType, timeoutMs));
+ }
+
+ /// Reads server segment sizes with compression summaries and optional per-column details.
+ public Map<String, List<SegmentSizeInfo>> getSegmentSizeInfoFromServers(BiMap<String, String> serverEndPoints,
+ String tableNameWithType, int timeoutMs, boolean includeColumnCompressionStats) {
+ return extractSegmentSizeInfo(getTableSizeInfoFromServers(serverEndPoints, tableNameWithType, timeoutMs,
+ includeColumnCompressionStats));
+ }
+
+ private static Map<String, List<SegmentSizeInfo>> extractSegmentSizeInfo(
+ Map<String, TableSizeInfo> tableSizeInfoMap) {
+ Map<String, List<SegmentSizeInfo>> result = new HashMap<>();
+ for (Map.Entry<String, TableSizeInfo> entry : tableSizeInfoMap.entrySet()) {
+ result.put(entry.getKey(), entry.getValue().getSegments());
+ }
+ return result;
+ }
+
+ /// Reads versioned server table-size responses without compression statistics.
+ public Map<String, TableSizeInfo> getTableSizeInfoFromServers(BiMap<String, String> serverEndPoints,
+ String tableNameWithType, int timeoutMs) {
+ return getTableSizeInfoFromServers(serverEndPoints, tableNameWithType, timeoutMs, false, false);
+ }
+
+ /// Reads versioned server table-size responses with compression summaries and optional per-column details.
+ public Map<String, TableSizeInfo> getTableSizeInfoFromServers(BiMap<String, String> serverEndPoints,
+ String tableNameWithType, int timeoutMs, boolean includeColumnCompressionStats) {
+ return getTableSizeInfoFromServers(serverEndPoints, tableNameWithType, timeoutMs, true,
+ includeColumnCompressionStats);
+ }
+
+ private Map<String, TableSizeInfo> getTableSizeInfoFromServers(BiMap<String, String> serverEndPoints,
+ String tableNameWithType, int timeoutMs, boolean includeCompressionStats, boolean includeColumnCompressionStats) {
int numServers = serverEndPoints.size();
LOGGER.info("Reading segment sizes from {} servers for table: {} with timeout: {}ms", numServers, tableNameWithType,
timeoutMs);
List<String> serverUrls = new ArrayList<>(numServers);
BiMap<String, String> endpointsToServers = serverEndPoints.inverse();
+ boolean requestCompressionStats = includeCompressionStats || includeColumnCompressionStats;
for (String endpoint : endpointsToServers.keySet()) {
- String tableSizeUri = endpoint + "/table/" + tableNameWithType + "/size";
+ String tableSizeUri = endpoint + "/table/" + tableNameWithType + "/size"
+ + (requestCompressionStats ? "?includeCompressionStats=true" : "")
+ + (includeColumnCompressionStats ? "&includeColumnCompressionStats=true" : "");
serverUrls.add(tableSizeUri);
}
@@ -68,12 +106,12 @@ public class ServerTableSizeReader {
CompletionServiceHelper.CompletionServiceResponse serviceResponse =
completionServiceHelper.doMultiGetRequest(serverUrls, tableNameWithType, false, timeoutMs,
"get segment size info from servers");
- Map<String, List<SegmentSizeInfo>> serverToSegmentSizeInfoListMap = new HashMap<>();
+ Map<String, TableSizeInfo> serverToTableSizeInfoMap = new HashMap<>();
int failedParses = 0;
for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
try {
TableSizeInfo tableSizeInfo = JsonUtils.stringToObject(streamResponse.getValue(), TableSizeInfo.class);
- serverToSegmentSizeInfoListMap.put(streamResponse.getKey(), tableSizeInfo.getSegments());
+ serverToTableSizeInfoMap.put(streamResponse.getKey(), tableSizeInfo);
} catch (IOException e) {
failedParses++;
LOGGER.error("Unable to parse server {} response due to an error: ", streamResponse.getKey(), e);
@@ -82,6 +120,6 @@ public class ServerTableSizeReader {
if (failedParses != 0) {
LOGGER.warn("Failed to parse {} / {} segment size info responses from servers.", failedParses, serverUrls.size());
}
- return serverToSegmentSizeInfoListMap;
+ return serverToTableSizeInfoMap;
}
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
index 59954502003..5533eaaeb63 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
@@ -87,13 +87,15 @@ public class TableSize {
@ApiParam(value = "Provide detailed information") @DefaultValue("true") @QueryParam("verbose") boolean verbose,
@ApiParam(value = "Include replaced segments") @DefaultValue("true")
@QueryParam("includeReplacedSegments") boolean includeReplacedSegments,
+ @ApiParam(value = "Include per-column compression stats in response (default false to avoid large responses)")
+ @DefaultValue("false") @QueryParam("includeColumnCompressionStats") boolean includeColumnCompressionStats,
@Context HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
TableSizeReader.TableSizeDetails tableSizeDetails = null;
try {
tableSizeDetails =
_tableSizeReader.getTableSizeDetails(tableName, _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000,
- includeReplacedSegments);
+ includeReplacedSegments, includeColumnCompressionStats);
if (!verbose) {
if (tableSizeDetails._offlineSegments != null) {
tableSizeDetails._offlineSegments._segments = new HashMap<>();
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index 2d06ea9d976..883499ef799 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -122,8 +122,9 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
try {
TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType);
updateTableConfigMetrics(tableNameWithType, tableConfig, context);
- updateSegmentMetrics(tableNameWithType, tableConfig, context);
- updateTableSizeMetrics(tableNameWithType);
+ if (updateSegmentMetrics(tableNameWithType, tableConfig, context)) {
+ updateTableSizeMetrics(tableNameWithType, tableConfig);
+ }
} catch (Exception e) {
LOGGER.error("Caught exception while updating segment status for table {}", tableNameWithType, e);
// Remove the metric for this table
@@ -256,16 +257,31 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
}
}
- private void updateTableSizeMetrics(String tableNameWithType)
+ void updateTableSizeMetrics(String tableNameWithType, TableConfig tableConfig)
throws InvalidConfigException {
- _tableSizeReader.getTableSizeDetails(tableNameWithType, TABLE_CHECKER_TIMEOUT_MS, true);
+ TableSizeReader.TableSizeDetails tableSizeDetails =
+ _tableSizeReader.getTableSizeDetails(tableNameWithType, TABLE_CHECKER_TIMEOUT_MS, true,
+ TableSizeReader.CompressionStatsMode.AGGREGATE_SUMMARY);
+ boolean compressionStatsEnabled = tableConfig != null && tableConfig.getIndexingConfig() != null
+ && tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ if (!compressionStatsEnabled || tableSizeDetails == null) {
+ _tableSizeReader.clearCompressionMetrics(tableNameWithType);
+ return;
+ }
+ TableSizeReader.TableSubTypeSizeDetails subTypeSizeDetails = tableConfig.getTableType() == TableType.OFFLINE
+ ? tableSizeDetails._offlineSegments : tableSizeDetails._realtimeSegments;
+ if (subTypeSizeDetails != null) {
+ _tableSizeReader.updateCompressionMetrics(tableNameWithType, subTypeSizeDetails);
+ } else {
+ _tableSizeReader.clearCompressionMetrics(tableNameWithType);
+ }
}
/**
* Runs a segment status pass over the given table.
* TODO: revisit the logic and reduce the ZK access
*/
- private void updateSegmentMetrics(String tableNameWithType, TableConfig tableConfig, Context context) {
+ private boolean updateSegmentMetrics(String tableNameWithType, TableConfig tableConfig, Context context) {
TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
ServerQueryInfoFetcher serverQueryInfoFetcher = new ServerQueryInfoFetcher(_pinotHelixResourceManager);
@@ -275,7 +291,7 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
if (idealState == null) {
LOGGER.warn("Table {} has null ideal state. Skipping segment status checks", tableNameWithType);
removeMetricsForTable(tableNameWithType);
- return;
+ return false;
}
if (!idealState.isEnabled()) {
@@ -284,7 +300,7 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
}
removeMetricsForTable(tableNameWithType);
context._disabledTables.add(tableNameWithType);
- return;
+ return false;
}
if (PinotLLCRealtimeSegmentManager.isTablePaused(idealState)) {
@@ -340,7 +356,7 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
_controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, 100);
_controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.SEGMENTS_WITH_LESS_REPLICAS, 0);
_controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.TABLE_COMPRESSED_SIZE, 0);
- return;
+ return true;
}
long evSnapshotTimestamp = System.currentTimeMillis();
@@ -537,6 +553,7 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
new MissingConsumingSegmentFinder(tableNameWithType, propertyStore, _controllerMetrics,
streamConfigs, idealState).findAndEmitMetrics(idealState);
}
+ return true;
}
private boolean isServerQueryable(ServerQueryInfo serverInfo) {
@@ -566,6 +583,7 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
_controllerMetrics.removeTableGauge(tableNameWithType, key, ControllerGauge.TABLE_TENANT_INFO);
}
}
+ _tableSizeReader.clearCompressionMetrics(tableNameWithType);
for (ControllerGauge metric : ControllerGauge.values()) {
if (!metric.isGlobal()) {
_controllerMetrics.removeTableGauge(tableNameWithType, metric);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
index a1a1b988af1..aede47792fd 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/CompletionServiceHelper.java
@@ -24,14 +24,16 @@ import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.CompletionService;
import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
-import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.pinot.common.http.MultiHttpRequest;
-import org.apache.pinot.common.http.MultiHttpRequestResponse;
+import org.apache.pinot.common.http.MultiHttpRequestBufferedResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,11 +82,20 @@ public class CompletionServiceHelper {
public CompletionServiceResponse doMultiGetRequest(List<String> serverURLs, String tableNameWithType,
boolean multiRequestPerServer, @Nullable Map<String, String> requestHeaders, int timeoutMs,
@Nullable String useCase) {
- // TODO: use some service other than completion service so that we know which server encounters the error
- CompletionService<MultiHttpRequestResponse> completionService =
- new MultiHttpRequest(_executor, _httpConnectionManager).executeGet(serverURLs, requestHeaders, timeoutMs);
+ MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(_executor, _httpConnectionManager).executeGetBuffered(serverURLs, requestHeaders,
+ timeoutMs);
+ return collectBufferedResponse(tableNameWithType, serverURLs.size(), execution, multiRequestPerServer, useCase);
+ }
- return collectResponse(tableNameWithType, serverURLs.size(), completionService, multiRequestPerServer, useCase);
+ /// Makes buffered, cancellable GET requests and returns once all complete or the deadline from the
+ /// [System#nanoTime()] clock is reached.
+ public CompletionServiceResponse doMultiGetRequestUntil(List<String> serverURLs, String tableNameWithType,
+ boolean multiRequestPerServer, int timeoutMs, long nanoTimeDeadline, @Nullable String useCase) {
+ MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(_executor, _httpConnectionManager).executeGetBuffered(serverURLs, null, timeoutMs);
+ return collectBufferedResponseUntil(tableNameWithType, serverURLs.size(), execution, multiRequestPerServer, useCase,
+ nanoTimeDeadline);
}
/**
@@ -105,59 +116,100 @@ public class CompletionServiceHelper {
public CompletionServiceResponse doMultiPostRequest(List<Pair<String, String>> serverURLsAndRequestBodies,
String tableNameWithType, boolean multiRequestPerServer, @Nullable Map<String, String> requestHeaders,
int timeoutMs, @Nullable String useCase) {
+ MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(_executor, _httpConnectionManager).createBufferedExecution(timeoutMs);
+ try {
+ for (Pair<String, String> requestAndBody : serverURLsAndRequestBodies) {
+ HttpPost request = new HttpPost(requestAndBody.getLeft());
+ request.setEntity(new StringEntity(requestAndBody.getRight()));
+ if (requestHeaders != null) {
+ requestHeaders.forEach(request::setHeader);
+ }
+ execution.submit(request);
+ }
+ } catch (RuntimeException | Error e) {
+ execution.close();
+ throw e;
+ }
+ return collectBufferedResponse(tableNameWithType, serverURLsAndRequestBodies.size(), execution,
+ multiRequestPerServer, useCase);
+ }
- CompletionService<MultiHttpRequestResponse> completionService =
- new MultiHttpRequest(_executor, _httpConnectionManager).executePost(serverURLsAndRequestBodies, requestHeaders,
- timeoutMs);
+ private CompletionServiceResponse collectBufferedResponse(String tableNameWithType, int size,
+ MultiHttpRequest.BufferedRequestExecution execution, boolean multiRequestPerServer, @Nullable String useCase) {
+ return collectBufferedResponse(tableNameWithType, size, execution, multiRequestPerServer, useCase, 0, false);
+ }
- return collectResponse(tableNameWithType, serverURLsAndRequestBodies.size(), completionService,
- multiRequestPerServer, useCase);
+ private CompletionServiceResponse collectBufferedResponseUntil(String tableNameWithType, int size,
+ MultiHttpRequest.BufferedRequestExecution execution, boolean multiRequestPerServer, @Nullable String useCase,
+ long nanoTimeDeadline) {
+ return collectBufferedResponse(tableNameWithType, size, execution, multiRequestPerServer, useCase, nanoTimeDeadline,
+ true);
}
- private CompletionServiceResponse collectResponse(String tableNameWithType, int size,
- CompletionService<MultiHttpRequestResponse> completionService, boolean multiRequestPerServer,
- @Nullable String useCase) {
+ private CompletionServiceResponse collectBufferedResponse(String tableNameWithType, int size,
+ MultiHttpRequest.BufferedRequestExecution execution, boolean multiRequestPerServer, @Nullable String useCase,
+ long nanoTimeDeadline, boolean enforceDeadline) {
CompletionServiceResponse completionServiceResponse = new CompletionServiceResponse();
-
- for (int i = 0; i < size; i++) {
- MultiHttpRequestResponse multiHttpRequestResponse = null;
- try {
- multiHttpRequestResponse = completionService.take().get();
- URI uri = multiHttpRequestResponse.getURI();
- String instance =
- _endpointsToServers.get(String.format("%s://%s:%d", uri.getScheme(), uri.getHost(), uri.getPort()));
- int statusCode = multiHttpRequestResponse.getResponse().getCode();
- if (statusCode >= 300) {
- String reason = multiHttpRequestResponse.getResponse().getReasonPhrase();
- LOGGER.error("Server: {} returned error: {}, reason: {} for uri: {}", instance, statusCode, reason, uri);
- completionServiceResponse._failedResponseCount++;
- continue;
- }
- String responseString = EntityUtils.toString(multiHttpRequestResponse.getResponse().getEntity());
- String key = multiRequestPerServer ? uri.toString() : instance;
- // If there are multiple requests to the same server with the same URI but different payloads,
- // we append a count value to the key to ensure each response is uniquely identified.
- // Otherwise, the map will store only the last response, overwriting previous ones.
- if (multiRequestPerServer) {
- int count = completionServiceResponse._instanceToRequestCount.compute(key, (k, v) -> v == null ? 1 : v + 1);
- key = key + "__" + count;
- }
- completionServiceResponse._httpResponses.put(key, responseString);
- } catch (Exception e) {
- String reason = useCase == null ? "" : String.format(" in '%s'", useCase);
- LOGGER.error("Connection error {}. Details: {}", reason, e.getMessage());
- completionServiceResponse._failedResponseCount++;
- } finally {
- if (multiHttpRequestResponse != null) {
- try {
- multiHttpRequestResponse.close();
- } catch (Exception e) {
- LOGGER.error("Connection close error. Details: {}", e.getMessage());
+ try (execution) {
+ for (int i = 0; i < size; i++) {
+ try {
+ Future<MultiHttpRequestBufferedResponse> completedFuture;
+ if (enforceDeadline) {
+ long remainingNanos = nanoTimeDeadline - System.nanoTime();
+ if (remainingNanos <= 0) {
+ completionServiceResponse._failedResponseCount += size - i;
+ break;
+ }
+ completedFuture = execution.poll(remainingNanos, TimeUnit.NANOSECONDS);
+ if (completedFuture == null) {
+ completionServiceResponse._failedResponseCount += size - i;
+ break;
+ }
+ } else {
+ completedFuture = execution.take();
}
+ MultiHttpRequestBufferedResponse response = completedFuture.get();
+ addResponse(completionServiceResponse, response.getURI(), response.getStatusCode(),
+ response.getReasonPhrase(), response.getResponseBody(), multiRequestPerServer);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ completionServiceResponse._failedResponseCount += size - i;
+ break;
+ } catch (Exception e) {
+ recordConnectionError(completionServiceResponse, useCase, e);
}
}
}
+ return finishResponse(tableNameWithType, size, completionServiceResponse);
+ }
+
+ private void addResponse(CompletionServiceResponse response, URI uri, int statusCode,
+ @Nullable String reasonPhrase, @Nullable String responseBody, boolean multiRequestPerServer) {
+ String instance =
+ _endpointsToServers.get(String.format("%s://%s:%d", uri.getScheme(), uri.getHost(), uri.getPort()));
+ if (statusCode >= 300) {
+ LOGGER.error("Server: {} returned error: {}, reason: {} for uri: {}", instance, statusCode, reasonPhrase, uri);
+ response._failedResponseCount++;
+ return;
+ }
+ String key = multiRequestPerServer ? uri.toString() : instance;
+ // Multiple requests can use the same URI with different POST bodies, so preserve every response with a suffix.
+ if (multiRequestPerServer) {
+ int count = response._instanceToRequestCount.compute(key, (k, v) -> v == null ? 1 : v + 1);
+ key = key + "__" + count;
+ }
+ response._httpResponses.put(key, responseBody != null ? responseBody : "");
+ }
+
+ private static void recordConnectionError(CompletionServiceResponse response, @Nullable String useCase, Exception e) {
+ String reason = useCase == null ? "" : String.format(" in '%s'", useCase);
+ LOGGER.error("Connection error {}. Details: {}", reason, e.getMessage());
+ response._failedResponseCount++;
+ }
+ private static CompletionServiceResponse finishResponse(String tableNameWithType, int size,
+ CompletionServiceResponse completionServiceResponse) {
int numServerRequestsResponded = completionServiceResponse._httpResponses.size();
if (numServerRequestsResponded != size) {
LOGGER.warn("Finished reading information for table: {} with {}/{} server-request responses", tableNameWithType,
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerCompressionStatsReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerCompressionStatsReader.java
new file mode 100644
index 00000000000..4133cbde189
--- /dev/null
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerCompressionStatsReader.java
@@ -0,0 +1,648 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.util;
+
+import com.google.common.collect.BiMap;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.io.HttpClientConnectionManager;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.hc.core5.util.Timeout;
+import org.apache.pinot.common.compression.ColumnCompressionStatsAccumulator;
+import org.apache.pinot.common.http.MultiHttpRequest;
+import org.apache.pinot.common.http.MultiHttpRequestBufferedResponse;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.common.restlet.resources.CompressionStatsSummary;
+import org.apache.pinot.common.restlet.resources.SegmentCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ServerCompressionStatsResponse;
+import org.apache.pinot.common.restlet.resources.TableSegments;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.UrlBuilderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/// Reads per-segment compression statistics from deterministic preferred replicas with bounded batching and fallback.
+///
+/// One instance may be called concurrently. Each call owns its scheduling state and HTTP client while sharing the
+/// injected executor and connection manager. Requests containing segments with another candidate use a proportional
+/// slice of the remaining deadline. Final-candidate requests keep the overall deadline, and the scheduler reserves
+/// capacity for both classes so neither can starve the other.
+final class ServerCompressionStatsReader {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ServerCompressionStatsReader.class);
+ private static final int MAX_SEGMENTS_PER_REQUEST = 500;
+ private static final int MAX_SUMMARY_REQUESTS_IN_FLIGHT = 8;
+ private static final int MAX_DETAIL_REQUESTS_IN_FLIGHT = 4;
+ private static final int ESTIMATED_COLUMNS_WHEN_UNFILTERED = 1_000;
+ private static final int REPLICA_ROTATION_SALT = 0x9e3779b9;
+
+ private final Executor _executor;
+ private final HttpClientConnectionManager _connectionManager;
+
+ ServerCompressionStatsReader(Executor executor, HttpClientConnectionManager connectionManager) {
+ _executor = executor;
+ _connectionManager = connectionManager;
+ }
+
+ CompressionStatsResult read(String tableNameWithType, Map<String, List<String>> serverToSegments,
+ BiMap<String, String> serverToEndpoints, @Nullable List<String> columns,
+ boolean includeColumnCompressionStats, long deadlineNanos) {
+ return read(tableNameWithType, serverToSegments, serverToEndpoints, columns, includeColumnCompressionStats,
+ deadlineNanos, false);
+ }
+
+ CompressionStatsResult readWithSelectedSegments(String tableNameWithType,
+ Map<String, List<String>> serverToSegments, BiMap<String, String> serverToEndpoints,
+ @Nullable List<String> columns, boolean includeColumnCompressionStats, long deadlineNanos) {
+ return read(tableNameWithType, serverToSegments, serverToEndpoints, columns, includeColumnCompressionStats,
+ deadlineNanos, true);
+ }
+
+ private CompressionStatsResult read(String tableNameWithType, Map<String, List<String>> serverToSegments,
+ BiMap<String, String> serverToEndpoints, @Nullable List<String> columns,
+ boolean includeColumnCompressionStats, long deadlineNanos, boolean includeSelectedSegments) {
+ SegmentStates segmentStates = createSegmentStates(serverToSegments, serverToEndpoints, deadlineNanos);
+ Collection<SegmentState> states = segmentStates._states;
+ CompressionStatsAggregation aggregation =
+ new CompressionStatsAggregation(segmentStates._totalSegments, includeColumnCompressionStats,
+ includeSelectedSegments);
+ if (states.isEmpty() || System.nanoTime() >= deadlineNanos) {
+ return aggregation.result();
+ }
+
+ int segmentsPerRequest = segmentsPerRequest(columns, includeColumnCompressionStats);
+ int maxRequestsInFlight = includeColumnCompressionStats
+ ? MAX_DETAIL_REQUESTS_IN_FLIGHT : MAX_SUMMARY_REQUESTS_IN_FLIGHT;
+ Deque<RequestBatch> retryableBatches = new ArrayDeque<>();
+ Deque<RequestBatch> finalBatches = new ArrayDeque<>();
+ enqueue(states, retryableBatches, finalBatches, segmentsPerRequest);
+ Map<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> activeRequests = new HashMap<>();
+ boolean preferFinal = true;
+
+ int executionTimeoutMs = toTimeoutMs(deadlineNanos - System.nanoTime());
+ try (MultiHttpRequest.BufferedRequestExecution execution =
+ new MultiHttpRequest(_executor, _connectionManager).createBufferedExecution(executionTimeoutMs)) {
+ while (System.nanoTime() < deadlineNanos
+ && (!retryableBatches.isEmpty() || !finalBatches.isEmpty() || !activeRequests.isEmpty())) {
+ preferFinal = fillAvailableSlots(tableNameWithType, columns, includeColumnCompressionStats, deadlineNanos,
+ retryableBatches, finalBatches, execution, activeRequests, maxRequestsInFlight, preferFinal);
+ if (activeRequests.isEmpty()) {
+ break;
+ }
+
+ long now = System.nanoTime();
+ long nextDeadline = Math.min(deadlineNanos, activeRequests.values().stream()
+ .mapToLong(request -> request._deadlineNanos).min().orElse(deadlineNanos));
+ long waitNanos = Math.max(1, nextDeadline - now);
+ Future<MultiHttpRequestBufferedResponse> completed = execution.poll(waitNanos, TimeUnit.NANOSECONDS);
+ if (completed != null) {
+ ActiveRequest active = activeRequests.remove(completed);
+ if (active != null && !processCompleted(completed, active, aggregation, retryableBatches, finalBatches,
+ segmentsPerRequest)) {
+ break;
+ }
+ }
+ expireRequests(System.nanoTime(), activeRequests, retryableBatches, finalBatches, execution,
+ segmentsPerRequest);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ CompressionStatsResult result = aggregation.result();
+ if (result._summary != null && result._summary.isPartialCoverage()) {
+ LOGGER.warn("Finished reading compression statistics for table: {} with {}/{} covered segments",
+ tableNameWithType, result._summary.getSegmentsWithCompleteStats(), result._summary.getTotalSegments());
+ } else {
+ LOGGER.info("Finished reading compression statistics for table: {}", tableNameWithType);
+ }
+ return result;
+ }
+
+ private boolean fillAvailableSlots(String tableNameWithType, @Nullable List<String> columns,
+ boolean includeColumnCompressionStats, long deadlineNanos, Deque<RequestBatch> retryableBatches,
+ Deque<RequestBatch> finalBatches, MultiHttpRequest.BufferedRequestExecution execution,
+ Map<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> activeRequests, int maxRequestsInFlight,
+ boolean preferFinal) {
+ int reservedRequestsPerClass = maxRequestsInFlight / 2;
+ while (activeRequests.size() < maxRequestsInFlight) {
+ int activeFinal = 0;
+ int activeRetryable = 0;
+ for (ActiveRequest request : activeRequests.values()) {
+ if (request._batch.isFinalAttempt()) {
+ activeFinal++;
+ } else {
+ activeRetryable++;
+ }
+ }
+ RequestBatch batch;
+ if (!finalBatches.isEmpty() && !retryableBatches.isEmpty()) {
+ if (activeFinal >= reservedRequestsPerClass) {
+ batch = retryableBatches.pollFirst();
+ } else if (activeRetryable >= reservedRequestsPerClass) {
+ batch = finalBatches.pollFirst();
+ } else if (preferFinal) {
+ batch = finalBatches.pollFirst();
+ preferFinal = false;
+ } else {
+ batch = retryableBatches.pollFirst();
+ preferFinal = true;
+ }
+ } else if (!retryableBatches.isEmpty()) {
+ batch = retryableBatches.pollFirst();
+ } else if (!finalBatches.isEmpty()) {
+ batch = finalBatches.pollFirst();
+ } else {
+ break;
+ }
+ if (!submit(batch, tableNameWithType, columns, includeColumnCompressionStats, deadlineNanos,
+ execution, activeRequests)) {
+ break;
+ }
+ }
+ return preferFinal;
+ }
+
+ private boolean submit(RequestBatch batch, String tableNameWithType, @Nullable List<String> columns,
+ boolean includeColumnCompressionStats, long overallDeadlineNanos,
+ MultiHttpRequest.BufferedRequestExecution execution,
+ Map<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> activeRequests) {
+ long now = System.nanoTime();
+ long remainingNanos = overallDeadlineNanos - now;
+ if (remainingNanos <= 0) {
+ return false;
+ }
+ long requestBudgetNanos = batch.isFinalAttempt() ? remainingNanos
+ : Math.max(1, remainingNanos / batch._attemptsRemaining);
+ long requestDeadlineNanos = Math.min(overallDeadlineNanos, now + requestBudgetNanos);
+ int timeoutMs = toTimeoutMs(requestBudgetNanos);
+ String url = buildUrl(tableNameWithType, columns, includeColumnCompressionStats,
+ batch._endpoint);
+ HttpPost request = new HttpPost(url);
+ request.setHeader("Content-Type", "application/json");
+ request.setConfig(RequestConfig.custom()
+ .setConnectionRequestTimeout(Timeout.of(timeoutMs, TimeUnit.MILLISECONDS))
+ .setResponseTimeout(Timeout.of(timeoutMs, TimeUnit.MILLISECONDS))
+ .build());
+ try {
+ request.setEntity(new StringEntity(JsonUtils.objectToString(new TableSegments(batch.segmentNames()))));
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to serialize compression segment batch", e);
+ }
+ Future<MultiHttpRequestBufferedResponse> future = execution.submit(request);
+ activeRequests.put(future, new ActiveRequest(batch, requestDeadlineNanos));
+ return true;
+ }
+
+ private boolean processCompleted(Future<MultiHttpRequestBufferedResponse> completed, ActiveRequest active,
+ CompressionStatsAggregation aggregation, Deque<RequestBatch> retryableBatches,
+ Deque<RequestBatch> finalBatches, int segmentsPerRequest) {
+ try {
+ MultiHttpRequestBufferedResponse response = completed.get();
+ if (response.getStatusCode() == 413 && active._batch._states.size() > 1) {
+ split(active._batch, retryableBatches, finalBatches);
+ return true;
+ }
+ if (response.getStatusCode() >= 300) {
+ LOGGER.error("Server returned error: {}, reason: {} for uri: {}", response.getStatusCode(),
+ response.getReasonPhrase(), response.getURI());
+ retry(active._batch, retryableBatches, finalBatches, segmentsPerRequest);
+ return true;
+ }
+ if (response.getResponseBody() == null) {
+ throw new IOException("Response body is empty");
+ }
+ ServerCompressionStatsResponse statsResponse =
+ JsonUtils.stringToObject(response.getResponseBody(), ServerCompressionStatsResponse.class);
+ if (statsResponse.getMetadataVersion() < ServerCompressionStatsResponse.CURRENT_METADATA_VERSION) {
+ retry(active._batch, retryableBatches, finalBatches, segmentsPerRequest);
+ return true;
+ }
+ Map<String, SegmentState> statesByName = active._batch.statesByName();
+ for (SegmentCompressionStatsContribution contribution : statsResponse.getSegmentCompressionStats()) {
+ SegmentState state = statesByName.get(contribution.getSegmentName());
+ if (state == null || state._complete) {
+ continue;
+ }
+ aggregation.add(state, contribution);
+ }
+ retry(active._batch, retryableBatches, finalBatches, segmentsPerRequest);
+ return true;
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (CancellationException | ExecutionException | IOException e) {
+ Throwable cause = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e;
+ LOGGER.error("Connection error in 'compression statistics'. Details: {}", cause.getMessage());
+ retry(active._batch, retryableBatches, finalBatches, segmentsPerRequest);
+ return true;
+ }
+ }
+
+ private static void expireRequests(long nowNanos,
+ Map<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> activeRequests,
+ Deque<RequestBatch> retryableBatches, Deque<RequestBatch> finalBatches,
+ MultiHttpRequest.BufferedRequestExecution execution, int segmentsPerRequest) {
+ List<Map.Entry<Future<MultiHttpRequestBufferedResponse>, ActiveRequest>> expired = new ArrayList<>();
+ for (Map.Entry<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> entry : activeRequests.entrySet()) {
+ if (entry.getValue()._deadlineNanos <= nowNanos) {
+ expired.add(entry);
+ }
+ }
+ for (Map.Entry<Future<MultiHttpRequestBufferedResponse>, ActiveRequest> entry : expired) {
+ activeRequests.remove(entry.getKey());
+ execution.cancel(entry.getKey());
+ retry(entry.getValue()._batch, retryableBatches, finalBatches, segmentsPerRequest);
+ }
+ }
+
+ private static void retry(RequestBatch batch, Deque<RequestBatch> retryableBatches,
+ Deque<RequestBatch> finalBatches, int segmentsPerRequest) {
+ List<SegmentState> nextAttempts = new ArrayList<>();
+ for (SegmentState state : batch._states) {
+ if (!state._complete && ++state._candidateIndex < state.candidateCount()) {
+ nextAttempts.add(state);
+ }
+ }
+ enqueue(nextAttempts, retryableBatches, finalBatches, segmentsPerRequest);
+ }
+
+ private static void enqueue(Iterable<SegmentState> states, Deque<RequestBatch> retryableBatches,
+ Deque<RequestBatch> finalBatches, int segmentsPerRequest) {
+ Map<RequestKey, List<SegmentState>> grouped = new TreeMap<>();
+ for (SegmentState state : states) {
+ if (state._complete || state._candidateIndex >= state.candidateCount()) {
+ continue;
+ }
+ int attemptsRemaining = state.candidateCount() - state._candidateIndex;
+ String server = state.currentServer();
+ grouped.computeIfAbsent(new RequestKey(server, attemptsRemaining), ignored -> new ArrayList<>()).add(state);
+ }
+ for (Map.Entry<RequestKey, List<SegmentState>> entry : grouped.entrySet()) {
+ List<SegmentState> groupedStates = entry.getValue();
+ groupedStates.sort(Comparator.comparing(state -> state._segmentName));
+ for (int start = 0; start < groupedStates.size(); start += segmentsPerRequest) {
+ int end = Math.min(start + segmentsPerRequest, groupedStates.size());
+ RequestBatch batch = new RequestBatch(entry.getKey()._attemptsRemaining,
+ new ArrayList<>(groupedStates.subList(start, end)));
+ (batch.isFinalAttempt() ? finalBatches : retryableBatches).addLast(batch);
+ }
+ }
+ }
+
+ private static void split(RequestBatch batch, Deque<RequestBatch> retryableBatches,
+ Deque<RequestBatch> finalBatches) {
+ int midpoint = batch._states.size() / 2;
+ Deque<RequestBatch> destination = batch.isFinalAttempt() ? finalBatches : retryableBatches;
+ destination.addFirst(new RequestBatch(batch._attemptsRemaining,
+ new ArrayList<>(batch._states.subList(midpoint, batch._states.size()))));
+ destination.addFirst(new RequestBatch(batch._attemptsRemaining,
+ new ArrayList<>(batch._states.subList(0, midpoint))));
+ }
+
+ private static int segmentsPerRequest(@Nullable List<String> columns, boolean includeColumnCompressionStats) {
+ if (!includeColumnCompressionStats) {
+ return MAX_SEGMENTS_PER_REQUEST;
+ }
+ int estimatedColumns = ESTIMATED_COLUMNS_WHEN_UNFILTERED;
+ if (columns != null && !columns.isEmpty() && !columns.contains("*")) {
+ estimatedColumns = Math.max(1, new HashSet<>(columns).size());
+ }
+ return Math.max(1, Math.min(MAX_SEGMENTS_PER_REQUEST,
+ ServerCompressionStatsResponse.MAX_COLUMN_CONTRIBUTIONS_PER_RESPONSE / estimatedColumns));
+ }
+
+ private static int toTimeoutMs(long timeoutNanos) {
+ return (int) Math.min(Integer.MAX_VALUE,
+ Math.max(1, TimeUnit.NANOSECONDS.toMillis(timeoutNanos) + 1));
+ }
+
+ private static SegmentStates createSegmentStates(Map<String, List<String>> serverToSegments,
+ BiMap<String, String> serverToEndpoints, long deadlineNanos) {
+ Map<String, SegmentState> states = new HashMap<>();
+ Set<String> segmentsWithoutState = null;
+ boolean collectCandidates = System.nanoTime() < deadlineNanos;
+ int segmentsUntilDeadlineCheck = 1_024;
+ for (Map.Entry<String, List<String>> entry : serverToSegments.entrySet()) {
+ boolean endpointAvailable = serverToEndpoints.containsKey(entry.getKey());
+ for (String segment : entry.getValue()) {
+ if (collectCandidates) {
+ SegmentState state = states.computeIfAbsent(segment,
+ ignored -> new SegmentState(segment, serverToEndpoints));
+ if (endpointAvailable) {
+ state.addCandidate(entry.getKey());
+ }
+ if (--segmentsUntilDeadlineCheck == 0) {
+ collectCandidates = System.nanoTime() < deadlineNanos;
+ segmentsUntilDeadlineCheck = 1_024;
+ }
+ } else if (!states.containsKey(segment)) {
+ if (segmentsWithoutState == null) {
+ segmentsWithoutState = new HashSet<>();
+ }
+ segmentsWithoutState.add(segment);
+ }
+ }
+ }
+ if (collectCandidates) {
+ for (SegmentState state : states.values()) {
+ state.finishCandidates();
+ }
+ }
+ int totalSegments = states.size() + (segmentsWithoutState != null ? segmentsWithoutState.size() : 0);
+ return new SegmentStates(states.values(), totalSegments);
+ }
+
+ private static String buildUrl(String tableNameWithType, @Nullable List<String> columns,
+ boolean includeColumnCompressionStats, String endpoint) {
+ String columnsParam = UrlBuilderUtils.generateColumnsParam(columns);
+ StringBuilder url = new StringBuilder(String.format("%s/tables/%s/compression-stats", endpoint,
+ URLEncoder.encode(tableNameWithType, StandardCharsets.UTF_8)));
+ if (columnsParam != null) {
+ url.append('?').append(columnsParam);
+ }
+ if (includeColumnCompressionStats) {
+ url.append(columnsParam != null ? '&' : '?').append("includeColumnCompressionStats=true");
+ }
+ return url.toString();
+ }
+
+ /// Owns controller-side replica selection and aggregation for one read. Segment state and its selected columns stay
+ /// together so retries cannot update one without the other.
+ private static final class CompressionStatsAggregation {
+ private final int _totalSegments;
+ private final boolean _includeColumnStats;
+ private final boolean _includeSelectedSegments;
+ private final Map<String, ColumnCompressionStatsAccumulator> _columnAccumulators = new HashMap<>();
+ private final Map<String, SelectedSegmentCompressionStats> _selectedSegments = new HashMap<>();
+ private long _uncompressedValueSizeInBytes;
+ private long _forwardIndexAndDictionaryStorageSizeInBytes;
+ private int _segmentsWithCompleteStats;
+
+ private CompressionStatsAggregation(int totalSegments, boolean includeColumnStats,
+ boolean includeSelectedSegments) {
+ _totalSegments = totalSegments;
+ _includeColumnStats = includeColumnStats;
+ _includeSelectedSegments = includeSelectedSegments;
+ }
+
+ private void add(SegmentState state, SegmentCompressionStatsContribution contribution) {
+ boolean selectedCompleteContribution = contribution.isComplete()
+ && contribution.getUncompressedValueSizeInBytes() >= 0
+ && contribution.getForwardIndexAndDictionaryStorageSizeInBytes() >= 0;
+ if (selectedCompleteContribution) {
+ _uncompressedValueSizeInBytes += contribution.getUncompressedValueSizeInBytes();
+ _forwardIndexAndDictionaryStorageSizeInBytes +=
+ contribution.getForwardIndexAndDictionaryStorageSizeInBytes();
+ _segmentsWithCompleteStats++;
+ if (_includeSelectedSegments) {
+ _selectedSegments.put(state._segmentName,
+ new SelectedSegmentCompressionStats(state.currentServer(), contribution));
+ }
+ }
+
+ Map<String, ColumnCompressionStatsContribution> columnStats = contribution.getColumnCompressionStats();
+ if (_includeColumnStats && columnStats != null) {
+ Set<String> selectedColumns = state._selectedColumns;
+ for (Map.Entry<String, ColumnCompressionStatsContribution> entry : columnStats.entrySet()) {
+ if (!isUsable(entry.getValue())) {
+ continue;
+ }
+ if (selectedColumns == null && !selectedCompleteContribution) {
+ selectedColumns = new HashSet<>();
+ state._selectedColumns = selectedColumns;
+ }
+ if (selectedColumns == null || selectedColumns.add(entry.getKey())) {
+ _columnAccumulators.computeIfAbsent(entry.getKey(), ignored -> new ColumnCompressionStatsAccumulator())
+ .add(entry.getValue());
+ }
+ }
+ }
+
+ if (selectedCompleteContribution) {
+ state._complete = true;
+ state._selectedColumns = null;
+ }
+ }
+
+ private CompressionStatsResult result() {
+ double ratio = _forwardIndexAndDictionaryStorageSizeInBytes > 0
+ ? (double) _uncompressedValueSizeInBytes / _forwardIndexAndDictionaryStorageSizeInBytes : 0;
+ CompressionStatsSummary summary = new CompressionStatsSummary(_uncompressedValueSizeInBytes,
+ _forwardIndexAndDictionaryStorageSizeInBytes, ratio, _segmentsWithCompleteStats, _totalSegments,
+ _segmentsWithCompleteStats < _totalSegments);
+ if (!_includeColumnStats || _columnAccumulators.isEmpty()) {
+ return new CompressionStatsResult(summary, null, _selectedSegments);
+ }
+ List<ColumnCompressionStatsInfo> columnStats = new ArrayList<>(_columnAccumulators.size());
+ for (Map.Entry<String, ColumnCompressionStatsAccumulator> entry : _columnAccumulators.entrySet()) {
+ columnStats.add(entry.getValue().toColumnCompressionStatsInfo(entry.getKey()));
+ }
+ columnStats.sort(Comparator.comparing(ColumnCompressionStatsInfo::getColumn));
+ return new CompressionStatsResult(summary, columnStats, _selectedSegments);
+ }
+
+ private static boolean isUsable(ColumnCompressionStatsContribution candidate) {
+ return candidate.getUncompressedValueSizeInBytes() >= 0
+ && candidate.getForwardIndexAndDictionaryStorageSizeInBytes() >= 0
+ && candidate.getNumSegments() > 0 && !candidate.getEncodingBreakdown().isEmpty();
+ }
+ }
+
+ /// Immutable aggregate plus optional per-segment selections from one compression read.
+ static final class CompressionStatsResult {
+ @Nullable
+ private final CompressionStatsSummary _summary;
+ @Nullable
+ private final List<ColumnCompressionStatsInfo> _columnStats;
+ private final Map<String, SelectedSegmentCompressionStats> _selectedSegments;
+
+ private CompressionStatsResult(@Nullable CompressionStatsSummary summary,
+ @Nullable List<ColumnCompressionStatsInfo> columnStats,
+ Map<String, SelectedSegmentCompressionStats> selectedSegments) {
+ _summary = summary;
+ _columnStats = columnStats;
+ _selectedSegments = Map.copyOf(selectedSegments);
+ }
+
+ @Nullable
+ CompressionStatsSummary getSummary() {
+ return _summary;
+ }
+
+ @Nullable
+ List<ColumnCompressionStatsInfo> getColumnStats() {
+ return _columnStats;
+ }
+
+ Map<String, SelectedSegmentCompressionStats> getSelectedSegments() {
+ return _selectedSegments;
+ }
+ }
+
+ /// Complete statistics selected from one source replica for a logical segment.
+ static final class SelectedSegmentCompressionStats {
+ private final String _server;
+ private final SegmentCompressionStatsContribution _contribution;
+
+ private SelectedSegmentCompressionStats(String server,
+ SegmentCompressionStatsContribution contribution) {
+ _server = server;
+ _contribution = contribution;
+ }
+
+ String getServer() {
+ return _server;
+ }
+
+ SegmentCompressionStatsContribution getContribution() {
+ return _contribution;
+ }
+ }
+
+ private static final class SegmentState {
+ private final String _segmentName;
+ private final List<String> _candidates = new ArrayList<>();
+ private final BiMap<String, String> _serverToEndpoints;
+ private int _candidateOffset;
+ private int _candidateIndex;
+ private boolean _complete;
+ @Nullable
+ private Set<String> _selectedColumns;
+
+ private SegmentState(String segmentName, BiMap<String, String> serverToEndpoints) {
+ _segmentName = segmentName;
+ _serverToEndpoints = serverToEndpoints;
+ }
+
+ private void addCandidate(String candidate) {
+ if (_candidates.isEmpty() || !_candidates.get(_candidates.size() - 1).equals(candidate)) {
+ _candidates.add(candidate);
+ }
+ }
+
+ private void finishCandidates() {
+ _candidates.sort(String::compareTo);
+ if (_candidates.size() > 1) {
+ _candidateOffset = Math.floorMod(_segmentName.hashCode() ^ REPLICA_ROTATION_SALT, _candidates.size());
+ }
+ }
+
+ private int candidateCount() {
+ return _candidates.size();
+ }
+
+ private String currentServer() {
+ return _candidates.get((_candidateOffset + _candidateIndex) % _candidates.size());
+ }
+
+ private String endpoint() {
+ return _serverToEndpoints.get(currentServer());
+ }
+ }
+
+ private static final class SegmentStates {
+ private final Collection<SegmentState> _states;
+ private final int _totalSegments;
+
+ private SegmentStates(Collection<SegmentState> states, int totalSegments) {
+ _states = states;
+ _totalSegments = totalSegments;
+ }
+ }
+
+ private static final class RequestKey implements Comparable<RequestKey> {
+ private final String _server;
+ private final int _attemptsRemaining;
+
+ private RequestKey(String server, int attemptsRemaining) {
+ _server = server;
+ _attemptsRemaining = attemptsRemaining;
+ }
+
+ @Override
+ public int compareTo(RequestKey other) {
+ int serverComparison = _server.compareTo(other._server);
+ return serverComparison != 0 ? serverComparison : Integer.compare(_attemptsRemaining, other._attemptsRemaining);
+ }
+ }
+
+ private static final class RequestBatch {
+ private final String _endpoint;
+ private final int _attemptsRemaining;
+ private final List<SegmentState> _states;
+
+ private RequestBatch(int attemptsRemaining, List<SegmentState> states) {
+ _endpoint = states.get(0).endpoint();
+ _attemptsRemaining = attemptsRemaining;
+ _states = states;
+ }
+
+ private boolean isFinalAttempt() {
+ return _attemptsRemaining == 1;
+ }
+
+ private List<String> segmentNames() {
+ List<String> names = new ArrayList<>(_states.size());
+ for (SegmentState state : _states) {
+ names.add(state._segmentName);
+ }
+ return names;
+ }
+
+ private Map<String, SegmentState> statesByName() {
+ Map<String, SegmentState> states = new HashMap<>();
+ for (SegmentState state : _states) {
+ states.put(state._segmentName, state);
+ }
+ return states;
+ }
+ }
+
+ private static final class ActiveRequest {
+ private final RequestBatch _batch;
+ private final long _deadlineNanos;
+
+ private ActiveRequest(RequestBatch batch, long deadlineNanos) {
+ _batch = batch;
+ _deadlineNanos = deadlineNanos;
+ }
+ }
+}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
index a079f6af06e..05cf61ae425 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -34,6 +34,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.ws.rs.client.ClientBuilder;
@@ -42,6 +43,7 @@ import javax.ws.rs.core.Response;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
+import org.apache.pinot.common.restlet.resources.ServerTableMetadataInfo;
import org.apache.pinot.common.restlet.resources.TableMetadataInfo;
import org.apache.pinot.common.restlet.resources.TableSegments;
import org.apache.pinot.common.restlet.resources.ValidDocIdsBitmapResponse;
@@ -92,50 +94,65 @@ public class ServerSegmentMetadataReader {
* - If table does not have replica groups, send requests to a minimal set of servers hosting all segments of the
* table.
*/
+ /// Reads aggregate metadata without compression statistics.
public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWithType,
- BiMap<String, String> serverEndPoints, List<String> columns, int numReplica, int timeoutMs) {
+ BiMap<String, String> serverEndPoints, @Nullable List<String> columns, int numReplica, int timeoutMs) {
+ return getAggregatedTableMetadataFromServer(tableNameWithType, serverEndPoints, columns, numReplica, timeoutMs,
+ false, false, Map.of());
+ }
+
+ public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWithType,
+ BiMap<String, String> serverEndPoints, @Nullable List<String> columns, int numReplica, int timeoutMs,
+ boolean compressionStatsEnabled, boolean includeColumnCompressionStats,
+ Map<String, List<String>> serverToSegmentsMap) {
int numServers = serverEndPoints.size();
LOGGER.info("Reading aggregated segment metadata from {} servers for table: {} with timeout: {}ms", numServers,
tableNameWithType, timeoutMs);
+ long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMs);
List<String> serverUrls = new ArrayList<>(numServers);
BiMap<String, String> endpointsToServers = serverEndPoints.inverse();
for (String endpoint : endpointsToServers.keySet()) {
- String serverUrl = generateAggregateSegmentMetadataServerURL(tableNameWithType, columns, endpoint);
+ String serverUrl = generateAggregateSegmentMetadataServerURL(tableNameWithType, columns, endpoint, false);
serverUrls.add(serverUrl);
}
// Helper service to run a http get call to the server
CompletionServiceHelper completionServiceHelper =
new CompletionServiceHelper(_executor, _connectionManager, endpointsToServers);
- CompletionServiceHelper.CompletionServiceResponse serviceResponse =
- completionServiceHelper.doMultiGetRequest(serverUrls, tableNameWithType, false, timeoutMs);
+ CompletionServiceHelper.CompletionServiceResponse serviceResponse = compressionStatsEnabled
+ ? completionServiceHelper.doMultiGetRequestUntil(serverUrls, tableNameWithType, false, timeoutMs, deadlineNanos,
+ "aggregate segment metadata")
+ : completionServiceHelper.doMultiGetRequest(serverUrls, tableNameWithType, false, timeoutMs,
+ "aggregate segment metadata");
long totalDiskSizeInBytes = 0;
int totalNumSegments = 0;
long totalNumRows = 0;
int failedParses = 0;
- final Map<String, Double> columnLengthMap = new HashMap<>();
- final Map<String, Double> columnCardinalityMap = new HashMap<>();
- final Map<String, Double> maxNumMultiValuesMap = new HashMap<>();
- final Map<String, Map<String, Double>> columnIndexSizeMap = new HashMap<>();
- final Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap = new HashMap<>();
+ Map<String, Double> columnLengthMap = new HashMap<>();
+ Map<String, Double> columnCardinalityMap = new HashMap<>();
+ Map<String, Double> maxNumMultiValuesMap = new HashMap<>();
+ Map<String, Map<String, Double>> columnIndexSizeMap = new HashMap<>();
+ Map<Integer, Map<String, Long>> partitionToServerPrimaryKeyCountMap = new HashMap<>();
for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
try {
- TableMetadataInfo tableMetadataInfo =
- JsonUtils.stringToObject(streamResponse.getValue(), TableMetadataInfo.class);
+ ServerTableMetadataInfo tableMetadataInfo =
+ JsonUtils.stringToObject(streamResponse.getValue(), ServerTableMetadataInfo.class);
totalDiskSizeInBytes += tableMetadataInfo.getDiskSizeInBytes();
totalNumRows += tableMetadataInfo.getNumRows();
totalNumSegments += tableMetadataInfo.getNumSegments();
tableMetadataInfo.getColumnLengthMap().forEach((k, v) -> columnLengthMap.merge(k, v, Double::sum));
tableMetadataInfo.getColumnCardinalityMap().forEach((k, v) -> columnCardinalityMap.merge(k, v, Double::sum));
tableMetadataInfo.getMaxNumMultiValuesMap().forEach((k, v) -> maxNumMultiValuesMap.merge(k, v, Double::sum));
- tableMetadataInfo.getColumnIndexSizeMap().forEach((k, v) -> columnIndexSizeMap.merge(k, v, (l, r) -> {
- for (Map.Entry<String, Double> e : r.entrySet()) {
- l.put(e.getKey(), l.getOrDefault(e.getKey(), 0d) + e.getValue());
+ tableMetadataInfo.getColumnIndexSizeMap().forEach((column, indexSizes) -> {
+ Map<String, Double> aggregateIndexSizes = columnIndexSizeMap.get(column);
+ if (aggregateIndexSizes == null) {
+ columnIndexSizeMap.put(column, new HashMap<>(indexSizes));
+ } else {
+ indexSizes.forEach((index, size) -> aggregateIndexSizes.merge(index, size, Double::sum));
}
- return l;
- }));
+ });
tableMetadataInfo.getPartitionToServerPrimaryKeyCountMap().forEach(
(partition, serverToPrimaryKeyCount) -> partitionToServerPrimaryKeyCountMap.merge(partition,
new HashMap<>(serverToPrimaryKeyCount), (l, r) -> {
@@ -165,9 +182,15 @@ public class ServerSegmentMetadataReader {
totalNumSegments /= numReplica;
totalNumRows /= numReplica;
+ ServerCompressionStatsReader.CompressionStatsResult compressionStats = compressionStatsEnabled
+ ? new ServerCompressionStatsReader(_executor, _connectionManager).read(tableNameWithType,
+ serverToSegmentsMap, serverEndPoints, columns, includeColumnCompressionStats, deadlineNanos) : null;
+
TableMetadataInfo aggregateTableMetadataInfo =
new TableMetadataInfo(tableNameWithType, totalDiskSizeInBytes, totalNumSegments, totalNumRows, columnLengthMap,
- columnCardinalityMap, maxNumMultiValuesMap, columnIndexSizeMap, partitionToServerPrimaryKeyCountMap);
+ columnCardinalityMap, maxNumMultiValuesMap, columnIndexSizeMap, partitionToServerPrimaryKeyCountMap,
+ compressionStats != null ? compressionStats.getColumnStats() : null,
+ compressionStats != null ? compressionStats.getSummary() : null);
if (failedParses != 0) {
LOGGER.warn("Failed to parse {} / {} aggregated segment metadata responses from servers.", failedParses,
serverUrls.size());
@@ -458,11 +481,20 @@ public class ServerSegmentMetadataReader {
}
private String generateAggregateSegmentMetadataServerURL(String tableNameWithType, @Nullable List<String> columns,
- String endpoint) {
+ String endpoint, boolean includeColumnCompressionStats) {
tableNameWithType = encode(tableNameWithType);
String columnsParam = UrlBuilderUtils.generateColumnsParam(columns);
String url = String.format("%s/tables/%s/metadata", endpoint, tableNameWithType);
- return columnsParam != null ? url + "?" + columnsParam : url;
+ StringBuilder sb = new StringBuilder(url);
+ if (columnsParam != null) {
+ sb.append("?").append(columnsParam);
+ if (includeColumnCompressionStats) {
+ sb.append("&includeColumnCompressionStats=true");
+ }
+ } else if (includeColumnCompressionStats) {
+ sb.append("?includeColumnCompressionStats=true");
+ }
+ return sb.toString();
}
public String generateSegmentMetadataServerURL(String tableNameWithType, String segmentName,
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
index 7219fa2ec62..2824ef823ef 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
@@ -254,14 +254,19 @@ public class TableMetadataReader {
return JsonUtils.objectToJsonNode(new HashMap<String, String>());
}
- /**
- * This method retrieves the aggregated segment metadata for a given table.
- * Currently supports only OFFLINE tables.
- * @return a map of segmentName to its metadata
- */
- public JsonNode getAggregateTableMetadata(String tableNameWithType, List<String> columns, int numReplica,
+ /// Retrieves aggregated segment metadata for an offline table without compression statistics.
+ public JsonNode getAggregateTableMetadata(String tableNameWithType, @Nullable List<String> columns, int numReplica,
int timeoutMs)
throws InvalidConfigException {
+ return getAggregateTableMetadata(tableNameWithType, columns, numReplica, timeoutMs, false, false);
+ }
+
+ /// Retrieves aggregated segment metadata for an offline table with explicit compression-statistics controls.
+ ///
+ /// @return aggregate metadata as JSON
+ public JsonNode getAggregateTableMetadata(String tableNameWithType, @Nullable List<String> columns, int numReplica,
+ int timeoutMs, boolean compressionStatsEnabled, boolean includeColumnCompressionStats)
+ throws InvalidConfigException {
final Map<String, List<String>> serverToSegments =
_pinotHelixResourceManager.getServerToSegmentsMap(tableNameWithType);
BiMap<String, String> endpoints =
@@ -271,7 +276,7 @@ public class TableMetadataReader {
TableMetadataInfo aggregateTableMetadataInfo =
serverSegmentMetadataReader.getAggregatedTableMetadataFromServer(tableNameWithType, endpoints, columns,
- numReplica, timeoutMs);
+ numReplica, timeoutMs, compressionStatsEnabled, includeColumnCompressionStats, serverToSegments);
return JsonUtils.objectToJsonNode(aggregateTableMetadataInfo);
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
index bfbf7a4e4e7..97c820bc757 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
@@ -19,6 +19,7 @@
package org.apache.pinot.controller.util;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.BiMap;
@@ -27,14 +28,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnegative;
import javax.annotation.Nullable;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
+import org.apache.pinot.common.compression.ColumnCompressionStatsAccumulator;
import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.metadata.ZKMetadataProvider;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.common.restlet.resources.CompressionStatsSummary;
+import org.apache.pinot.common.restlet.resources.SegmentCompressionStatsContribution;
import org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
+import org.apache.pinot.common.restlet.resources.TableSizeInfo;
import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.api.resources.ServerTableSizeReader;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
@@ -51,11 +58,31 @@ public class TableSizeReader {
private static final Logger LOGGER = LoggerFactory.getLogger(TableSizeReader.class);
public static final long DEFAULT_SIZE_WHEN_MISSING_OR_ERROR = -1L;
+ /// Selects which compression fields are returned by a table-size read.
+ public enum CompressionStatsMode {
+ NONE(false, false, false),
+ AGGREGATE_SUMMARY(true, false, false),
+ SUMMARY_WITH_SEGMENT_DETAILS(true, false, true),
+ COLUMNS_WITH_SEGMENT_DETAILS(false, true, true),
+ SUMMARY_AND_COLUMNS_WITH_SEGMENT_DETAILS(true, true, true);
+
+ private final boolean _includeSummary;
+ private final boolean _includeColumns;
+ private final boolean _includeSelectedSegments;
+
+ CompressionStatsMode(boolean includeSummary, boolean includeColumns, boolean includeSelectedSegments) {
+ _includeSummary = includeSummary;
+ _includeColumns = includeColumns;
+ _includeSelectedSegments = includeSelectedSegments;
+ }
+ }
+
private final Executor _executor;
private final HttpClientConnectionManager _connectionManager;
private final PinotHelixResourceManager _helixResourceManager;
private final ControllerMetrics _controllerMetrics;
private final LeadControllerManager _leadControllerManager;
+ private final Object _compressionMetricsLock = new Object();
public TableSizeReader(Executor executor, HttpClientConnectionManager connectionManager,
ControllerMetrics controllerMetrics, PinotHelixResourceManager helixResourceManager,
@@ -67,23 +94,42 @@ public class TableSizeReader {
_leadControllerManager = leadControllerManager;
}
- /**
- * Get the table size.
- * This one polls all servers in parallel for segment sizes. In the response,
- * reported size indicates actual sizes collected from servers. For errors,
- * we use the size of largest segment as an estimate.
- * Returns null if the table is not found.
- * @param tableName table name without type
- * @param timeoutMsec timeout in milliseconds for reading table sizes from server
- * @param includeReplacedSegments include replaced segments in table size calculation
- * @return
- */
+ /// Reads table size without collecting compression statistics.
+ ///
+ /// This method polls all servers in parallel for segment sizes. Reported size contains the sizes returned by
+ /// servers; missing replica sizes are estimated from the largest reported replica of the same segment.
+ ///
+ /// @param tableName table name without type
+ /// @param timeoutMsec timeout in milliseconds for reading table sizes from servers
+ /// @param includeReplacedSegments whether replaced segments are included in table size calculation
+ /// @return table size details, or `null` when the table does not exist
@Nullable
public TableSizeDetails getTableSizeDetails(String tableName, @Nonnegative int timeoutMsec,
boolean includeReplacedSegments)
throws InvalidConfigException {
+ return getTableSizeDetails(tableName, timeoutMsec, includeReplacedSegments, CompressionStatsMode.NONE);
+ }
+
+ /// Reads table size with compression summaries and optional per-column details.
+ @Nullable
+ public TableSizeDetails getTableSizeDetails(String tableName, @Nonnegative int timeoutMsec,
+ boolean includeReplacedSegments, boolean includeColumnCompressionStats)
+ throws InvalidConfigException {
+ return getTableSizeDetails(tableName, timeoutMsec, includeReplacedSegments,
+ includeColumnCompressionStats ? CompressionStatsMode.SUMMARY_AND_COLUMNS_WITH_SEGMENT_DETAILS
+ : CompressionStatsMode.SUMMARY_WITH_SEGMENT_DETAILS);
+ }
+
+ /// Reads table size with an explicit compression response mode.
+ @Nullable
+ public TableSizeDetails getTableSizeDetails(String tableName, @Nonnegative int timeoutMsec,
+ boolean includeReplacedSegments, CompressionStatsMode compressionStatsMode)
+ throws InvalidConfigException {
Preconditions.checkNotNull(tableName, "Table name should not be null");
+ Preconditions.checkNotNull(compressionStatsMode, "Compression stats mode should not be null");
Preconditions.checkArgument(timeoutMsec > 0, "Timeout value must be greater than 0");
+ boolean includeCompressionStats = compressionStatsMode._includeSummary;
+ boolean includeColumnCompressionStats = compressionStatsMode._includeColumns;
TableConfig offlineTableConfig =
ZKMetadataProvider.getOfflineTableConfig(_helixResourceManager.getPropertyStore(), tableName);
@@ -101,7 +147,12 @@ public class TableSizeReader {
TableSizeDetails tableSizeDetails = new TableSizeDetails(tableName);
if (hasRealtimeTableConfig) {
String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName);
- tableSizeDetails._realtimeSegments = getTableSubtypeSize(realtimeTableName, timeoutMsec, includeReplacedSegments);
+ boolean compressionStatsCollectionEnabled = compressionStatsMode != CompressionStatsMode.NONE
+ && isCompressionStatsEnabled(realtimeTableConfig);
+ tableSizeDetails._realtimeSegments =
+ getTableSubtypeSize(realtimeTableName, timeoutMsec, includeReplacedSegments,
+ compressionStatsCollectionEnabled, includeColumnCompressionStats,
+ compressionStatsMode._includeSelectedSegments);
// taking max(0,value) as values as set to -1 if all the segments are in error
tableSizeDetails._reportedSizeInBytes += Math.max(tableSizeDetails._realtimeSegments._reportedSizeInBytes, 0L);
tableSizeDetails._estimatedSizeInBytes += Math.max(tableSizeDetails._realtimeSegments._estimatedSizeInBytes, 0L);
@@ -123,11 +174,24 @@ public class TableSizeReader {
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
emitMetrics(realtimeTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER, largestSegmentSizeOnServer);
+ } else {
+ removeMetric(realtimeTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER);
+ }
+ if (!includeColumnCompressionStats) {
+ tableSizeDetails._realtimeSegments._columnCompressionStats = null;
+ }
+ if (!includeCompressionStats) {
+ tableSizeDetails._realtimeSegments._compressionStats = null;
}
}
if (hasOfflineTableConfig) {
String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName);
- tableSizeDetails._offlineSegments = getTableSubtypeSize(offlineTableName, timeoutMsec, includeReplacedSegments);
+ boolean compressionStatsCollectionEnabled = compressionStatsMode != CompressionStatsMode.NONE
+ && isCompressionStatsEnabled(offlineTableConfig);
+ tableSizeDetails._offlineSegments =
+ getTableSubtypeSize(offlineTableName, timeoutMsec, includeReplacedSegments,
+ compressionStatsCollectionEnabled, includeColumnCompressionStats,
+ compressionStatsMode._includeSelectedSegments);
// taking max(0,value) as values as set to -1 if all the segments are in error
tableSizeDetails._reportedSizeInBytes += Math.max(tableSizeDetails._offlineSegments._reportedSizeInBytes, 0L);
tableSizeDetails._estimatedSizeInBytes += Math.max(tableSizeDetails._offlineSegments._estimatedSizeInBytes, 0L);
@@ -149,6 +213,14 @@ public class TableSizeReader {
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
emitMetrics(offlineTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER, largestSegmentSizeOnServer);
+ } else {
+ removeMetric(offlineTableName, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER);
+ }
+ if (!includeColumnCompressionStats) {
+ tableSizeDetails._offlineSegments._columnCompressionStats = null;
+ }
+ if (!includeCompressionStats) {
+ tableSizeDetails._offlineSegments._compressionStats = null;
}
}
@@ -163,12 +235,64 @@ public class TableSizeReader {
return tableSizeDetails;
}
+ /// Publishes compression gauges for one typed table. The periodic segment status checker is the sole caller so a
+ /// concurrent REST size request cannot republish gauges from a stale table-config snapshot.
+ public void updateCompressionMetrics(String tableNameWithType, TableSubTypeSizeDetails subTypeDetails) {
+ synchronized (_compressionMetricsLock) {
+ if (!_leadControllerManager.isLeaderForTable(tableNameWithType)) {
+ return;
+ }
+ CompressionStatsSummary stats = subTypeDetails._compressionStats;
+ if (stats != null && stats.getSegmentsWithCompleteStats() > 0
+ && stats.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes() > 0) {
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA,
+ stats.getUncompressedValueSizePerReplicaInBytes());
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA,
+ stats.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes());
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT,
+ Math.round(stats.getCompressionRatio() * 100));
+ } else {
+ clearCompressionMetricsLocked(tableNameWithType);
+ }
+ }
+ }
+
+ /// Removes all compression gauges for the table from this controller.
+ public void clearCompressionMetrics(String tableNameWithType) {
+ synchronized (_compressionMetricsLock) {
+ clearCompressionMetricsLocked(tableNameWithType);
+ }
+ }
+
+ private void clearCompressionMetricsLocked(String tableNameWithType) {
+ _controllerMetrics.removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA);
+ _controllerMetrics.removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA);
+ _controllerMetrics.removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT);
+ }
+
private void emitMetrics(String tableNameWithType, ControllerGauge controllerGauge, long value) {
if (_leadControllerManager.isLeaderForTable(tableNameWithType)) {
_controllerMetrics.setValueOfTableGauge(tableNameWithType, controllerGauge, value);
}
}
+ private void removeMetric(String tableNameWithType, ControllerGauge controllerGauge) {
+ if (_leadControllerManager.isLeaderForTable(tableNameWithType)) {
+ _controllerMetrics.removeTableGauge(tableNameWithType, controllerGauge);
+ }
+ }
+
+ private static boolean isCompressionStatsEnabled(@Nullable TableConfig tableConfig) {
+ return tableConfig != null && tableConfig.getIndexingConfig() != null
+ && tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ }
+
//
// Reported size below indicates the sizes actually reported by servers on successful responses.
// Estimated sizes indicates the size estimated size with approximated calculations for errored servers
@@ -222,6 +346,16 @@ public class TableSizeReader {
@JsonProperty("reportedSizePerReplicaInBytes")
public long _reportedSizePerReplicaInBytes = 0;
+ @Nullable
+ @JsonProperty("compressionStats")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public CompressionStatsSummary _compressionStats;
+
+ @Nullable
+ @JsonProperty("columnCompressionStats")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public List<ColumnCompressionStatsInfo> _columnCompressionStats;
+
@JsonProperty("segments")
public Map<String, SegmentSizeDetails> _segments = new HashMap<>();
}
@@ -240,25 +374,44 @@ public class TableSizeReader {
@JsonProperty("serverInfo")
public Map<String, SegmentSizeInfo> _serverInfo = new HashMap<>();
+
+ public SegmentSizeDetails() {
+ }
}
+ /// Reads one typed table size without collecting compression statistics.
public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int timeoutMs,
boolean includeReplacedSegments)
throws InvalidConfigException {
+ return getTableSubtypeSize(tableNameWithType, timeoutMs, includeReplacedSegments, false, false, false);
+ }
+
+ public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int timeoutMs,
+ boolean includeReplacedSegments, boolean includeColumnCompressionStats)
+ throws InvalidConfigException {
+ return getTableSubtypeSize(tableNameWithType, timeoutMs, includeReplacedSegments, true,
+ includeColumnCompressionStats, true);
+ }
+
+ private TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int timeoutMs,
+ boolean includeReplacedSegments, boolean compressionStatsEnabled, boolean includeColumnCompressionStats,
+ boolean includeSelectedCompressionStats)
+ throws InvalidConfigException {
+ long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMs);
Map<String, List<String>> serverToSegmentsMap = _helixResourceManager.getServerToSegmentsMap(tableNameWithType,
null, includeReplacedSegments);
ServerTableSizeReader serverTableSizeReader = new ServerTableSizeReader(_executor, _connectionManager);
BiMap<String, String> endpoints = _helixResourceManager.getDataInstanceAdminEndpoints(serverToSegmentsMap.keySet());
- Map<String, List<SegmentSizeInfo>> serverToSegmentSizeInfoListMap =
- serverTableSizeReader.getSegmentSizeInfoFromServers(endpoints, tableNameWithType, timeoutMs);
+ Map<String, TableSizeInfo> serverToTableSizeInfoMap =
+ serverTableSizeReader.getTableSizeInfoFromServers(endpoints, tableNameWithType, timeoutMs);
TableSubTypeSizeDetails subTypeSizeDetails = new TableSubTypeSizeDetails();
Map<String, SegmentSizeDetails> segmentToSizeDetailsMap = subTypeSizeDetails._segments;
Map<String, Map<String, SegmentSizeInfo>> serverToSegmentMap = new HashMap<>();
- for (Map.Entry<String, List<SegmentSizeInfo>> entry : serverToSegmentSizeInfoListMap.entrySet()) {
+ for (Map.Entry<String, TableSizeInfo> entry : serverToTableSizeInfoMap.entrySet()) {
Map<String, SegmentSizeInfo> segmentMap = new HashMap<>();
- for (SegmentSizeInfo info : entry.getValue()) {
+ for (SegmentSizeInfo info : entry.getValue().getSegments()) {
segmentMap.put(info.getSegmentName(), info);
}
serverToSegmentMap.put(entry.getKey(), segmentMap);
@@ -296,7 +449,7 @@ public class TableSizeReader {
for (Map.Entry<String, SegmentSizeDetails> entry : segmentToSizeDetailsMap.entrySet()) {
String segment = entry.getKey();
SegmentSizeDetails sizeDetails = entry.getValue();
- // Iterate over all segment size info, update reported size, track max segment size and number of errored servers
+ // Iterate over all segment size info: update reported size, track max segment size and count errored servers.
sizeDetails._maxReportedSizePerReplicaInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
int errors = 0;
for (SegmentSizeInfo sizeInfo : sizeDetails._serverInfo.values()) {
@@ -308,6 +461,7 @@ public class TableSizeReader {
errors++;
}
}
+
// Update estimated size, track segments that are missing from all servers
if (errors != sizeDetails._serverInfo.size()) {
// Use max segment size from other servers to estimate the segment size not reported
@@ -326,6 +480,21 @@ public class TableSizeReader {
}
}
+ if (compressionStatsEnabled) {
+ ServerCompressionStatsReader compressionStatsReader =
+ new ServerCompressionStatsReader(_executor, _connectionManager);
+ ServerCompressionStatsReader.CompressionStatsResult compressionStats = includeSelectedCompressionStats
+ ? compressionStatsReader.readWithSelectedSegments(tableNameWithType, serverToSegmentsMap, endpoints, null,
+ includeColumnCompressionStats, deadlineNanos)
+ : compressionStatsReader.read(tableNameWithType, serverToSegmentsMap, endpoints, null,
+ includeColumnCompressionStats, deadlineNanos);
+ subTypeSizeDetails._compressionStats = compressionStats.getSummary();
+ subTypeSizeDetails._columnCompressionStats = compressionStats.getColumnStats();
+ if (includeSelectedCompressionStats) {
+ attachSelectedCompressionStats(segmentToSizeDetailsMap, compressionStats.getSelectedSegments());
+ }
+ }
+
// Update metrics for missing segments
if (subTypeSizeDetails._missingSegments > 0) {
int numSegments = segmentToSizeDetailsMap.size();
@@ -346,4 +515,41 @@ public class TableSizeReader {
return subTypeSizeDetails;
}
+
+ private static void attachSelectedCompressionStats(Map<String, SegmentSizeDetails> segmentToSizeDetailsMap,
+ Map<String, ServerCompressionStatsReader.SelectedSegmentCompressionStats> selectedSegments) {
+ for (Map.Entry<String, ServerCompressionStatsReader.SelectedSegmentCompressionStats> entry
+ : selectedSegments.entrySet()) {
+ SegmentSizeDetails segmentSizeDetails = segmentToSizeDetailsMap.get(entry.getKey());
+ if (segmentSizeDetails == null) {
+ continue;
+ }
+ ServerCompressionStatsReader.SelectedSegmentCompressionStats selected = entry.getValue();
+ SegmentSizeInfo existing = segmentSizeDetails._serverInfo.get(selected.getServer());
+ if (existing == null) {
+ continue;
+ }
+ SegmentCompressionStatsContribution contribution = selected.getContribution();
+ segmentSizeDetails._serverInfo.put(selected.getServer(),
+ new SegmentSizeInfo(entry.getKey(), existing.getDiskSizeInBytes(),
+ contribution.getUncompressedValueSizeInBytes(),
+ contribution.getForwardIndexAndDictionaryStorageSizeInBytes(),
+ toColumnCompressionStatsInfo(contribution)));
+ }
+ }
+
+ @Nullable
+ private static Map<String, ColumnCompressionStatsInfo> toColumnCompressionStatsInfo(
+ SegmentCompressionStatsContribution contribution) {
+ if (contribution.getColumnCompressionStats() == null) {
+ return null;
+ }
+ Map<String, ColumnCompressionStatsInfo> result = new HashMap<>();
+ contribution.getColumnCompressionStats().forEach((column, columnContribution) -> {
+ ColumnCompressionStatsAccumulator accumulator = new ColumnCompressionStatsAccumulator();
+ accumulator.add(columnContribution);
+ result.put(column, accumulator.toColumnCompressionStatsInfo(column));
+ });
+ return result;
+ }
}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderCompressionStatsTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderCompressionStatsTest.java
new file mode 100644
index 00000000000..0c1d49e8f08
--- /dev/null
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderCompressionStatsTest.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
+import org.apache.pinot.common.restlet.resources.TableSizeInfo;
+import org.apache.pinot.controller.api.resources.ServerTableSizeReader;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Tests that [ServerTableSizeReader] deserializes [SegmentSizeInfo] compression-statistics fields.
+public class ServerTableSizeReaderCompressionStatsTest {
+ private static final String URI_PATH = "/table/";
+ private static final int TIMEOUT_MSEC = 5000;
+ private final ExecutorService _executor = Executors.newFixedThreadPool(2);
+ private final PoolingHttpClientConnectionManager _connectionManager = new PoolingHttpClientConnectionManager();
+ private HttpServer _serverWithStats;
+ private HttpServer _serverWithoutStats;
+ private HttpServer _serverError;
+ private int _portWithStats;
+ private int _portWithoutStats;
+ private int _portError;
+ private final AtomicReference<String> _statsQuery = new AtomicReference<>();
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ // Server with compression stats
+ Map<String, ColumnCompressionStatsInfo> colStats = new HashMap<>();
+ colStats.put("col_a", rawStats("col_a", 10000, 2000, ChunkCompressionType.LZ4));
+ colStats.put("col_b", rawStats("col_b", 20000, 5000, ChunkCompressionType.ZSTANDARD));
+
+ List<SegmentSizeInfo> statsSegments = List.of(
+ new SegmentSizeInfo("s1", 50000, 30000, 7000, colStats),
+ new SegmentSizeInfo("s2", 40000, 15000, 3000));
+ TableSizeInfo statsTable =
+ new TableSizeInfo("testTable", 90000, statsSegments, TableSizeInfo.CURRENT_METADATA_VERSION);
+
+ _serverWithStats = startServer(createRecordingHandler(200, statsTable, _statsQuery));
+ _portWithStats = _serverWithStats.getAddress().getPort();
+
+ // Server without compression stats (backward compat)
+ List<SegmentSizeInfo> noStatsSegments = List.of(new SegmentSizeInfo("s3", 60000));
+ TableSizeInfo noStatsTable = new TableSizeInfo("testTable", 60000, noStatsSegments);
+ _serverWithoutStats = startServer(createHandler(200, noStatsTable));
+ _portWithoutStats = _serverWithoutStats.getAddress().getPort();
+
+ // Server returning 500
+ _serverError = startServer(createHandler(500, null));
+ _portError = _serverError.getAddress().getPort();
+ }
+
+ @AfterClass
+ public void tearDown() {
+ if (_serverWithStats != null) {
+ _serverWithStats.stop(0);
+ }
+ if (_serverWithoutStats != null) {
+ _serverWithoutStats.stop(0);
+ }
+ if (_serverError != null) {
+ _serverError.stop(0);
+ }
+ _executor.shutdownNow();
+ try {
+ _executor.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ _connectionManager.close();
+ }
+
+ @Test
+ public void testDeserializesNewFields() {
+ ServerTableSizeReader reader = new ServerTableSizeReader(_executor, _connectionManager);
+ BiMap<String, String> endpoints = HashBiMap.create();
+ endpoints.put("server0", "http://localhost:" + _portWithStats);
+
+ Map<String, List<SegmentSizeInfo>> result =
+ reader.getSegmentSizeInfoFromServers(endpoints, "testTable", TIMEOUT_MSEC, true);
+ assertEquals(result.size(), 1);
+
+ List<SegmentSizeInfo> segments = result.get("server0");
+ assertNotNull(segments);
+ assertEquals(segments.size(), 2);
+
+ // s1 has compression stats
+ SegmentSizeInfo s1 = segments.get(0);
+ assertEquals(s1.getSegmentName(), "s1");
+ assertEquals(s1.getDiskSizeInBytes(), 50000);
+ assertEquals(s1.getCompressionStatsUncompressedValueSizeInBytes(), 30000);
+ assertEquals(s1.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), 7000);
+ Map<String, ColumnCompressionStatsInfo> colStats = s1.getColumnCompressionStats();
+ assertNotNull(colStats);
+ assertEquals(colStats.size(), 2);
+ assertEquals(colStats.get("col_a").getColumn(), "col_a");
+ assertEquals(colStats.get("col_a").getUncompressedValueSizeInBytes(), 10000);
+ assertEquals(colStats.get("col_a").getForwardIndexAndDictionaryStorageSizeInBytes(), 2000);
+ assertEquals(colStats.get("col_a").getEncodingBreakdown().get(0).getChunkCompressionType(),
+ ChunkCompressionType.LZ4);
+ assertTrue(_statsQuery.get().contains("includeCompressionStats=true"));
+ assertTrue(_statsQuery.get().contains("includeColumnCompressionStats=true"));
+
+ // s2 has summary stats but no column details.
+ SegmentSizeInfo s2 = segments.get(1);
+ assertEquals(s2.getCompressionStatsUncompressedValueSizeInBytes(), 15000);
+ }
+
+ @Test
+ public void testBackwardCompatWithoutNewFields() {
+ ServerTableSizeReader reader = new ServerTableSizeReader(_executor, _connectionManager);
+ BiMap<String, String> endpoints = HashBiMap.create();
+ endpoints.put("server1", "http://localhost:" + _portWithoutStats);
+
+ Map<String, List<SegmentSizeInfo>> result =
+ reader.getSegmentSizeInfoFromServers(endpoints, "testTable", TIMEOUT_MSEC, true);
+ assertEquals(result.size(), 1);
+
+ List<SegmentSizeInfo> segments = result.get("server1");
+ assertNotNull(segments);
+ assertEquals(segments.size(), 1);
+
+ SegmentSizeInfo s3 = segments.get(0);
+ assertEquals(s3.getSegmentName(), "s3");
+ assertEquals(s3.getDiskSizeInBytes(), 60000);
+ // Default values for missing fields (-1 indicates not available)
+ assertEquals(s3.getCompressionStatsUncompressedValueSizeInBytes(), -1);
+ assertEquals(s3.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(), -1);
+ }
+
+ @Test
+ public void testErrorServerExcluded() {
+ ServerTableSizeReader reader = new ServerTableSizeReader(_executor, _connectionManager);
+ BiMap<String, String> endpoints = HashBiMap.create();
+ endpoints.put("server0", "http://localhost:" + _portWithStats);
+ endpoints.put("server_err", "http://localhost:" + _portError);
+
+ Map<String, List<SegmentSizeInfo>> result =
+ reader.getSegmentSizeInfoFromServers(endpoints, "testTable", TIMEOUT_MSEC, true);
+ // Error server should be excluded
+ assertTrue(result.containsKey("server0"));
+ assertFalse(result.containsKey("server_err"));
+ }
+
+ private HttpHandler createHandler(int status, TableSizeInfo tableSize) {
+ return httpExchange -> {
+ byte[] json = (tableSize != null ? JsonUtils.objectToString(tableSize) : "error")
+ .getBytes(StandardCharsets.UTF_8);
+ httpExchange.sendResponseHeaders(status, json.length);
+ try (OutputStream responseBody = httpExchange.getResponseBody()) {
+ responseBody.write(json);
+ }
+ };
+ }
+
+ private HttpHandler createRecordingHandler(int status, TableSizeInfo tableSize, AtomicReference<String> query) {
+ HttpHandler delegate = createHandler(status, tableSize);
+ return exchange -> {
+ query.set(exchange.getRequestURI().getQuery());
+ delegate.handle(exchange);
+ };
+ }
+
+ private HttpServer startServer(HttpHandler handler)
+ throws IOException {
+ HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
+ server.createContext(URI_PATH, handler);
+ server.start();
+ return server;
+ }
+
+ private static ColumnCompressionStatsInfo rawStats(String column, long rawSize,
+ long forwardIndexAndDictionaryStorageSize, ChunkCompressionType chunkCompressionType) {
+ return ColumnCompressionStatsInfo.builder(column)
+ .withUncompressedValueSizeInBytes(rawSize)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(forwardIndexAndDictionaryStorageSize)
+ .withCompressionRatio((double) rawSize / forwardIndexAndDictionaryStorageSize)
+ .withObservedIndexes(List.of("forward_index"))
+ .withEncodingBreakdown(List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, chunkCompressionType, 1, rawSize, forwardIndexAndDictionaryStorageSize)))
+ .withNumSegments(1)
+ .build();
+ }
+}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderTest.java
index a5c0d6537c1..f2e3d73ce0e 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ServerTableSizeReaderTest.java
@@ -26,12 +26,14 @@ import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
+import java.net.ServerSocket;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
import org.apache.pinot.common.restlet.resources.TableSizeInfo;
@@ -50,7 +52,7 @@ import static org.testng.Assert.assertTrue;
public class ServerTableSizeReaderTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ServerTableSizeReader.class);
- private static final int SERVER_PORT_START = 10000;
+ private static final String LOOPBACK_HOST = "127.0.0.1";
private static final String URI_PATH = "/table/";
private static final String TABLE_NAME = "myTable";
private static final int TIMEOUT_MSEC = 5000;
@@ -61,6 +63,7 @@ public class ServerTableSizeReaderTest {
private final List<HttpServer> _servers = new ArrayList<>();
private final List<String> _serverList = new ArrayList<>();
private final List<String> _endpointList = new ArrayList<>();
+ private final AtomicInteger _compressionRequests = new AtomicInteger();
private List<Integer> _server1Segments;
private List<Integer> _server2Segments;
private TableSizeInfo _tableInfo1;
@@ -72,7 +75,7 @@ public class ServerTableSizeReaderTest {
throws IOException {
for (int i = 0; i < SERVER_COUNT; i++) {
_serverList.add("server_" + i);
- _endpointList.add("http://localhost:" + (SERVER_PORT_START + i));
+ _endpointList.add(null);
}
_server1Segments = Arrays.asList(1, 3, 5);
@@ -81,11 +84,12 @@ public class ServerTableSizeReaderTest {
_tableInfo1 = createTableSizeInfo(TABLE_NAME, _server1Segments);
_tableInfo2 = createTableSizeInfo(TABLE_NAME, _server2Segments);
_tableInfo3 = createTableSizeInfo(TABLE_NAME, new ArrayList<Integer>());
- _servers.add(startServer(SERVER_PORT_START, createHandler(200, _tableInfo1, 0)));
- _servers.add(startServer(SERVER_PORT_START + 1, createHandler(200, _tableInfo2, 0)));
- _servers.add(startServer(SERVER_PORT_START + 3, createHandler(500, null, 0)));
- _servers.add(startServer(SERVER_PORT_START + 4, createHandler(200, null, TIMEOUT_MSEC * 20)));
- _servers.add(startServer(SERVER_PORT_START + 5, createHandler(200, _tableInfo3, 0)));
+ startServer(0, createHandler(200, _tableInfo1, 0));
+ startServer(1, createHandler(200, _tableInfo2, 0));
+ _endpointList.set(2, endpoint(findUnusedPort()));
+ startServer(3, createHandler(500, null, 0));
+ startServer(4, createHandler(200, null, TIMEOUT_MSEC + 1000));
+ startServer(5, createHandler(200, _tableInfo3, 0));
}
@AfterClass
@@ -117,6 +121,10 @@ public class ServerTableSizeReaderTest {
@Override
public void handle(HttpExchange httpExchange)
throws IOException {
+ String query = httpExchange.getRequestURI().getQuery();
+ if (query != null && query.contains("includeCompressionStats=true")) {
+ _compressionRequests.incrementAndGet();
+ }
if (sleepTimeMs > 0) {
try {
Thread.sleep(sleepTimeMs);
@@ -133,21 +141,30 @@ public class ServerTableSizeReaderTest {
};
}
- private HttpServer startServer(int port, HttpHandler handler)
+ private void startServer(int serverIndex, HttpHandler handler)
throws IOException {
- final HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);
+ HttpServer server = HttpServer.create(new InetSocketAddress(LOOPBACK_HOST, 0), 0);
server.createContext(URI_PATH, handler);
- new Thread(new Runnable() {
- @Override
- public void run() {
- server.start();
- }
- }).start();
- return server;
+ server.start();
+ _servers.add(server);
+ _endpointList.set(serverIndex, endpoint(server.getAddress().getPort()));
+ }
+
+ private static int findUnusedPort()
+ throws IOException {
+ try (ServerSocket socket = new ServerSocket()) {
+ socket.bind(new InetSocketAddress(LOOPBACK_HOST, 0));
+ return socket.getLocalPort();
+ }
+ }
+
+ private static String endpoint(int port) {
+ return "http://" + LOOPBACK_HOST + ":" + port;
}
@Test
public void testServerSizeReader() {
+ _compressionRequests.set(0);
ServerTableSizeReader reader = new ServerTableSizeReader(_executor, _httpConnectionManager);
BiMap<String, String> endpoints = HashBiMap.create();
for (int i = 0; i < 2; i++) {
@@ -164,6 +181,8 @@ public class ServerTableSizeReaderTest {
assertEquals(serverSizes.get(_serverList.get(0)), _tableInfo1.getSegments());
assertEquals(serverSizes.get(_serverList.get(1)), _tableInfo2.getSegments());
assertEquals(serverSizes.get(_serverList.get(5)), _tableInfo3.getSegments());
+ assertEquals(_compressionRequests.get(), 0,
+ "The legacy size-reader overload must not request compression statistics");
}
@Test
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableMetadataReaderCompressionTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableMetadataReaderCompressionTest.java
new file mode 100644
index 00000000000..3927fdbaf48
--- /dev/null
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableMetadataReaderCompressionTest.java
@@ -0,0 +1,1128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.common.restlet.resources.SegmentCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ServerCompressionStatsResponse;
+import org.apache.pinot.common.restlet.resources.ServerTableMetadataInfo;
+import org.apache.pinot.common.restlet.resources.TableMetadataInfo;
+import org.apache.pinot.common.restlet.resources.TableSegments;
+import org.apache.pinot.controller.util.CompletionServiceHelper;
+import org.apache.pinot.controller.util.ServerSegmentMetadataReader;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies bounded controller fan-out, replica fallback, and aggregation for table metadata compression statistics.
+public class TableMetadataReaderCompressionTest {
+ private static final int TIMEOUT_MSEC = 10_000;
+
+ private final ExecutorService _executor = Executors.newFixedThreadPool(8);
+ private final PoolingHttpClientConnectionManager _connectionManager = new PoolingHttpClientConnectionManager();
+ private HttpServer _httpServer0;
+ private HttpServer _httpServer1;
+ private int _portServer0;
+ private int _portServer1;
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ _connectionManager.setDefaultMaxPerRoute(8);
+ List<SegmentCompressionStatsContribution> contributions = List.of(
+ segment("segment0", true, 10_000, 2_000,
+ rawStats("col_a", 4_000, 1_000, ChunkCompressionType.LZ4),
+ rawStats("col_b", 6_000, 1_000, ChunkCompressionType.ZSTANDARD)),
+ segment("segment1", true, 10_000, 2_000,
+ rawStats("col_a", 4_000, 1_000, ChunkCompressionType.LZ4),
+ rawStats("col_b", 6_000, 1_000, ChunkCompressionType.ZSTANDARD)),
+ segment("segment2", true, 10_000, 3_000,
+ rawStats("col_a", 4_000, 1_000, ChunkCompressionType.LZ4),
+ rawStats("col_b", 6_000, 2_000, ChunkCompressionType.ZSTANDARD)));
+ ServerCompressionStatsResponse compressionResponse = compressionResponse(
+ List.of("segment0", "segment1", "segment2"), contributions);
+ _httpServer0 = startServer(createHandler(currentInfo(), compressionResponse));
+ _httpServer1 = startServer(createHandler(currentInfo(), compressionResponse));
+ _portServer0 = _httpServer0.getAddress().getPort();
+ _portServer1 = _httpServer1.getAddress().getPort();
+ }
+
+ @AfterClass
+ public void tearDown() {
+ if (_httpServer0 != null) {
+ _httpServer0.stop(0);
+ }
+ if (_httpServer1 != null) {
+ _httpServer1.stop(0);
+ }
+ _executor.shutdownNow();
+ _connectionManager.close();
+ }
+
+ @Test
+ public void testCurrentProtocolAggregatesPerReplica() {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(_portServer0, _portServer1), null, 2, TIMEOUT_MSEC, true, true,
+ assignment(2, "segment0", "segment1", "segment2"));
+
+ assertEquals(result.getDiskSizeInBytes(), 50_000L);
+ assertEquals(result.getNumSegments(), 3L);
+ assertEquals(result.getColumnIndexSizeMap().get("col_a").get("forward"), 2_000d);
+ assertNotNull(result.getCompressionStats());
+ assertEquals(result.getCompressionStats().getUncompressedValueSizePerReplicaInBytes(), 30_000L);
+ assertEquals(result.getCompressionStats().getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 7_000L);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+
+ List<ColumnCompressionStatsInfo> columns = result.getColumnCompressionStats();
+ assertNotNull(columns);
+ assertEquals(columns.size(), 2);
+ assertEquals(columns.get(0).getColumn(), "col_a");
+ assertEquals(columns.get(0).getEncodingBreakdown().get(0).getChunkCompressionType(), ChunkCompressionType.LZ4);
+ assertEquals(columns.get(1).getEncodingBreakdown().get(0).getChunkCompressionType(),
+ ChunkCompressionType.ZSTANDARD);
+ }
+
+ @Test
+ public void testIndexSizeAggregationWithSingleServer() {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(_portServer0), null, 1, TIMEOUT_MSEC, false, false,
+ assignment(1, "segment0", "segment1", "segment2"));
+
+ assertEquals(result.getColumnIndexSizeMap().get("col_a").get("forward"), 2_000d);
+ }
+
+ @Test
+ public void testCurrentReplicaCoversSegmentsWhenAnotherReplicaIsUnsupported()
+ throws IOException {
+ TableMetadataInfo legacy = new TableMetadataInfo("testTable_OFFLINE", 50_000, 3, 1_000,
+ Map.of("col_a", 4.0), Map.of("col_a", 50.0), Map.of(), Map.of(), Map.of());
+ HttpServer legacyServer = startServer(createLegacyHandler(legacy));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(legacyServer.getAddress().getPort(), _portServer0), null, 2, TIMEOUT_MSEC,
+ true, true, assignment(2, "segment0", "segment1", "segment2"));
+
+ assertEquals(result.getDiskSizeInBytes(), 50_000L);
+ assertNotNull(result.getCompressionStats());
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 3);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 3);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ assertNotNull(result.getColumnCompressionStats());
+ } finally {
+ legacyServer.stop(0);
+ }
+ }
+
+ @Test
+ public void testCurrentReplicaCoversSegmentsWhenPreferredReplicaUsesOlderProtocol()
+ throws IOException {
+ List<SegmentCompressionStatsContribution> staleContributions = List.of(
+ segment("segment0", true, 1_000_000, 1),
+ segment("segment1", true, 1_000_000, 1),
+ segment("segment2", true, 1_000_000, 1));
+ ServerCompressionStatsResponse staleResponse = new ServerCompressionStatsResponse(0, staleContributions);
+ AtomicInteger staleRequests = new AtomicInteger();
+ HttpServer staleServer = startServer(createCountingHandler(currentInfo(), staleResponse, staleRequests));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(staleServer.getAddress().getPort(), _portServer0), null, 2, TIMEOUT_MSEC,
+ true, true, assignment(2, "segment0", "segment1", "segment2"));
+
+ assertTrue(staleRequests.get() > 0);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 3);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 3);
+ assertEquals(result.getCompressionStats().getUncompressedValueSizePerReplicaInBytes(), 30_000L);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ staleServer.stop(0);
+ }
+ }
+
+ @Test
+ public void testOlderProtocolOnlyProducesExplicitPartialCoverage()
+ throws IOException {
+ ServerCompressionStatsResponse staleResponse = new ServerCompressionStatsResponse(0,
+ List.of(segment("segment0", true, 1_000_000, 1)));
+ HttpServer staleServer = startServer(createHandler(currentInfo(), staleResponse));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(staleServer.getAddress().getPort()), null, 1, TIMEOUT_MSEC,
+ true, true, assignment(1, "segment0"));
+
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 0);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 1);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+ assertNull(result.getColumnCompressionStats());
+ } finally {
+ staleServer.stop(0);
+ }
+ }
+
+ @Test
+ public void testUnsupportedCompressionEndpointProducesExplicitPartialCoverage()
+ throws IOException {
+ TableMetadataInfo legacy = new TableMetadataInfo("testTable_OFFLINE", 50_000, 3, 1_000,
+ Map.of("col_a", 4.0), Map.of("col_a", 50.0), Map.of(), Map.of(), Map.of());
+ HttpServer legacyServer = startServer(createLegacyHandler(legacy));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(legacyServer.getAddress().getPort()), null, 1, TIMEOUT_MSEC,
+ true, true, assignment(1, "segment0", "segment1", "segment2"));
+
+ assertEquals(result.getDiskSizeInBytes(), 50_000L);
+ assertNotNull(result.getCompressionStats());
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 0);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 3);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+ assertNull(result.getColumnCompressionStats());
+ } finally {
+ legacyServer.stop(0);
+ }
+ }
+
+ @Test
+ public void testIncludeColumnStatsFalseDoesNotRequestOrReturnColumns()
+ throws IOException {
+ AtomicReference<String> query = new AtomicReference<>();
+ ServerCompressionStatsResponse summaryOnly = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 10_000, 2_000)));
+ HttpServer server = startServer(createRecordingHandler(currentInfo(), summaryOnly, query));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, TIMEOUT_MSEC, true, false,
+ assignment(1, "segment0"));
+
+ assertNotNull(result.getCompressionStats());
+ assertNull(result.getColumnCompressionStats());
+ assertFalse(query.get() != null && query.get().contains("includeColumnCompressionStats=true"));
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testCompressionFlagOffSuppressesCompression()
+ throws IOException {
+ AtomicInteger compressionRequests = new AtomicInteger();
+ HttpServer server = startServer(createCountingHandler(currentInfo(), completeResponse(List.of("segment0")),
+ compressionRequests));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, TIMEOUT_MSEC, false, true,
+ assignment(1, "segment0"));
+
+ assertEquals(result.getDiskSizeInBytes(), currentInfo().getDiskSizeInBytes());
+ assertNull(result.getCompressionStats());
+ assertNull(result.getColumnCompressionStats());
+ assertEquals(compressionRequests.get(), 0,
+ "A disabled table must complete ordinary metadata collection without compression fan-out");
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testDictionaryContributionKeepsEncodingSeparateFromCodec()
+ throws IOException {
+ ColumnCompressionStatsContribution dictionary = new ColumnCompressionStatsContribution("dict_col", 8_000,
+ 2_000, List.of("dictionary", "forward_index"),
+ List.of(new ColumnCompressionStatsContribution.EncodingContribution(
+ EncodingType.DICTIONARY, null, 1, 8_000, 2_000)), 1);
+ ServerCompressionStatsResponse response = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 8_000, 2_000, dictionary)));
+ HttpServer server = startServer(createHandler(currentInfo(), response));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, TIMEOUT_MSEC, true, true,
+ assignment(1, "segment0"));
+ ColumnCompressionStatsInfo column = result.getColumnCompressionStats().get(0);
+ assertEquals(column.getEncodingBreakdown().get(0).getEncoding(), EncodingType.DICTIONARY);
+ assertNull(column.getEncodingBreakdown().get(0).getChunkCompressionType());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testReplicaSelectionMatchesTableSizePolicy()
+ throws IOException {
+ ServerCompressionStatsResponse first = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 120, 20,
+ rawStats("col_a", 120, 20, ChunkCompressionType.LZ4))));
+ ServerCompressionStatsResponse second = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 100, 25,
+ rawStats("col_a", 100, 25, ChunkCompressionType.LZ4))));
+ AtomicInteger firstRequests = new AtomicInteger();
+ AtomicInteger secondRequests = new AtomicInteger();
+ HttpServer firstServer = startServer(createCountingHandler(currentInfo(), first, firstRequests));
+ HttpServer secondServer = startServer(createCountingHandler(currentInfo(), second, secondRequests));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(firstServer.getAddress().getPort(), secondServer.getAddress().getPort()),
+ null, 2, TIMEOUT_MSEC, true, true, assignment(2, "segment0"));
+
+ assertEquals(result.getCompressionStats().getUncompressedValueSizePerReplicaInBytes(), 120L);
+ assertEquals(result.getCompressionStats().getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 20L);
+ assertEquals(result.getColumnCompressionStats().get(0).getUncompressedValueSizeInBytes(), 120L);
+ assertEquals(result.getColumnCompressionStats().get(0).getForwardIndexAndDictionaryStorageSizeInBytes(), 20L);
+ assertEquals(firstRequests.get(), 1);
+ assertEquals(secondRequests.get(), 0, "A complete first replica must avoid replica-multiplied fanout");
+ } finally {
+ firstServer.stop(0);
+ secondServer.stop(0);
+ }
+ }
+
+ @Test
+ public void testIncompleteReplicaColumnsAreNotDoubleCountedOnCompleteFallback()
+ throws IOException {
+ ServerCompressionStatsResponse incomplete = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", false, -1, -1,
+ rawStats("col_a", 100, 25, ChunkCompressionType.LZ4))));
+ ServerCompressionStatsResponse complete = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 500, 100,
+ rawStats("col_a", 200, 50, ChunkCompressionType.LZ4),
+ rawStats("col_b", 300, 50, ChunkCompressionType.ZSTANDARD))));
+ HttpServer preferred = startServer(createHandler(currentInfo(), incomplete));
+ HttpServer fallback = startServer(createHandler(currentInfo(), complete));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(preferred.getAddress().getPort(), fallback.getAddress().getPort()), null, 2,
+ TIMEOUT_MSEC, true, true, assignment(2, "segment0"));
+
+ assertEquals(result.getCompressionStats().getUncompressedValueSizePerReplicaInBytes(), 500L);
+ assertEquals(result.getCompressionStats().getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 100L);
+ assertEquals(result.getColumnCompressionStats().get(0).getColumn(), "col_a");
+ assertEquals(result.getColumnCompressionStats().get(0).getUncompressedValueSizeInBytes(), 100L);
+ assertEquals(result.getColumnCompressionStats().get(1).getColumn(), "col_b");
+ assertEquals(result.getColumnCompressionStats().get(1).getUncompressedValueSizeInBytes(), 300L);
+ } finally {
+ preferred.stop(0);
+ fallback.stop(0);
+ }
+ }
+
+ @Test
+ public void testReplicaPreferenceDistributesSegmentsAcrossServers()
+ throws IOException {
+ List<String> segments = new ArrayList<>();
+ for (int i = 0; i < 1_000; i++) {
+ segments.add("balancedSegment" + i);
+ }
+ AtomicInteger firstRequests = new AtomicInteger();
+ AtomicInteger secondRequests = new AtomicInteger();
+ HttpServer first = startServer(createEchoingCompressionHandler(currentInfo(), firstRequests));
+ HttpServer second = startServer(createEchoingCompressionHandler(currentInfo(), secondRequests));
+ try {
+ Map<String, List<String>> assignment = new LinkedHashMap<>();
+ assignment.put("server0", segments);
+ assignment.put("server1", segments);
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(first.getAddress().getPort(), second.getAddress().getPort()), null, 2,
+ TIMEOUT_MSEC, true, false, assignment);
+
+ assertTrue(firstRequests.get() > 0);
+ assertTrue(secondRequests.get() > 0);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), segments.size());
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ first.stop(0);
+ second.stop(0);
+ }
+ }
+
+ @Test
+ public void testStalledPreferredReplicaFallsBackWithinDeadline()
+ throws Exception {
+ int timeoutMs = 2_000;
+ ServerCompressionStatsResponse complete = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 100, 25,
+ rawStats("col_a", 100, 25, ChunkCompressionType.LZ4))));
+ AtomicInteger preferredRequests = new AtomicInteger();
+ AtomicInteger fallbackRequests = new AtomicInteger();
+ CountDownLatch preferredStarted = new CountDownLatch(1);
+ CountDownLatch releasePreferred = new CountDownLatch(1);
+ CountDownLatch fallbackStarted = new CountDownLatch(1);
+ HttpServer preferred = startServer(createBlockingCompressionHandler(currentInfo(), complete,
+ preferredRequests, preferredStarted, releasePreferred));
+ HttpServer fallback = startServer(createSignalingHandler(currentInfo(), complete,
+ fallbackRequests, fallbackStarted));
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future<TableMetadataInfo> resultFuture = caller.submit(() -> reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(preferred.getAddress().getPort(), fallback.getAddress().getPort()), null, 2,
+ timeoutMs, true, true, assignment(2, "segment0")));
+
+ assertTrue(preferredStarted.await(5, TimeUnit.SECONDS));
+ assertTrue(fallbackStarted.await(5, TimeUnit.SECONDS),
+ "A stalled preferred replica must leave time for a healthy fallback");
+ TableMetadataInfo result = resultFuture.get(5, TimeUnit.SECONDS);
+ assertEquals(preferredRequests.get(), 1);
+ assertEquals(fallbackRequests.get(), 1);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 1);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ releasePreferred.countDown();
+ preferred.stop(0);
+ fallback.stop(0);
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testFinalReplicaKeepsOverallDeadlineWhileAnotherSegmentFallsBack()
+ throws Exception {
+ int timeoutMs = 2_500;
+ ServerCompressionStatsResponse preferredResponse = compressionResponse(List.of("segment0", "segment1"),
+ List.of(segment("segment0", true, 100, 25), segment("segment1", true, 200, 50)));
+ ServerCompressionStatsResponse fallbackResponse = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 100, 25)));
+ AtomicInteger preferredRequests = new AtomicInteger();
+ AtomicInteger fallbackRequests = new AtomicInteger();
+ CountDownLatch preferredStarted = new CountDownLatch(1);
+ CountDownLatch releasePreferred = new CountDownLatch(1);
+ CountDownLatch fallbackStarted = new CountDownLatch(1);
+ HttpServer preferred = startServer(createBlockingCompressionHandler(currentInfo(), preferredResponse,
+ preferredRequests, preferredStarted, releasePreferred));
+ HttpServer fallback = startServer(createSignalingHandler(currentInfo(), fallbackResponse, fallbackRequests,
+ fallbackStarted));
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Map<String, List<String>> assignment = new LinkedHashMap<>();
+ assignment.put("server0", List.of("segment0", "segment1"));
+ assignment.put("server1", List.of("segment0"));
+
+ Future<TableMetadataInfo> resultFuture = caller.submit(() -> reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(preferred.getAddress().getPort(), fallback.getAddress().getPort()), null, 2,
+ timeoutMs, true, false, assignment));
+
+ assertTrue(preferredStarted.await(5, TimeUnit.SECONDS));
+ assertTrue(fallbackStarted.await(5, TimeUnit.SECONDS),
+ "The retryable segment must fall back before the final replica is released");
+ releasePreferred.countDown();
+ TableMetadataInfo result = resultFuture.get(5, TimeUnit.SECONDS);
+
+ assertEquals(preferredRequests.get(), 2,
+ "Final and retryable segment groups use independent request deadlines");
+ assertEquals(fallbackRequests.get(), 1,
+ "The retryable segment must fall back without cancelling the final replica request");
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 2);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 2);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ releasePreferred.countDown();
+ preferred.stop(0);
+ fallback.stop(0);
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testIncompleteSegmentRemainsInCoverageDenominator()
+ throws IOException {
+ ServerCompressionStatsResponse response = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", false, -1, -1,
+ rawStats("col_a", 100, 25, ChunkCompressionType.LZ4))));
+ HttpServer server = startServer(createHandler(currentInfo(), response));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, TIMEOUT_MSEC, true, true,
+ assignment(1, "segment0"));
+
+ assertNotNull(result.getCompressionStats());
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 0);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 1);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testSegmentMissingFromEveryServerRemainsInCoverageDenominator()
+ throws IOException {
+ ServerCompressionStatsResponse response = compressionResponse(List.of("segment0"), List.of());
+ HttpServer server = startServer(createHandler(currentInfo(), response));
+ try {
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, TIMEOUT_MSEC, true, false,
+ assignment(1, "segment0"));
+
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 0);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 1);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testCompressionFanoutUsesRemainingMetadataTimeout()
+ throws Exception {
+ int timeoutMs = 500;
+ AtomicInteger compressionRequests = new AtomicInteger();
+ CountDownLatch metadataStarted = new CountDownLatch(1);
+ CountDownLatch releaseMetadata = new CountDownLatch(1);
+ ServerCompressionStatsResponse response = compressionResponse(List.of("segment0"),
+ List.of(segment("segment0", true, 100, 25)));
+ ExecutorService serverExecutor = Executors.newFixedThreadPool(2);
+ HttpServer server = startServer(exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ compressionRequests.incrementAndGet();
+ writeJson(exchange, response);
+ } else {
+ metadataStarted.countDown();
+ try {
+ releaseMetadata.await();
+ writeJson(exchange, currentInfo());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while holding metadata response", e);
+ }
+ }
+ }, serverExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future<TableMetadataInfo> resultFuture = caller.submit(() -> reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, timeoutMs, true, false,
+ assignment(1, "segment0")));
+
+ assertTrue(metadataStarted.await(5, TimeUnit.SECONDS));
+ TableMetadataInfo result = resultFuture.get(5, TimeUnit.SECONDS);
+ assertEquals(compressionRequests.get(), 0,
+ "Compression collection must not start a second timeout window after metadata fanout");
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), 0);
+ assertEquals(result.getCompressionStats().getTotalSegments(), 1);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ releaseMetadata.countDown();
+ server.stop(0);
+ serverExecutor.shutdownNow();
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testMetadataDeadlineIncludesExecutorQueueTime()
+ throws Exception {
+ AtomicInteger serverRequests = new AtomicInteger();
+ HttpServer server = startServer(exchange -> {
+ serverRequests.incrementAndGet();
+ writeJson(exchange, currentInfo());
+ });
+ CountDownLatch blockerStarted = new CountDownLatch(1);
+ CountDownLatch releaseBlocker = new CountDownLatch(1);
+ ThreadPoolExecutor saturatedExecutor =
+ new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+ saturatedExecutor.submit(() -> {
+ blockerStarted.countDown();
+ try {
+ releaseBlocker.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ assertTrue(blockerStarted.await(5, TimeUnit.SECONDS));
+ Future<TableMetadataInfo> resultFuture = caller.submit(() ->
+ new ServerSegmentMetadataReader(saturatedExecutor, _connectionManager)
+ .getAggregatedTableMetadataFromServer("testTable_OFFLINE", endpoints(server.getAddress().getPort()),
+ null, 1, 200, true, false, assignment(1, "segment0")));
+
+ TableMetadataInfo result = resultFuture.get(2, TimeUnit.SECONDS);
+ assertTrue(result.getCompressionStats().isPartialCoverage());
+
+ releaseBlocker.countDown();
+ saturatedExecutor.shutdown();
+ assertTrue(saturatedExecutor.awaitTermination(5, TimeUnit.SECONDS));
+ assertEquals(serverRequests.get(), 0, "Cancelled queued HTTP work must not execute after the deadline");
+ } finally {
+ releaseBlocker.countDown();
+ saturatedExecutor.shutdownNow();
+ caller.shutdownNow();
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testLegacyCollectionDoesNotApplyFanoutDeadlineToQueuedRequests()
+ throws Exception {
+ AtomicInteger serverRequests = new AtomicInteger();
+ HttpServer server = startServer(exchange -> {
+ serverRequests.incrementAndGet();
+ writeJson(exchange, currentInfo());
+ });
+ CountDownLatch blockerStarted = new CountDownLatch(1);
+ CountDownLatch releaseBlocker = new CountDownLatch(1);
+ CountDownLatch requestQueued = new CountDownLatch(1);
+ ThreadPoolExecutor requestExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>()) {
+ @Override
+ public void execute(Runnable command) {
+ super.execute(command);
+ if (getQueue().contains(command)) {
+ requestQueued.countDown();
+ }
+ }
+ };
+ requestExecutor.submit(() -> {
+ blockerStarted.countDown();
+ try {
+ releaseBlocker.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+ ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ assertTrue(blockerStarted.await(5, TimeUnit.SECONDS));
+ String endpoint = "http://localhost:" + server.getAddress().getPort();
+ BiMap<String, String> serverEndpoints = HashBiMap.create();
+ serverEndpoints.put("server0", endpoint);
+
+ Future<CompletionServiceHelper.CompletionServiceResponse> resultFuture = caller.submit(() ->
+ new CompletionServiceHelper(requestExecutor, _connectionManager, serverEndpoints.inverse())
+ .doMultiGetRequest(List.of(endpoint + "/tables/testTable_OFFLINE/metadata"),
+ "testTable_OFFLINE", false, 500));
+ assertTrue(requestQueued.await(5, TimeUnit.SECONDS));
+ scheduler.schedule(releaseBlocker::countDown, 750, TimeUnit.MILLISECONDS);
+ CompletionServiceHelper.CompletionServiceResponse result = resultFuture.get(5, TimeUnit.SECONDS);
+
+ assertEquals(result._failedResponseCount, 0);
+ assertEquals(result._httpResponses.size(), 1);
+ assertEquals(serverRequests.get(), 1,
+ "Legacy callers must allow queued work to start after the per-request timeout interval");
+ } finally {
+ releaseBlocker.countDown();
+ requestExecutor.shutdownNow();
+ scheduler.shutdownNow();
+ caller.shutdownNow();
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testDeadlineIncludesResponseBody()
+ throws Exception {
+ CountDownLatch bodyStarted = new CountDownLatch(1);
+ CountDownLatch releaseBody = new CountDownLatch(1);
+ ExecutorService serverExecutor = Executors.newSingleThreadExecutor();
+ HttpServer server = startServer(exchange -> {
+ exchange.sendResponseHeaders(200, 2);
+ try (OutputStream responseBody = exchange.getResponseBody()) {
+ responseBody.write('{');
+ responseBody.flush();
+ bodyStarted.countDown();
+ try {
+ releaseBody.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ responseBody.write('}');
+ }
+ }, serverExecutor);
+ ExecutorService requestExecutor = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ String endpoint = "http://localhost:" + server.getAddress().getPort();
+ BiMap<String, String> serverEndpoints = HashBiMap.create();
+ serverEndpoints.put("server0", endpoint);
+ Future<CompletionServiceHelper.CompletionServiceResponse> resultFuture = caller.submit(() ->
+ new CompletionServiceHelper(requestExecutor, _connectionManager, serverEndpoints.inverse())
+ .doMultiGetRequestUntil(List.of(endpoint + "/tables/testTable_OFFLINE/metadata"),
+ "testTable_OFFLINE", false, 5_000, System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(200),
+ "deadline test"));
+
+ assertTrue(bodyStarted.await(5, TimeUnit.SECONDS));
+ CompletionServiceHelper.CompletionServiceResponse result = resultFuture.get(2, TimeUnit.SECONDS);
+ assertEquals(result._failedResponseCount, 1);
+ assertTrue(result._httpResponses.isEmpty());
+ requestExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS);
+ } finally {
+ releaseBody.countDown();
+ requestExecutor.shutdownNow();
+ caller.shutdownNow();
+ server.stop(0);
+ serverExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testInterruptedCollectionReleasesBufferedConnection()
+ throws Exception {
+ CountDownLatch bodyStarted = new CountDownLatch(1);
+ CountDownLatch releaseBody = new CountDownLatch(1);
+ ExecutorService serverExecutor = Executors.newSingleThreadExecutor();
+ HttpServer server = startServer(exchange -> {
+ exchange.sendResponseHeaders(200, 2);
+ try (OutputStream responseBody = exchange.getResponseBody()) {
+ responseBody.write('{');
+ responseBody.flush();
+ bodyStarted.countDown();
+ try {
+ releaseBody.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ responseBody.write('}');
+ }
+ }, serverExecutor);
+ ExecutorService requestExecutor = Executors.newSingleThreadExecutor();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+ try {
+ String endpoint = "http://localhost:" + server.getAddress().getPort();
+ BiMap<String, String> serverEndpoints = HashBiMap.create();
+ serverEndpoints.put("server0", endpoint);
+ Future<CompletionServiceHelper.CompletionServiceResponse> resultFuture = caller.submit(() ->
+ new CompletionServiceHelper(requestExecutor, connectionManager, serverEndpoints.inverse())
+ .doMultiGetRequest(List.of(endpoint + "/tables/testTable_OFFLINE/metadata"),
+ "testTable_OFFLINE", false, 5_000));
+
+ assertTrue(bodyStarted.await(5, TimeUnit.SECONDS));
+ assertTrue(resultFuture.cancel(true));
+ caller.submit(() -> { }).get(2, TimeUnit.SECONDS);
+ requestExecutor.submit(() -> { }).get(2, TimeUnit.SECONDS);
+ assertEquals(connectionManager.getTotalStats().getLeased(), 0);
+ } finally {
+ releaseBody.countDown();
+ requestExecutor.shutdownNow();
+ caller.shutdownNow();
+ connectionManager.close();
+ server.stop(0);
+ serverExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testCompressionRequestsUseBoundedBatchesAndRollingConcurrency()
+ throws Exception {
+ int numSegments = 4_001;
+ List<String> segments = new ArrayList<>(numSegments);
+ for (int i = 0; i < numSegments; i++) {
+ segments.add("segment" + i);
+ }
+ AtomicInteger requests = new AtomicInteger();
+ AtomicInteger activeRequests = new AtomicInteger();
+ AtomicInteger maxActiveRequests = new AtomicInteger();
+ AtomicBoolean oversizedBatch = new AtomicBoolean();
+ AtomicBoolean initialWindowTimedOut = new AtomicBoolean();
+ CountDownLatch initialWindowStarted = new CountDownLatch(8);
+ CountDownLatch blockedRequestStarted = new CountDownLatch(1);
+ CountDownLatch ninthRequestStarted = new CountDownLatch(1);
+ CountDownLatch releaseBlockedRequest = new CountDownLatch(1);
+ ExecutorService serverExecutor = Executors.newFixedThreadPool(9);
+ HttpServer server = startServer(exchange -> {
+ if (!exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ writeJson(exchange, currentInfo());
+ return;
+ }
+ int requestNumber = requests.incrementAndGet();
+ int active = activeRequests.incrementAndGet();
+ maxActiveRequests.accumulateAndGet(active, Math::max);
+ try {
+ TableSegments requested = JsonUtils.stringToObject(
+ new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8), TableSegments.class);
+ List<String> requestedSegments = requested.getSegments();
+ oversizedBatch.compareAndSet(false, requestedSegments.size() > 500);
+ if (requestNumber <= 8) {
+ initialWindowStarted.countDown();
+ if (!initialWindowStarted.await(5, TimeUnit.SECONDS)) {
+ initialWindowTimedOut.set(true);
+ }
+ }
+ if (requestNumber == 1) {
+ blockedRequestStarted.countDown();
+ if (!releaseBlockedRequest.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting to release the blocked compression request");
+ }
+ } else if (requestNumber == 9) {
+ ninthRequestStarted.countDown();
+ }
+ List<SegmentCompressionStatsContribution> contributions = new ArrayList<>(requestedSegments.size());
+ for (String segmentName : requestedSegments) {
+ contributions.add(segment(segmentName, true, 1, 1));
+ }
+ writeJson(exchange, new ServerCompressionStatsResponse(
+ ServerCompressionStatsResponse.CURRENT_METADATA_VERSION, contributions));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while synchronizing compression request wave", e);
+ } finally {
+ activeRequests.decrementAndGet();
+ }
+ }, serverExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future<TableMetadataInfo> resultFuture = caller.submit(() -> reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), null, 1, 60_000, true, false,
+ Map.of("server0", segments)));
+
+ assertTrue(blockedRequestStarted.await(5, TimeUnit.SECONDS));
+ assertTrue(ninthRequestStarted.await(5, TimeUnit.SECONDS),
+ "A completed request must immediately free an in-flight slot while another request is stalled");
+ releaseBlockedRequest.countDown();
+ TableMetadataInfo result = resultFuture.get(10, TimeUnit.SECONDS);
+
+ assertEquals(requests.get(), 9);
+ assertFalse(oversizedBatch.get());
+ assertFalse(initialWindowTimedOut.get());
+ assertEquals(maxActiveRequests.get(), 8);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), numSegments);
+ assertEquals(result.getCompressionStats().getTotalSegments(), numSegments);
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ releaseBlockedRequest.countDown();
+ server.stop(0);
+ serverExecutor.shutdownNow();
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testOversizedDetailResponseIsSplitWithoutChangingReplica()
+ throws Exception {
+ List<String> segments = List.of("segment0", "segment1", "segment2", "segment3", "segment4", "segment5",
+ "segment6", "segment7");
+ AtomicInteger requests = new AtomicInteger();
+ AtomicInteger largestSuccessfulBatch = new AtomicInteger();
+ HttpServer server = startServer(exchange -> {
+ if (!exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ writeJson(exchange, currentInfo());
+ return;
+ }
+ requests.incrementAndGet();
+ TableSegments requested = JsonUtils.stringToObject(
+ new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8), TableSegments.class);
+ if (requested.getSegments().size() > 3) {
+ exchange.sendResponseHeaders(413, -1);
+ exchange.close();
+ return;
+ }
+ largestSuccessfulBatch.accumulateAndGet(requested.getSegments().size(), Math::max);
+ writeJson(exchange, completeResponse(requested.getSegments()));
+ });
+ try {
+ List<String> columns = List.of("column0", "column1", "column2", "column3", "column4", "column5",
+ "column6", "column7", "column8", "column9");
+ TableMetadataInfo result = reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), columns, 1, TIMEOUT_MSEC, true, true,
+ Map.of("server0", segments));
+
+ assertEquals(requests.get(), 7);
+ assertTrue(largestSuccessfulBatch.get() <= 3);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), segments.size());
+ assertFalse(result.getCompressionStats().isPartialCoverage());
+ } finally {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testDetailRequestsBoundSegmentColumnContributionsAndConcurrency()
+ throws Exception {
+ List<String> segments = new ArrayList<>();
+ for (int i = 0; i < 401; i++) {
+ segments.add("detailSegment" + i);
+ }
+ List<String> columns = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ columns.add("column" + i);
+ }
+ AtomicInteger requests = new AtomicInteger();
+ AtomicInteger activeRequests = new AtomicInteger();
+ AtomicInteger maxActiveRequests = new AtomicInteger();
+ AtomicBoolean oversizedBatch = new AtomicBoolean();
+ CountDownLatch initialWindowStarted = new CountDownLatch(4);
+ CountDownLatch fifthRequestStarted = new CountDownLatch(1);
+ CountDownLatch releaseFirstRequest = new CountDownLatch(1);
+ ExecutorService serverExecutor = Executors.newFixedThreadPool(5);
+ HttpServer server = startServer(exchange -> {
+ if (!exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ writeJson(exchange, currentInfo());
+ return;
+ }
+ int requestNumber = requests.incrementAndGet();
+ int active = activeRequests.incrementAndGet();
+ maxActiveRequests.accumulateAndGet(active, Math::max);
+ try {
+ TableSegments requested = JsonUtils.stringToObject(
+ new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8), TableSegments.class);
+ oversizedBatch.compareAndSet(false, (long) requested.getSegments().size() * columns.size()
+ > ServerCompressionStatsResponse.MAX_COLUMN_CONTRIBUTIONS_PER_RESPONSE);
+ if (requestNumber <= 4) {
+ initialWindowStarted.countDown();
+ if (!initialWindowStarted.await(5, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for the detail request window");
+ }
+ }
+ if (requestNumber == 1) {
+ if (!releaseFirstRequest.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting to release the first detail request");
+ }
+ } else if (requestNumber == 5) {
+ fifthRequestStarted.countDown();
+ }
+ writeJson(exchange, completeResponse(requested.getSegments()));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while synchronizing detail requests", e);
+ } finally {
+ activeRequests.decrementAndGet();
+ }
+ }, serverExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future<TableMetadataInfo> resultFuture = caller.submit(() -> reader().getAggregatedTableMetadataFromServer(
+ "testTable_OFFLINE", endpoints(server.getAddress().getPort()), columns, 1, 60_000, true, true,
+ Map.of("server0", segments)));
+
+ assertTrue(fifthRequestStarted.await(5, TimeUnit.SECONDS));
+ releaseFirstRequest.countDown();
+ TableMetadataInfo result = resultFuture.get(10, TimeUnit.SECONDS);
+
+ assertEquals(requests.get(), 5);
+ assertFalse(oversizedBatch.get());
+ assertEquals(maxActiveRequests.get(), 4);
+ assertEquals(result.getCompressionStats().getSegmentsWithCompleteStats(), segments.size());
+ } finally {
+ releaseFirstRequest.countDown();
+ server.stop(0);
+ serverExecutor.shutdownNow();
+ caller.shutdownNow();
+ }
+ }
+
+ private ServerSegmentMetadataReader reader() {
+ return new ServerSegmentMetadataReader(_executor, _connectionManager);
+ }
+
+ private static ServerTableMetadataInfo currentInfo() {
+ return ServerTableMetadataInfo.builder("testTable_OFFLINE")
+ .withDiskSizeInBytes(50_000)
+ .withNumSegments(3)
+ .withNumRows(1_000)
+ .withColumnLengthMap(Map.of("col_a", 4.0, "col_b", 100.0))
+ .withColumnCardinalityMap(Map.of("col_a", 50.0, "col_b", 200.0))
+ .withColumnIndexSizeMap(Map.of("col_a", Map.of("forward", 6_000d)))
+ .build();
+ }
+
+ private static ServerCompressionStatsResponse completeResponse(List<String> segments) {
+ List<SegmentCompressionStatsContribution> contributions = new ArrayList<>(segments.size());
+ for (String segment : segments) {
+ contributions.add(segment(segment, true, 1, 1));
+ }
+ return new ServerCompressionStatsResponse(ServerCompressionStatsResponse.CURRENT_METADATA_VERSION,
+ contributions);
+ }
+
+ private static ServerCompressionStatsResponse compressionResponse(List<String> expectedSegments,
+ List<SegmentCompressionStatsContribution> contributions) {
+ List<SegmentCompressionStatsContribution> responseContributions = new ArrayList<>(expectedSegments.size()
+ + contributions.size());
+ for (String segment : expectedSegments) {
+ responseContributions.add(new SegmentCompressionStatsContribution(segment, false, -1, -1, null));
+ }
+ responseContributions.addAll(contributions);
+ return new ServerCompressionStatsResponse(ServerCompressionStatsResponse.CURRENT_METADATA_VERSION,
+ responseContributions);
+ }
+
+ private static ColumnCompressionStatsContribution rawStats(String column, long rawSize,
+ long forwardIndexAndDictionaryStorageSize, ChunkCompressionType chunkCompressionType) {
+ return new ColumnCompressionStatsContribution(column, rawSize, forwardIndexAndDictionaryStorageSize,
+ List.of("forward_index"),
+ List.of(new ColumnCompressionStatsContribution.EncodingContribution(
+ EncodingType.RAW, chunkCompressionType, 1, rawSize, forwardIndexAndDictionaryStorageSize)), 1);
+ }
+
+ private static SegmentCompressionStatsContribution segment(String segmentName, boolean complete,
+ long rawSize, long forwardIndexAndDictionaryStorageSize, ColumnCompressionStatsContribution... columns) {
+ Map<String, ColumnCompressionStatsContribution> columnMap = new LinkedHashMap<>();
+ for (ColumnCompressionStatsContribution column : columns) {
+ columnMap.put(column.getColumn(), column);
+ }
+ return new SegmentCompressionStatsContribution(segmentName, complete, rawSize,
+ forwardIndexAndDictionaryStorageSize, columnMap);
+ }
+
+ private static BiMap<String, String> endpoints(int... ports) {
+ BiMap<String, String> endpoints = HashBiMap.create();
+ for (int i = 0; i < ports.length; i++) {
+ endpoints.put("server" + i, "http://localhost:" + ports[i]);
+ }
+ return endpoints;
+ }
+
+ private static Map<String, List<String>> assignment(int numServers, String... segments) {
+ Map<String, List<String>> assignment = new LinkedHashMap<>();
+ for (int i = 0; i < numServers; i++) {
+ assignment.put("server" + i, List.of(segments));
+ }
+ return assignment;
+ }
+
+ private static HttpHandler createHandler(Object metadata, Object compressionStats) {
+ return exchange -> writeJson(exchange,
+ exchange.getRequestURI().getPath().endsWith("/compression-stats") ? compressionStats : metadata);
+ }
+
+ private static HttpHandler createLegacyHandler(Object metadata) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ exchange.sendResponseHeaders(404, -1);
+ exchange.close();
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static HttpHandler createRecordingHandler(Object metadata, Object compressionStats,
+ AtomicReference<String> query) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ query.set(exchange.getRequestURI().getQuery());
+ writeJson(exchange, compressionStats);
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static HttpHandler createCountingHandler(Object metadata, Object compressionStats, AtomicInteger requests) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ requests.incrementAndGet();
+ writeJson(exchange, compressionStats);
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static HttpHandler createEchoingCompressionHandler(Object metadata, AtomicInteger requests) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ requests.incrementAndGet();
+ TableSegments requested = JsonUtils.stringToObject(
+ new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8), TableSegments.class);
+ writeJson(exchange, completeResponse(requested.getSegments()));
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static HttpHandler createBlockingCompressionHandler(Object metadata, Object compressionStats,
+ AtomicInteger requests, CountDownLatch started, CountDownLatch release) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ requests.incrementAndGet();
+ started.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while holding compression response", e);
+ }
+ writeJson(exchange, compressionStats);
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static HttpHandler createSignalingHandler(Object metadata, Object compressionStats, AtomicInteger requests,
+ CountDownLatch started) {
+ return exchange -> {
+ if (exchange.getRequestURI().getPath().endsWith("/compression-stats")) {
+ requests.incrementAndGet();
+ started.countDown();
+ writeJson(exchange, compressionStats);
+ } else {
+ writeJson(exchange, metadata);
+ }
+ };
+ }
+
+ private static void writeJson(HttpExchange exchange, Object info)
+ throws IOException {
+ byte[] json = JsonUtils.objectToString(info).getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, json.length);
+ try (OutputStream responseBody = exchange.getResponseBody()) {
+ responseBody.write(json);
+ }
+ }
+
+ private static HttpServer startServer(HttpHandler handler)
+ throws IOException {
+ return startServer(handler, null);
+ }
+
+ private static HttpServer startServer(HttpHandler handler, ExecutorService executor)
+ throws IOException {
+ HttpServer server = HttpServer.create(new InetSocketAddress(0), 0);
+ server.createContext("/tables/", handler);
+ if (executor != null) {
+ server.setExecutor(executor);
+ }
+ server.start();
+ return server;
+ }
+}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderCompressionStatsTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderCompressionStatsTest.java
new file mode 100644
index 00000000000..97e825ddfd7
--- /dev/null
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderCompressionStatsTest.java
@@ -0,0 +1,682 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.api;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.sun.net.httpserver.HttpHandler;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.helix.AccessOption;
+import org.apache.helix.store.zk.ZkHelixPropertyStore;
+import org.apache.pinot.common.exception.InvalidConfigException;
+import org.apache.pinot.common.metrics.ControllerGauge;
+import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.common.metrics.MetricValueUtils;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.ColumnCompressionStatsInfo;
+import org.apache.pinot.common.restlet.resources.CompressionStatsSummary;
+import org.apache.pinot.common.restlet.resources.SegmentCompressionStatsContribution;
+import org.apache.pinot.common.restlet.resources.SegmentSizeInfo;
+import org.apache.pinot.common.restlet.resources.ServerCompressionStatsResponse;
+import org.apache.pinot.common.restlet.resources.TableSegments;
+import org.apache.pinot.common.restlet.resources.TableSizeInfo;
+import org.apache.pinot.common.utils.config.TableConfigSerDeUtils;
+import org.apache.pinot.controller.LeadControllerManager;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.util.TableSizeReader;
+import org.apache.pinot.controller.utils.FakeHttpServer;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.spi.config.table.FieldConfig.EncodingType;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.metrics.PinotMetricUtils;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.mockito.ArgumentMatchers;
+import org.mockito.InOrder;
+import org.mockito.stubbing.Answer;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.*;
+
+
+/// Tests compression-statistics aggregation in [TableSizeReader].
+public class TableSizeReaderCompressionStatsTest {
+ private static final String URI_PATH = "/table/";
+ private static final int TIMEOUT_MSEC = 10000;
+ private static final int NUM_REPLICAS = 2;
+
+ private final ExecutorService _executor = Executors.newFixedThreadPool(4);
+ private final PoolingHttpClientConnectionManager _connectionManager = new PoolingHttpClientConnectionManager();
+ private final ControllerMetrics _controllerMetrics =
+ new ControllerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
+ private final Map<String, FakeCompressionServer> _serverMap = new HashMap<>();
+ private PinotHelixResourceManager _helix;
+ private LeadControllerManager _leadControllerManager;
+
+ @BeforeClass
+ public void setUp()
+ throws IOException {
+ _helix = mock(PinotHelixResourceManager.class);
+ _leadControllerManager = mock(LeadControllerManager.class);
+
+ TableConfig tableConfig =
+ new TableConfigBuilder(TableType.OFFLINE).setTableName("compressionTable").setNumReplicas(NUM_REPLICAS).build();
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+
+ TableConfig flagOffTableConfig =
+ new TableConfigBuilder(TableType.OFFLINE).setTableName("flagOffTable").setNumReplicas(NUM_REPLICAS).build();
+ // compressionStatsEnabled defaults to false; do not enable it.
+
+ ZkHelixPropertyStore mockPropertyStore = mock(ZkHelixPropertyStore.class);
+
+ when(mockPropertyStore.get(ArgumentMatchers.anyString(), ArgumentMatchers.eq(null),
+ ArgumentMatchers.eq(AccessOption.PERSISTENT))).thenAnswer((Answer) invocationOnMock -> {
+ String path = (String) invocationOnMock.getArguments()[0];
+ if (path.contains("offline_OFFLINE")) {
+ return TableConfigSerDeUtils.toZNRecord(tableConfig);
+ }
+ if (path.contains("flagOffTable_OFFLINE")) {
+ return TableConfigSerDeUtils.toZNRecord(flagOffTableConfig);
+ }
+ return null;
+ });
+
+ when(_helix.getPropertyStore()).thenReturn(mockPropertyStore);
+ when(_helix.getNumReplicas(any(TableConfig.class))).thenReturn(NUM_REPLICAS);
+ when(_leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
+
+ // server0: segment s1 and s2 with compression stats
+ Map<String, ColumnCompressionStatsInfo> s1ColStats = new HashMap<>();
+ s1ColStats.put("col_a", rawStats("col_a", 10000, 2000, ChunkCompressionType.LZ4));
+ s1ColStats.put("col_b", rawStats("col_b", 20000, 5000, ChunkCompressionType.ZSTANDARD));
+
+ Map<String, ColumnCompressionStatsInfo> s2ColStats = new HashMap<>();
+ s2ColStats.put("col_a", rawStats("col_a", 15000, 3000, ChunkCompressionType.LZ4));
+
+ List<SegmentSizeInfo> server0Sizes = List.of(
+ new SegmentSizeInfo("s1", 50000, 30000, 7000, s1ColStats),
+ new SegmentSizeInfo("s2", 40000, 15000, 3000, s2ColStats));
+ FakeCompressionServer s0 = new FakeCompressionServer(List.of("s1", "s2"), server0Sizes);
+ startServer(s0, server0Sizes, TableSizeInfo.CURRENT_METADATA_VERSION);
+ _serverMap.put("server0", s0);
+
+ // server1: segment s1 and s2 (replica) with same stats
+ List<SegmentSizeInfo> server1Sizes = List.of(
+ new SegmentSizeInfo("s1", 50000, 30000, 7000, s1ColStats),
+ new SegmentSizeInfo("s2", 40000, 15000, 3000, s2ColStats));
+ FakeCompressionServer s1 = new FakeCompressionServer(List.of("s1", "s2"), server1Sizes);
+ startServer(s1, server1Sizes, TableSizeInfo.CURRENT_METADATA_VERSION);
+ _serverMap.put("server1", s1);
+
+ // server2: segment s3 without compression stats (old server)
+ List<SegmentSizeInfo> server2Sizes = List.of(new SegmentSizeInfo("s3", 60000));
+ FakeCompressionServer s2 = new FakeCompressionServer(List.of("s3"), server2Sizes);
+ startServer(s2, server2Sizes, 0);
+ _serverMap.put("server2", s2);
+
+ addServer("server3", List.of("s4"), List.of(new SegmentSizeInfo("s4", 1_000, 120, 20,
+ Map.of("col_a", rawStats("col_a", 120, 20, ChunkCompressionType.LZ4)))),
+ TableSizeInfo.CURRENT_METADATA_VERSION);
+ addServer("server4", List.of("s4"), List.of(new SegmentSizeInfo("s4", 1_000, 100, 25,
+ Map.of("col_a", rawStats("col_a", 100, 25, ChunkCompressionType.LZ4)))),
+ TableSizeInfo.CURRENT_METADATA_VERSION);
+ addServer("server5", List.of("s3"), List.of(new SegmentSizeInfo("s3", 1_000, 80, 20,
+ Map.of("col_a", rawStats("col_a", 80, 20, ChunkCompressionType.LZ4)))),
+ TableSizeInfo.CURRENT_METADATA_VERSION);
+ addServer("server6", List.of("s5"), List.of(new SegmentSizeInfo("s5", 1_000, 0, 0)),
+ TableSizeInfo.CURRENT_METADATA_VERSION);
+ }
+
+ @AfterClass
+ public void tearDown() {
+ for (FakeCompressionServer server : _serverMap.values()) {
+ server.stop();
+ }
+ _executor.shutdownNow();
+ _connectionManager.close();
+ }
+
+ private void addServer(String name, List<String> segments, List<SegmentSizeInfo> sizes, int metadataVersion)
+ throws IOException {
+ FakeCompressionServer server = new FakeCompressionServer(segments, sizes);
+ startServer(server, sizes, metadataVersion);
+ _serverMap.put(name, server);
+ }
+
+ private void startServer(FakeCompressionServer server, List<SegmentSizeInfo> sizes, int metadataVersion)
+ throws IOException {
+ server.start(URI_PATH, createHandler(200, sizes, metadataVersion));
+ server._httpServer.createContext("/tables/", createCompressionHandler(server, sizes, metadataVersion));
+ }
+
+ private HttpHandler createHandler(int status, List<SegmentSizeInfo> segmentSizes, int metadataVersion) {
+ return httpExchange -> {
+ long tableSizeInBytes = 0;
+ boolean includeCompressionStats = httpExchange.getRequestURI().getQuery() != null
+ && httpExchange.getRequestURI().getQuery().contains("includeCompressionStats=true");
+ boolean includeColumnCompressionStats = httpExchange.getRequestURI().getQuery() != null
+ && httpExchange.getRequestURI().getQuery().contains("includeColumnCompressionStats=true");
+ List<SegmentSizeInfo> responseSegments = new ArrayList<>(segmentSizes.size());
+ for (SegmentSizeInfo segmentSize : segmentSizes) {
+ tableSizeInBytes += segmentSize.getDiskSizeInBytes();
+ responseSegments.add(includeCompressionStats
+ ? new SegmentSizeInfo(segmentSize.getSegmentName(), segmentSize.getDiskSizeInBytes(),
+ segmentSize.getCompressionStatsUncompressedValueSizeInBytes(),
+ segmentSize.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes(),
+ includeColumnCompressionStats ? segmentSize.getColumnCompressionStats() : null)
+ : new SegmentSizeInfo(segmentSize.getSegmentName(), segmentSize.getDiskSizeInBytes()));
+ }
+ TableSizeInfo tableInfo =
+ new TableSizeInfo("compressionTable", tableSizeInBytes, responseSegments, metadataVersion);
+ byte[] json = JsonUtils.objectToString(tableInfo).getBytes(StandardCharsets.UTF_8);
+ httpExchange.sendResponseHeaders(status, json.length);
+ try (OutputStream responseBody = httpExchange.getResponseBody()) {
+ responseBody.write(json);
+ }
+ };
+ }
+
+ private HttpHandler createCompressionHandler(FakeCompressionServer server, List<SegmentSizeInfo> segmentSizes,
+ int metadataVersion) {
+ return httpExchange -> {
+ server._compressionRequests.incrementAndGet();
+ TableSegments request = JsonUtils.stringToObject(
+ new String(httpExchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8), TableSegments.class);
+ Map<String, SegmentSizeInfo> sizesBySegment = new HashMap<>();
+ for (SegmentSizeInfo segmentSize : segmentSizes) {
+ sizesBySegment.put(segmentSize.getSegmentName(), segmentSize);
+ }
+ boolean includeColumnStats = httpExchange.getRequestURI().getQuery() != null
+ && httpExchange.getRequestURI().getQuery().contains("includeColumnCompressionStats=true");
+ List<SegmentCompressionStatsContribution> contributions = new ArrayList<>();
+ for (String segmentName : request.getSegments()) {
+ SegmentSizeInfo size = sizesBySegment.get(segmentName);
+ if (size == null) {
+ contributions.add(new SegmentCompressionStatsContribution(segmentName, false, -1, -1, null));
+ continue;
+ }
+ long uncompressedValueSize = size.getCompressionStatsUncompressedValueSizeInBytes();
+ long storageSize = size.getCompressionStatsForwardIndexAndDictionaryStorageSizeInBytes();
+ contributions.add(new SegmentCompressionStatsContribution(segmentName,
+ uncompressedValueSize >= 0 && storageSize >= 0, uncompressedValueSize, storageSize,
+ includeColumnStats && size.getColumnCompressionStats() != null
+ ? toContributions(size.getColumnCompressionStats()) : null));
+ }
+ ServerCompressionStatsResponse response = new ServerCompressionStatsResponse(metadataVersion, contributions);
+ byte[] json = JsonUtils.objectToString(response).getBytes(StandardCharsets.UTF_8);
+ httpExchange.sendResponseHeaders(200, json.length);
+ try (OutputStream responseBody = httpExchange.getResponseBody()) {
+ responseBody.write(json);
+ }
+ };
+ }
+
+ private static class FakeCompressionServer extends FakeHttpServer {
+ final List<String> _segments;
+ final List<SegmentSizeInfo> _sizes;
+ final AtomicInteger _compressionRequests = new AtomicInteger();
+
+ FakeCompressionServer(List<String> segments, List<SegmentSizeInfo> sizes) {
+ _segments = segments;
+ _sizes = sizes;
+ }
+ }
+
+ private TableSizeReader createReader(String[] servers)
+ throws InvalidConfigException {
+ when(_helix.getServerToSegmentsMap(anyString(), any(), anyBoolean())).thenAnswer(
+ (Answer<Map<String, List<String>>>) invocation -> {
+ Map<String, List<String>> map = new HashMap<>();
+ for (String server : servers) {
+ map.put(server, _serverMap.get(server)._segments);
+ }
+ return map;
+ });
+
+ when(_helix.getDataInstanceAdminEndpoints(ArgumentMatchers.anySet())).thenAnswer(
+ (Answer<BiMap<String, String>>) invocation -> {
+ BiMap<String, String> endpoints = HashBiMap.create(servers.length);
+ for (String server : servers) {
+ endpoints.put(server, _serverMap.get(server)._endpoint);
+ }
+ return endpoints;
+ });
+
+ return new TableSizeReader(_executor, _connectionManager, _controllerMetrics, _helix, _leadControllerManager);
+ }
+
+ private TableSizeReader.TableSizeDetails testRunner(String[] servers, String table)
+ throws InvalidConfigException {
+ TableSizeReader reader = createReader(servers);
+ TableSizeReader.TableSizeDetails details = reader.getTableSizeDetails(table, TIMEOUT_MSEC, true, true);
+ reader.updateCompressionMetrics(TableNameBuilder.OFFLINE.tableNameWithType(table), details._offlineSegments);
+ return details;
+ }
+
+ private TableSizeReader.TableSizeDetails testRunner(String[] servers, String table,
+ boolean includeColumnCompressionStats)
+ throws InvalidConfigException {
+ TableSizeReader reader = createReader(servers);
+ TableSizeReader.TableSizeDetails details =
+ reader.getTableSizeDetails(table, TIMEOUT_MSEC, true, includeColumnCompressionStats);
+ reader.updateCompressionMetrics(TableNameBuilder.OFFLINE.tableNameWithType(table), details._offlineSegments);
+ return details;
+ }
+
+ @Test
+ public void testCompressionStatsAggregation()
+ throws InvalidConfigException {
+ String[] servers = {"server0", "server1"};
+ TableSizeReader.TableSizeDetails details = testRunner(servers, "offline");
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails);
+
+ // s1: rawFwdIdx=30000, compressedFwdIdx=7000 (first complete replica)
+ // s2: rawFwdIdx=15000, compressedFwdIdx=3000 (first complete replica)
+ // Total per replica: raw=45000, compressed=10000
+ CompressionStatsSummary cs = offlineDetails._compressionStats;
+ assertNotNull(cs);
+ assertEquals(cs.getUncompressedValueSizePerReplicaInBytes(), 45000);
+ assertEquals(cs.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 10000);
+
+ // Compression ratio = 45000 / 10000 = 4.5
+ assertEquals(cs.getCompressionRatio(), 4.5, 0.01);
+
+ // Both segments have stats
+ assertEquals(cs.getSegmentsWithCompleteStats(), 2);
+ assertEquals(cs.getTotalSegments(), 2);
+ assertFalse(cs.isPartialCoverage());
+
+ // Verify compression metrics emitted
+ String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType("offline");
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA), 45000);
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA), 10000);
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT), 450);
+
+ // Verify per-column compression stats aggregation (now top-level on TableSubTypeSizeDetails)
+ // s1: col_a(raw=10000, compressed=2000), col_b(raw=20000, compressed=5000)
+ // s2: col_a(raw=15000, compressed=3000)
+ // Aggregated: col_a: raw=10000+15000=25000, compressed=2000+3000=5000
+ // col_b: raw=20000, compressed=5000
+ List<ColumnCompressionStatsInfo> colStats = offlineDetails._columnCompressionStats;
+ assertNotNull(colStats, "Per-column compression stats should be present");
+ assertFalse(colStats.isEmpty(), "Per-column compression stats should not be empty");
+ // List is sorted by column name: col_a, col_b
+ ColumnCompressionStatsInfo colA = colStats.get(0);
+ assertEquals(colA.getColumn(), "col_a");
+ assertEquals(colA.getUncompressedValueSizeInBytes(), 25000);
+ assertEquals(colA.getForwardIndexAndDictionaryStorageSizeInBytes(), 5000);
+ assertEquals(colA.getCompressionRatio(), 5.0, 0.01);
+ assertEquals(colA.getEncodingBreakdown().get(0).getChunkCompressionType(), ChunkCompressionType.LZ4);
+
+ ColumnCompressionStatsInfo colB = colStats.get(1);
+ assertEquals(colB.getColumn(), "col_b");
+ assertEquals(colB.getUncompressedValueSizeInBytes(), 20000);
+ assertEquals(colB.getForwardIndexAndDictionaryStorageSizeInBytes(), 5000);
+ assertEquals(colB.getCompressionRatio(), 4.0, 0.01);
+ assertEquals(colB.getEncodingBreakdown().get(0).getChunkCompressionType(), ChunkCompressionType.ZSTANDARD);
+
+ for (String segment : List.of("s1", "s2")) {
+ List<SegmentSizeInfo> selectedReplicaStats = offlineDetails._segments.get(segment)._serverInfo.values().stream()
+ .filter(info -> info.getCompressionStatsUncompressedValueSizeInBytes() >= 0)
+ .toList();
+ assertEquals(selectedReplicaStats.size(), 1,
+ "Bounded replica selection must attach detailed stats only to the selected server");
+ Map<String, ColumnCompressionStatsInfo> selectedColumns = selectedReplicaStats.get(0).getColumnCompressionStats();
+ assertNotNull(selectedColumns);
+ assertTrue(selectedColumns.values().stream().allMatch(info -> info.getCompressionRatio() > 0));
+ }
+ }
+
+ @Test
+ public void testPartialCompressionCoverage()
+ throws InvalidConfigException {
+ // Mix of servers with and without compression stats
+ String[] servers = {"server0", "server1", "server2"};
+ TableSizeReader.TableSizeDetails details = testRunner(servers, "offline");
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails);
+
+ // s1 and s2 have compression stats, s3 does not
+ CompressionStatsSummary cs = offlineDetails._compressionStats;
+ assertNotNull(cs);
+ assertEquals(cs.getSegmentsWithCompleteStats(), 2);
+ assertEquals(cs.getTotalSegments(), 3);
+ assertTrue(cs.isPartialCoverage());
+
+ // Compression ratio still computed from segments that have stats
+ assertEquals(cs.getUncompressedValueSizePerReplicaInBytes(), 45000);
+ assertEquals(cs.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 10000);
+ assertEquals(cs.getCompressionRatio(), 4.5, 0.01);
+ }
+
+ @Test
+ public void testNoCompressionStats()
+ throws InvalidConfigException {
+ // The legacy response contributes to the coverage denominator but not to tracked sizes.
+ String[] servers = {"server2"};
+ TableSizeReader.TableSizeDetails details = testRunner(servers, "offline");
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails);
+
+ assertNotNull(offlineDetails._compressionStats);
+ assertEquals(offlineDetails._compressionStats.getSegmentsWithCompleteStats(), 0);
+ assertEquals(offlineDetails._compressionStats.getTotalSegments(), 1);
+ assertTrue(offlineDetails._compressionStats.isPartialCoverage());
+ }
+
+ @Test
+ public void testZeroEligibleColumnsCountAsCompleteCoverage()
+ throws InvalidConfigException {
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails =
+ testRunner(new String[]{"server6"}, "offline")._offlineSegments;
+
+ assertNotNull(offlineDetails._compressionStats);
+ assertEquals(offlineDetails._compressionStats.getSegmentsWithCompleteStats(), 1);
+ assertEquals(offlineDetails._compressionStats.getTotalSegments(), 1);
+ assertFalse(offlineDetails._compressionStats.isPartialCoverage());
+ assertEquals(offlineDetails._compressionStats.getUncompressedValueSizePerReplicaInBytes(), 0L);
+ assertEquals(offlineDetails._compressionStats.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 0L);
+ assertEquals(offlineDetails._compressionStats.getCompressionRatio(), 0.0);
+ }
+
+ @Test
+ public void testStaleMetricsClearedWhenNoStats()
+ throws InvalidConfigException {
+ // First run with servers that have stats to emit metrics
+ String[] serversWithStats = {"server0", "server1"};
+ testRunner(serversWithStats, "offline");
+
+ String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType("offline");
+ // Verify metrics were emitted
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT), 450);
+
+ // Now run with only an old server; stale metrics should be cleared.
+ String[] serversNoStats = {"server2"};
+ testRunner(serversNoStats, "offline");
+
+ // Metrics should be cleared (0 means removed)
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT), 0);
+ }
+
+ @Test
+ public void testCompressionStatsSummaryPresentWhenColumnStatsExcluded()
+ throws InvalidConfigException {
+ // Regression test: compressionStats summary must be present even when includeColumnCompressionStats=false.
+ // Previously, the summary was gated on perColumnMax which is only populated when servers are
+ // called with includeColumnCompressionStats=true. This caused the summary to be null for the default case.
+ String[] servers = {"server0", "server1"};
+ TableSizeReader.TableSizeDetails details = testRunner(servers, "offline", false);
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails);
+
+ // Summary must remain available when column details are not requested.
+ assertNotNull(offlineDetails._compressionStats,
+ "compressionStats summary should be present even when includeColumnCompressionStats=false");
+ assertTrue(offlineDetails._compressionStats.getUncompressedValueSizePerReplicaInBytes() > 0,
+ "uncompressedValueSizePerReplicaInBytes should be > 0");
+ assertTrue(offlineDetails._compressionStats.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes() > 0,
+ "forwardIndexAndDictionaryStorageSizePerReplicaInBytes should be > 0");
+ assertTrue(offlineDetails._compressionStats.getSegmentsWithCompleteStats() > 0,
+ "segmentsWithCompleteStats should be > 0");
+
+ // Per-column list must still be null (not requested)
+ assertNull(offlineDetails._columnCompressionStats,
+ "columnCompressionStats should be null when includeColumnCompressionStats=false");
+ long selectedReplicas = offlineDetails._segments.values().stream()
+ .flatMap(segment -> segment._serverInfo.values().stream())
+ .filter(info -> info.getCompressionStatsUncompressedValueSizeInBytes() >= 0)
+ .peek(info -> assertNull(info.getColumnCompressionStats()))
+ .count();
+ assertEquals(selectedReplicas, 2L);
+ }
+
+ @Test
+ public void testColumnCompressionStatsPresentWhenSummaryExcluded()
+ throws InvalidConfigException {
+ TableSizeReader.TableSizeDetails details = createReader(new String[]{"server0", "server1"})
+ .getTableSizeDetails("offline", TIMEOUT_MSEC, true,
+ TableSizeReader.CompressionStatsMode.COLUMNS_WITH_SEGMENT_DETAILS);
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNull(offlineDetails._compressionStats);
+ assertNotNull(offlineDetails._columnCompressionStats);
+ assertEquals(offlineDetails._columnCompressionStats.size(), 2);
+ assertEquals(offlineDetails._columnCompressionStats.get(0).getColumn(), "col_a");
+ assertEquals(offlineDetails._columnCompressionStats.get(1).getColumn(), "col_b");
+ }
+
+ @Test
+ public void testAggregateOnlyModeDoesNotRetainSelectedSegments()
+ throws InvalidConfigException {
+ TableSizeReader.TableSizeDetails details = createReader(new String[]{"server0", "server1"})
+ .getTableSizeDetails("offline", TIMEOUT_MSEC, true, TableSizeReader.CompressionStatsMode.AGGREGATE_SUMMARY);
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails._compressionStats);
+ assertEquals(offlineDetails._compressionStats.getSegmentsWithCompleteStats(), 2);
+ assertNull(offlineDetails._columnCompressionStats);
+ assertTrue(offlineDetails._segments.values().stream()
+ .flatMap(segment -> segment._serverInfo.values().stream())
+ .allMatch(info -> info.getCompressionStatsUncompressedValueSizeInBytes() < 0));
+ }
+
+ @Test
+ public void testCompressionMetricUpdateAndCleanupAreAtomic()
+ throws Exception {
+ String[] servers = {"server0", "server1"};
+ createReader(servers);
+ ControllerMetrics metrics = mock(ControllerMetrics.class);
+ CountDownLatch firstUpdateStarted = new CountDownLatch(1);
+ CountDownLatch continueUpdate = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ firstUpdateStarted.countDown();
+ assertTrue(continueUpdate.await(10, TimeUnit.SECONDS));
+ return null;
+ }).when(metrics).setValueOfTableGauge(anyString(),
+ ArgumentMatchers.eq(ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA),
+ ArgumentMatchers.anyLong());
+ TableSizeReader reader = new TableSizeReader(_executor, _connectionManager, metrics, _helix,
+ _leadControllerManager);
+ String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType("offline");
+
+ TableSizeReader.TableSizeDetails details = reader.getTableSizeDetails("offline", TIMEOUT_MSEC, true, true);
+ Future<?> update = _executor.submit(
+ () -> reader.updateCompressionMetrics(tableNameWithType, details._offlineSegments));
+ assertTrue(firstUpdateStarted.await(10, TimeUnit.SECONDS));
+ Future<?> cleanup = _executor.submit(() -> reader.clearCompressionMetrics(tableNameWithType));
+ continueUpdate.countDown();
+ update.get(10, TimeUnit.SECONDS);
+ cleanup.get(10, TimeUnit.SECONDS);
+
+ InOrder order = inOrder(metrics);
+ order.verify(metrics).setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA, 45_000);
+ order.verify(metrics).setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA, 10_000);
+ order.verify(metrics).setValueOfTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT, 450);
+ order.verify(metrics).removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA);
+ order.verify(metrics).removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA);
+ order.verify(metrics).removeTableGauge(tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT);
+ }
+
+ @Test
+ public void testTableSizeReadDoesNotPublishCompressionMetrics()
+ throws Exception {
+ String[] servers = {"server0", "server1"};
+ createReader(servers);
+ ControllerMetrics metrics = mock(ControllerMetrics.class);
+ TableSizeReader reader = new TableSizeReader(_executor, _connectionManager, metrics, _helix,
+ _leadControllerManager);
+
+ reader.getTableSizeDetails("offline", TIMEOUT_MSEC, true, true);
+
+ verify(metrics, never()).setValueOfTableGauge(anyString(),
+ ArgumentMatchers.eq(ControllerGauge.TABLE_COMPRESSION_STATS_UNCOMPRESSED_VALUE_SIZE_PER_REPLICA),
+ ArgumentMatchers.anyLong());
+ verify(metrics, never()).setValueOfTableGauge(anyString(),
+ ArgumentMatchers.eq(
+ ControllerGauge.TABLE_COMPRESSION_STATS_FORWARD_INDEX_AND_DICTIONARY_STORAGE_SIZE_PER_REPLICA),
+ ArgumentMatchers.anyLong());
+ verify(metrics, never()).setValueOfTableGauge(anyString(),
+ ArgumentMatchers.eq(ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT),
+ ArgumentMatchers.anyLong());
+ }
+
+ @Test
+ public void testNonLeaderDoesNotPublishCompressionMetrics() {
+ LeadControllerManager leadControllerManager = mock(LeadControllerManager.class);
+ when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(false);
+ ControllerMetrics metrics = mock(ControllerMetrics.class);
+ TableSizeReader reader = new TableSizeReader(_executor, _connectionManager, metrics, _helix,
+ leadControllerManager);
+ TableSizeReader.TableSubTypeSizeDetails details = new TableSizeReader.TableSubTypeSizeDetails();
+ details._compressionStats = new CompressionStatsSummary(100, 25, 4.0, 1, 1, false);
+
+ reader.updateCompressionMetrics("offline_OFFLINE", details);
+
+ verifyNoInteractions(metrics);
+ }
+
+ @Test
+ public void testCompressionStatsNullWhenFlagOff()
+ throws InvalidConfigException {
+ // Use servers with compression stats but with compressionStatsEnabled=false on the table config
+ String[] servers = {"server0", "server1"};
+ for (String server : servers) {
+ _serverMap.get(server)._compressionRequests.set(0);
+ }
+ TableSizeReader.TableSizeDetails details = testRunner(servers, "flagOffTable");
+
+ TableSizeReader.TableSubTypeSizeDetails offlineDetails = details._offlineSegments;
+ assertNotNull(offlineDetails);
+ assertTrue(offlineDetails._reportedSizeInBytes > 0,
+ "A disabled table must still complete ordinary size collection");
+
+ // compressionStats should be null when the flag is OFF (suppressed from JSON via @JsonInclude NON_NULL)
+ assertNull(offlineDetails._compressionStats,
+ "compressionStats should be null when compressionStatsEnabled is false");
+
+ // columnCompressionStats should also be null when flag is OFF
+ assertNull(offlineDetails._columnCompressionStats,
+ "columnCompressionStats should be null when compressionStatsEnabled is false");
+
+ // Verify no compression metrics were emitted for this table
+ String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType("flagOffTable");
+ assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
+ ControllerGauge.TABLE_COMPRESSION_STATS_RATIO_PERCENT), 0);
+ for (String server : servers) {
+ assertEquals(_serverMap.get(server)._compressionRequests.get(), 0,
+ "A disabled table must not fan out to the compression endpoint");
+ }
+ }
+
+ @Test
+ public void testReplicaSelectionKeepsSummaryAndColumnPairCoherent()
+ throws InvalidConfigException {
+ TableSizeReader.TableSizeDetails details = testRunner(new String[]{"server3", "server4"}, "offline");
+ CompressionStatsSummary summary = details._offlineSegments._compressionStats;
+ assertEquals(summary.getUncompressedValueSizePerReplicaInBytes(), 120L);
+ assertEquals(summary.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 20L);
+ ColumnCompressionStatsInfo column = details._offlineSegments._columnCompressionStats.get(0);
+ assertEquals(column.getUncompressedValueSizeInBytes(), 120L);
+ assertEquals(column.getForwardIndexAndDictionaryStorageSizeInBytes(), 20L);
+ }
+
+ @Test
+ public void testReplicaSelectionFallsBackFromLegacyServer()
+ throws InvalidConfigException {
+ TableSizeReader.TableSizeDetails details = testRunner(new String[]{"server2", "server5"}, "offline");
+ CompressionStatsSummary summary = details._offlineSegments._compressionStats;
+
+ assertEquals(summary.getSegmentsWithCompleteStats(), 1);
+ assertEquals(summary.getUncompressedValueSizePerReplicaInBytes(), 80L);
+ assertEquals(summary.getForwardIndexAndDictionaryStorageSizePerReplicaInBytes(), 20L);
+ assertFalse(summary.isPartialCoverage());
+ }
+
+ private static ColumnCompressionStatsInfo rawStats(String column, long rawSize,
+ long forwardIndexAndDictionaryStorageSize, ChunkCompressionType chunkCompressionType) {
+ return ColumnCompressionStatsInfo.builder(column)
+ .withUncompressedValueSizeInBytes(rawSize)
+ .withForwardIndexAndDictionaryStorageSizeInBytes(forwardIndexAndDictionaryStorageSize)
+ .withCompressionRatio((double) rawSize / forwardIndexAndDictionaryStorageSize)
+ .withEncodingBreakdown(List.of(new ColumnCompressionStatsInfo.EncodingBreakdownEntry(
+ EncodingType.RAW, chunkCompressionType, 1, rawSize, forwardIndexAndDictionaryStorageSize)))
+ .withNumSegments(1)
+ .build();
+ }
+
+ private static Map<String, ColumnCompressionStatsContribution> toContributions(
+ Map<String, ColumnCompressionStatsInfo> columnStats) {
+ Map<String, ColumnCompressionStatsContribution> contributions = new HashMap<>();
+ columnStats.forEach((column, info) -> {
+ List<ColumnCompressionStatsContribution.EncodingContribution> breakdown = new ArrayList<>();
+ for (ColumnCompressionStatsInfo.EncodingBreakdownEntry entry : info.getEncodingBreakdown()) {
+ breakdown.add(new ColumnCompressionStatsContribution.EncodingContribution(entry.getEncoding(),
+ entry.getChunkCompressionType(), entry.getNumSegments(), entry.getUncompressedValueSizeInBytes(),
+ entry.getForwardIndexAndDictionaryStorageSizeInBytes()));
+ }
+ contributions.put(column, new ColumnCompressionStatsContribution(column,
+ info.getUncompressedValueSizeInBytes(), info.getForwardIndexAndDictionaryStorageSizeInBytes(),
+ info.getObservedIndexes(), breakdown, info.getNumSegments()));
+ });
+ return contributions;
+ }
+}
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
index e4edc5e3ace..8808c96e192 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
@@ -347,6 +347,8 @@ public class TableSizeReaderTest {
public void testGetTableSubTypeSizeAllErrors() throws InvalidConfigException {
final String[] servers = {"server2", "server5"};
String table = "offline";
+ String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER, 123L);
TableSizeReader.TableSizeDetails tableSizeDetails = testRunner(servers, table);
TableSizeReader.TableSubTypeSizeDetails offlineSizes = tableSizeDetails._offlineSegments;
assertNotNull(offlineSizes);
@@ -355,7 +357,6 @@ public class TableSizeReaderTest {
assertEquals(offlineSizes._reportedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
assertEquals(tableSizeDetails._estimatedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
assertEquals(tableSizeDetails._reportedSizePerReplicaInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
- String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
assertEquals(MetricValueUtils.getTableGaugeValue(_controllerMetrics, tableNameWithType,
ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 100);
assertEquals(MetricValueUtils
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
index 0066e482c28..8df09a39a23 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
@@ -57,13 +57,17 @@ import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
@@ -148,11 +152,15 @@ public class SegmentStatusCheckerTest {
}
private void runSegmentStatusChecker(PinotHelixResourceManager resourceManager, int waitForPushTimeInSeconds) {
+ runSegmentStatusChecker(resourceManager, waitForPushTimeInSeconds, mock(TableSizeReader.class));
+ }
+
+ private void runSegmentStatusChecker(PinotHelixResourceManager resourceManager, int waitForPushTimeInSeconds,
+ TableSizeReader tableSizeReader) {
LeadControllerManager leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
ControllerConf controllerConf = mock(ControllerConf.class);
when(controllerConf.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(waitForPushTimeInSeconds);
- TableSizeReader tableSizeReader = mock(TableSizeReader.class);
SegmentStatusChecker segmentStatusChecker =
new SegmentStatusChecker(resourceManager, leadControllerManager, controllerConf, _controllerMetrics,
tableSizeReader);
@@ -160,6 +168,63 @@ public class SegmentStatusCheckerTest {
segmentStatusChecker.run();
}
+ @DataProvider(name = "compressionMetricLifecycle")
+ public Object[][] compressionMetricLifecycle() {
+ return new Object[][]{
+ {TableType.OFFLINE, true, true, false},
+ {TableType.REALTIME, true, true, false},
+ {TableType.OFFLINE, false, true, false},
+ {TableType.REALTIME, false, true, false},
+ {TableType.OFFLINE, false, true, true},
+ {TableType.REALTIME, true, false, true},
+ {TableType.OFFLINE, true, true, true}
+ };
+ }
+
+ @Test(dataProvider = "compressionMetricLifecycle")
+ public void testCompressionMetricLifecycle(TableType tableType, boolean enabled, boolean tableSizePresent,
+ boolean subtypeMissing)
+ throws Exception {
+ TableConfigBuilder tableConfigBuilder = new TableConfigBuilder(tableType).setTableName(RAW_TABLE_NAME)
+ .setCompressionStatsEnabled(enabled);
+ if (tableType == TableType.REALTIME) {
+ tableConfigBuilder.setTimeColumnName("timeColumn").setStreamConfigs(getStreamConfigMap());
+ }
+ TableConfig tableConfig = tableConfigBuilder.build();
+ String tableNameWithType = tableType == TableType.OFFLINE ? OFFLINE_TABLE_NAME : REALTIME_TABLE_NAME;
+
+ TableSizeReader.TableSubTypeSizeDetails subtypeDetails = new TableSizeReader.TableSubTypeSizeDetails();
+ TableSizeReader.TableSizeDetails tableSizeDetails = null;
+ if (tableSizePresent) {
+ tableSizeDetails = new TableSizeReader.TableSizeDetails(RAW_TABLE_NAME);
+ if (!subtypeMissing) {
+ if (tableType == TableType.OFFLINE) {
+ tableSizeDetails._offlineSegments = subtypeDetails;
+ } else {
+ tableSizeDetails._realtimeSegments = subtypeDetails;
+ }
+ }
+ }
+
+ TableSizeReader tableSizeReader = mock(TableSizeReader.class);
+ when(tableSizeReader.getTableSizeDetails(tableNameWithType, 30_000, true,
+ TableSizeReader.CompressionStatsMode.AGGREGATE_SUMMARY)).thenReturn(tableSizeDetails);
+ SegmentStatusChecker checker = new SegmentStatusChecker(mock(PinotHelixResourceManager.class),
+ mock(LeadControllerManager.class), mock(ControllerConf.class), _controllerMetrics, tableSizeReader);
+
+ checker.updateTableSizeMetrics(tableNameWithType, tableConfig);
+
+ verify(tableSizeReader).getTableSizeDetails(tableNameWithType, 30_000, true,
+ TableSizeReader.CompressionStatsMode.AGGREGATE_SUMMARY);
+ if (enabled && tableSizePresent && !subtypeMissing) {
+ verify(tableSizeReader).updateCompressionMetrics(tableNameWithType, subtypeDetails);
+ verify(tableSizeReader, never()).clearCompressionMetrics(tableNameWithType);
+ } else {
+ verify(tableSizeReader).clearCompressionMetrics(tableNameWithType);
+ verify(tableSizeReader, never()).updateCompressionMetrics(anyString(), any());
+ }
+ }
+
private void verifyControllerMetrics(String tableNameWithType, int expectedReplicationFromConfig,
int expectedNumSegmentsIncludingReplaced, int expectedNumSegment, int expectedNumReplicas,
int expectedPercentOfReplicas, int expectedSegmentsInErrorState, int expectedPercentSegmentsAvailable,
@@ -689,7 +754,8 @@ public class SegmentStatusCheckerTest {
}
@Test
- public void disabledTableTest() {
+ public void disabledTableTest()
+ throws Exception {
IdealState idealState = new IdealState(OFFLINE_TABLE_NAME);
// disable table in idealstate
idealState.enable(false);
@@ -703,9 +769,12 @@ public class SegmentStatusCheckerTest {
when(resourceManager.getAllTables()).thenReturn(List.of(OFFLINE_TABLE_NAME));
when(resourceManager.getTableIdealState(OFFLINE_TABLE_NAME)).thenReturn(idealState);
- runSegmentStatusChecker(resourceManager, 0);
+ TableSizeReader tableSizeReader = mock(TableSizeReader.class);
+ runSegmentStatusChecker(resourceManager, 0, tableSizeReader);
assertEquals(MetricValueUtils.getGlobalGaugeValue(_controllerMetrics, ControllerGauge.DISABLED_TABLE_COUNT), 1);
verifyControllerMetricsNotExist();
+ verify(tableSizeReader, never()).getTableSizeDetails(anyString(), anyInt(), anyBoolean(),
+ any(TableSizeReader.CompressionStatsMode.class));
}
@Test
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsOfflineIngestionIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsOfflineIngestionIntegrationTest.java
new file mode 100644
index 00000000000..453f64bb76c
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsOfflineIngestionIntegrationTest.java
@@ -0,0 +1,402 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Integration test that validates compression stats tracking end-to-end for offline batch ingestion.
+///
+/// Creates an offline table with `compressionStatsEnabled=true`, ingests data from Avro files with several raw
+/// (no-dictionary) columns using LZ4 compression, and then verifies that the controller's
+/// `GET /tables/{table}/size` API response includes valid uncompressed-value sizes, forward-index and dictionary
+/// storage sizes, compression ratios, and segment coverage.
+@Test(suiteName = "CustomClusterIntegrationTest")
+public class CompressionStatsOfflineIngestionIntegrationTest extends CustomDataQueryClusterIntegrationTest {
+ // Raw columns that will have compression stats tracked.
+ // These are metric/dimension columns from the default On_Time schema that support raw encoding.
+ private static final List<String> RAW_COLUMNS =
+ List.of("ActualElapsedTime", "ArrDelay", "DepDelay", "CRSDepTime");
+
+ @Override
+ public String getTableName() {
+ return "compressionStatsOfflineTest";
+ }
+
+ @Override
+ public String getTimeColumnName() {
+ return "DaysSinceEpoch";
+ }
+
+ @Override
+ protected long getCountStarResult() {
+ return DEFAULT_COUNT_STAR_RESULT;
+ }
+
+ @Override
+ public Schema createSchema() {
+ try {
+ Schema schema = createSchema(getSchemaFileName());
+ schema.setSchemaName(getTableName());
+ return schema;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public List<File> createAvroFiles()
+ throws Exception {
+ return unpackAvroData(_tempDir);
+ }
+
+ @Override
+ protected List<String> getNoDictionaryColumns() {
+ return new ArrayList<>(RAW_COLUMNS);
+ }
+
+ @Override
+ protected List<FieldConfig> getFieldConfigs() {
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ for (String column : RAW_COLUMNS) {
+ fieldConfigs.add(
+ new FieldConfig(column, FieldConfig.EncodingType.RAW, List.of(),
+ FieldConfig.CompressionCodec.LZ4, null));
+ }
+ return fieldConfigs;
+ }
+
+ @Override
+ public TableConfig createOfflineTableConfig() {
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(getTableName())
+ .setTimeColumnName(getTimeColumnName())
+ .setNoDictionaryColumns(getNoDictionaryColumns())
+ .setFieldConfigList(getFieldConfigs())
+ .setNumReplicas(getNumReplicas())
+ .build();
+
+ // Enable compression stats tracking
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setCompressionStatsEnabled(true);
+
+ return tableConfig;
+ }
+
+ @Test
+ public void testCompressionStatsInTableSizeApi()
+ throws Exception {
+ // Call the controller table size API
+ String response = sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + getTableName() + "/size");
+ JsonNode tableSizeJson = JsonUtils.stringToJsonNode(response);
+
+ // Verify top-level structure
+ assertNotNull(tableSizeJson.get("tableName"), "Response should have tableName");
+ assertTrue(tableSizeJson.get("reportedSizeInBytes").asLong() > 0,
+ "reportedSizeInBytes should be > 0");
+
+ // Get offline segment details
+ JsonNode offlineSegments = tableSizeJson.get("offlineSegments");
+ assertNotNull(offlineSegments, "offlineSegments should be present");
+
+ // Verify compression stats are nested under compressionStats object
+ JsonNode compressionStatsNode = offlineSegments.get("compressionStats");
+ assertNotNull(compressionStatsNode, "compressionStats should be present");
+
+ long uncompressedValueSizePerReplicaInBytes =
+ compressionStatsNode.get("uncompressedValueSizePerReplicaInBytes").asLong();
+ long forwardIndexAndDictionaryStorageSizePerReplicaInBytes =
+ compressionStatsNode.get("forwardIndexAndDictionaryStorageSizePerReplicaInBytes").asLong();
+ double compressionRatio = compressionStatsNode.get("compressionRatio").asDouble();
+ int segmentsWithCompleteStats = compressionStatsNode.get("segmentsWithCompleteStats").asInt();
+ int totalSegments = compressionStatsNode.get("totalSegments").asInt();
+ boolean partialCoverage = compressionStatsNode.get("partialCoverage").asBoolean();
+
+ // Uncompressed value size should be > 0 (we have 4 raw columns across 12 segments).
+ assertTrue(uncompressedValueSizePerReplicaInBytes > 0,
+ "uncompressedValueSizePerReplicaInBytes should be > 0, got: " + uncompressedValueSizePerReplicaInBytes);
+
+ // Forward-index and dictionary storage size should be > 0.
+ assertTrue(forwardIndexAndDictionaryStorageSizePerReplicaInBytes > 0,
+ "forwardIndexAndDictionaryStorageSizePerReplicaInBytes should be > 0, got: "
+ + forwardIndexAndDictionaryStorageSizePerReplicaInBytes);
+
+ assertEquals(compressionRatio,
+ (double) uncompressedValueSizePerReplicaInBytes / forwardIndexAndDictionaryStorageSizePerReplicaInBytes, 1e-9,
+ "compressionRatio should equal uncompressedValueSize / forwardIndexAndDictionaryStorageSize");
+
+ // Uncompressed value size should be >= encoded value storage size for these numeric columns.
+ assertTrue(uncompressedValueSizePerReplicaInBytes >= forwardIndexAndDictionaryStorageSizePerReplicaInBytes,
+ "uncompressedValueSize (" + uncompressedValueSizePerReplicaInBytes
+ + ") should be >= forwardIndexAndDictionaryStorageSize ("
+ + forwardIndexAndDictionaryStorageSizePerReplicaInBytes + ")");
+
+ // Compression ratio = raw / compressed, should be >= 1.0
+ assertTrue(compressionRatio >= 1.0,
+ "compressionRatio should be >= 1.0, got: " + compressionRatio);
+
+ // All 12 segments should have compression stats since compressionStatsEnabled=true
+ assertEquals(totalSegments, 12, "totalSegments should be 12");
+ assertEquals(segmentsWithCompleteStats, 12,
+ "segmentsWithCompleteStats should equal totalSegments (all segments built with stats enabled)");
+
+ // No partial coverage since all segments have stats
+ assertFalse(partialCoverage,
+ "partialCoverage should be false when all segments have stats");
+ }
+
+ @Test
+ public void testPerSegmentCompressionStats()
+ throws Exception {
+ // Call table size API with verbose=true (default) to get per-segment details
+ String response = sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + getTableName()
+ + "/size?verbose=true&includeColumnCompressionStats=true");
+ JsonNode tableSizeJson = JsonUtils.stringToJsonNode(response);
+
+ JsonNode offlineSegments = tableSizeJson.get("offlineSegments");
+ JsonNode segments = offlineSegments.get("segments");
+ assertNotNull(segments, "segments map should be present in verbose response");
+
+ // Each segment should have server info with compression stats
+ int segmentsChecked = 0;
+ int serverStatsChecked = 0;
+ int columnStatsChecked = 0;
+ var fieldNames = segments.fieldNames();
+ while (fieldNames.hasNext()) {
+ String segmentName = fieldNames.next();
+ JsonNode segmentDetails = segments.get(segmentName);
+ JsonNode serverInfo = segmentDetails.get("serverInfo");
+ assertNotNull(serverInfo, "serverInfo should be present for segment: " + segmentName);
+
+ // The bounded controller fan-out selects one complete replica and attaches stats only to that server entry.
+ boolean selectedReplicaFound = false;
+ var serverNames = serverInfo.fieldNames();
+ while (serverNames.hasNext()) {
+ String serverName = serverNames.next();
+ JsonNode sizeInfo = serverInfo.get(serverName);
+ long diskSize = sizeInfo.get("diskSizeInBytes").asLong();
+ if (sizeInfo.has("compressionStatsUncompressedValueSizeInBytes")) {
+ assertTrue(diskSize > 0, "The selected stats-bearing replica should also report segment size");
+ assertTrue(sizeInfo.has("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes"),
+ "Server info should have compressionStatsForwardIndexAndDictionaryStorageSizeInBytes for segment "
+ + segmentName);
+ // Segment should have uncompressed-value and encoded-storage sizes.
+ long rawSize = sizeInfo.get("compressionStatsUncompressedValueSizeInBytes").asLong();
+ long compressedSize = sizeInfo.get("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes").asLong();
+ assertTrue(rawSize > 0,
+ "compressionStatsUncompressedValueSizeInBytes should be > 0 for segment " + segmentName);
+ assertTrue(compressedSize > 0,
+ "compressionStatsForwardIndexAndDictionaryStorageSizeInBytes should be > 0 for segment " + segmentName);
+ serverStatsChecked++;
+
+ // Verify all requested raw columns have compression statistics.
+ JsonNode columnStats = sizeInfo.get("columnCompressionStats");
+ assertNotNull(columnStats, "columnCompressionStats should be present for segment " + segmentName);
+ for (String col : RAW_COLUMNS) {
+ assertTrue(columnStats.has(col), "Missing compression stats for raw column " + col);
+ JsonNode colInfo = columnStats.get(col);
+ long columnUncompressedSize = colInfo.get("uncompressedValueSizeInBytes").asLong();
+ long columnStorageSize = colInfo.get("forwardIndexAndDictionaryStorageSizeInBytes").asLong();
+ assertTrue(columnUncompressedSize > 0,
+ "Per-column uncompressed value size should be > 0 for " + col);
+ assertTrue(columnStorageSize > 0,
+ "Per-column forward-index and dictionary storage size should be > 0 for " + col);
+ assertEquals(colInfo.get("compressionRatio").asDouble(),
+ (double) columnUncompressedSize / columnStorageSize, 1e-9,
+ "Per-column compression ratio should equal uncompressed size / storage size for " + col);
+ JsonNode encoding = colInfo.get("encodingBreakdown").get(0);
+ assertEquals(encoding.get("encoding").asText(), "RAW");
+ assertEquals(encoding.get("chunkCompressionType").asText(), "LZ4",
+ "Chunk compression type should be LZ4 for " + col);
+ // Encoding is represented by encodingBreakdown; the old hasDictionary field is not emitted.
+ assertFalse(colInfo.has("hasDictionary"),
+ "hasDictionary field should no longer be present in the response for " + col);
+ columnStatsChecked++;
+ }
+ selectedReplicaFound = true;
+ }
+ }
+ assertTrue(selectedReplicaFound, "Each complete segment should expose one selected stats-bearing replica");
+ segmentsChecked++;
+ }
+ assertTrue(segmentsChecked > 0, "Should have checked at least one segment");
+ assertTrue(serverStatsChecked > 0, "At least one immutable segment must expose compression statistics");
+ assertTrue(columnStatsChecked > 0, "At least one raw column must expose compression statistics");
+ }
+
+ @Test
+ public void testColumnCompressionStatsAreOptIn()
+ throws Exception {
+ String sizeApi = "http://localhost:" + getControllerPort() + "/tables/" + getTableName()
+ + "/size?verbose=true";
+ assertSizeColumnStatsPresence(JsonUtils.stringToJsonNode(sendGetRequest(sizeApi)), false);
+ assertSizeColumnStatsPresence(
+ JsonUtils.stringToJsonNode(sendGetRequest(sizeApi + "&includeColumnCompressionStats=true")), true);
+
+ String metadataApi = "http://localhost:" + getControllerPort() + "/tables/" + getTableName()
+ + "/metadata?type=OFFLINE";
+ JsonNode metadataWithoutColumns = JsonUtils.stringToJsonNode(sendGetRequest(metadataApi));
+ assertNotNull(metadataWithoutColumns.get("compressionStats"));
+ assertFalse(metadataWithoutColumns.has("columnCompressionStats"));
+
+ JsonNode metadataWithColumns =
+ JsonUtils.stringToJsonNode(sendGetRequest(metadataApi + "&includeColumnCompressionStats=true"));
+ assertNotNull(metadataWithColumns.get("compressionStats"));
+ JsonNode columnCompressionStats = metadataWithColumns.get("columnCompressionStats");
+ assertNotNull(columnCompressionStats);
+ assertTrue(columnCompressionStats.isArray());
+ assertTrue(columnCompressionStats.size() > 0);
+
+ String filteredColumn = RAW_COLUMNS.get(0);
+ JsonNode filteredMetadata = JsonUtils.stringToJsonNode(sendGetRequest(
+ metadataApi + "&columns=" + filteredColumn + "&includeColumnCompressionStats=true"));
+ JsonNode filteredColumnStats = filteredMetadata.get("columnCompressionStats");
+ assertNotNull(filteredColumnStats);
+ assertEquals(filteredColumnStats.size(), 1);
+ assertEquals(filteredColumnStats.get(0).get("column").asText(), filteredColumn);
+ }
+
+ private static void assertSizeColumnStatsPresence(JsonNode tableSizeJson, boolean expected) {
+ JsonNode segments = tableSizeJson.get("offlineSegments").get("segments");
+ int statsBearingReplicas = 0;
+ var segmentNames = segments.fieldNames();
+ while (segmentNames.hasNext()) {
+ JsonNode serverInfo = segments.get(segmentNames.next()).get("serverInfo");
+ var serverNames = serverInfo.fieldNames();
+ while (serverNames.hasNext()) {
+ JsonNode sizeInfo = serverInfo.get(serverNames.next());
+ if (sizeInfo.has("compressionStatsUncompressedValueSizeInBytes")) {
+ assertEquals(sizeInfo.has("columnCompressionStats"), expected);
+ statsBearingReplicas++;
+ }
+ }
+ }
+ assertTrue(statsBearingReplicas > 0, "The opt-in check must inspect at least one stats-bearing replica");
+ }
+
+ @Test
+ public void testCompressionStatsDisabledTable()
+ throws Exception {
+ // Create a second table WITHOUT compressionStatsEnabled
+ String noStatsTableName = "compressionStatsDisabledTest";
+ Schema schema = createSchema();
+ schema.setSchemaName(noStatsTableName);
+ addSchema(schema);
+
+ TableConfig noStatsConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(noStatsTableName)
+ .setTimeColumnName(getTimeColumnName())
+ .setNoDictionaryColumns(getNoDictionaryColumns())
+ .setFieldConfigList(getFieldConfigs())
+ .setNumReplicas(getNumReplicas())
+ .build();
+ // compressionStatsEnabled defaults to false; do NOT set it
+
+ addTableConfig(noStatsConfig);
+
+ // Build and upload segments for the no-stats table
+ File noStatsSegmentDir = new File(_tempDir, "noStatsSegmentDir");
+ File noStatsTarDir = new File(_tempDir, "noStatsTarDir");
+ TestUtils.ensureDirectoriesExistAndEmpty(noStatsSegmentDir, noStatsTarDir);
+
+ List<File> avroFiles = unpackAvroData(_tempDir);
+ ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, noStatsConfig, schema, 0,
+ noStatsSegmentDir, noStatsTarDir);
+ uploadSegments(noStatsTableName, noStatsTarDir);
+
+ // Wait for docs to load
+ TestUtils.waitForCondition(aVoid -> {
+ try {
+ return getCurrentCountStarResult(noStatsTableName) == DEFAULT_COUNT_STAR_RESULT;
+ } catch (Exception e) {
+ return false;
+ }
+ }, 600_000L, "Failed to load documents for no-stats table");
+
+ try {
+ // Query table size
+ String response = sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + noStatsTableName + "/size");
+ JsonNode tableSizeJson = JsonUtils.stringToJsonNode(response);
+
+ JsonNode offlineSegments = tableSizeJson.get("offlineSegments");
+ assertNotNull(offlineSegments);
+
+ // compressionStats should be absent (null/suppressed) since compressionStatsEnabled was false
+ JsonNode compressionStatsNode = offlineSegments.get("compressionStats");
+ assertNull(compressionStatsNode,
+ "compressionStats should be absent when compressionStatsEnabled is false");
+
+ // Enable tracking after the original segments were built, then add one newly tracked segment. This models a
+ // rolling table upgrade where legacy and current segments coexist.
+ noStatsConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+ updateTableConfig(noStatsConfig);
+
+ File trackedSegmentDir = new File(_tempDir, "trackedSegmentDir");
+ File trackedTarDir = new File(_tempDir, "trackedTarDir");
+ TestUtils.ensureDirectoriesExistAndEmpty(trackedSegmentDir, trackedTarDir);
+ int legacySegmentCount = avroFiles.size();
+ ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles.subList(0, 1), noStatsConfig, schema,
+ legacySegmentCount, trackedSegmentDir, trackedTarDir);
+ uploadSegments(noStatsTableName, trackedTarDir);
+
+ TestUtils.waitForCondition(aVoid -> {
+ try {
+ JsonNode stats = JsonUtils.stringToJsonNode(sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + noStatsTableName + "/size"))
+ .path("offlineSegments").path("compressionStats");
+ return stats.path("totalSegments").asInt() == legacySegmentCount + 1
+ && stats.path("segmentsWithCompleteStats").asInt() == 1;
+ } catch (Exception e) {
+ return false;
+ }
+ }, 600_000L, "Failed to observe mixed legacy and tracked compression-statistics coverage");
+
+ JsonNode mixedCompressionStats = JsonUtils.stringToJsonNode(sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + noStatsTableName + "/size"))
+ .path("offlineSegments").path("compressionStats");
+ assertEquals(mixedCompressionStats.get("totalSegments").asInt(), legacySegmentCount + 1);
+ assertEquals(mixedCompressionStats.get("segmentsWithCompleteStats").asInt(), 1);
+ assertTrue(mixedCompressionStats.get("partialCoverage").asBoolean());
+ assertTrue(mixedCompressionStats.get("uncompressedValueSizePerReplicaInBytes").asLong() > 0);
+ assertTrue(mixedCompressionStats.get("forwardIndexAndDictionaryStorageSizePerReplicaInBytes").asLong() > 0);
+ } finally {
+ // Clean up the second table even if assertions fail
+ dropOfflineTable(noStatsTableName);
+ }
+ }
+}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsRealtimeIngestionIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsRealtimeIngestionIntegrationTest.java
new file mode 100644
index 00000000000..85c75295972
--- /dev/null
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CompressionStatsRealtimeIngestionIntegrationTest.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Integration test that validates compression stats tracking end-to-end for realtime (Kafka) ingestion.
+///
+/// Creates a realtime table with `compressionStatsEnabled=true`, pushes data from Avro files into Kafka with several
+/// raw (no-dictionary) columns using LZ4 compression, waits for all documents to be consumed, and then verifies that
+/// the controller's `GET /tables/{table}/size` API response includes valid compression statistics for the completed
+/// (COMPLETED) segments.
+@Test(suiteName = "CustomClusterIntegrationTest")
+public class CompressionStatsRealtimeIngestionIntegrationTest extends CustomDataQueryClusterIntegrationTest {
+
+ // Raw columns that will have compression stats tracked.
+ // These are metric/dimension columns from the default On_Time schema that support raw encoding.
+ private static final List<String> RAW_COLUMNS =
+ List.of("ActualElapsedTime", "ArrDelay", "DepDelay", "CRSDepTime");
+
+ @Override
+ public String getTableName() {
+ return "compressionStatsRealtimeTest";
+ }
+
+ @Override
+ public String getTimeColumnName() {
+ return "DaysSinceEpoch";
+ }
+
+ @Override
+ protected String getSortedColumn() {
+ return null;
+ }
+
+ @Override
+ protected long getCountStarResult() {
+ return DEFAULT_COUNT_STAR_RESULT;
+ }
+
+ @Override
+ public boolean isRealtimeTable() {
+ return true;
+ }
+
+ @Override
+ public Schema createSchema() {
+ try {
+ Schema schema = createSchema(getSchemaFileName());
+ schema.setSchemaName(getTableName());
+ return schema;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public List<File> createAvroFiles()
+ throws Exception {
+ return unpackAvroData(_tempDir);
+ }
+
+ @Override
+ protected List<String> getNoDictionaryColumns() {
+ return new ArrayList<>(RAW_COLUMNS);
+ }
+
+ @Override
+ protected List<FieldConfig> getFieldConfigs() {
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ for (String column : RAW_COLUMNS) {
+ fieldConfigs.add(
+ new FieldConfig(column, FieldConfig.EncodingType.RAW, List.of(),
+ FieldConfig.CompressionCodec.LZ4, null));
+ }
+ return fieldConfigs;
+ }
+
+ @Override
+ protected TableConfig createRealtimeTableConfig(File sampleAvroFile) {
+ TableConfig tableConfig = super.createRealtimeTableConfig(sampleAvroFile);
+
+ // Enable compression stats tracking
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setCompressionStatsEnabled(true);
+
+ return tableConfig;
+ }
+
+ @Test
+ public void testCompressionStatsInTableSizeApiForRealtimeTable()
+ throws Exception {
+ // Call the controller table size API
+ String response = sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + getTableName() + "/size");
+ JsonNode tableSizeJson = JsonUtils.stringToJsonNode(response);
+
+ // Verify top-level structure
+ assertNotNull(tableSizeJson.get("tableName"), "Response should have tableName");
+ assertTrue(tableSizeJson.get("reportedSizeInBytes").asLong() >= 0,
+ "reportedSizeInBytes should be >= 0");
+
+ // Get realtime segment details
+ JsonNode realtimeSegments = tableSizeJson.get("realtimeSegments");
+ assertNotNull(realtimeSegments, "realtimeSegments should be present");
+
+ // Verify compression stats are nested under compressionStats object
+ JsonNode compressionStatsNode = realtimeSegments.get("compressionStats");
+ assertNotNull(compressionStatsNode, "compressionStats should be present");
+ assertTrue(compressionStatsNode.has("uncompressedValueSizePerReplicaInBytes"),
+ "compressionStats should have uncompressedValueSizePerReplicaInBytes");
+ assertTrue(compressionStatsNode.has("forwardIndexAndDictionaryStorageSizePerReplicaInBytes"),
+ "compressionStats should have forwardIndexAndDictionaryStorageSizePerReplicaInBytes");
+ assertTrue(compressionStatsNode.has("compressionRatio"),
+ "compressionStats should have compressionRatio");
+ assertTrue(compressionStatsNode.has("segmentsWithCompleteStats"),
+ "compressionStats should have segmentsWithCompleteStats");
+ assertTrue(compressionStatsNode.has("totalSegments"),
+ "compressionStats should have totalSegments");
+
+ long uncompressedValueSizePerReplicaInBytes =
+ compressionStatsNode.get("uncompressedValueSizePerReplicaInBytes").asLong();
+ long forwardIndexAndDictionaryStorageSizePerReplicaInBytes =
+ compressionStatsNode.get("forwardIndexAndDictionaryStorageSizePerReplicaInBytes").asLong();
+ double compressionRatio = compressionStatsNode.get("compressionRatio").asDouble();
+ int segmentsWithCompleteStats = compressionStatsNode.get("segmentsWithCompleteStats").asInt();
+ int totalSegments = compressionStatsNode.get("totalSegments").asInt();
+
+ // Total segments should be > 0 (at least consuming segments exist)
+ assertTrue(totalSegments > 0,
+ "totalSegments should be > 0, got: " + totalSegments);
+
+ assertTrue(segmentsWithCompleteStats > 0,
+ "At least one committed realtime segment must contribute compression stats");
+ assertTrue(uncompressedValueSizePerReplicaInBytes > 0,
+ "uncompressedValueSizePerReplicaInBytes should be > 0 when segments have stats, got: "
+ + uncompressedValueSizePerReplicaInBytes);
+ assertTrue(forwardIndexAndDictionaryStorageSizePerReplicaInBytes > 0,
+ "forwardIndexAndDictionaryStorageSizePerReplicaInBytes should be > 0 when segments have stats, got: "
+ + forwardIndexAndDictionaryStorageSizePerReplicaInBytes);
+ assertEquals(compressionRatio,
+ (double) uncompressedValueSizePerReplicaInBytes / forwardIndexAndDictionaryStorageSizePerReplicaInBytes, 1e-9,
+ "compressionRatio should equal uncompressedValueSize / forwardIndexAndDictionaryStorageSize");
+ assertTrue(uncompressedValueSizePerReplicaInBytes >= forwardIndexAndDictionaryStorageSizePerReplicaInBytes,
+ "uncompressedValueSize (" + uncompressedValueSizePerReplicaInBytes
+ + ") should be >= forwardIndexAndDictionaryStorageSize ("
+ + forwardIndexAndDictionaryStorageSizePerReplicaInBytes + ")");
+ assertTrue(compressionRatio >= 1.0,
+ "compressionRatio should be >= 1.0, got: " + compressionRatio);
+ }
+
+ @Test
+ public void testPerSegmentCompressionStatsForRealtimeTable()
+ throws Exception {
+ // Call table size API with verbose=true to get per-segment details
+ String response = sendGetRequest(
+ "http://localhost:" + getControllerPort() + "/tables/" + getTableName() + "/size?verbose=true");
+ JsonNode tableSizeJson = JsonUtils.stringToJsonNode(response);
+
+ JsonNode realtimeSegments = tableSizeJson.get("realtimeSegments");
+ assertNotNull(realtimeSegments, "realtimeSegments should be present");
+
+ JsonNode segments = realtimeSegments.get("segments");
+ assertNotNull(segments, "segments map should be present in verbose response");
+
+ // At least one segment should exist
+ assertTrue(segments.size() > 0, "Should have at least one segment");
+
+ // Iterate segments and validate structure
+ int segmentsChecked = 0;
+ int statsBearingSegments = 0;
+ var fieldNames = segments.fieldNames();
+ while (fieldNames.hasNext()) {
+ String segmentName = fieldNames.next();
+ JsonNode segmentDetails = segments.get(segmentName);
+ JsonNode serverInfo = segmentDetails.get("serverInfo");
+ assertNotNull(serverInfo, "serverInfo should be present for segment: " + segmentName);
+
+ var serverNames = serverInfo.fieldNames();
+ while (serverNames.hasNext()) {
+ String serverName = serverNames.next();
+ JsonNode sizeInfo = serverInfo.get(serverName);
+ long diskSize = sizeInfo.get("diskSizeInBytes").asLong();
+ // Only the immutable replica selected by bounded compression fan-out carries summary fields.
+ if (sizeInfo.has("compressionStatsUncompressedValueSizeInBytes")) {
+ assertTrue(diskSize > 0, "The selected stats-bearing replica should also report segment size");
+ assertTrue(sizeInfo.has("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes"),
+ "Server info should have compressionStatsForwardIndexAndDictionaryStorageSizeInBytes for segment "
+ + segmentName);
+ assertTrue(sizeInfo.get("compressionStatsUncompressedValueSizeInBytes").asLong() > 0,
+ "Uncompressed value size should be positive for segment " + segmentName);
+ assertTrue(sizeInfo.get("compressionStatsForwardIndexAndDictionaryStorageSizeInBytes").asLong() > 0,
+ "Forward-index storage size should be positive for segment " + segmentName);
+ statsBearingSegments++;
+ }
+ }
+ segmentsChecked++;
+ }
+ assertTrue(segmentsChecked > 0, "Should have checked at least one segment");
+ assertTrue(statsBearingSegments > 0, "At least one completed realtime segment must expose compression stats");
+ }
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/BaseChunkForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/BaseChunkForwardIndexWriter.java
index 8b3d22aef40..ad35bc020ec 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/BaseChunkForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/BaseChunkForwardIndexWriter.java
@@ -69,8 +69,11 @@ public abstract class BaseChunkForwardIndexWriter implements Closeable {
protected int _chunkSize;
protected long _dataOffset;
+ protected long _uncompressedValueSize;
+ protected boolean _trackUncompressedValueSize = false;
private final int _headerEntryChunkOffsetSize;
+ private final long _initialDataOffset;
/**
* Constructor for the class.
@@ -95,6 +98,7 @@ public abstract class BaseChunkForwardIndexWriter implements Closeable {
_chunkCompressor = ChunkCompressorFactory.getCompressor(compressionType);
_headerEntryChunkOffsetSize = version == 2 ? Integer.BYTES : Long.BYTES;
_dataOffset = writeHeader(compressionType, totalDocs, numDocsPerChunk, sizeOfEntry, version);
+ _initialDataOffset = _dataOffset;
_chunkBuffer = ByteBuffer.allocateDirect(_chunkSize);
int maxCompressedChunkSize = _chunkCompressor.maxCompressedSize(_chunkSize); // may exceed original chunk size
_compressedBuffer = ByteBuffer.allocateDirect(maxCompressedChunkSize);
@@ -196,4 +200,16 @@ public abstract class BaseChunkForwardIndexWriter implements Closeable {
_dataOffset += sizeToWrite;
_chunkBuffer.clear();
}
+
+ /// Returns the total uncompressed size of data written so far, or `-1` when tracking is disabled.
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _trackUncompressedValueSize ? _uncompressedValueSize : -1;
+ }
+
+ /// Enables uncompressed-size tracking before the first value is written.
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ Preconditions.checkState(_chunkBuffer.position() == 0 && _dataOffset == _initialDataOffset,
+ "Uncompressed-size tracking must be enabled before writing values");
+ _trackUncompressedValueSize = true;
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
index 8b517a84f9c..18bba3652c0 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
@@ -77,8 +77,18 @@ public class FixedByteChunkForwardIndexWriter extends BaseChunkForwardIndexWrite
flushChunkIfNeeded();
}
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ // Include in-flight bytes from the current unflushed chunk so callers reading
+ // before close() (e.g., writeMetadata()) get the correct total.
+ return _trackUncompressedValueSize ? _uncompressedValueSize + _chunkDataOffset : -1;
+ }
+
@Override
protected void writeChunk() {
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += _chunkDataOffset;
+ }
super.writeChunk();
_chunkDataOffset = 0;
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
index f04481dc04e..f51d11dceff 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriter.java
@@ -150,6 +150,9 @@ public class VarByteChunkForwardIndexWriter extends BaseChunkForwardIndexWriter
* </ul>
*/
protected void writeChunk() {
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += _chunkDataOffSet - _chunkHeaderSize;
+ }
// For partially filled chunks, we still need to clear the offsets for remaining rows, as we reuse this buffer.
for (int i = _chunkHeaderOffset; i < _chunkHeaderSize; i += Integer.BYTES) {
_chunkBuffer.putInt(i, 0);
@@ -161,4 +164,9 @@ public class VarByteChunkForwardIndexWriter extends BaseChunkForwardIndexWriter
_chunkHeaderOffset = 0;
_chunkDataOffSet = _chunkHeaderSize;
}
+
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _trackUncompressedValueSize ? _uncompressedValueSize + _chunkDataOffSet - _chunkHeaderSize : -1;
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
index c6b30c03848..40f308b62dc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkForwardIndexWriterV4.java
@@ -92,6 +92,8 @@ public class VarByteChunkForwardIndexWriterV4 implements VarByteChunkWriter {
private int _nextDocId = 0;
private int _metadataSize = 0;
private long _chunkOffset = 0;
+ private long _uncompressedValueSize = 0;
+ private boolean _trackUncompressedValueSize = false;
public VarByteChunkForwardIndexWriterV4(File file, ChunkCompressionType compressionType, int chunkSize)
throws IOException {
@@ -202,6 +204,9 @@ public class VarByteChunkForwardIndexWriterV4 implements VarByteChunkWriter {
buffer = ByteBuffer.wrap(bytes);
}
try {
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += bytes.length;
+ }
_nextDocId++;
write(buffer, true);
} finally {
@@ -224,6 +229,10 @@ public class VarByteChunkForwardIndexWriterV4 implements VarByteChunkWriter {
[------16-bytes-----][----a+b+c bytes----------]
*/
int numDocs = _nextDocId - _docIdOffset;
+ int logicalDataSize = _chunkBuffer.position() - Integer.BYTES - numDocs * Integer.BYTES;
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += logicalDataSize;
+ }
_chunkBuffer.putInt(0, numDocs);
// collect offsets
int[] offsets = new int[numDocs];
@@ -332,4 +341,20 @@ public class VarByteChunkForwardIndexWriterV4 implements VarByteChunkWriter {
FileUtils.deleteQuietly(_dataBuffer);
_chunkCompressor.close();
}
+
+ /// Returns the total uncompressed size of data written so far, or `-1` when tracking is disabled.
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ if (!_trackUncompressedValueSize) {
+ return -1;
+ }
+ int bufferedDocs = _nextDocId - _docIdOffset;
+ return _uncompressedValueSize + _chunkBuffer.position() - Integer.BYTES - bufferedDocs * Integer.BYTES;
+ }
+
+ /// Enables uncompressed-size tracking before the first value is written.
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ Preconditions.checkState(_nextDocId == 0,
+ "Uncompressed-size tracking must be enabled before writing values");
+ _trackUncompressedValueSize = true;
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
index 80739ca63d5..6159c116920 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/VarByteChunkWriter.java
@@ -42,4 +42,10 @@ public interface VarByteChunkWriter extends Closeable {
void putStringMV(String[] values);
void putBytesMV(byte[][] values);
+
+ /// Returns the total uncompressed size of data written so far, or `-1` when unavailable.
+ long getRawForwardIndexUncompressedValueSizeInBytes();
+
+ /// Enables uncompressed-size tracking before the first value is written.
+ void enableRawForwardIndexUncompressedValueSizeTracking();
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
index a978221ef69..2bb4a44a61a 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/BaseSegmentCreator.java
@@ -47,6 +47,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVe
import org.apache.pinot.segment.local.segment.index.converter.SegmentFormatConverterFactory;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexPlugin;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.local.segment.index.forward.CompressionStatsMetadata;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.invertedindex.MultiColumnTextIndexHandler;
import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
@@ -169,13 +170,18 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
Preconditions.checkState(dictEnabledColumn,
"Cannot create inverted index for raw index column: %s", columnName);
}
- IndexCreationContext.Common context = getIndexCreationContext(fieldSpec, dictEnabledColumn);
-
FieldIndexConfigs config = adaptConfig(columnName, originalConfig, columnStatistics, _config, dictEnabledColumn);
+ IndexCreationContext.Common context =
+ getIndexCreationContext(fieldSpec, dictEnabledColumn, _config.isCompressionStatsEnabled());
SegmentDictionaryCreator dictionaryCreator = null;
if (dictEnabledColumn) {
- dictionaryCreator = getDictionaryCreator(columnName, originalConfig, context);
+ ForwardIndexConfig forwardIndexConfig = config.getConfig(StandardIndexes.forward());
+ boolean trackDictionaryValues = _config.isCompressionStatsEnabled() && forwardIndexConfig.isEnabled()
+ && forwardIndexConfig.getEncodingType() == FieldConfig.EncodingType.DICTIONARY;
+ IndexCreationContext.Common dictionaryContext = trackDictionaryValues == _config.isCompressionStatsEnabled()
+ ? context : getIndexCreationContext(fieldSpec, true, trackDictionaryValues);
+ dictionaryCreator = getDictionaryCreator(columnName, originalConfig, dictionaryContext);
}
List<IndexCreator> indexCreators = getIndexCreatorsByColumn(fieldSpec, context, config, dictEnabledColumn);
@@ -184,7 +190,8 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
indexCreators, getNullValueCreator(fieldSpec));
}
- private IndexCreationContext.Common getIndexCreationContext(FieldSpec fieldSpec, boolean dictEnabledColumn) {
+ private IndexCreationContext.Common getIndexCreationContext(FieldSpec fieldSpec, boolean dictEnabledColumn,
+ boolean compressionStatsEnabled) {
ColumnStatistics columnStats = _columnStatisticsMap.get(fieldSpec.getName());
return new IndexCreationContext.Builder(_indexDir, _config.getTableConfig(), columnStats, dictEnabledColumn)
.withOnHeap(_config.isOnHeap())
@@ -196,6 +203,7 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
.withMutableSegmentCompacted(_config.isMutableSegmentCompacted())
.withMutableToImmutableDocIdMap(_config.getMutableToImmutableDocIdMap())
.withContinueOnError(_config.isContinueOnError())
+ .withCompressionStatsEnabled(compressionStatsEnabled)
.build();
}
@@ -552,6 +560,36 @@ public abstract class BaseSegmentCreator implements SegmentCreator {
hasDictionary, dictionaryElementSize, fwdConfig.getEncodingType(), false);
}
+ if (_config.isCompressionStatsEnabled()) {
+ for (Map.Entry<String, ColumnIndexCreators> entry : _colIndexes.entrySet()) {
+ String column = entry.getKey();
+ ColumnIndexCreators colCreators = entry.getValue();
+ ForwardIndexCreator fwdCreator = colCreators.getForwardIndexCreator();
+ CompressionStatsMetadata compressionMetadata = CompressionStatsMetadata.unavailable();
+ if (fwdCreator != null && !fwdCreator.isDictionaryEncoded()) {
+ compressionMetadata = CompressionStatsMetadata.forRawForwardIndex(
+ fwdCreator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ fwdCreator.getRawForwardIndexChunkCompressionType());
+ } else if (fwdCreator != null) {
+ SegmentDictionaryCreator dictCreator = colCreators.getDictionaryCreator();
+ if (dictCreator != null) {
+ FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
+ DataType storedType = fieldSpec != null ? fieldSpec.getDataType().getStoredType() : null;
+ long uncompressedValueSizeInBytes;
+ if (storedType != null && storedType.isFixedWidth()) {
+ ColumnStatistics columnStatistics = _columnStatisticsMap.get(column);
+ int totalValues = columnStatistics != null ? columnStatistics.getTotalNumberOfEntries() : _totalDocs;
+ uncompressedValueSizeInBytes = (long) totalValues * storedType.size();
+ } else {
+ uncompressedValueSizeInBytes = dictCreator.getTotalVariableLengthUncompressedValueSizeInBytes();
+ }
+ compressionMetadata = CompressionStatsMetadata.forDictionary(uncompressedValueSizeInBytes);
+ }
+ }
+ compressionMetadata.applyTo(properties, column);
+ }
+ }
+
SegmentZKPropsConfig segmentZKPropsConfig = _config.getSegmentZKPropsConfig();
if (segmentZKPropsConfig != null) {
properties.setProperty(Realtime.START_OFFSET, segmentZKPropsConfig.getStartOffset());
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnIndexCreators.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnIndexCreators.java
index 0ef96b7b17b..7b4c0b6f619 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnIndexCreators.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/ColumnIndexCreators.java
@@ -27,6 +27,7 @@ import org.apache.pinot.common.utils.FileUtils;
import org.apache.pinot.segment.local.segment.creator.impl.nullvalue.NullValueVectorCreator;
import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
import org.apache.pinot.segment.spi.index.IndexCreator;
+import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec;
@@ -92,6 +93,17 @@ public class ColumnIndexCreators implements Closeable {
return _indexConfigs;
}
+ /// Returns the [ForwardIndexCreator] for this column, or `null` if no forward index was created.
+ @Nullable
+ public ForwardIndexCreator getForwardIndexCreator() {
+ for (IndexCreator creator : _indexCreators) {
+ if (creator instanceof ForwardIndexCreator) {
+ return (ForwardIndexCreator) creator;
+ }
+ }
+ return null;
+ }
+
public void seal() throws IOException {
if (_isSealed) {
return;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreator.java
index 471495173c7..f084978f889 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreator.java
@@ -20,6 +20,8 @@ package org.apache.pinot.segment.local.segment.creator.impl;
import java.io.IOException;
import java.math.BigDecimal;
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec;
@@ -55,6 +57,17 @@ class SameValueForwardIndexCreator implements ForwardIndexCreator {
return _delegate.getValueType();
}
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _delegate.getRawForwardIndexUncompressedValueSizeInBytes();
+ }
+
+ @Nullable
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _delegate.getRawForwardIndexChunkCompressionType();
+ }
+
@Override
public void putInt(int value) {
_delegate.putInt((int) _actualValue);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
index 3b580147201..d1b4769d670 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentDictionaryCreator.java
@@ -34,12 +34,14 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.io.util.FixedByteValueReaderWriter;
import org.apache.pinot.segment.local.io.util.VarLengthValueWriter;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.spi.creator.IndexCreationContext;
import org.apache.pinot.segment.spi.index.IndexCreator;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +59,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
*/
public class SegmentDictionaryCreator implements IndexCreator {
private static final Logger LOGGER = LoggerFactory.getLogger(SegmentDictionaryCreator.class);
+ private static final int MAX_CACHED_SERIALIZED_LENGTHS = 4 * 1024 * 1024 / Integer.BYTES;
private final String _columnName;
private final DataType _storedType;
@@ -69,20 +72,56 @@ public class SegmentDictionaryCreator implements IndexCreator {
private Double2IntOpenHashMap _doubleValueToIndexMap;
private Object2IntOpenHashMap<Object> _objectValueToIndexMap;
private int _numBytesPerEntry = 0;
+ private final boolean _trackUncompressedValueBytes;
+ // Cache lengths computed while serializing the dictionary, capped at 4 MiB per column to avoid cardinality-sized
+ // allocations for very large dictionaries.
+ @Nullable
+ private int[] _serializedValueLengthByDictId;
+ /// Accumulated uncompressed serialized column-value bytes across all rows indexed. Populated only when compression
+ /// stats are enabled
+ /// and for variable-length types (STRING, BYTES, BIG_DECIMAL); 0 for fixed-width types.
+ private long _totalUncompressedValueSizeInBytes;
+
+ /// Controls whether dictionary creation tracks uncompressed column-value bytes.
+ public enum UncompressedValueSizeTracking {
+ ENABLED,
+ DISABLED;
+
+ public static UncompressedValueSizeTracking fromEnabled(boolean enabled) {
+ return enabled ? ENABLED : DISABLED;
+ }
+ }
public SegmentDictionaryCreator(String columnName, DataType storedType, File indexFile,
boolean useVarLengthDictionary) {
+ this(columnName, storedType, indexFile, useVarLengthDictionary, UncompressedValueSizeTracking.DISABLED);
+ }
+
+ public SegmentDictionaryCreator(String columnName, DataType storedType, File indexFile,
+ boolean useVarLengthDictionary, UncompressedValueSizeTracking tracking) {
_columnName = columnName;
_storedType = storedType;
_dictionaryFile = indexFile;
_useVarLengthDictionary = useVarLengthDictionary;
+ _trackUncompressedValueBytes = tracking == UncompressedValueSizeTracking.ENABLED;
}
public SegmentDictionaryCreator(FieldSpec fieldSpec, File indexDir, boolean useVarLengthDictionary) {
+ this(fieldSpec, indexDir, useVarLengthDictionary, UncompressedValueSizeTracking.DISABLED);
+ }
+
+ public SegmentDictionaryCreator(FieldSpec fieldSpec, File indexDir, boolean useVarLengthDictionary,
+ UncompressedValueSizeTracking tracking) {
_columnName = fieldSpec.getName();
_storedType = fieldSpec.getDataType().getStoredType();
_dictionaryFile = new File(indexDir, _columnName + DictionaryIndexType.getFileExtension());
_useVarLengthDictionary = useVarLengthDictionary;
+ _trackUncompressedValueBytes = tracking == UncompressedValueSizeTracking.ENABLED;
+ }
+
+ public SegmentDictionaryCreator(IndexCreationContext context, boolean useVarLengthDictionary) {
+ this(context.getFieldSpec(), context.getIndexDir(), useVarLengthDictionary,
+ UncompressedValueSizeTracking.fromEnabled(context.isCompressionStatsEnabled()));
}
@Override
@@ -187,6 +226,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
numValues = sortedBigDecimals.length;
Preconditions.checkState(numValues > 0);
_objectValueToIndexMap = new Object2IntOpenHashMap<>(numValues);
+ initializeSerializedLengthCache(numValues);
// Get the maximum length of all entries
byte[][] sortedBigDecimalBytes = new byte[numValues][];
@@ -195,6 +235,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
_objectValueToIndexMap.put(value, i);
byte[] valueBytes = BigDecimalUtils.serialize(value);
sortedBigDecimalBytes[i] = valueBytes;
+ cacheSerializedLength(i, valueBytes.length);
_numBytesPerEntry = Math.max(_numBytesPerEntry, valueBytes.length);
}
@@ -208,6 +249,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
numValues = sortedStrings.length;
Preconditions.checkState(numValues > 0);
_objectValueToIndexMap = new Object2IntOpenHashMap<>(numValues);
+ initializeSerializedLengthCache(numValues);
// Get the maximum length of all entries
byte[][] sortedStringBytes = new byte[numValues][];
@@ -216,6 +258,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
_objectValueToIndexMap.put(value, i);
byte[] valueBytes = value.getBytes(UTF_8);
sortedStringBytes[i] = valueBytes;
+ cacheSerializedLength(i, valueBytes.length);
_numBytesPerEntry = Math.max(_numBytesPerEntry, valueBytes.length);
}
@@ -229,6 +272,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
numValues = sortedBytes.length;
Preconditions.checkState(numValues > 0);
_objectValueToIndexMap = new Object2IntOpenHashMap<>(numValues);
+ initializeSerializedLengthCache(numValues);
// Get the maximum length of all entries
byte[][] sortedByteArrays = new byte[numValues][];
@@ -236,6 +280,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
ByteArray value = sortedBytes[i];
sortedByteArrays[i] = value.getBytes();
_objectValueToIndexMap.put(value, i);
+ cacheSerializedLength(i, value.getBytes().length);
_numBytesPerEntry = Math.max(_numBytesPerEntry, value.getBytes().length);
}
@@ -304,6 +349,11 @@ public class SegmentDictionaryCreator implements IndexCreator {
return _numBytesPerEntry;
}
+ /// Returns value bytes observed through variable-length dictionary lookups, or `0` when tracking is disabled.
+ public long getTotalVariableLengthUncompressedValueSizeInBytes() {
+ return _totalUncompressedValueSizeInBytes;
+ }
+
public int indexOfSV(Object value) {
switch (_storedType) {
case INT:
@@ -316,9 +366,9 @@ public class SegmentDictionaryCreator implements IndexCreator {
return _doubleValueToIndexMap.get((double) value);
case BIG_DECIMAL:
case STRING:
- return _objectValueToIndexMap.getInt(value);
+ return indexOfVariableLengthValue(value);
case BYTES:
- return _objectValueToIndexMap.getInt(new ByteArray((byte[]) value));
+ return indexOfVariableLengthValue(new ByteArray((byte[]) value));
default:
throw new UnsupportedOperationException("Unsupported data type : " + _storedType);
}
@@ -356,14 +406,14 @@ public class SegmentDictionaryCreator implements IndexCreator {
* Get dictionary index for a String value.
*/
public int indexOfSV(String value) {
- return _objectValueToIndexMap.getInt(value);
+ return indexOfVariableLengthValue(value);
}
/**
* Get dictionary index for a byte array value.
*/
public int indexOfSV(byte[] value) {
- return _objectValueToIndexMap.getInt(new ByteArray(value));
+ return indexOfVariableLengthValue(new ByteArray(value));
}
public int[] indexOfMV(int[] values) {
@@ -400,28 +450,85 @@ public class SegmentDictionaryCreator implements IndexCreator {
public int[] indexOfMV(BigDecimal[] values) {
int[] indexes = new int[values.length];
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < values.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(values[i]);
+ }
+ return indexes;
+ }
for (int i = 0; i < values.length; i++) {
- indexes[i] = _objectValueToIndexMap.getInt(values[i]);
+ indexes[i] = trackUncompressedValueSize(values[i], _objectValueToIndexMap.getInt(values[i]));
}
return indexes;
}
public int[] indexOfMV(String[] values) {
int[] indexes = new int[values.length];
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < values.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(values[i]);
+ }
+ return indexes;
+ }
for (int i = 0; i < values.length; i++) {
- indexes[i] = _objectValueToIndexMap.getInt(values[i]);
+ indexes[i] = trackUncompressedValueSize(values[i], _objectValueToIndexMap.getInt(values[i]));
}
return indexes;
}
public int[] indexOfMV(byte[][] values) {
int[] indexes = new int[values.length];
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < values.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(new ByteArray(values[i]));
+ }
+ return indexes;
+ }
for (int i = 0; i < values.length; i++) {
- indexes[i] = _objectValueToIndexMap.getInt(new ByteArray(values[i]));
+ ByteArray value = new ByteArray(values[i]);
+ indexes[i] = trackUncompressedValueSize(value, _objectValueToIndexMap.getInt(value));
}
return indexes;
}
+ private int indexOfVariableLengthValue(Object value) {
+ int dictId = _objectValueToIndexMap.getInt(value);
+ return _trackUncompressedValueBytes ? trackUncompressedValueSize(value, dictId) : dictId;
+ }
+
+ private int trackUncompressedValueSize(Object value, int dictId) {
+ if (_serializedValueLengthByDictId != null) {
+ _totalUncompressedValueSizeInBytes += _serializedValueLengthByDictId[dictId];
+ return dictId;
+ }
+ switch (_storedType) {
+ case BIG_DECIMAL:
+ _totalUncompressedValueSizeInBytes += BigDecimalUtils.byteSize((BigDecimal) value);
+ break;
+ case STRING:
+ _totalUncompressedValueSizeInBytes += Utf8Utils.encodedLengthWithReplacement((String) value);
+ break;
+ case BYTES:
+ _totalUncompressedValueSizeInBytes += ((ByteArray) value).getBytes().length;
+ break;
+ default:
+ throw new IllegalStateException("Cannot track uncompressed size for data type: " + _storedType);
+ }
+ return dictId;
+ }
+
+ private void initializeSerializedLengthCache(int cardinality) {
+ if (_trackUncompressedValueBytes && cardinality <= MAX_CACHED_SERIALIZED_LENGTHS) {
+ _serializedValueLengthByDictId = new int[cardinality];
+ }
+ }
+
+ private void cacheSerializedLength(int dictId, int length) {
+ if (_serializedValueLengthByDictId != null) {
+ _serializedValueLengthByDictId[dictId] = length;
+ }
+ }
+
public int[] indexOfMV(Object value) {
Object[] multiValues = (Object[]) value;
int[] indexes = new int[multiValues.length];
@@ -448,14 +555,37 @@ public class SegmentDictionaryCreator implements IndexCreator {
}
break;
case BIG_DECIMAL:
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < multiValues.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(multiValues[i]);
+ }
+ break;
+ }
+ for (int i = 0; i < multiValues.length; i++) {
+ indexes[i] = trackUncompressedValueSize(multiValues[i], _objectValueToIndexMap.getInt(multiValues[i]));
+ }
+ break;
case STRING:
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < multiValues.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(multiValues[i]);
+ }
+ break;
+ }
for (int i = 0; i < multiValues.length; i++) {
- indexes[i] = _objectValueToIndexMap.getInt(multiValues[i]);
+ indexes[i] = trackUncompressedValueSize(multiValues[i], _objectValueToIndexMap.getInt(multiValues[i]));
}
break;
case BYTES:
+ if (!_trackUncompressedValueBytes) {
+ for (int i = 0; i < multiValues.length; i++) {
+ indexes[i] = _objectValueToIndexMap.getInt(new ByteArray((byte[]) multiValues[i]));
+ }
+ break;
+ }
for (int i = 0; i < multiValues.length; i++) {
- indexes[i] = _objectValueToIndexMap.getInt(new ByteArray((byte[]) multiValues[i]));
+ ByteArray bytes = new ByteArray((byte[]) multiValues[i]);
+ indexes[i] = trackUncompressedValueSize(bytes, _objectValueToIndexMap.getInt(bytes));
}
break;
default:
@@ -473,6 +603,7 @@ public class SegmentDictionaryCreator implements IndexCreator {
_floatValueToIndexMap = null;
_doubleValueToIndexMap = null;
_objectValueToIndexMap = null;
+ _serializedValueLengthByDictId = null;
}
@Override
@@ -482,5 +613,6 @@ public class SegmentDictionaryCreator implements IndexCreator {
@Override
public void close() {
+ postIndexingCleanup();
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV1.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV1.java
index c681265ffb9..d76eb26b2a1 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV1.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV1.java
@@ -40,7 +40,6 @@ import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPre
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.creator.ColumnStatistics;
-import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec;
@@ -57,7 +56,7 @@ import org.apache.pinot.spi.data.FieldSpec;
* </ul>
*/
-public class CLPForwardIndexCreatorV1 implements ForwardIndexCreator {
+public class CLPForwardIndexCreatorV1 implements CompressionStatsTrackingForwardIndexCreator {
public static final byte[] MAGIC_BYTES = "CLP.v1".getBytes(StandardCharsets.UTF_8);
private final String _column;
private final int _numDocs;
@@ -77,6 +76,8 @@ public class CLPForwardIndexCreatorV1 implements ForwardIndexCreator {
private final File _logTypeFwdIndexFile;
private final File _dictVarsFwdIndexFile;
private final File _encodedVarsFwdIndexFile;
+ private boolean _trackUncompressedValueSize;
+ private long _uncompressedValueSize;
public CLPForwardIndexCreatorV1(File baseIndexDir, String column, int numDocs, ColumnStatistics columnStatistics)
throws IOException {
@@ -144,6 +145,21 @@ public class CLPForwardIndexCreatorV1 implements ForwardIndexCreator {
return FieldSpec.DataType.STRING;
}
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _trackUncompressedValueSize ? _uncompressedValueSize : -1;
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return ChunkCompressionType.PASS_THROUGH;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _trackUncompressedValueSize = true;
+ }
+
@Override
public void putBigDecimal(BigDecimal value) {
throw new UnsupportedOperationException("Non string types are not supported");
@@ -157,6 +173,9 @@ public class CLPForwardIndexCreatorV1 implements ForwardIndexCreator {
try {
_clpMessageEncoder.encodeMessage(value, _clpEncodedMessage);
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += _clpEncodedMessage.getMessage().length;
+ }
logtype = _clpEncodedMessage.getLogTypeAsString();
dictVars = _clpEncodedMessage.getDictionaryVarsAsStrings();
encodedVars = _clpEncodedMessage.getEncodedVarsAsBoxedLongs();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV2.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV2.java
index b8ff2a103e8..2378988e99b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV2.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CLPForwardIndexCreatorV2.java
@@ -91,7 +91,7 @@ import org.slf4j.LoggerFactory;
* @see VarByteChunkForwardIndexWriterV5
* @see ForwardIndexCreator
*/
-public class CLPForwardIndexCreatorV2 implements ForwardIndexCreator {
+public class CLPForwardIndexCreatorV2 implements CompressionStatsTrackingForwardIndexCreator {
public static final Logger LOGGER = LoggerFactory.getLogger(CLPForwardIndexCreatorV2.class);
public static final byte[] MAGIC_BYTES = "CLP.v2".getBytes(StandardCharsets.UTF_8);
@@ -126,6 +126,8 @@ public class CLPForwardIndexCreatorV2 implements ForwardIndexCreator {
private final BytesOffHeapMutableDictionary _mutableLogtypeDict;
private final BytesOffHeapMutableDictionary _mutableDictVarDict;
private final ChunkCompressionType _chunkCompressionType;
+ private boolean _trackUncompressedValueSize;
+ private long _uncompressedValueSize;
/**
* Initializes a forward index creator for the given column using the provided base directory, column statistics and
@@ -327,11 +329,17 @@ public class CLPForwardIndexCreatorV2 implements ForwardIndexCreator {
EncodedMessage encodedMessage = _clpEncodedMessage;
try {
_clpMessageEncoder.encodeMessage(value, encodedMessage);
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += encodedMessage.getMessage().length;
+ }
} catch (IOException e) {
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += value.getBytes(StandardCharsets.UTF_8).length;
+ }
// Encode a fail-to-encode message if CLP encoding fails
encodedMessage = _failToEncodeClpEncodedMessage;
} finally {
- appendEncodedMessage(encodedMessage);
+ appendEncodedMessageInternal(encodedMessage);
}
}
@@ -341,6 +349,13 @@ public class CLPForwardIndexCreatorV2 implements ForwardIndexCreator {
* @param clpEncodedMessage The encoded message to append, must not be null.
*/
public void appendEncodedMessage(EncodedMessage clpEncodedMessage) {
+ if (_trackUncompressedValueSize) {
+ _uncompressedValueSize += clpEncodedMessage.getMessage().length;
+ }
+ appendEncodedMessageInternal(clpEncodedMessage);
+ }
+
+ private void appendEncodedMessageInternal(EncodedMessage clpEncodedMessage) {
if (_isClpEncoded) {
// Logtype
_logtypeIdFwdIndex.putInt(_mutableLogtypeDict.index(clpEncodedMessage.getLogtype()));
@@ -470,6 +485,22 @@ public class CLPForwardIndexCreatorV2 implements ForwardIndexCreator {
_dataFile.close();
}
+ /// Returns the original UTF-8 message bytes written to this CLP index, or `-1` when tracking is disabled.
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _trackUncompressedValueSize ? _uncompressedValueSize : -1;
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _trackUncompressedValueSize = true;
+ }
+
@Override
public boolean isDictionaryEncoded() {
return false;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CompressionStatsTrackingForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CompressionStatsTrackingForwardIndexCreator.java
new file mode 100644
index 00000000000..6550f6553e6
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/CompressionStatsTrackingForwardIndexCreator.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator.impl.fwd;
+
+import javax.annotation.Nullable;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
+
+
+/// Internal capability configured by the creator factory before any values are written.
+///
+/// Creators have a single-threaded build lifecycle. Enabling and writing concurrently is not supported.
+public interface CompressionStatsTrackingForwardIndexCreator extends ForwardIndexCreator {
+ /// Enables tracking before the first value is written.
+ void enableRawForwardIndexUncompressedValueSizeTracking();
+
+ /// Returns the tracked uncompressed serialized value size, or `-1` when tracking is disabled.
+ @Override
+ long getRawForwardIndexUncompressedValueSizeInBytes();
+
+ /// Returns the raw forward-index chunk-compression type, or null when tracking is disabled.
+ @Override
+ @Nullable
+ ChunkCompressionType getRawForwardIndexChunkCompressionType();
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
index feddb9b8ece..5ba42f42162 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueFixedByteRawIndexCreator.java
@@ -28,7 +28,6 @@ import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkWriter;
import org.apache.pinot.segment.spi.V1Constants.Indexes;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
-import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -36,10 +35,11 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
* Raw (non-dictionary-encoded) forward index creator for multi-value column of fixed length data type (INT, LONG,
* FLOAT, DOUBLE).
*/
-public class MultiValueFixedByteRawIndexCreator implements ForwardIndexCreator {
+public class MultiValueFixedByteRawIndexCreator implements CompressionStatsTrackingForwardIndexCreator {
private final VarByteChunkWriter _indexWriter;
private final DataType _valueType;
+ private final ChunkCompressionType _chunkCompressionType;
/**
* Create a var-byte raw index creator for the given column
@@ -105,6 +105,7 @@ public class MultiValueFixedByteRawIndexCreator implements ForwardIndexCreator {
}
}
_valueType = valueType;
+ _chunkCompressionType = compressionType;
}
@Override
@@ -147,4 +148,19 @@ public class MultiValueFixedByteRawIndexCreator implements ForwardIndexCreator {
throws IOException {
_indexWriter.close();
}
+
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _indexWriter.getRawForwardIndexUncompressedValueSizeInBytes();
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _indexWriter.enableRawForwardIndexUncompressedValueSizeTracking();
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
index 01e345d6068..691bf5a2879 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/MultiValueVarByteRawIndexCreator.java
@@ -29,7 +29,6 @@ import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkWriter;
import org.apache.pinot.segment.spi.V1Constants.Indexes;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
-import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -37,10 +36,11 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
* Raw (non-dictionary-encoded) forward index creator for multi-value column of variable length data type (STRING,
* BYTES).
*/
-public class MultiValueVarByteRawIndexCreator implements ForwardIndexCreator {
+public class MultiValueVarByteRawIndexCreator implements CompressionStatsTrackingForwardIndexCreator {
private final VarByteChunkWriter _indexWriter;
private final DataType _valueType;
+ private final ChunkCompressionType _chunkCompressionType;
/**
* Create a var-byte raw index creator for the given column
@@ -96,6 +96,7 @@ public class MultiValueVarByteRawIndexCreator implements ForwardIndexCreator {
_indexWriter = new VarByteChunkForwardIndexWriterV4(file, compressionType, chunkSize);
}
_valueType = valueType;
+ _chunkCompressionType = compressionType;
}
@Override
@@ -134,6 +135,21 @@ public class MultiValueVarByteRawIndexCreator implements ForwardIndexCreator {
_indexWriter.close();
}
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _indexWriter.getRawForwardIndexUncompressedValueSizeInBytes();
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _indexWriter.enableRawForwardIndexUncompressedValueSizeTracking();
+ }
+
/**
* The actual content in an MV array is prepended with 2 prefixes:
* 1. elementLengthStoragePrefixInBytes - bytes required to store the length of each element in the largest array
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
index 453519c8a69..5610d3e20fc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueFixedByteRawIndexCreator.java
@@ -24,7 +24,6 @@ import org.apache.pinot.segment.local.io.writer.impl.FixedByteChunkForwardIndexW
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
-import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -32,9 +31,10 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
* Raw (non-dictionary-encoded) forward index creator for single-value column of fixed length data type (INT, LONG,
* FLOAT, DOUBLE).
*/
-public class SingleValueFixedByteRawIndexCreator implements ForwardIndexCreator {
+public class SingleValueFixedByteRawIndexCreator implements CompressionStatsTrackingForwardIndexCreator {
private final FixedByteChunkForwardIndexWriter _indexWriter;
private final DataType _valueType;
+ private final ChunkCompressionType _chunkCompressionType;
/**
* Constructor for the class
@@ -72,6 +72,7 @@ public class SingleValueFixedByteRawIndexCreator implements ForwardIndexCreator
new FixedByteChunkForwardIndexWriter(file, compressionType, totalDocs, targetDocsPerChunk, valueType.size(),
writerVersion);
_valueType = valueType;
+ _chunkCompressionType = compressionType;
}
@Override
@@ -114,4 +115,19 @@ public class SingleValueFixedByteRawIndexCreator implements ForwardIndexCreator
throws IOException {
_indexWriter.close();
}
+
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _indexWriter.getRawForwardIndexUncompressedValueSizeInBytes();
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _indexWriter.enableRawForwardIndexUncompressedValueSizeTracking();
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
index 69028d4a244..47d34cb1691 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/fwd/SingleValueVarByteRawIndexCreator.java
@@ -29,17 +29,17 @@ import org.apache.pinot.segment.local.io.writer.impl.VarByteChunkWriter;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
-import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
import org.apache.pinot.spi.data.FieldSpec.DataType;
/**
* Raw (non-dictionary-encoded) forward index creator for single-value column of variable length data type (BIG_DECIMAL,
* STRING, BYTES).
*/
-public class SingleValueVarByteRawIndexCreator implements ForwardIndexCreator {
+public class SingleValueVarByteRawIndexCreator implements CompressionStatsTrackingForwardIndexCreator {
private final VarByteChunkWriter _indexWriter;
private final DataType _valueType;
+ private final ChunkCompressionType _chunkCompressionType;
/**
* Create a var-byte raw index creator for the given column
@@ -94,6 +94,7 @@ public class SingleValueVarByteRawIndexCreator implements ForwardIndexCreator {
_indexWriter = new VarByteChunkForwardIndexWriterV4(file, compressionType, chunkSize);
}
_valueType = valueType;
+ _chunkCompressionType = compressionType;
}
@VisibleForTesting
@@ -137,4 +138,19 @@ public class SingleValueVarByteRawIndexCreator implements ForwardIndexCreator {
throws IOException {
_indexWriter.close();
}
+
+ @Override
+ public long getRawForwardIndexUncompressedValueSizeInBytes() {
+ return _indexWriter.getRawForwardIndexUncompressedValueSizeInBytes();
+ }
+
+ @Override
+ public ChunkCompressionType getRawForwardIndexChunkCompressionType() {
+ return _chunkCompressionType;
+ }
+
+ @Override
+ public void enableRawForwardIndexUncompressedValueSizeTracking() {
+ _indexWriter.enableRawForwardIndexUncompressedValueSizeTracking();
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
index 66016a1a2c1..502998cf69c 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
@@ -255,7 +255,7 @@ public class DictionaryIndexType
@Override
public SegmentDictionaryCreator createIndexCreator(IndexCreationContext context, DictionaryIndexConfig indexConfig) {
boolean useVarLengthDictionary = shouldUseVarLengthDictionary(context, indexConfig);
- return new SegmentDictionaryCreator(context.getFieldSpec(), context.getIndexDir(), useVarLengthDictionary);
+ return new SegmentDictionaryCreator(context, useVarLengthDictionary);
}
public boolean shouldUseVarLengthDictionary(IndexCreationContext context, DictionaryIndexConfig indexConfig) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/CompressionStatsMetadata.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/CompressionStatsMetadata.java
new file mode 100644
index 00000000000..42329efa424
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/CompressionStatsMetadata.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.forward;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.FORWARD_INDEX_DICTIONARY_ENCODED_UNCOMPRESSED_VALUE_SIZE_IN_BYTES;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.FORWARD_INDEX_RAW_CHUNK_COMPRESSION_TYPE;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.FORWARD_INDEX_RAW_UNCOMPRESSED_VALUE_SIZE_IN_BYTES;
+import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.getKeyFor;
+
+
+/// Immutable, thread-safe value object for a column's persisted compression-statistics metadata.
+///
+/// Applying a value always updates all three related keys, preventing stale raw and dictionary states from being mixed
+/// after an encoding transition.
+public final class CompressionStatsMetadata {
+ private static final CompressionStatsMetadata UNAVAILABLE = new CompressionStatsMetadata(null, null, null);
+
+ private final Long _rawForwardIndexUncompressedValueSizeInBytes;
+ private final ChunkCompressionType _rawForwardIndexChunkCompressionType;
+ private final Long _dictionaryEncodedUncompressedValueSizeInBytes;
+
+ private CompressionStatsMetadata(@Nullable Long forwardIndexUncompressedValueSizeInBytes,
+ @Nullable ChunkCompressionType chunkCompressionType, @Nullable Long dictionaryUncompressedValueSizeInBytes) {
+ _rawForwardIndexUncompressedValueSizeInBytes = forwardIndexUncompressedValueSizeInBytes;
+ _rawForwardIndexChunkCompressionType = chunkCompressionType;
+ _dictionaryEncodedUncompressedValueSizeInBytes = dictionaryUncompressedValueSizeInBytes;
+ }
+
+ /// Creates metadata for a raw forward index, or unavailable metadata when either required value is absent.
+ public static CompressionStatsMetadata forRawForwardIndex(long uncompressedValueSizeInBytes,
+ @Nullable ChunkCompressionType chunkCompressionType) {
+ return uncompressedValueSizeInBytes >= 0 && chunkCompressionType != null
+ ? new CompressionStatsMetadata(uncompressedValueSizeInBytes, chunkCompressionType, null) : UNAVAILABLE;
+ }
+
+ /// Creates metadata for a dictionary-encoded column, or unavailable metadata when the size was not tracked.
+ public static CompressionStatsMetadata forDictionary(long uncompressedValueSizeInBytes) {
+ return uncompressedValueSizeInBytes >= 0
+ ? new CompressionStatsMetadata(null, null, uncompressedValueSizeInBytes) : UNAVAILABLE;
+ }
+
+ /// Returns the canonical metadata value that clears all compression-statistics keys.
+ public static CompressionStatsMetadata unavailable() {
+ return UNAVAILABLE;
+ }
+
+ /// Applies this state to a loader metadata-update map. Null values instruct the metadata updater to remove keys.
+ public void applyTo(Map<String, String> metadataProperties, String column) {
+ metadataProperties.put(getKeyFor(column, FORWARD_INDEX_RAW_UNCOMPRESSED_VALUE_SIZE_IN_BYTES),
+ stringValue(_rawForwardIndexUncompressedValueSizeInBytes));
+ metadataProperties.put(getKeyFor(column, FORWARD_INDEX_RAW_CHUNK_COMPRESSION_TYPE),
+ _rawForwardIndexChunkCompressionType != null ? _rawForwardIndexChunkCompressionType.name() : null);
+ metadataProperties.put(getKeyFor(column, FORWARD_INDEX_DICTIONARY_ENCODED_UNCOMPRESSED_VALUE_SIZE_IN_BYTES),
+ stringValue(_dictionaryEncodedUncompressedValueSizeInBytes));
+ }
+
+ /// Applies this state directly to a segment metadata properties configuration.
+ public void applyTo(PropertiesConfiguration properties, String column) {
+ setOrClear(properties, getKeyFor(column, FORWARD_INDEX_RAW_UNCOMPRESSED_VALUE_SIZE_IN_BYTES),
+ _rawForwardIndexUncompressedValueSizeInBytes);
+ setOrClear(properties, getKeyFor(column, FORWARD_INDEX_RAW_CHUNK_COMPRESSION_TYPE),
+ _rawForwardIndexChunkCompressionType != null ? _rawForwardIndexChunkCompressionType.name() : null);
+ setOrClear(properties, getKeyFor(column, FORWARD_INDEX_DICTIONARY_ENCODED_UNCOMPRESSED_VALUE_SIZE_IN_BYTES),
+ _dictionaryEncodedUncompressedValueSizeInBytes);
+ }
+
+ @Nullable
+ private static String stringValue(@Nullable Long value) {
+ return value != null ? String.valueOf(value) : null;
+ }
+
+ private static void setOrClear(PropertiesConfiguration properties, String key, @Nullable Object value) {
+ if (value != null) {
+ properties.setProperty(key, value);
+ } else {
+ properties.clearProperty(key);
+ }
+ }
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
index eaf669be34e..fedf2bacbcf 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexCreatorFactory.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.CLPForwardIndexCreatorV1;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.CLPForwardIndexCreatorV2;
+import org.apache.pinot.segment.local.segment.creator.impl.fwd.CompressionStatsTrackingForwardIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.MultiValueEntryDictForwardIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.MultiValueFixedByteRawIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.MultiValueUnsortedForwardIndexCreator;
@@ -73,37 +74,41 @@ public class ForwardIndexCreatorFactory {
} else {
// Raw forward index
DataType storedType = fieldSpec.getDataType().getStoredType();
+ ForwardIndexCreator creator;
if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLP) {
// CLP (V1) uses hard-coded chunk compressor which is set to `PassThrough`
- return new CLPForwardIndexCreatorV1(indexDir, columnName, numTotalDocs, context.getColumnStatistics());
- }
- if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2) {
- // Use the default chunk compression codec for CLP, currently configured to use ZStandard
- return new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics());
- }
- if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2_ZSTD) {
- return new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics(), ChunkCompressionType.ZSTANDARD);
- }
- if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2_LZ4) {
- return new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics(), ChunkCompressionType.LZ4);
- }
- ChunkCompressionType chunkCompressionType = indexConfig.getChunkCompressionType();
- if (chunkCompressionType == null) {
- chunkCompressionType = ForwardIndexType.getDefaultCompressionType(fieldSpec.getFieldType());
- }
- boolean deriveNumDocsPerChunk = indexConfig.isDeriveNumDocsPerChunk();
- int writerVersion = indexConfig.getRawIndexWriterVersion();
- int targetMaxChunkSize = indexConfig.getTargetMaxChunkSizeBytes();
- int targetDocsPerChunk = indexConfig.getTargetDocsPerChunk();
- if (fieldSpec.isSingleValueField()) {
- return getRawIndexCreatorForSVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
- context.getLengthOfLongestElement(), deriveNumDocsPerChunk, writerVersion, targetMaxChunkSize,
- targetDocsPerChunk);
+ creator = new CLPForwardIndexCreatorV1(indexDir, columnName, numTotalDocs, context.getColumnStatistics());
+ } else if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2) {
+ // Use the default CLP chunk compression type, currently ZSTANDARD.
+ creator = new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics());
+ } else if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2_ZSTD) {
+ creator = new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics(), ChunkCompressionType.ZSTANDARD);
+ } else if (indexConfig.getCompressionCodec() == FieldConfig.CompressionCodec.CLPV2_LZ4) {
+ creator = new CLPForwardIndexCreatorV2(indexDir, context.getColumnStatistics(), ChunkCompressionType.LZ4);
} else {
- return getRawIndexCreatorForMVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
- context.getMaxNumberOfMultiValues(), deriveNumDocsPerChunk, writerVersion,
- context.getMaxRowLengthInBytes(), targetMaxChunkSize, targetDocsPerChunk);
+ ChunkCompressionType chunkCompressionType = indexConfig.getChunkCompressionType();
+ if (chunkCompressionType == null) {
+ chunkCompressionType = ForwardIndexType.getDefaultCompressionType(fieldSpec.getFieldType());
+ }
+ boolean deriveNumDocsPerChunk = indexConfig.isDeriveNumDocsPerChunk();
+ int writerVersion = indexConfig.getRawIndexWriterVersion();
+ int targetMaxChunkSize = indexConfig.getTargetMaxChunkSizeBytes();
+ int targetDocsPerChunk = indexConfig.getTargetDocsPerChunk();
+ if (fieldSpec.isSingleValueField()) {
+ creator = getRawIndexCreatorForSVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
+ context.getLengthOfLongestElement(), deriveNumDocsPerChunk, writerVersion, targetMaxChunkSize,
+ targetDocsPerChunk);
+ } else {
+ creator = getRawIndexCreatorForMVColumn(indexDir, chunkCompressionType, columnName, storedType, numTotalDocs,
+ context.getMaxNumberOfMultiValues(), deriveNumDocsPerChunk, writerVersion,
+ context.getMaxRowLengthInBytes(), targetMaxChunkSize, targetDocsPerChunk);
+ }
+ }
+ if (context.isCompressionStatsEnabled()
+ && creator instanceof CompressionStatsTrackingForwardIndexCreator trackingCreator) {
+ trackingCreator.enableRawForwardIndexUncompressedValueSizeTracking();
}
+ return creator;
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
index 1df5608e596..70dc3cb09c5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ForwardIndexHandler.java
@@ -21,6 +21,7 @@ package org.apache.pinot.segment.local.segment.index.loader;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.File;
+import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
@@ -43,6 +44,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.stats.MapColumnPreInd
import org.apache.pinot.segment.local.segment.creator.impl.stats.NoDictColumnStatisticsCollector;
import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.local.segment.index.forward.CompressionStatsMetadata;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
import org.apache.pinot.segment.local.utils.ClusterConfigForTable;
import org.apache.pinot.segment.spi.ColumnMetadata;
@@ -72,6 +74,8 @@ import org.apache.pinot.spi.data.ComplexFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
+import org.apache.pinot.spi.utils.Utf8Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,10 +146,14 @@ public class ForwardIndexHandler extends BaseIndexHandler {
String column = entry.getKey();
List<Operation> operations = entry.getValue();
- // Backfill missing 1.6.0-era stats for this column before dispatching ops so the per-op handlers can read
- // them off `ColumnMetadata` when they construct their forward-index creator. No-op when the column is
- // fixed-width, already has the stats, or has no forward index on disk.
- backfillMissingStats(column, segmentWriter);
+ boolean needsShapeMetadata = operations.contains(Operation.ENABLE_DICT_FORWARD_INDEX)
+ || operations.contains(Operation.ENABLE_RAW_FORWARD_INDEX)
+ || operations.contains(Operation.CHANGE_INDEX_COMPRESSION_TYPE)
+ || (operations.contains(Operation.DISABLE_DICTIONARY) && !forwardIndexDisabledColumns.contains(column)
+ && isForwardIndexDictionaryEncoded(column));
+ if (needsShapeMetadata) {
+ backfillMissingStats(column, segmentWriter);
+ }
for (Operation operation : operations) {
switch (operation) {
@@ -218,6 +226,19 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
}
+ @Override
+ public void postUpdateIndicesCleanup(SegmentDirectory.Writer segmentWriter)
+ throws Exception {
+ super.postUpdateIndicesCleanup(segmentWriter);
+ if (!_tmpForwardIndexColumns.isEmpty()) {
+ Map<String, String> metadataProperties = new HashMap<>();
+ for (String column : _tmpForwardIndexColumns) {
+ CompressionStatsMetadata.unavailable().applyTo(metadataProperties, column);
+ }
+ SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
+ }
+ }
+
@VisibleForTesting
Map<String, List<Operation>> computeOperations(SegmentDirectory.Reader segmentReader)
throws Exception {
@@ -434,6 +455,9 @@ public class ForwardIndexHandler extends BaseIndexHandler {
metadataProperties.put(getKeyFor(column, HAS_DICTIONARY), String.valueOf(false));
metadataProperties.put(getKeyFor(column, FORWARD_INDEX_ENCODING), EncodingType.RAW.name());
metadataProperties.put(getKeyFor(column, DICTIONARY_ELEMENT_SIZE), String.valueOf(0));
+ if (!segmentWriter.hasIndexFor(column, StandardIndexes.forward())) {
+ CompressionStatsMetadata.unavailable().applyTo(metadataProperties, column);
+ }
// TODO: See https://github.com/apache/pinot/pull/16921 for details
// TODO: Remove the property after 1.6.0 release
// metadataProperties.put(getKeyFor(column, BITS_PER_ELEMENT), null);
@@ -574,18 +598,35 @@ public class ForwardIndexHandler extends BaseIndexHandler {
// the correct encoding for the new forward index. The `IndexCreationContext.Builder` constructor pulls
// length stats from `ColumnMetadata`; `updateIndices` has already backfilled them for this column.
IndexReaderFactory<ForwardIndexReader> readerFactory = StandardIndexes.forward().getReaderFactory();
+ CompressionStatsMetadata rawCompressionMetadata = CompressionStatsMetadata.unavailable();
+ long dictionaryUncompressedValueSize = -1;
try (ForwardIndexReader<?> reader = readerFactory.createIndexReader(segmentWriter, _fieldIndexConfigs.get(column),
columnMetadata)) {
- IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata)
+ .withCompressionStatsEnabled(_tableConfig.getIndexingConfig().isCompressionStatsEnabled())
+ .build();
ForwardIndexConfig config = _fieldIndexConfigs.get(column).getConfig(StandardIndexes.forward());
- try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, config)) {
+ boolean trackDictionarySize = isDictionaryEncodedForwardIndex
+ && _tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ try (Dictionary dictionary = trackDictionarySize ? DictionaryIndexType.read(segmentWriter, columnMetadata) : null;
+ ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, config)) {
if (!reader.getStoredType().equals(creator.getValueType())) {
// Creator stored type should match reader stored type for raw columns. We do not support changing datatypes.
throw new UnsupportedOperationException(
"Unsupported operation to change datatype for column=" + column + " from " + reader.getStoredType()
+ " to " + creator.getValueType());
}
- forwardIndexRewriteHelper(column, columnMetadata, reader, creator, columnMetadata.getTotalDocs(), null, null);
+ if (dictionary != null) {
+ dictionaryUncompressedValueSize = forwardIndexReadDictWriteDictAndTrack(columnMetadata, reader, creator,
+ dictionary);
+ } else {
+ forwardIndexRewriteHelper(column, columnMetadata, reader, creator, columnMetadata.getTotalDocs(), null, null);
+ }
+ if (!isDictionaryEncodedForwardIndex) {
+ rawCompressionMetadata = CompressionStatsMetadata.forRawForwardIndex(
+ creator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ creator.getRawForwardIndexChunkCompressionType());
+ }
}
}
@@ -597,12 +638,24 @@ public class ForwardIndexHandler extends BaseIndexHandler {
segmentWriter.removeIndex(column, StandardIndexes.forward());
LoaderUtils.writeIndexToV3Format(segmentWriter, column, fwdIndexFile, StandardIndexes.forward());
+ CompressionStatsMetadata compressionMetadata = CompressionStatsMetadata.unavailable();
+ if (isDictionaryEncodedForwardIndex) {
+ compressionMetadata = CompressionStatsMetadata.forDictionary(dictionaryUncompressedValueSize);
+ } else if (_tableConfig.getIndexingConfig().isCompressionStatsEnabled()) {
+ compressionMetadata = rawCompressionMetadata;
+ }
+ Map<String, String> compressionMetadataProperties = new HashMap<>();
+ compressionMetadata.applyTo(compressionMetadataProperties, column);
+ SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, compressionMetadataProperties);
+
// Delete the marker file.
FileUtils.deleteQuietly(inProgress);
LOGGER.info("Created forward index for segment: {}, column: {}", segmentName, column);
}
+
+
private void forwardIndexRewriteHelper(String column, ColumnMetadata existingColumnMetadata,
ForwardIndexReader<?> reader, ForwardIndexCreator creator, int numDocs,
@Nullable SegmentDictionaryCreator dictionaryCreator, @Nullable Dictionary dictionaryReader) {
@@ -629,17 +682,52 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
}
- private <C extends ForwardIndexReaderContext> void forwardIndexReadDictWriteDictHelper(ForwardIndexReader<C> reader,
+ private static <C extends ForwardIndexReaderContext> long forwardIndexReadDictWriteDictAndTrack(
+ ColumnMetadata columnMetadata, ForwardIndexReader<C> reader, ForwardIndexCreator creator,
+ Dictionary dictionary) {
+ DataType storedType = dictionary.getValueType().getStoredType();
+ C readerContext = reader.createContext();
+ int numDocs = columnMetadata.getTotalDocs();
+ if (storedType.isFixedWidth()) {
+ long numEntries = forwardIndexReadDictWriteDictHelper(reader, creator, numDocs);
+ return numEntries * storedType.size();
+ }
+ long uncompressedValueSizeInBytes = 0;
+ if (reader.isSingleValue()) {
+ for (int docId = 0; docId < numDocs; docId++) {
+ int dictId = reader.getDictId(docId, readerContext);
+ creator.putDictId(dictId);
+ uncompressedValueSizeInBytes += dictionary.getValueSize(dictId);
+ }
+ } else {
+ for (int docId = 0; docId < numDocs; docId++) {
+ int[] dictIds = reader.getDictIdMV(docId, readerContext);
+ creator.putDictIdMV(dictIds);
+ for (int dictId : dictIds) {
+ uncompressedValueSizeInBytes += dictionary.getValueSize(dictId);
+ }
+ }
+ }
+ return uncompressedValueSizeInBytes;
+ }
+
+ private static <C extends ForwardIndexReaderContext> long forwardIndexReadDictWriteDictHelper(
+ ForwardIndexReader<C> reader,
ForwardIndexCreator creator, int numDocs) {
C readerContext = reader.createContext();
if (reader.isSingleValue()) {
for (int i = 0; i < numDocs; i++) {
creator.putDictId(reader.getDictId(i, readerContext));
}
+ return numDocs;
} else {
+ long numEntries = 0;
for (int i = 0; i < numDocs; i++) {
- creator.putDictIdMV(reader.getDictIdMV(i, readerContext));
+ int[] dictIds = reader.getDictIdMV(i, readerContext);
+ creator.putDictIdMV(dictIds);
+ numEntries += dictIds.length;
}
+ return numEntries;
}
}
@@ -949,7 +1037,9 @@ public class ForwardIndexHandler extends BaseIndexHandler {
boolean useVarLength = dictConf.isUseVarLengthDictionary() || DictionaryIndexType.shouldUseVarLengthDictionary(
reader.getStoredType(), statsCollector) || (optimizeDictionaryType
&& DictionaryIndexType.optimizeTypeShouldUseVarLengthDictionary(reader.getStoredType(), statsCollector));
- dictionaryCreator = new SegmentDictionaryCreator(fieldSpec, segmentMetadata.getIndexDir(), useVarLength);
+ boolean compressionStatsEnabled = _tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ dictionaryCreator = new SegmentDictionaryCreator(fieldSpec, segmentMetadata.getIndexDir(), useVarLength,
+ SegmentDictionaryCreator.UncompressedValueSizeTracking.fromEnabled(compressionStatsEnabled));
dictionaryCreator.build(statsCollector.getUniqueValuesSet());
LOGGER.info("Built dictionary. Rewriting dictionary enabled forward index for segment={} and column={}",
@@ -984,6 +1074,15 @@ public class ForwardIndexHandler extends BaseIndexHandler {
metadataProperties.put(getKeyFor(column, CARDINALITY), String.valueOf(cardinality));
metadataProperties.put(getKeyFor(column, BITS_PER_ELEMENT),
String.valueOf(PinotDataBitSet.getNumBitsPerValue(cardinality - 1)));
+ CompressionStatsMetadata compressionMetadata = CompressionStatsMetadata.unavailable();
+ if (_tableConfig.getIndexingConfig().isCompressionStatsEnabled()) {
+ DataType storedType = fieldSpec.getDataType().getStoredType();
+ long uncompressedValueSize = storedType.isFixedWidth()
+ ? (long) statsCollector.getTotalNumberOfEntries() * storedType.size()
+ : dictionaryCreator.getTotalVariableLengthUncompressedValueSizeInBytes();
+ compressionMetadata = CompressionStatsMetadata.forDictionary(uncompressedValueSize);
+ }
+ compressionMetadata.applyTo(metadataProperties, column);
SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
// We remove indexes that have to be rewritten when a dictEnabled is toggled. Note that the respective index
@@ -1054,6 +1153,12 @@ public class ForwardIndexHandler extends BaseIndexHandler {
metadataProperties.put(getKeyFor(column, CARDINALITY), String.valueOf(dictionaryCardinality));
metadataProperties.put(getKeyFor(column, BITS_PER_ELEMENT),
String.valueOf(PinotDataBitSet.getNumBitsPerValue(dictionaryCardinality - 1)));
+ CompressionStatsMetadata compressionMetadata = _tableConfig.getIndexingConfig().isCompressionStatsEnabled()
+ ? CompressionStatsMetadata.forRawForwardIndex(
+ existingColMetadata.getRawForwardIndexUncompressedValueSizeInBytes(),
+ existingColMetadata.getRawForwardIndexChunkCompressionType())
+ : CompressionStatsMetadata.unavailable();
+ compressionMetadata.applyTo(metadataProperties, column);
SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
// Secondary indexes that require dictionary semantics should be rebuilt against the new shared dictionary.
@@ -1098,7 +1203,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
LOGGER.info("Creating raw forward index for segment={} and column={}", segmentName, column);
- rewriteDictToRawForwardIndex(existingColMetadata, segmentWriter, indexDir);
+ CompressionStatsMetadata compressionMetadata =
+ rewriteDictToRawForwardIndex(existingColMetadata, segmentWriter, indexDir);
// Remove dictionary and forward index
segmentWriter.removeIndex(column, StandardIndexes.forward());
@@ -1114,6 +1220,7 @@ public class ForwardIndexHandler extends BaseIndexHandler {
// TODO: See https://github.com/apache/pinot/pull/16921 for details
// TODO: Remove the property after 1.6.0 release
// metadataProperties.put(getKeyFor(column, BITS_PER_ELEMENT), null);
+ compressionMetadata.applyTo(metadataProperties, column);
SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
// Remove range index, inverted index and FST index.
@@ -1126,9 +1233,10 @@ public class ForwardIndexHandler extends BaseIndexHandler {
}
/// Flip a dict-encoded forward index to RAW while keeping the dictionary intact. Used when an enabled
- /// secondary index (e.g. inverted, FST, IFST) requires the dictionary to remain after the encoding flip —
- /// the secondary index continues to operate against the same dict ids, so it does not need to be removed
- /// or rebuilt. Compare with {@link #disableDictionaryAndCreateRawForwardIndex}, which performs the same
+ /// secondary index (e.g. inverted, FST, IFST) requires the dictionary to remain after the encoding flip.
+ /// The secondary index continues to operate against the same dict ids, so it does not need to be removed
+ /// or rebuilt. Compare with [#disableDictionaryAndCreateRawForwardIndex(String, SegmentDirectory.Writer)], which
+ /// performs the same
/// rewrite and additionally drops the dictionary plus all dict-dependent indexes.
private void convertDictForwardToRawKeepingDictionary(String column, SegmentDirectory.Writer segmentWriter)
throws Exception {
@@ -1150,7 +1258,8 @@ public class ForwardIndexHandler extends BaseIndexHandler {
LOGGER.info("Converting dict-encoded forward index to raw (keeping dictionary) for segment={} column={}",
segmentName, column);
- rewriteDictToRawForwardIndex(existingColMetadata, segmentWriter, indexDir);
+ CompressionStatsMetadata compressionMetadata =
+ rewriteDictToRawForwardIndex(existingColMetadata, segmentWriter, indexDir);
// Remove the old dict-encoded forward index file and write the new raw forward index in its place.
// Crucially we do NOT remove the dictionary or any dict-dependent secondary indexes — the dictionary stays
@@ -1160,25 +1269,34 @@ public class ForwardIndexHandler extends BaseIndexHandler {
Map<String, String> metadataProperties = new HashMap<>();
metadataProperties.put(getKeyFor(column, FORWARD_INDEX_ENCODING), EncodingType.RAW.name());
+ compressionMetadata.applyTo(metadataProperties, column);
SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
FileUtils.deleteQuietly(inProgress);
LOGGER.info("Converted forward index to raw (dictionary kept) for segment: {}, column: {}", segmentName, column);
}
- private void rewriteDictToRawForwardIndex(ColumnMetadata columnMetadata, SegmentDirectory.Writer segmentWriter,
- File indexDir)
+ /// Rewrites a dictionary-encoded forward index as a raw forward index.
+ ///
+ /// @return compression metadata reported by the creator that built the new raw forward index
+ private CompressionStatsMetadata rewriteDictToRawForwardIndex(ColumnMetadata columnMetadata,
+ SegmentDirectory.Writer segmentWriter, File indexDir)
throws Exception {
String column = columnMetadata.getColumnName();
FieldIndexConfigs indexConfigs = _fieldIndexConfigs.get(column);
try (ForwardIndexReader<?> forwardIndex = StandardIndexes.forward().getReaderFactory()
.createIndexReader(segmentWriter, indexConfigs, columnMetadata);
Dictionary dictionary = DictionaryIndexType.read(segmentWriter, columnMetadata)) {
- IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata).build();
+ IndexCreationContext context = new IndexCreationContext.Builder(indexDir, _tableConfig, columnMetadata)
+ .withCompressionStatsEnabled(_tableConfig.getIndexingConfig().isCompressionStatsEnabled())
+ .build();
ForwardIndexConfig config = indexConfigs.getConfig(StandardIndexes.forward());
try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, config)) {
forwardIndexRewriteHelper(column, columnMetadata, forwardIndex, creator, columnMetadata.getTotalDocs(), null,
dictionary);
+ return CompressionStatsMetadata.forRawForwardIndex(
+ creator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ creator.getRawForwardIndexChunkCompressionType());
}
}
}
@@ -1212,34 +1330,179 @@ public class ForwardIndexHandler extends BaseIndexHandler {
return;
}
boolean dictionaryEncoded = forwardIndex.isDictionaryEncoded();
- try (Dictionary dictionary = dictionaryEncoded ? DictionaryIndexType.read(segmentWriter, columnMetadata) : null;
- PinotSegmentColumnReader columnReader = new PinotSegmentColumnReader(column, forwardIndex, dictionary, null,
- columnMetadata.getMaxNumberOfMultiValues())) {
- AbstractColumnStatisticsCollector statsCollector = getStatsCollector(column, storedType, false);
- int numDocs = columnMetadata.getTotalDocs();
- assert numDocs > 0;
- for (int i = 0; i < numDocs; i++) {
- statsCollector.collect(columnReader.getValue(i));
- }
+ try (Dictionary dictionary = dictionaryEncoded ? DictionaryIndexType.read(segmentWriter, columnMetadata) : null) {
+ ShapeStats shapeStats = collectShapeStats(column, columnMetadata, forwardIndex, dictionary);
Map<String, String> metadataProperties = new HashMap<>();
metadataProperties.put(getKeyFor(column, FORWARD_INDEX_ENCODING),
dictionaryEncoded ? EncodingType.DICTIONARY.name() : EncodingType.RAW.name());
metadataProperties.put(getKeyFor(column, LENGTH_OF_SHORTEST_ELEMENT),
- String.valueOf(statsCollector.getLengthOfShortestElement()));
+ String.valueOf(shapeStats._lengthOfShortestElement));
metadataProperties.put(getKeyFor(column, LENGTH_OF_LONGEST_ELEMENT),
- String.valueOf(statsCollector.getLengthOfLongestElement()));
+ String.valueOf(shapeStats._lengthOfLongestElement));
if (storedType == DataType.STRING) {
- metadataProperties.put(getKeyFor(column, IS_ASCII), String.valueOf(statsCollector.isAscii()));
+ metadataProperties.put(getKeyFor(column, IS_ASCII), String.valueOf(shapeStats._isAscii));
}
if (!singleValue) {
metadataProperties.put(getKeyFor(column, MAX_ROW_LENGTH_IN_BYTES),
- String.valueOf(statsCollector.getMaxRowLengthInBytes()));
+ String.valueOf(shapeStats._maxRowLengthInBytes));
}
SegmentMetadataUtils.updateMetadataProperties(_segmentDirectory, metadataProperties);
}
}
}
+ private <C extends ForwardIndexReaderContext> ShapeStats collectShapeStats(String column,
+ ColumnMetadata columnMetadata, ForwardIndexReader<C> forwardIndex, @Nullable Dictionary dictionary)
+ throws IOException {
+ if (dictionary != null) {
+ return collectDictionaryShapeStats(columnMetadata, forwardIndex, dictionary);
+ }
+ DataType storedType = columnMetadata.getDataType().getStoredType();
+ if (storedType != DataType.STRING && storedType != DataType.BYTES && storedType != DataType.BIG_DECIMAL) {
+ return collectShapeStatsWithCollector(column, columnMetadata, forwardIndex);
+ }
+
+ ShapeStats stats = new ShapeStats();
+ C context = forwardIndex.createContext();
+ int numDocs = columnMetadata.getTotalDocs();
+ boolean singleValue = columnMetadata.isSingleValue();
+ int maxNumMultiValues = Math.max(columnMetadata.getMaxNumberOfMultiValues(), 1);
+ String[] stringBuffer = !singleValue && storedType == DataType.STRING
+ ? new String[maxNumMultiValues] : null;
+ byte[][] bytesBuffer = !singleValue && storedType == DataType.BYTES
+ ? new byte[maxNumMultiValues][] : null;
+ BigDecimal[] bigDecimalBuffer = !singleValue && storedType == DataType.BIG_DECIMAL
+ ? new BigDecimal[maxNumMultiValues] : null;
+ for (int docId = 0; docId < numDocs; docId++) {
+ int rowLength = 0;
+ switch (storedType) {
+ case STRING:
+ if (singleValue) {
+ String value = forwardIndex.getString(docId, context);
+ rowLength = Utf8Utils.encodedLengthWithReplacement(value);
+ stats.addElement(rowLength, isAscii(value));
+ } else {
+ int numValues = forwardIndex.getStringMV(docId, stringBuffer, context);
+ for (int i = 0; i < numValues; i++) {
+ int valueLength = Utf8Utils.encodedLengthWithReplacement(stringBuffer[i]);
+ rowLength += valueLength;
+ stats.addElement(valueLength, isAscii(stringBuffer[i]));
+ }
+ }
+ break;
+ case BYTES:
+ if (singleValue) {
+ rowLength = forwardIndex.getBytes(docId, context).length;
+ stats.addElement(rowLength, true);
+ } else {
+ int numValues = forwardIndex.getBytesMV(docId, bytesBuffer, context);
+ for (int i = 0; i < numValues; i++) {
+ rowLength += bytesBuffer[i].length;
+ stats.addElement(bytesBuffer[i].length, true);
+ }
+ }
+ break;
+ case BIG_DECIMAL:
+ if (singleValue) {
+ rowLength = BigDecimalUtils.byteSize(forwardIndex.getBigDecimal(docId, context));
+ stats.addElement(rowLength, true);
+ } else {
+ int numValues = forwardIndex.getBigDecimalMV(docId, bigDecimalBuffer, context);
+ for (int i = 0; i < numValues; i++) {
+ int valueLength = BigDecimalUtils.byteSize(bigDecimalBuffer[i]);
+ rowLength += valueLength;
+ stats.addElement(valueLength, true);
+ }
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unsupported variable-width type: " + storedType);
+ }
+ if (!singleValue) {
+ stats._maxRowLengthInBytes = Math.max(stats._maxRowLengthInBytes, rowLength);
+ }
+ }
+ stats.finish();
+ return stats;
+ }
+
+ private static <C extends ForwardIndexReaderContext> ShapeStats collectDictionaryShapeStats(
+ ColumnMetadata columnMetadata, ForwardIndexReader<C> forwardIndex, Dictionary dictionary) {
+ ShapeStats stats = new ShapeStats();
+ boolean stringValues = dictionary.getValueType().getStoredType() == DataType.STRING;
+ for (int dictId = 0; dictId < dictionary.length(); dictId++) {
+ stats.addElement(dictionary.getValueSize(dictId), !stringValues || isAscii((String) dictionary.get(dictId)));
+ }
+ if (!columnMetadata.isSingleValue()) {
+ int[] dictIdBuffer = new int[Math.max(columnMetadata.getMaxNumberOfMultiValues(), 1)];
+ C context = forwardIndex.createContext();
+ for (int docId = 0; docId < columnMetadata.getTotalDocs(); docId++) {
+ int numValues = forwardIndex.getDictIdMV(docId, dictIdBuffer, context);
+ int rowLength = 0;
+ for (int i = 0; i < numValues; i++) {
+ rowLength += dictionary.getValueSize(dictIdBuffer[i]);
+ }
+ stats._maxRowLengthInBytes = Math.max(stats._maxRowLengthInBytes, rowLength);
+ }
+ }
+ stats.finish();
+ return stats;
+ }
+
+ private ShapeStats collectShapeStatsWithCollector(String column, ColumnMetadata columnMetadata,
+ ForwardIndexReader<?> forwardIndex)
+ throws IOException {
+ try (PinotSegmentColumnReader columnReader = new PinotSegmentColumnReader(column, forwardIndex, null, null,
+ columnMetadata.getMaxNumberOfMultiValues())) {
+ AbstractColumnStatisticsCollector statsCollector =
+ getStatsCollector(column, columnMetadata.getDataType().getStoredType(), false);
+ for (int docId = 0; docId < columnMetadata.getTotalDocs(); docId++) {
+ statsCollector.collect(columnReader.getValue(docId));
+ }
+ return new ShapeStats(statsCollector.getLengthOfShortestElement(), statsCollector.getLengthOfLongestElement(),
+ statsCollector.isAscii(), statsCollector.getMaxRowLengthInBytes());
+ }
+ }
+
+ private static boolean isAscii(String value) {
+ for (int i = 0; i < value.length(); i++) {
+ if (value.charAt(i) > 0x7f) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static final class ShapeStats {
+ private int _lengthOfShortestElement = Integer.MAX_VALUE;
+ private int _lengthOfLongestElement;
+ private boolean _isAscii = true;
+ private int _maxRowLengthInBytes;
+
+ private ShapeStats() {
+ }
+
+ private ShapeStats(int lengthOfShortestElement, int lengthOfLongestElement, boolean isAscii,
+ int maxRowLengthInBytes) {
+ _lengthOfShortestElement = lengthOfShortestElement;
+ _lengthOfLongestElement = lengthOfLongestElement;
+ _isAscii = isAscii;
+ _maxRowLengthInBytes = maxRowLengthInBytes;
+ }
+
+ private void addElement(int sizeInBytes, boolean isAscii) {
+ _lengthOfShortestElement = Math.min(_lengthOfShortestElement, sizeInBytes);
+ _lengthOfLongestElement = Math.max(_lengthOfLongestElement, sizeInBytes);
+ _isAscii &= isAscii;
+ }
+
+ private void finish() {
+ if (_lengthOfShortestElement == Integer.MAX_VALUE) {
+ _lengthOfShortestElement = 0;
+ }
+ }
+ }
+
/// `requireUniqueValues=true` forces a per-type collector even when the no-dict optimization would apply.
/// Callers building a dictionary out of the raw values must opt in — `_fieldIndexConfigs` may still report
/// `dictionary=disabled` when the dictionary requirement was derived from a secondary-index need rather
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index 521d4e822c9..bc29adf67bf 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.local.segment.index.forward.CompressionStatsMetadata;
import org.apache.pinot.segment.local.segment.index.inverted.InvertedIndexType;
import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader;
import org.apache.pinot.segment.spi.ColumnMetadata;
@@ -264,25 +265,33 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
DataType storedType = _columnMetadata.getStoredType();
boolean backfillStats =
!storedType.isFixedWidth() && _columnMetadata.getLengthOfShortestElement() < 0;
+ boolean compressionStatsEnabled = isCompressionStatsEnabled();
int lengthOfShortestElement = Integer.MAX_VALUE;
int lengthOfLongestElement = 0;
boolean isAscii = storedType == DataType.STRING;
+ long dictionaryUncompressedValueSize = compressionStatsEnabled && storedType.isFixedWidth()
+ ? (long) _numDocs * storedType.size() : 0;
for (int dictId = 0; dictId < _cardinality; dictId++) {
ImmutableRoaringBitmap docIdsBitmap = invertedIndexReader.getDocIds(dictId);
int finalDictId = dictId;
docIdsBitmap.stream().forEach(docId -> putInt(_forwardIndexValueBuffer, docId, finalDictId));
+ int valueSize = backfillStats || (compressionStatsEnabled && !storedType.isFixedWidth())
+ ? dictionary.getValueSize(dictId) : 0;
if (backfillStats) {
- int valueSize = dictionary.getValueSize(dictId);
lengthOfShortestElement = Math.min(lengthOfShortestElement, valueSize);
lengthOfLongestElement = Math.max(lengthOfLongestElement, valueSize);
if (isAscii) {
isAscii = valueSize == dictionary.getStringValue(dictId).length();
}
}
+ if (compressionStatsEnabled && !storedType.isFixedWidth()) {
+ dictionaryUncompressedValueSize += (long) valueSize * docIdsBitmap.getCardinality();
+ }
}
IndexCreationContext.Builder builder =
- new IndexCreationContext.Builder(_segmentMetadata.getIndexDir(), _tableConfig, _columnMetadata);
+ new IndexCreationContext.Builder(_segmentMetadata.getIndexDir(), _tableConfig, _columnMetadata)
+ .withCompressionStatsEnabled(compressionStatsEnabled);
if (backfillStats) {
builder.withLengthOfShortestElement(lengthOfShortestElement);
builder.withLengthOfLongestElement(lengthOfLongestElement);
@@ -290,7 +299,7 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
// NOTE: this method closes buffers and removes files
- writeToForwardIndex(dictionary, builder.build());
+ CompressionStatsMetadata rawCompressionMetadata = writeToForwardIndex(dictionary, builder.build());
// Setup and return the metadata properties to update
Map<String, String> metadataProperties = new HashMap<>();
@@ -312,6 +321,7 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
metadataProperties.put(getKeyFor(_columnName, IS_ASCII), String.valueOf(isAscii));
}
}
+ addCompressionMetadata(metadataProperties, rawCompressionMetadata, dictionaryUncompressedValueSize);
return metadataProperties;
}
}
@@ -365,16 +375,21 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
// the source segment is missing them, so we can backfill those metadata keys without a second dictionary scan.
DataType storedType = _columnMetadata.getStoredType();
boolean isFixedWidth = storedType.isFixedWidth();
+ boolean compressionStatsEnabled = isCompressionStatsEnabled();
int fixedSize = isFixedWidth ? storedType.size() : 0;
int maxRowLengthInBytes = isFixedWidth ? maxNumberOfMultiValues * fixedSize : 0;
boolean backfillStats = !isFixedWidth && _columnMetadata.getLengthOfShortestElement() < 0;
int lengthOfShortestElement = Integer.MAX_VALUE;
int lengthOfLongestElement = 0;
boolean isAscii = storedType == DataType.STRING;
+ long dictionaryUncompressedValueSize = 0;
for (int dictId = 0; dictId < _cardinality; dictId++) {
ImmutableRoaringBitmap docIdsBitmap = invertedIndexReader.getDocIds(dictId);
PeekableIntIterator intIterator = docIdsBitmap.getIntIterator();
int valueSize = isFixedWidth ? fixedSize : dictionary.getValueSize(dictId);
+ if (compressionStatsEnabled) {
+ dictionaryUncompressedValueSize += (long) valueSize * docIdsBitmap.getCardinality();
+ }
if (backfillStats) {
lengthOfShortestElement = Math.min(lengthOfShortestElement, valueSize);
lengthOfLongestElement = Math.max(lengthOfLongestElement, valueSize);
@@ -403,14 +418,15 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
IndexCreationContext.Builder builder = new IndexCreationContext.Builder(indexDir, _tableConfig, _columnMetadata)
.withTotalNumberOfEntries(_nextValueId)
.withMaxNumberOfMultiValues(maxNumberOfMultiValues)
- .withMaxRowLengthInBytes(maxRowLengthInBytes);
+ .withMaxRowLengthInBytes(maxRowLengthInBytes)
+ .withCompressionStatsEnabled(compressionStatsEnabled);
if (backfillStats) {
builder.withLengthOfShortestElement(lengthOfShortestElement);
builder.withLengthOfLongestElement(lengthOfLongestElement);
builder.withAscii(isAscii);
}
- writeToForwardIndex(dictionary, builder.build());
+ CompressionStatsMetadata rawCompressionMetadata = writeToForwardIndex(dictionary, builder.build());
// Setup and return the metadata properties to update
Map<String, String> metadataProperties = new HashMap<>();
@@ -437,11 +453,12 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
}
}
+ addCompressionMetadata(metadataProperties, rawCompressionMetadata, dictionaryUncompressedValueSize);
return metadataProperties;
}
}
- private void writeToForwardIndex(Dictionary dictionary, IndexCreationContext context)
+ private CompressionStatsMetadata writeToForwardIndex(Dictionary dictionary, IndexCreationContext context)
throws IOException {
try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, _forwardIndexConfig)) {
if (creator.isDictionaryEncoded()) {
@@ -601,6 +618,10 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
throw new IllegalStateException("Invalid type" + creator.getValueType() + " cannot create forward index");
}
}
+ return creator.isDictionaryEncoded() ? CompressionStatsMetadata.unavailable()
+ : CompressionStatsMetadata.forRawForwardIndex(
+ creator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ creator.getRawForwardIndexChunkCompressionType());
} catch (Exception e) {
throw new IOException(String.format(
"Cannot create the forward index from inverted index for column %s", _columnName), e);
@@ -611,6 +632,29 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
}
}
+ private boolean isCompressionStatsEnabled() {
+ return _tableConfig.getIndexingConfig() != null
+ && _tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ }
+
+ private void addCompressionMetadata(Map<String, String> metadataProperties,
+ CompressionStatsMetadata rawCompressionMetadata, long dictionaryUncompressedValueSizeInBytes) {
+ if (_isTemporaryForwardIndex) {
+ return;
+ }
+ CompressionStatsMetadata compressionMetadata = CompressionStatsMetadata.unavailable();
+ if (!isCompressionStatsEnabled()) {
+ compressionMetadata.applyTo(metadataProperties, _columnName);
+ return;
+ }
+ if (_forwardIndexConfig.getEncodingType() == EncodingType.DICTIONARY) {
+ compressionMetadata = CompressionStatsMetadata.forDictionary(dictionaryUncompressedValueSizeInBytes);
+ } else {
+ compressionMetadata = rawCompressionMetadata;
+ }
+ compressionMetadata.applyTo(metadataProperties, _columnName);
+ }
+
private static void putInt(PinotDataBuffer buffer, long index, int value) {
buffer.putInt(index << 2, value);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index d88b3b90c60..193596209b7 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -50,6 +50,7 @@ import org.apache.pinot.segment.local.segment.creator.impl.stats.MapColumnPreInd
import org.apache.pinot.segment.local.segment.creator.impl.stats.NoDictColumnStatisticsCollector;
import org.apache.pinot.segment.local.segment.creator.impl.stats.StringColumnPreIndexStatsCollector;
import org.apache.pinot.segment.local.segment.index.dictionary.DictionaryIndexType;
+import org.apache.pinot.segment.local.segment.index.forward.CompressionStatsMetadata;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderUtils;
import org.apache.pinot.segment.local.segment.readers.PinotSegmentColumnReader;
@@ -472,7 +473,8 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
// We always create a dictionary for default value columns.
// We will have only one value in the dictionary.
int dictionaryElementSize;
- try (SegmentDictionaryCreator creator = new SegmentDictionaryCreator(fieldSpec, _indexDir, false)) {
+ try (SegmentDictionaryCreator creator = new SegmentDictionaryCreator(fieldSpec, _indexDir, false,
+ SegmentDictionaryCreator.UncompressedValueSizeTracking.fromEnabled(isCompressionStatsEnabled()))) {
creator.build(sortedArray);
dictionaryElementSize = creator.getNumBytesPerEntry();
}
@@ -529,6 +531,10 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
// Add the column metadata information to the metadata properties.
BaseSegmentCreator.addColumnMetadataInfo(_segmentProperties, column, columnStatistics, totalDocs, fieldSpec, true,
dictionaryElementSize, FieldConfig.EncodingType.DICTIONARY, true);
+ DataType storedType = fieldSpec.getDataType().getStoredType();
+ long uncompressedValueSizeInBytes = (long) columnStatistics.getTotalNumberOfEntries()
+ * (storedType.isFixedWidth() ? storedType.size() : dictionaryElementSize);
+ putDictionaryCompressionStats(column, uncompressedValueSizeInBytes);
}
private boolean isNullable(FieldSpec fieldSpec) {
@@ -1089,7 +1095,8 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
// Create dictionary
try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(fieldSpec, _indexDir,
- useVarLengthDictionary)) {
+ useVarLengthDictionary,
+ SegmentDictionaryCreator.UncompressedValueSizeTracking.fromEnabled(isCompressionStatsEnabled()))) {
dictionaryCreator.build(columnStatistics.getUniqueValuesSet());
int numDocs = outputValues.length;
@@ -1112,6 +1119,11 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
// Add the column metadata
BaseSegmentCreator.addColumnMetadataInfo(_segmentProperties, column, columnStatistics, numDocs, fieldSpec, true,
dictionaryCreator.getNumBytesPerEntry(), FieldConfig.EncodingType.DICTIONARY, true);
+ DataType storedType = fieldSpec.getDataType().getStoredType();
+ long uncompressedValueSizeInBytes = storedType.isFixedWidth()
+ ? (long) columnStatistics.getTotalNumberOfEntries() * storedType.size()
+ : dictionaryCreator.getTotalVariableLengthUncompressedValueSizeInBytes();
+ putDictionaryCompressionStats(column, uncompressedValueSizeInBytes);
}
}
}
@@ -1127,6 +1139,7 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
int numDocs = outputValues.length;
boolean isSingleValue = fieldSpec.isSingleValueField();
+ CompressionStatsMetadata compressionMetadata;
try (ForwardIndexCreator forwardIndexCreator
= getForwardIndexCreator(columnStatistics, column, false)) {
if (isSingleValue) {
@@ -1192,19 +1205,30 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
}
}
forwardIndexCreator.seal();
+ compressionMetadata = CompressionStatsMetadata.forRawForwardIndex(
+ forwardIndexCreator.getRawForwardIndexUncompressedValueSizeInBytes(),
+ forwardIndexCreator.getRawForwardIndexChunkCompressionType());
}
// Add the column metadata
BaseSegmentCreator.addColumnMetadataInfo(_segmentProperties, column, columnStatistics, numDocs, fieldSpec, false,
0, FieldConfig.EncodingType.RAW, true);
+ compressionMetadata.applyTo(_segmentProperties, column);
}
private ForwardIndexCreator getForwardIndexCreator(ColumnStatistics columnStatistics,
String column, boolean hasDictionary)
throws Exception {
IndexCreationContext indexCreationContext =
- new IndexCreationContext.Builder(_indexDir, _tableConfig, columnStatistics, hasDictionary).build();
+ new IndexCreationContext.Builder(_indexDir, _tableConfig, columnStatistics, hasDictionary)
+ .withCompressionStatsEnabled(isCompressionStatsEnabled())
+ .build();
+ ForwardIndexConfig forwardIndexConfig = getForwardIndexConfig(column, hasDictionary);
+ return StandardIndexes.forward().createIndexCreator(indexCreationContext, forwardIndexConfig);
+ }
+
+ private ForwardIndexConfig getForwardIndexConfig(String column, boolean hasDictionary) {
ForwardIndexConfig forwardIndexConfig = null;
FieldIndexConfigs fieldIndexConfig = _indexLoadingConfig.getFieldIndexConfig(column);
if (fieldIndexConfig != null) {
@@ -1214,7 +1238,19 @@ public abstract class BaseDefaultColumnHandler implements DefaultColumnHandler {
forwardIndexConfig = ForwardIndexConfig.getDefault(
hasDictionary ? FieldConfig.EncodingType.DICTIONARY : FieldConfig.EncodingType.RAW);
}
- return StandardIndexes.forward().createIndexCreator(indexCreationContext, forwardIndexConfig);
+ return forwardIndexConfig;
+ }
+
+ private boolean isCompressionStatsEnabled() {
+ return _tableConfig.getIndexingConfig() != null
+ && _tableConfig.getIndexingConfig().isCompressionStatsEnabled();
+ }
+
+ private void putDictionaryCompressionStats(String column, long uncompressedValueSizeInBytes) {
+ CompressionStatsMetadata compressionMetadata = isCompressionStatsEnabled()
+ ? CompressionStatsMetadata.forDictionary(uncompressedValueSizeInBytes)
+ : CompressionStatsMetadata.unavailable();
+ compressionMetadata.applyTo(_segmentProperties, column);
}
@SuppressWarnings("rawtypes")
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/SegmentGeneratorConfigPropagationTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/SegmentGeneratorConfigPropagationTest.java
new file mode 100644
index 00000000000..e7f21b35e6a
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/SegmentGeneratorConfigPropagationTest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator;
+
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies that `compressionStatsEnabled` propagates from [TableConfig] to [SegmentGeneratorConfig].
+public class SegmentGeneratorConfigPropagationTest {
+
+ /// Verifies explicit enablement is propagated.
+ @Test
+ public void testCompressionStatsEnabledPropagation() {
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testTable")
+ .build();
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+
+ Schema schema = new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addSingleValueDimension("col1", DataType.INT)
+ .build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ assertTrue(config.isCompressionStatsEnabled(),
+ "compressionStatsEnabled should be true when explicitly enabled on TableConfig");
+ }
+
+ /// Verifies the default remains disabled.
+ @Test
+ public void testCompressionStatsDisabledByDefault() {
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testTable")
+ .build();
+
+ Schema schema = new Schema.SchemaBuilder()
+ .setSchemaName("testTable")
+ .addSingleValueDimension("col1", DataType.INT)
+ .build();
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ assertFalse(config.isCompressionStatsEnabled(),
+ "compressionStatsEnabled should be false by default when not set on TableConfig");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreatorTest.java
new file mode 100644
index 00000000000..9e6c60abbd3
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/SameValueForwardIndexCreatorTest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator.impl;
+
+import org.apache.pinot.segment.spi.index.creator.ForwardIndexCreator;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+
+/// Verifies that [SameValueForwardIndexCreator] delegates compression tracking metadata.
+public class SameValueForwardIndexCreatorTest {
+ @Test
+ public void testCompressionStatsTrackingDelegates() {
+ ForwardIndexCreator delegate = mock(ForwardIndexCreator.class);
+ when(delegate.getRawForwardIndexUncompressedValueSizeInBytes()).thenReturn(42L);
+ SameValueForwardIndexCreator creator = new SameValueForwardIndexCreator("default", delegate);
+
+ assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(), 42L);
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java
index a739e183e70..c109c1d3bff 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorTest.java
@@ -19,9 +19,10 @@
package org.apache.pinot.segment.local.segment.index.creator;
import java.io.File;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.PinotBuffersAfterMethodCheckRule;
import org.apache.pinot.segment.local.segment.creator.impl.fwd.CLPForwardIndexCreatorV1;
@@ -73,33 +74,23 @@ public class CLPForwardIndexCreatorTest implements PinotBuffersAfterMethodCheckR
"2023/10/27 16:35:10.607 INFO [ControllerResponseFilter] [grizzly-http-server-6] Handled request from 0.0"
+ ".0.0 GET https://pinot-pinot-broker-headless.managed.svc.cluster.local:8093/tables, content-type "
+ "application/json status code 200 OK");
+ logLines.add("2023/10/27 16:35:11.001 INFO Received caf\u00e9 event from \u6771\u4eac");
logLines.add("null");
- Schema schema = new Schema();
- schema.addField(new DimensionFieldSpec("column1", FieldSpec.DataType.STRING, true));
- TableConfig tableConfig =
- new TableConfig("mytable", TableType.REALTIME.name(), new SegmentsValidationAndRetentionConfig(),
- new TenantConfig(null, null, null), new IndexingConfig(), new TableCustomConfig(null), null, null, null,
- null, null, null, null, null, null, null, null, false, null, null, null, null);
- List<FieldConfig> fieldConfigList = new ArrayList<>();
- fieldConfigList.add(new FieldConfig("column1", FieldConfig.EncodingType.RAW, Collections.EMPTY_LIST,
- FieldConfig.CompressionCodec.CLP, Collections.EMPTY_MAP));
- tableConfig.setFieldConfigList(fieldConfigList);
- StatsCollectorConfig statsCollectorConfig = new StatsCollectorConfig(tableConfig, schema, null);
- StringColumnPreIndexStatsCollector statsCollector =
- new StringColumnPreIndexStatsCollector("column1", statsCollectorConfig);
- for (String logLine : logLines) {
- statsCollector.collect(logLine);
- }
- statsCollector.seal();
+ StringColumnPreIndexStatsCollector statsCollector = createStatsCollector(logLines);
+ long expectedRawBytes = logLines.stream()
+ .mapToLong(logLine -> logLine.getBytes(StandardCharsets.UTF_8).length)
+ .sum();
File indexFile = new File(TEMP_DIR, "column1" + V1Constants.Indexes.RAW_SV_FORWARD_INDEX_FILE_EXTENSION);
try (CLPForwardIndexCreatorV1 clpForwardIndexCreatorV1 =
new CLPForwardIndexCreatorV1(TEMP_DIR, "column1", logLines.size(), statsCollector)) {
-
+ clpForwardIndexCreatorV1.enableRawForwardIndexUncompressedValueSizeTracking();
for (String logLine : logLines) {
clpForwardIndexCreatorV1.putString(logLine);
}
+ Assert.assertEquals(clpForwardIndexCreatorV1.getRawForwardIndexUncompressedValueSizeInBytes(), expectedRawBytes,
+ "CLP V1 should report the exact original UTF-8 message bytes");
clpForwardIndexCreatorV1.seal();
}
@@ -112,6 +103,41 @@ public class CLPForwardIndexCreatorTest implements PinotBuffersAfterMethodCheckR
}
}
+ @Test
+ public void testCompressionStatsDisabledReturnsUnavailable()
+ throws Exception {
+ List<String> logLines = List.of("plain message", "caf\u00e9 event from \u6771\u4eac");
+ StringColumnPreIndexStatsCollector statsCollector = createStatsCollector(logLines);
+ try (CLPForwardIndexCreatorV1 creator =
+ new CLPForwardIndexCreatorV1(TEMP_DIR, "column1", logLines.size(), statsCollector)) {
+ for (String logLine : logLines) {
+ creator.putString(logLine);
+ }
+ Assert.assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(), -1L,
+ "CLP V1 should return the unavailable sentinel when tracking is disabled");
+ creator.seal();
+ }
+ }
+
+ private static StringColumnPreIndexStatsCollector createStatsCollector(List<String> logLines) {
+ Schema schema = new Schema();
+ schema.addField(new DimensionFieldSpec("column1", FieldSpec.DataType.STRING, true));
+ TableConfig tableConfig =
+ new TableConfig("mytable", TableType.REALTIME.name(), new SegmentsValidationAndRetentionConfig(),
+ new TenantConfig(null, null, null), new IndexingConfig(), new TableCustomConfig(null), null, null, null,
+ null, null, null, null, null, null, null, null, false, null, null, null, null);
+ tableConfig.setFieldConfigList(List.of(new FieldConfig("column1", FieldConfig.EncodingType.RAW, List.of(),
+ FieldConfig.CompressionCodec.CLP, Map.of())));
+ StatsCollectorConfig statsCollectorConfig = new StatsCollectorConfig(tableConfig, schema, null);
+ StringColumnPreIndexStatsCollector statsCollector =
+ new StringColumnPreIndexStatsCollector("column1", statsCollectorConfig);
+ for (String logLine : logLines) {
+ statsCollector.collect(logLine);
+ }
+ statsCollector.seal();
+ return statsCollector;
+ }
+
@AfterClass
public void tearDown()
throws Exception {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorV2Test.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorV2Test.java
index 52b8292b497..f2d028e0b54 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorV2Test.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CLPForwardIndexCreatorV2Test.java
@@ -24,6 +24,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
@@ -173,4 +174,48 @@ public class CLPForwardIndexCreatorV2Test implements PinotBuffersAfterClassCheck
File indexFile = new File(TEMP_DIR, COLUMN_NAME + V1Constants.Indexes.RAW_SV_FORWARD_INDEX_FILE_EXTENSION);
return indexFile.length();
}
+
+ @Test
+ public void testGetUncompressedValueSizeAfterWriting()
+ throws IOException {
+ try (CLPMutableForwardIndexV2 mutable = new CLPMutableForwardIndexV2(COLUMN_NAME, _memoryManager)) {
+ for (int i = 0; i < _logMessages.size(); i++) {
+ mutable.setString(i, _logMessages.get(i));
+ }
+ TestUtils.ensureDirectoriesExistAndEmpty(TEMP_DIR);
+ CLPForwardIndexCreatorV2 creator =
+ new CLPForwardIndexCreatorV2(TEMP_DIR, mutable, ChunkCompressionType.ZSTANDARD);
+ creator.enableRawForwardIndexUncompressedValueSizeTracking();
+ for (int i = 0; i < _logMessages.size(); i++) {
+ creator.putString(mutable.getString(i));
+ }
+ creator.seal();
+ creator.close();
+ long expectedRawBytes = _logMessages.stream()
+ .mapToLong(message -> message.getBytes(StandardCharsets.UTF_8).length)
+ .sum();
+ Assert.assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(), expectedRawBytes,
+ "CLP V2 should report original UTF-8 message bytes");
+ }
+ }
+
+ @Test
+ public void testGetUncompressedValueSizeDisabledReturnsUnavailable()
+ throws IOException {
+ try (CLPMutableForwardIndexV2 mutable = new CLPMutableForwardIndexV2(COLUMN_NAME, _memoryManager)) {
+ for (int i = 0; i < _logMessages.size(); i++) {
+ mutable.setString(i, _logMessages.get(i));
+ }
+ TestUtils.ensureDirectoriesExistAndEmpty(TEMP_DIR);
+ CLPForwardIndexCreatorV2 creator =
+ new CLPForwardIndexCreatorV2(TEMP_DIR, mutable, ChunkCompressionType.ZSTANDARD);
+ for (int i = 0; i < _logMessages.size(); i++) {
+ creator.putString(mutable.getString(i));
+ }
+ creator.seal();
+ creator.close();
+ Assert.assertEquals(creator.getRawForwardIndexUncompressedValueSizeInBytes(), -1L,
+ "getRawForwardIndexUncompressedValueSizeInBytes() should be unavailable when tracking is disabled");
+ }
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsCornerCaseTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsCornerCaseTest.java
new file mode 100644
index 00000000000..5ae8ec63b39
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsCornerCaseTest.java
@@ -0,0 +1,294 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.creator;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Exercises compression-statistics corner cases, including dictionary-only segments, disabled tracking, old segments,
+/// and configuration serialization.
+public class CompressionStatsCornerCaseTest {
+ private static final File TEMP_DIR =
+ new File(FileUtils.getTempDirectory(), CompressionStatsCornerCaseTest.class.getSimpleName());
+ private static final String RAW_TABLE_NAME = "compressionCornerCase";
+ private static final String SEGMENT_NAME = "cornerCaseSegment";
+ private static final int NUM_ROWS = 5000;
+ private static final Random RANDOM = new Random(42);
+
+ private static final String INT_RAW_COL = "intRawCol";
+ private static final String STRING_RAW_COL = "stringRawCol";
+ private static final String DICT_COL = "dictCol";
+
+ @BeforeMethod
+ public void setUp() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+
+ /// Builds a segment with the requested compression-statistics and chunk-compression configuration.
+ private File buildSegment(boolean compressionStatsEnabled, String compressionCodec)
+ throws Exception {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(INT_RAW_COL, DataType.INT)
+ .addSingleValueDimension(STRING_RAW_COL, DataType.STRING)
+ .addSingleValueDimension(DICT_COL, DataType.STRING)
+ .build();
+
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ if (compressionCodec != null) {
+ FieldConfig.CompressionCodec codec = FieldConfig.CompressionCodec.valueOf(compressionCodec);
+ fieldConfigs.add(new FieldConfig(INT_RAW_COL, FieldConfig.EncodingType.RAW, List.of(), codec, null));
+ fieldConfigs.add(new FieldConfig(STRING_RAW_COL, FieldConfig.EncodingType.RAW, List.of(), codec, null));
+ }
+
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(List.of(INT_RAW_COL, STRING_RAW_COL))
+ .setFieldConfigList(fieldConfigs)
+ .build();
+
+ if (compressionStatsEnabled) {
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+ }
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(SEGMENT_NAME);
+
+ List<GenericRow> rows = generateTestData();
+ SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+
+ return new File(TEMP_DIR, SEGMENT_NAME);
+ }
+
+ private List<GenericRow> generateTestData() {
+ List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue(INT_RAW_COL, RANDOM.nextInt(100000));
+ row.putValue(STRING_RAW_COL, RandomStringUtils.secure().nextAlphanumeric(20 + RANDOM.nextInt(80)));
+ row.putValue(DICT_COL, "value_" + (i % 100));
+ rows.add(row);
+ }
+ return rows;
+ }
+
+ @Test
+ public void testAllDictionaryColumnsNoCrash()
+ throws Exception {
+ // When ALL columns are dictionary-encoded, compression stats should gracefully produce no stats.
+ // This tests division-by-zero safety: forward-index storage size zero produces ratio zero.
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(DICT_COL, DataType.STRING)
+ .addSingleValueDimension("dictCol2", DataType.INT)
+ .build();
+
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(RAW_TABLE_NAME)
+ .build();
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(SEGMENT_NAME);
+
+ List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue(DICT_COL, "value_" + (i % 50));
+ row.putValue("dictCol2", i % 100);
+ rows.add(row);
+ }
+
+ SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+
+ File segmentDir = new File(TEMP_DIR, SEGMENT_NAME);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+
+ for (String colName : schema.getColumnNames()) {
+ ColumnMetadata colMeta = metadata.getColumnMetadataFor(colName);
+ assertTrue(colMeta.hasDictionary(), colName + " should have dictionary");
+ assertEquals(colMeta.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ colName + " should not have uncompressed forward index size");
+ assertNull(colMeta.getRawForwardIndexChunkCompressionType(),
+ colName + " should not have a forward-index chunk compression type");
+ }
+ }
+
+ @Test
+ public void testFlagOffThenOnProducesStats()
+ throws Exception {
+ // Flag OFF: no stats persisted
+ File segmentDirOff = buildSegment(false, "LZ4");
+ SegmentMetadataImpl metadataOff = new SegmentMetadataImpl(segmentDirOff);
+
+ ColumnMetadata rawMetaOff = metadataOff.getColumnMetadataFor(INT_RAW_COL);
+ assertFalse(rawMetaOff.hasDictionary());
+ assertEquals(rawMetaOff.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ "Flag OFF should not track uncompressed size");
+ assertNull(rawMetaOff.getRawForwardIndexChunkCompressionType(),
+ "Flag OFF should not track the chunk compression type");
+
+ // Clean up and rebuild with flag ON
+ FileUtils.deleteQuietly(TEMP_DIR);
+
+ // Flag ON: stats should be persisted
+ File segmentDirOn = buildSegment(true, "LZ4");
+ SegmentMetadataImpl metadataOn = new SegmentMetadataImpl(segmentDirOn);
+
+ ColumnMetadata rawMetaOn = metadataOn.getColumnMetadataFor(INT_RAW_COL);
+ assertFalse(rawMetaOn.hasDictionary());
+ assertTrue(rawMetaOn.getRawForwardIndexUncompressedValueSizeInBytes() > 0,
+ "Flag ON should track uncompressed size");
+ assertEquals(rawMetaOn.getRawForwardIndexChunkCompressionType(), ChunkCompressionType.LZ4,
+ "Flag ON should track the chunk compression type");
+ }
+
+ @Test
+ public void testOldSegmentWithoutStatsIsBackwardCompatible()
+ throws Exception {
+ // Simulate an "old" segment: raw columns but no compressionStatsEnabled, no field configs
+ File segmentDir = buildSegment(false, null);
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+
+ ColumnMetadata rawMeta = metadata.getColumnMetadataFor(INT_RAW_COL);
+ assertNotNull(rawMeta);
+ assertFalse(rawMeta.hasDictionary());
+ assertEquals(rawMeta.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ "Old segment should return INDEX_NOT_FOUND for uncompressed size");
+ assertNull(rawMeta.getRawForwardIndexChunkCompressionType(),
+ "Old segment should return null for the chunk compression type");
+
+ ColumnMetadata dictMeta = metadata.getColumnMetadataFor(DICT_COL);
+ assertNotNull(dictMeta);
+ assertTrue(dictMeta.hasDictionary());
+ assertEquals(dictMeta.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE);
+ assertNull(dictMeta.getRawForwardIndexChunkCompressionType());
+ assertEquals(dictMeta.getDictionaryEncodedUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ "Dictionary uncompressed-value size must remain unavailable when tracking is disabled");
+ }
+
+ @Test
+ public void testTrackedZeroByteRawColumnRemainsAvailable()
+ throws Exception {
+ String column = "emptyRaw";
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(column, DataType.STRING)
+ .build();
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(List.of(column))
+ .build();
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName("zeroByteSegment");
+ GenericRow row = new GenericRow();
+ row.putValue(column, "");
+ SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(List.of(row)));
+ driver.build();
+
+ ColumnMetadata metadata = new SegmentMetadataImpl(new File(TEMP_DIR, "zeroByteSegment"))
+ .getColumnMetadataFor(column);
+ assertEquals(metadata.getRawForwardIndexUncompressedValueSizeInBytes(), 0L,
+ "A tracked empty value is available and must not be confused with an untracked column");
+ assertEquals(metadata.getRawForwardIndexChunkCompressionType(), ChunkCompressionType.LZ4);
+ }
+
+ @Test
+ public void testIndexingConfigJsonRoundTrip()
+ throws Exception {
+ IndexingConfig original = new IndexingConfig();
+ original.setCompressionStatsEnabled(true);
+
+ String json = JsonUtils.objectToString(original);
+ assertTrue(json.contains("compressionStatsEnabled"),
+ "JSON should contain compressionStatsEnabled field");
+
+ IndexingConfig deserialized = JsonUtils.stringToObject(json, IndexingConfig.class);
+ assertTrue(deserialized.isCompressionStatsEnabled(),
+ "Deserialized config should have compressionStatsEnabled=true");
+
+ original.setCompressionStatsEnabled(false);
+ json = JsonUtils.objectToString(original);
+ deserialized = JsonUtils.stringToObject(json, IndexingConfig.class);
+ assertFalse(deserialized.isCompressionStatsEnabled(),
+ "Deserialized config should have compressionStatsEnabled=false");
+ }
+
+ @Test
+ public void testTableConfigJsonRoundTripWithCompressionStats()
+ throws Exception {
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testTable")
+ .setNoDictionaryColumns(List.of("col1", "col2"))
+ .build();
+ tableConfig.getIndexingConfig().setCompressionStatsEnabled(true);
+
+ String json = JsonUtils.objectToString(tableConfig);
+ TableConfig deserialized = JsonUtils.stringToObject(json, TableConfig.class);
+
+ assertTrue(deserialized.getIndexingConfig().isCompressionStatsEnabled(),
+ "Table config round-trip should preserve compressionStatsEnabled");
+ assertEquals(deserialized.getIndexingConfig().getNoDictionaryColumns(), List.of("col1", "col2"),
+ "Table config round-trip should preserve noDictionaryColumns");
+ }
+
+ @Test
+ public void testOldIndexingConfigJsonWithoutFieldDeserializes()
+ throws Exception {
+ String oldJson = "{\"noDictionaryColumns\":[\"col1\"]}";
+ IndexingConfig deserialized = JsonUtils.stringToObject(oldJson, IndexingConfig.class);
+ assertFalse(deserialized.isCompressionStatsEnabled(),
+ "Missing compressionStatsEnabled in IndexingConfig should default to false");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsSegmentCreationTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsSegmentCreationTest.java
new file mode 100644
index 00000000000..dd57a9fcca0
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/creator/CompressionStatsSegmentCreationTest.java
@@ -0,0 +1,281 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.creator;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
+import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
+import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+
+/// Tests segment-creation tracking and persistence when `compressionStatsEnabled` is set in the table config.
+public class CompressionStatsSegmentCreationTest {
+ private static final File TEMP_DIR =
+ new File(FileUtils.getTempDirectory(), CompressionStatsSegmentCreationTest.class.getSimpleName());
+ private static final String RAW_TABLE_NAME = "compressionStatsTable";
+ private static final String SEGMENT_NAME = "compressionStatsSegment";
+ private static final int NUM_ROWS = 5000;
+ private static final Random RANDOM = new Random(42);
+
+ private static final String INT_RAW_COL = "intRawCol";
+ private static final String STRING_RAW_COL = "stringRawCol";
+ private static final String DICT_COL = "dictCol";
+
+ @BeforeMethod
+ public void setUp() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ FileUtils.deleteQuietly(TEMP_DIR);
+ }
+
+ private List<GenericRow> generateTestData() {
+ List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ GenericRow row = new GenericRow();
+ row.putValue(INT_RAW_COL, RANDOM.nextInt(100000));
+ row.putValue(STRING_RAW_COL, RandomStringUtils.secure().nextAlphanumeric(20 + RANDOM.nextInt(80)));
+ row.putValue(DICT_COL, "value_" + (i % 100));
+ rows.add(row);
+ }
+ return rows;
+ }
+
+ private File buildSegment(boolean compressionStatsEnabled, String compressionCodec)
+ throws Exception {
+ Schema schema = new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME)
+ .addSingleValueDimension(INT_RAW_COL, DataType.INT)
+ .addSingleValueDimension(STRING_RAW_COL, DataType.STRING)
+ .addSingleValueDimension(DICT_COL, DataType.STRING)
+ .build();
+
+ List<FieldConfig> fieldConfigs = new ArrayList<>();
+ if (compressionCodec != null) {
+ FieldConfig.CompressionCodec codec = FieldConfig.CompressionCodec.valueOf(compressionCodec);
+ fieldConfigs.add(new FieldConfig(INT_RAW_COL, FieldConfig.EncodingType.RAW, List.of(), codec, null));
+ fieldConfigs.add(new FieldConfig(STRING_RAW_COL, FieldConfig.EncodingType.RAW, List.of(), codec, null));
+ }
+
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME)
+ .setNoDictionaryColumns(List.of(INT_RAW_COL, STRING_RAW_COL))
+ .setFieldConfigList(fieldConfigs)
+ .build();
+
+ if (compressionStatsEnabled) {
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setCompressionStatsEnabled(true);
+ }
+
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
+ config.setOutDir(TEMP_DIR.getAbsolutePath());
+ config.setSegmentName(SEGMENT_NAME);
+
+ List<GenericRow> rows = generateTestData();
+ SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+
+ return new File(TEMP_DIR, SEGMENT_NAME);
+ }
+
+ @Test
+ public void testCompressionStatsEnabled()
+ throws Exception {
+ File segmentDir = buildSegment(true, "LZ4");
+
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+
+ // Raw int column should have uncompressed size tracked
+ ColumnMetadata intMeta = metadata.getColumnMetadataFor(INT_RAW_COL);
+ assertNotNull(intMeta);
+ assertFalse(intMeta.hasDictionary());
+ long intUncompressedValueSize = intMeta.getRawForwardIndexUncompressedValueSizeInBytes();
+ assertTrue(intUncompressedValueSize > 0,
+ "Uncompressed size for raw int column should be > 0, got: " + intUncompressedValueSize);
+
+ // The uncompressed size reflects the total chunk buffer bytes written before compression.
+ // For fixed-width types this is the actual data chunked into chunk-buffer-sized blocks.
+ // It should be > 0 and in a reasonable range relative to the raw data size.
+ long rawDataSize = (long) NUM_ROWS * Integer.BYTES;
+ assertTrue(intUncompressedValueSize > 0 && intUncompressedValueSize <= rawDataSize * 2,
+ "Uncompressed int size " + intUncompressedValueSize + " should be > 0 and within 2x of raw data size "
+ + rawDataSize);
+
+ // The chunk compression type should be persisted.
+ assertEquals(intMeta.getRawForwardIndexChunkCompressionType(), ChunkCompressionType.LZ4);
+
+ // Raw string column should also have stats
+ ColumnMetadata stringMeta = metadata.getColumnMetadataFor(STRING_RAW_COL);
+ assertNotNull(stringMeta);
+ assertFalse(stringMeta.hasDictionary());
+ long stringUncompressedValueSize = stringMeta.getRawForwardIndexUncompressedValueSizeInBytes();
+ assertTrue(stringUncompressedValueSize > 0,
+ "Uncompressed size for raw string column should be > 0, got: " + stringUncompressedValueSize);
+ assertEquals(stringMeta.getRawForwardIndexChunkCompressionType(), ChunkCompressionType.LZ4);
+
+ // The compressed forward index size should be less than uncompressed for random string data
+ long stringCompressedSize = stringMeta.getIndexSizeFor(StandardIndexes.forward());
+ assertTrue(stringCompressedSize > 0, "Compressed forward index size should be > 0");
+ // Note: for LZ4, random data may not compress well, but the sizes should be trackable
+
+ // Verify compression ratio is meaningful
+ if (stringCompressedSize > 0 && stringUncompressedValueSize > 0) {
+ double ratio = (double) stringUncompressedValueSize / stringCompressedSize;
+ assertTrue(ratio > 0, "Compression ratio should be > 0, got: " + ratio);
+ }
+
+ // Dictionary-encoded column should NOT have uncompressed forward index stats
+ ColumnMetadata dictMeta = metadata.getColumnMetadataFor(DICT_COL);
+ assertNotNull(dictMeta);
+ assertTrue(dictMeta.hasDictionary());
+ assertEquals(dictMeta.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ "Dictionary-encoded column should not have uncompressed forward index size");
+ assertNull(dictMeta.getRawForwardIndexChunkCompressionType(),
+ "Dictionary-encoded column should not have a forward-index chunk compression type");
+ }
+
+ @Test
+ public void testCompressionStatsDisabled()
+ throws Exception {
+ File segmentDir = buildSegment(false, "LZ4");
+
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+
+ // When compressionStatsEnabled is false, no uncompressed size should be persisted
+ ColumnMetadata intMeta = metadata.getColumnMetadataFor(INT_RAW_COL);
+ assertNotNull(intMeta);
+ assertEquals(intMeta.getRawForwardIndexUncompressedValueSizeInBytes(), ColumnMetadata.UNAVAILABLE,
+ "Uncompressed size should not be tracked when compressionStatsEnabled is false");
+ assertNull(intMeta.getRawForwardIndexChunkCompressionType(),
+ "Chunk compression type should not be tracked when compressionStatsEnabled is false");
+ }
+
+ @Test
+ public void testCompressionStatsWithZstandard()
+ throws Exception {
+ File segmentDir = buildSegment(true, "ZSTANDARD");
+
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(segmentDir);
+
+ ColumnMetadata intMeta = metadata.getColumnMetadataFor(INT_RAW_COL);
... 4219 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org