You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/03/26 07:05:12 UTC
(pinot) branch master updated: Reduce memory allocation when computing length for STRING/BYTES (#17983)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9a4864e3993 Reduce memory allocation when computing length for STRING/BYTES (#17983)
9a4864e3993 is described below
commit 9a4864e3993e147640decb9377e70fbd70968340
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Thu Mar 26 00:05:05 2026 -0700
Reduce memory allocation when computing length for STRING/BYTES (#17983)
Auto-merged by automated PR review bot.
---
.../indexsegment/mutable/MutableSegmentImpl.java | 14 ++++------
.../converter/stats/MutableColumnStatistics.java | 4 +--
.../SchemaConformingTransformer.java | 5 ++--
.../stats/NoDictColumnStatisticsCollector.java | 4 +--
.../stats/StringColumnPreIndexStatsCollector.java | 6 ++---
...IndexAndDictionaryBasedForwardIndexCreator.java | 13 ++++------
.../local/startree/StarTreeBuilderUtils.java | 3 ++-
.../pinot/server/api/resources/TablesResource.java | 30 ++++++++++++----------
.../converter/DictionaryToRawIndexConverter.java | 4 +--
9 files changed, 39 insertions(+), 44 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
index ef35e6b62ed..1eda9456b18 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImpl.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.indexsegment.mutable;
import com.google.common.base.Preconditions;
+import com.google.common.base.Utf8;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
@@ -125,7 +126,6 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.pinot.spi.data.FieldSpec.DataType.BYTES;
import static org.apache.pinot.spi.data.FieldSpec.DataType.MAP;
import static org.apache.pinot.spi.data.FieldSpec.DataType.STRING;
@@ -1598,20 +1598,16 @@ public class MutableSegmentImpl implements MutableSegment {
switch (dataType) {
case STRING: {
- for (Object obj : values) {
- String value = (String) obj;
- int length = value.getBytes(UTF_8).length;
- rowLength += length;
+ for (Object value : values) {
+ rowLength += Utf8.encodedLength((String) value);
}
_varByteMVMaxRowLengthInBytes = Math.max(_varByteMVMaxRowLengthInBytes, rowLength);
break;
}
case BYTES: {
- for (Object obj : values) {
- ByteArray value = new ByteArray((byte[]) obj);
- int length = value.length();
- rowLength += length;
+ for (Object value : values) {
+ rowLength += ((byte[]) value).length;
}
_varByteMVMaxRowLengthInBytes = Math.max(_varByteMVMaxRowLengthInBytes, rowLength);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
index b68d11e0589..9bb751e3dcd 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/converter/stats/MutableColumnStatistics.java
@@ -19,7 +19,7 @@
package org.apache.pinot.segment.local.realtime.converter.stats;
import com.google.common.base.Preconditions;
-import java.nio.charset.StandardCharsets;
+import com.google.common.base.Utf8;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
@@ -116,7 +116,7 @@ public class MutableColumnStatistics implements ColumnStatistics {
break;
case STRING:
for (int i = 0; i < length; i++) {
- int elementLength = _dictionary.getStringValue(i).getBytes(StandardCharsets.UTF_8).length;
+ int elementLength = Utf8.encodedLength(_dictionary.getStringValue(i));
_minElementLength = Math.min(_minElementLength, elementLength);
_maxElementLength = Math.max(_maxElementLength, elementLength);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
index 6663bf4116a..3ce8ee2b7ec 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformer.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.recordtransformer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Preconditions;
-import java.nio.charset.StandardCharsets;
+import com.google.common.base.Utf8;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -141,8 +141,7 @@ public class SchemaConformingTransformer implements RecordTransformer {
unindexableExtrasFieldName != null ? getAndValidateExtrasFieldType(schema, unindexableExtrasFieldName) : null;
_mergedTextIndexFieldSpec = schema.getDimensionSpec(_transformerConfig.getMergedTextIndexField());
_schemaTree = validateSchemaAndCreateTree(schema, _transformerConfig);
- _jsonKeyValueSeparatorByteCount =
- _transformerConfig.getJsonKeyValueSeparator().getBytes(StandardCharsets.UTF_8).length;
+ _jsonKeyValueSeparatorByteCount = Utf8.encodedLength(_transformerConfig.getJsonKeyValueSeparator());
_continueOnError = ingestionConfig.isContinueOnError();
_serverMetrics = ServerMetrics.get();
_throttledLogger = new ThrottledLogger(_logger, ingestionConfig);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
index f047c56c479..8d68bf785c4 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/NoDictColumnStatisticsCollector.java
@@ -19,8 +19,8 @@
package org.apache.pinot.segment.local.segment.creator.impl.stats;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import com.google.common.base.Utf8;
import java.math.BigDecimal;
-import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang3.NotImplementedException;
@@ -171,7 +171,7 @@ public class NoDictColumnStatisticsCollector extends AbstractColumnStatisticsCol
return ((byte[]) value).length;
}
if (value instanceof CharSequence) {
- return ((CharSequence) value).toString().getBytes(StandardCharsets.UTF_8).length;
+ return Utf8.encodedLength((CharSequence) value);
}
if (value instanceof BigDecimal) {
return BigDecimalUtils.byteSize((BigDecimal) value);
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
index d43adbe3c75..0523fe7537b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StringColumnPreIndexStatsCollector.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.segment.creator.impl.stats;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Utf8;
import com.yscope.clp.compressorfrontend.BuiltInVariableHandlingRuleVersions;
import com.yscope.clp.compressorfrontend.EncodedMessage;
import com.yscope.clp.compressorfrontend.MessageEncoder;
@@ -30,7 +31,6 @@ import org.apache.pinot.segment.spi.creator.StatsCollectorConfig;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.data.FieldSpec;
-import static java.nio.charset.StandardCharsets.UTF_8;
public class StringColumnPreIndexStatsCollector extends AbstractColumnStatisticsCollector implements CLPStatsProvider {
@@ -63,7 +63,7 @@ public class StringColumnPreIndexStatsCollector extends AbstractColumnStatistics
_clpStatsCollector.collect(value);
}
- int length = value.getBytes(UTF_8).length;
+ int length = Utf8.encodedLength(value);
_minLength = Math.min(_minLength, length);
_maxLength = Math.max(_maxLength, length);
rowLength += length;
@@ -82,7 +82,7 @@ public class StringColumnPreIndexStatsCollector extends AbstractColumnStatistics
if (isPartitionEnabled()) {
updatePartition(value);
}
- int valueLength = value.getBytes(UTF_8).length;
+ int valueLength = Utf8.encodedLength(value);
_minLength = Math.min(_minLength, valueLength);
_maxLength = Math.max(_maxLength, valueLength);
_maxRowLength = _maxLength;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
index 9e267f90ef4..066695a0cb2 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.segment.index.loader;
import com.google.common.base.Preconditions;
+import com.google.common.base.Utf8;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
@@ -41,12 +42,10 @@ import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.segment.spi.utils.SegmentMetadataUtils;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.BigDecimalUtils;
-import org.apache.pinot.spi.utils.ByteArray;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.*;
@@ -392,12 +391,11 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
int updatedLengthOfLongestEntry;
switch (_storedType) {
case STRING:
- updatedLengthOfLongestEntry = Math.max(dictionary.getStringValue(dictId).getBytes(UTF_8).length,
+ updatedLengthOfLongestEntry = Math.max(Utf8.encodedLength(dictionary.getStringValue(dictId)),
lengthOfLongestEntry);
break;
case BYTES:
- ByteArray value = new ByteArray(dictionary.getBytesValue(dictId));
- updatedLengthOfLongestEntry = Math.max(value.length(), lengthOfLongestEntry);
+ updatedLengthOfLongestEntry = Math.max(dictionary.getBytesValue(dictId).length, lengthOfLongestEntry);
break;
case BIG_DECIMAL:
updatedLengthOfLongestEntry = Math.max(
@@ -414,13 +412,12 @@ public class InvertedIndexAndDictionaryBasedForwardIndexCreator implements AutoC
int curSizeOfRow = getInt(_forwardIndexMaxSizeBuffer, docId);
switch (_storedType) {
case STRING:
- int newSizeOfEntry = dictionary.getStringValue(dictId).length() + curSizeOfRow;
+ int newSizeOfEntry = Utf8.encodedLength(dictionary.getStringValue(dictId)) + curSizeOfRow;
putInt(_forwardIndexMaxSizeBuffer, docId, newSizeOfEntry);
maxRowLengthInBytes[0] = Math.max(newSizeOfEntry, maxRowLengthInBytes[0]);
break;
case BYTES:
- ByteArray value = new ByteArray(dictionary.getBytesValue(dictId));
- newSizeOfEntry = value.length() + curSizeOfRow;
+ newSizeOfEntry = dictionary.getBytesValue(dictId).length + curSizeOfRow;
putInt(_forwardIndexMaxSizeBuffer, docId, newSizeOfEntry);
maxRowLengthInBytes[0] = Math.max(newSizeOfEntry, maxRowLengthInBytes[0]);
break;
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
index ac376aed125..90f9a6ca2bc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.startree;
import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Utf8;
import java.io.File;
import java.io.IOException;
import java.nio.ByteOrder;
@@ -151,7 +152,7 @@ public class StarTreeBuilderUtils {
for (String dimension : dimensions) {
headerSizeInBytes += Integer.BYTES; // For dimension index
headerSizeInBytes += Integer.BYTES; // For length of dimension name
- headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name
+ headerSizeInBytes += Utf8.encodedLength(dimension); // For dimension name
}
headerSizeInBytes += Integer.BYTES; // For number of nodes.
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index aa6d7f8526c..f394c51097d 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -20,6 +20,7 @@ package org.apache.pinot.server.api.resources;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Preconditions;
+import com.google.common.base.Utf8;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiKeyAuthDefinition;
import io.swagger.annotations.ApiOperation;
@@ -32,7 +33,6 @@ import io.swagger.annotations.SwaggerDefinition;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
@@ -114,6 +114,7 @@ import org.apache.pinot.spi.stream.ConsumerPartitionState;
import org.apache.pinot.spi.stream.PartitionLagState;
import org.apache.pinot.spi.stream.StreamMetadataProvider;
import org.apache.pinot.spi.stream.StreamPartitionMsgOffset;
+import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateModel;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -245,26 +246,27 @@ public class TablesResource {
}
for (String column : columnSet) {
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataMap().get(column);
- int columnLength = 0;
- DataType storedDataType = columnMetadata.getDataType().getStoredType();
- if (storedDataType.isFixedWidth()) {
+ int columnLength;
+ DataType storedType = columnMetadata.getDataType().getStoredType();
+ if (storedType.isFixedWidth()) {
// For type of fixed width: INT, LONG, FLOAT, DOUBLE, BOOLEAN (stored as INT), TIMESTAMP (stored as LONG),
// set the columnLength as the fixed width.
- columnLength = storedDataType.size();
+ columnLength = storedType.size();
} else if (columnMetadata.hasDictionary()) {
// For type of variable width (String, Bytes), if it's stored using dictionary encoding, set the
// columnLength as the max length in dictionary.
columnLength = columnMetadata.getColumnMaxLength();
- } else if (storedDataType == DataType.STRING || storedDataType == DataType.BYTES) {
- // For type of variable width (String, Bytes), if it's stored using raw bytes, set the columnLength as
- // the length of the max value.
- if (columnMetadata.getMaxValue() != null) {
- String maxValueString = (String) columnMetadata.getMaxValue();
- columnLength = maxValueString.getBytes(StandardCharsets.UTF_8).length;
- }
} else {
- // For type of STRUCT, MAP, LIST, set the columnLength as DEFAULT_MAX_LENGTH (512).
- columnLength = FieldSpec.DEFAULT_MAX_LENGTH;
+ // For raw STRING/BYTES column, set the columnLength as the length of the max value.
+ Comparable<?> maxValue = columnMetadata.getMaxValue();
+ if (maxValue instanceof String) {
+ columnLength = Utf8.encodedLength((String) maxValue);
+ } else if (maxValue instanceof ByteArray) {
+ columnLength = ((ByteArray) maxValue).length();
+ } else {
+ // For type of STRUCT, MAP, LIST, set the columnLength as DEFAULT_MAX_LENGTH (512).
+ columnLength = FieldSpec.DEFAULT_MAX_LENGTH;
+ }
}
int columnCardinality = columnMetadata.getCardinality();
columnLengthMap.merge(column, (double) columnLength, Double::sum);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
index a9406a012e2..35b87faa75d 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/DictionaryToRawIndexConverter.java
@@ -19,6 +19,7 @@
package org.apache.pinot.tools.segment.converter;
import com.google.common.base.Preconditions;
+import com.google.common.base.Utf8;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
@@ -53,7 +54,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;
-import static java.nio.charset.StandardCharsets.UTF_8;
/**
@@ -390,7 +390,7 @@ public class DictionaryToRawIndexConverter {
int length = dictionary.length();
for (int dictId = 0; dictId < length; dictId++) {
String value = (String) dictionary.get(dictId);
- lengthOfLongestEntry = Math.max(lengthOfLongestEntry, value.getBytes(UTF_8).length);
+ lengthOfLongestEntry = Math.max(lengthOfLongestEntry, Utf8.encodedLength(value));
}
return lengthOfLongestEntry;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org