You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/10/03 01:14:12 UTC
(pinot) branch master updated: Use v4 as the default raw index version (#16943)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 77b8c8af677 Use v4 as the default raw index version (#16943)
77b8c8af677 is described below
commit 77b8c8af67730e9b9de35f89962a21e3229ee0ca
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Thu Oct 2 18:14:06 2025 -0700
Use v4 as the default raw index version (#16943)
---
.../tests/OfflineClusterIntegrationTest.java | 20 +++++++++-----------
.../pinot/segment/spi/index/ForwardIndexConfig.java | 11 +----------
2 files changed, 10 insertions(+), 21 deletions(-)
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index 71eafcc087e..02c16814488 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -1482,18 +1482,18 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
columnIndexSize = getColumnIndexSize(column);
assertFalse(columnIndexSize.has(StandardIndexes.DICTIONARY_ID));
assertTrue(columnIndexSize.has(StandardIndexes.FORWARD_ID));
- double v2rawIndexSize = columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble();
- assertTrue(v2rawIndexSize > forwardIndexSize);
+ double v4rawIndexSize = columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble();
+ assertTrue(v4rawIndexSize > forwardIndexSize);
// NOTE: Currently Pinot doesn't support directly changing raw index version, so we need to first reset it back to
// dictionary encoding.
// TODO: Support it
resetForwardIndex(dictionarySize, forwardIndexSize);
- // Convert 'DestCityName' to v4 raw index
+ // Convert 'DestCityName' to v2 raw index
List<FieldConfig> fieldConfigs = tableConfig.getFieldConfigList();
assertNotNull(fieldConfigs);
- ForwardIndexConfig forwardIndexConfig = new ForwardIndexConfig.Builder().withRawIndexWriterVersion(4).build();
+ ForwardIndexConfig forwardIndexConfig = new ForwardIndexConfig.Builder().withRawIndexWriterVersion(2).build();
ObjectNode indexes = JsonUtils.newObjectNode();
indexes.set("forward", forwardIndexConfig.toJsonNode());
FieldConfig fieldConfig =
@@ -1504,13 +1504,11 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
columnIndexSize = getColumnIndexSize(column);
assertFalse(columnIndexSize.has(StandardIndexes.DICTIONARY_ID));
assertTrue(columnIndexSize.has(StandardIndexes.FORWARD_ID));
- double v4RawIndexSize = columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble();
- assertTrue(v4RawIndexSize < v2rawIndexSize && v4RawIndexSize > forwardIndexSize);
+ double v2RawIndexSize = columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble();
+ assertTrue(v2RawIndexSize > v4rawIndexSize);
// Convert 'DestCityName' to SNAPPY compression
- forwardIndexConfig =
- new ForwardIndexConfig.Builder().withCompressionCodec(CompressionCodec.SNAPPY).withRawIndexWriterVersion(4)
- .build();
+ forwardIndexConfig = new ForwardIndexConfig.Builder().withCompressionCodec(CompressionCodec.SNAPPY).build();
indexes.set("forward", forwardIndexConfig.toJsonNode());
fieldConfig =
new FieldConfig.Builder(column).withEncodingType(FieldConfig.EncodingType.RAW).withIndexes(indexes).build();
@@ -1521,7 +1519,7 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
assertFalse(columnIndexSize.has(StandardIndexes.DICTIONARY_ID));
assertTrue(columnIndexSize.has(StandardIndexes.FORWARD_ID));
double v4SnappyRawIndexSize = columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble();
- assertTrue(v4SnappyRawIndexSize > v2rawIndexSize);
+ assertTrue(v4SnappyRawIndexSize > v4rawIndexSize);
// Removing FieldConfig should be no-op because compression is not explicitly set
fieldConfigs.remove(fieldConfigs.size() - 1);
@@ -1543,7 +1541,7 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
columnIndexSize = getColumnIndexSize(column);
assertFalse(columnIndexSize.has(StandardIndexes.DICTIONARY_ID));
assertTrue(columnIndexSize.has(StandardIndexes.FORWARD_ID));
- assertEquals(columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble(), v2rawIndexSize);
+ assertEquals(columnIndexSize.get(StandardIndexes.FORWARD_ID).asDouble(), v4rawIndexSize);
resetForwardIndex(dictionarySize, forwardIndexSize);
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
index a83c94e5801..c34f0661b32 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/ForwardIndexConfig.java
@@ -35,16 +35,7 @@ import org.apache.pinot.spi.utils.DataSizeUtils;
public class ForwardIndexConfig extends IndexConfig {
- @Deprecated
- public static final int DEFAULT_RAW_WRITER_VERSION = 2;
- @Deprecated
- public static final String DEFAULT_TARGET_MAX_CHUNK_SIZE = "1MB";
- @Deprecated
- public static final int DEFAULT_TARGET_MAX_CHUNK_SIZE_BYTES = 1024 * 1024;
- @Deprecated
- public static final int DEFAULT_TARGET_DOCS_PER_CHUNK = 1000;
-
- private static int _defaultRawIndexWriterVersion = 2;
+ private static int _defaultRawIndexWriterVersion = 4;
private static String _defaultTargetMaxChunkSize = "1MB";
private static int _defaultTargetMaxChunkSizeBytes = 1024 * 1024;
private static int _defaultTargetDocsPerChunk = 1000;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org