You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/03/22 05:19:29 UTC
(pinot) branch master updated: Add useRawBytes config to Murmur and Murmur3 partition functions (#17932)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1329e1f7f23 Add useRawBytes config to Murmur and Murmur3 partition functions (#17932)
1329e1f7f23 is described below
commit 1329e1f7f23bf64804ecd918567a1e5a644d9622
Author: Xiang Fu <xi...@apache.org>
AuthorDate: Sat Mar 21 22:19:21 2026 -0700
Add useRawBytes config to Murmur and Murmur3 partition functions (#17932)
* Add useRawBytes config to Murmur and Murmur3 partition functions
When partitioning on BYTES columns, the partition value is hex-encoded.
With useRawBytes=true in functionConfig, the hex string is decoded back
to raw bytes before hashing, ensuring partition assignment matches the
original byte values rather than treating the hex string as UTF-8 text.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
* Override getFunctionConfig() and restore backward-compatible constructor
- Murmur and Murmur3 partition functions now persist and return the
functionConfig via getFunctionConfig(), ensuring partition metadata
written to ZK retains the config so brokers can reconstruct the
function correctly.
- Re-introduce MurmurPartitionFunction(int) constructor delegating to
the new (int, Map) constructor to preserve SPI backward compatibility.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
* Defensive copy and @Nullable annotations for functionConfig
- Store functionConfig as Collections.unmodifiableMap() to prevent
external mutation from diverging with parsed fields.
- Annotate _functionConfig field and getFunctionConfig() with @Nullable
in both MurmurPartitionFunction and Murmur3PartitionFunction.
- Annotate Murmur3PartitionFunction constructor parameter with @Nullable.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
* Add factory end-to-end test for useRawBytes with Murmur/Murmur2/Murmur3
Verify that PartitionFunctionFactory.getPartitionFunction() correctly
wires functionConfig through to the partition function for all three
aliases, and that getFunctionConfig() roundtrips the config.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
---------
Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
.../spi/partition/Murmur3PartitionFunction.java | 25 +++++-
.../spi/partition/MurmurPartitionFunction.java | 30 ++++++-
.../spi/partition/PartitionFunctionFactory.java | 2 +-
.../spi/partition/PartitionFunctionTest.java | 95 ++++++++++++++++++++++
4 files changed, 148 insertions(+), 4 deletions(-)
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Murmur3PartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Murmur3PartitionFunction.java
index ba22c1a992f..9a0110f810e 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Murmur3PartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Murmur3PartitionFunction.java
@@ -19,8 +19,11 @@
package org.apache.pinot.segment.spi.partition;
import com.google.common.base.Preconditions;
+import java.util.Collections;
import java.util.Map;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -33,21 +36,27 @@ public class Murmur3PartitionFunction implements PartitionFunction {
private static final String NAME = "Murmur3";
private static final String SEED_KEY = "seed";
private static final String VARIANT_KEY = "variant";
+ private static final String USE_RAW_BYTES_KEY = "useRawBytes";
private final int _numPartitions;
+ @Nullable
+ private final Map<String, String> _functionConfig;
private final int _seed;
private final boolean _useX64;
+ private final boolean _useRawBytes;
/**
* Constructor for the class.
* @param numPartitions Number of partitions.
* @param functionConfig to extract configurations for the partition function.
*/
- public Murmur3PartitionFunction(int numPartitions, Map<String, String> functionConfig) {
+ public Murmur3PartitionFunction(int numPartitions, @Nullable Map<String, String> functionConfig) {
Preconditions.checkArgument(numPartitions > 0, "Number of partitions must be > 0");
_numPartitions = numPartitions;
+ _functionConfig = functionConfig != null ? Collections.unmodifiableMap(functionConfig) : null;
int seed = 0;
boolean useX64 = false;
+ boolean useRawBytes = false;
if (functionConfig != null) {
String seedString = functionConfig.get(SEED_KEY);
if (StringUtils.isNotEmpty(seedString)) {
@@ -62,13 +71,21 @@ public class Murmur3PartitionFunction implements PartitionFunction {
"Murmur3 variant must be either x86_32 or x64_32");
}
}
+ useRawBytes = Boolean.parseBoolean(functionConfig.get(USE_RAW_BYTES_KEY));
}
_seed = seed;
_useX64 = useX64;
+ _useRawBytes = useRawBytes;
}
@Override
public int getPartition(String value) {
+ if (_useRawBytes) {
+ byte[] bytes = BytesUtils.toBytes(value);
+ int hash = _useX64 ? MurmurHashFunctions.murmurHash3X64Bit32(bytes, _seed)
+ : MurmurHashFunctions.murmurHash3X86Bit32(bytes, _seed);
+ return (hash & Integer.MAX_VALUE) % _numPartitions;
+ }
int hash = _useX64 ? MurmurHashFunctions.murmurHash3X64Bit32(value, _seed)
: MurmurHashFunctions.murmurHash3X86Bit32(value.getBytes(UTF_8), _seed);
return (hash & Integer.MAX_VALUE) % _numPartitions;
@@ -84,6 +101,12 @@ public class Murmur3PartitionFunction implements PartitionFunction {
return _numPartitions;
}
+ @Nullable
+ @Override
+ public Map<String, String> getFunctionConfig() {
+ return _functionConfig;
+ }
+
// Keep it for backward-compatibility, use getName() instead
@Override
public String toString() {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
index 8251450c44b..6c64dee763a 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/MurmurPartitionFunction.java
@@ -19,6 +19,10 @@
package org.apache.pinot.segment.spi.partition;
import com.google.common.base.Preconditions;
+import java.util.Collections;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -29,20 +33,36 @@ import static java.nio.charset.StandardCharsets.UTF_8;
*/
public class MurmurPartitionFunction implements PartitionFunction {
private static final String NAME = "Murmur";
+ private static final String USE_RAW_BYTES_KEY = "useRawBytes";
private final int _numPartitions;
+ @Nullable
+ private final Map<String, String> _functionConfig;
+ private final boolean _useRawBytes;
/**
- * Constructor for the class.
+ * Constructor for backward compatibility.
* @param numPartitions Number of partitions.
*/
public MurmurPartitionFunction(int numPartitions) {
+ this(numPartitions, null);
+ }
+
+ /**
+ * Constructor for the class.
+ * @param numPartitions Number of partitions.
+ * @param functionConfig to extract configurations for the partition function.
+ */
+ public MurmurPartitionFunction(int numPartitions, @Nullable Map<String, String> functionConfig) {
Preconditions.checkArgument(numPartitions > 0, "Number of partitions must be > 0");
_numPartitions = numPartitions;
+ _functionConfig = functionConfig != null ? Collections.unmodifiableMap(functionConfig) : null;
+ _useRawBytes = functionConfig != null && Boolean.parseBoolean(functionConfig.get(USE_RAW_BYTES_KEY));
}
@Override
public int getPartition(String value) {
- return (MurmurHashFunctions.murmurHash2(value.getBytes(UTF_8)) & Integer.MAX_VALUE) % _numPartitions;
+ byte[] bytes = _useRawBytes ? BytesUtils.toBytes(value) : value.getBytes(UTF_8);
+ return (MurmurHashFunctions.murmurHash2(bytes) & Integer.MAX_VALUE) % _numPartitions;
}
@Override
@@ -55,6 +75,12 @@ public class MurmurPartitionFunction implements PartitionFunction {
return _numPartitions;
}
+ @Nullable
+ @Override
+ public Map<String, String> getFunctionConfig() {
+ return _functionConfig;
+ }
+
// Keep it for backward-compatibility, use getName() instead
@Override
public String toString() {
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
index 202b04843ca..5a2bc20f431 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/PartitionFunctionFactory.java
@@ -77,7 +77,7 @@ public class PartitionFunctionFactory {
case Murmur:
case Murmur2:
- return new MurmurPartitionFunction(numPartitions);
+ return new MurmurPartitionFunction(numPartitions, functionConfig);
case Murmur3:
return new Murmur3PartitionFunction(numPartitions, functionConfig);
diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
index 8bb44bb024f..42d7634376d 100644
--- a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
+++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/partition/PartitionFunctionTest.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
import org.testng.annotations.Test;
@@ -439,6 +440,100 @@ public class PartitionFunctionTest {
testPartitionFunction(murmurPartitionFunction, expectedPartitions);
}
+ @Test
+ public void testMurmurPartitionFunctionUseRawBytes() {
+ int numPartitions = 5;
+ Map<String, String> functionConfig = new HashMap<>();
+ functionConfig.put("useRawBytes", "true");
+ MurmurPartitionFunction partitionFunction = new MurmurPartitionFunction(numPartitions, functionConfig);
+
+ // When useRawBytes is true, the hex-encoded value should be decoded back to raw bytes before hashing.
+ // This means getPartition(hexString) should produce the same result as hashing the original raw bytes.
+ byte[] rawBytes = new byte[]{0x01, 0x02, 0x03, 0x04, 0x05};
+ String hexValue = BytesUtils.toHexString(rawBytes);
+
+ int expectedPartition =
+ (MurmurHashFunctions.murmurHash2(rawBytes) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(partitionFunction.getPartition(hexValue), expectedPartition);
+
+ // Without useRawBytes, the same hex string should be treated as UTF-8 text (different result).
+ MurmurPartitionFunction defaultPartitionFunction = new MurmurPartitionFunction(numPartitions, null);
+ int defaultPartition =
+ (MurmurHashFunctions.murmurHash2(hexValue.getBytes(UTF_8)) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(defaultPartitionFunction.getPartition(hexValue), defaultPartition);
+ }
+
+ @Test
+ public void testMurmur3PartitionFunctionUseRawBytes() {
+ int numPartitions = 5;
+ byte[] rawBytes = new byte[]{0x01, 0x02, 0x03, 0x04, 0x05};
+ String hexValue = BytesUtils.toHexString(rawBytes);
+
+ // Test x86_32 variant with useRawBytes
+ Map<String, String> functionConfig = new HashMap<>();
+ functionConfig.put("useRawBytes", "true");
+ Murmur3PartitionFunction partitionFunction = new Murmur3PartitionFunction(numPartitions, functionConfig);
+
+ int expectedPartition =
+ (MurmurHashFunctions.murmurHash3X86Bit32(rawBytes, 0) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(partitionFunction.getPartition(hexValue), expectedPartition);
+
+ // Test x64_32 variant with useRawBytes
+ functionConfig.put("variant", "x64_32");
+ Murmur3PartitionFunction partitionFunctionX64 = new Murmur3PartitionFunction(numPartitions, functionConfig);
+
+ int expectedPartitionX64 =
+ (MurmurHashFunctions.murmurHash3X64Bit32(rawBytes, 0) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(partitionFunctionX64.getPartition(hexValue), expectedPartitionX64);
+
+ // Test x86_32 variant with useRawBytes and non-zero seed
+ functionConfig.remove("variant");
+ functionConfig.put("seed", "9001");
+ Murmur3PartitionFunction partitionFunctionWithSeed = new Murmur3PartitionFunction(numPartitions, functionConfig);
+
+ int expectedPartitionWithSeed =
+ (MurmurHashFunctions.murmurHash3X86Bit32(rawBytes, 9001) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(partitionFunctionWithSeed.getPartition(hexValue), expectedPartitionWithSeed);
+
+ // Without useRawBytes, the same hex string should be treated as UTF-8 text.
+ Murmur3PartitionFunction defaultPartitionFunction = new Murmur3PartitionFunction(numPartitions, null);
+ int defaultPartition =
+ (MurmurHashFunctions.murmurHash3X86Bit32(hexValue.getBytes(UTF_8), 0) & Integer.MAX_VALUE) % numPartitions;
+ assertEquals(defaultPartitionFunction.getPartition(hexValue), defaultPartition);
+ }
+
+ @Test
+ public void testUseRawBytesThroughFactory() {
+ int numPartitions = 5;
+ byte[] rawBytes = new byte[]{0x01, 0x02, 0x03, 0x04, 0x05};
+ String hexValue = BytesUtils.toHexString(rawBytes);
+ int expectedPartition =
+ (MurmurHashFunctions.murmurHash2(rawBytes) & Integer.MAX_VALUE) % numPartitions;
+
+ Map<String, String> functionConfig = new HashMap<>();
+ functionConfig.put("useRawBytes", "true");
+
+ // Test Murmur alias through factory
+ PartitionFunction murmurFn =
+ PartitionFunctionFactory.getPartitionFunction("Murmur", numPartitions, functionConfig);
+ assertEquals(murmurFn.getPartition(hexValue), expectedPartition);
+ assertEquals(murmurFn.getFunctionConfig().get("useRawBytes"), "true");
+
+ // Test Murmur2 alias through factory
+ PartitionFunction murmur2Fn =
+ PartitionFunctionFactory.getPartitionFunction("Murmur2", numPartitions, functionConfig);
+ assertEquals(murmur2Fn.getPartition(hexValue), expectedPartition);
+ assertEquals(murmur2Fn.getFunctionConfig().get("useRawBytes"), "true");
+
+ // Test Murmur3 through factory
+ int expectedPartitionMurmur3 =
+ (MurmurHashFunctions.murmurHash3X86Bit32(rawBytes, 0) & Integer.MAX_VALUE) % numPartitions;
+ PartitionFunction murmur3Fn =
+ PartitionFunctionFactory.getPartitionFunction("Murmur3", numPartitions, functionConfig);
+ assertEquals(murmur3Fn.getPartition(hexValue), expectedPartitionMurmur3);
+ assertEquals(murmur3Fn.getFunctionConfig().get("useRawBytes"), "true");
+ }
+
@Test
public void testMurmur3PartitionFunctionEquivalence() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org