You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2025/12/13 00:43:03 UTC
(pinot) branch master updated: Fix ExpressionTransformer collection handling (#17363)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 624b90b6d15 Fix ExpressionTransformer collection handling (#17363)
624b90b6d15 is described below
commit 624b90b6d15073257221426ec74cd014145d994e
Author: Xiang Fu <xi...@apache.org>
AuthorDate: Fri Dec 12 16:42:56 2025 -0800
Fix ExpressionTransformer collection handling (#17363)
---
.../recordtransformer/ExpressionTransformer.java | 6 +--
.../ExpressionTransformerTest.java | 56 ++++++++++++++++++++++
2 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
index 7c62372f409..3e7649f663f 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformer.java
@@ -20,7 +20,7 @@ package org.apache.pinot.segment.local.recordtransformer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import java.util.Collections;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -154,7 +154,7 @@ public class ExpressionTransformer implements RecordTransformer {
_throttledLogger.warn("Caught exception while evaluation transform function for column: " + column, e);
record.markIncomplete();
}
- } else if (existingValue.getClass().isArray() || existingValue instanceof Collections
+ } else if (existingValue.getClass().isArray() || existingValue instanceof Collection
|| existingValue instanceof Map) {
try {
Object transformedValue = transformFunctionEvaluator.evaluate(record);
@@ -174,7 +174,7 @@ public class ExpressionTransformer implements RecordTransformer {
if (transformedValue.getClass() == existingValue.getClass()) {
return true;
}
- if (transformedValue instanceof Collections && existingValue instanceof Collections) {
+ if (transformedValue instanceof Collection && existingValue instanceof Collection) {
return true;
}
if (transformedValue instanceof Map && existingValue instanceof Map) {
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
index 70935130b92..c8601a71a6e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/recordtransformer/ExpressionTransformerTest.java
@@ -241,6 +241,34 @@ public class ExpressionTransformerTest {
Assert.assertEquals(genericRow.getValue("outgoing"), "123");
}
+ @Test
+ public void testExistingCollectionIsTransformedWhenIncompatibleType() {
+ Schema schema = new Schema.SchemaBuilder()
+ .addMultiValueDimension("rawBids", FieldSpec.DataType.INT)
+ .addMultiValueDimension("bids", FieldSpec.DataType.INT)
+ .build();
+
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ingestionConfig.setTransformConfigs(Collections.singletonList(
+ new TransformConfig("bids", "Groovy({rawBids.toArray()}, rawBids)")));
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testExistingCollectionIsTransformed")
+ .setIngestionConfig(ingestionConfig)
+ .build();
+
+ ExpressionTransformer expressionTransformer = new ExpressionTransformer(tableConfig, schema);
+ GenericRow genericRow = new GenericRow();
+ genericRow.putValue("rawBids", Arrays.asList(1, 2, 3));
+ // Simulate pre-existing collection value that should be overwritten by transform
+ genericRow.putValue("bids", Arrays.asList(10, 20));
+
+ expressionTransformer.transform(genericRow);
+
+ Object transformedValue = genericRow.getValue("bids");
+ Assert.assertTrue(transformedValue.getClass().isArray());
+ Assert.assertEquals(Arrays.asList((Object[]) transformedValue), Arrays.asList(1, 2, 3));
+ }
+
@Test
public void testTransformFunctionSortOrder() {
Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("a", FieldSpec.DataType.STRING)
@@ -430,4 +458,32 @@ public class ExpressionTransformerTest {
Assert.assertEquals(map.get("a"), 1);
Assert.assertEquals(map.get("b"), "x");
}
+
+ @Test
+ public void testJsonToArrayIngestionTransform() {
+ Schema schema = new Schema.SchemaBuilder()
+ .addSingleValueDimension("columnJson", FieldSpec.DataType.STRING)
+ .addMultiValueDimension("columnArray", FieldSpec.DataType.STRING)
+ .build();
+
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ingestionConfig.setTransformConfigs(Collections.singletonList(
+ new TransformConfig("columnArray", "jsonPathArray(columnJson, '$')")));
+ TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE)
+ .setTableName("testJsonToArrayIngestionTransform")
+ .setIngestionConfig(ingestionConfig)
+ .build();
+
+ ExpressionTransformer expressionTransformer = new ExpressionTransformer(tableConfig, schema);
+
+ GenericRow row = new GenericRow();
+ row.putValue("columnJson", "[\"a\",\"b\",\"c\"]");
+ // Pre-existing collection should be overwritten because transform returns an array (incompatible type)
+ row.putValue("columnArray", Arrays.asList("preExisting"));
+
+ expressionTransformer.transform(row);
+ Object transformedValue = row.getValue("columnArray");
+ Assert.assertTrue(transformedValue.getClass().isArray());
+ Assert.assertEquals(Arrays.asList((Object[]) transformedValue), Arrays.asList("a", "b", "c"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org