(pinot) branch master updated: Add explicit target data type constructor to DataTypeColumnTransformer (#18954)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new e55c18172a5 Add explicit target data type constructor to DataTypeColumnTransformer (#18954)
e55c18172a5 is described below

commit e55c18172a5ddfc313dc79420d86268c167bec66
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Thu Jul 9 23:49:56 2026 -0700

    Add explicit target data type constructor to DataTypeColumnTransformer (#18954)
---
 .../DataTypeColumnTransformer.java                 | 22 ++++++++++++++++------
 .../DataTypeColumnTransformerTest.java             | 11 +++++++++++
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
index f108aeb7926..5214a97d2c8 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformer.java
@@ -30,8 +30,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
+/// The `DataTypeColumnTransformer` is the column-major counterpart of the row-major `DataTypeTransformer`. It converts
+/// the values read from a [ColumnReader] to a target [PinotDataType]. It has two usages:
+/// - Schema columns: constructed from a [FieldSpec], it converts the values to the data type defined for the column in
+///   the [org.apache.pinot.spi.data.Schema].
+/// - Source fields: constructed from an explicit [PinotDataType] (see `IngestionConfig.sourceFieldConfigs`), it fixes
+///   the data type of a source field before other transformers (such as the expression transformer) consume it.
 public class DataTypeColumnTransformer implements ColumnTransformer {
-
   private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeColumnTransformer.class);
 
   private final PinotDataType _destDataType;
@@ -39,12 +44,17 @@ public class DataTypeColumnTransformer implements ColumnTransformer {
   private final boolean _continueOnError;
   private final ThrottledLogger _throttledLogger;
 
-  /**
-   * @param fieldSpec - The field spec for the column being created in Pinot.
-   * @param columnReader - The column reader to read the source data.
-   */
+  /// Creates a transformer that converts the values read from `columnReader` to the data type defined for the column
+  /// in the [org.apache.pinot.spi.data.Schema] (derived from the given [FieldSpec]).
   public DataTypeColumnTransformer(TableConfig tableConfig, FieldSpec fieldSpec, ColumnReader columnReader) {
-    _destDataType = PinotDataType.getPinotDataTypeForIngestion(fieldSpec);
+    this(tableConfig, PinotDataType.getPinotDataTypeForIngestion(fieldSpec), columnReader);
+  }
+
+  /// Creates a transformer that converts the values read from `columnReader` to the given [PinotDataType]. This is
+  /// useful for fixing the data type of a source field before other transformers (such as the expression transformer)
+  /// consume it.
+  public DataTypeColumnTransformer(TableConfig tableConfig, PinotDataType destDataType, ColumnReader columnReader) {
+    _destDataType = destDataType;
     _columnReader = columnReader;
     IngestionConfig ingestionConfig = tableConfig.getIngestionConfig();
     _continueOnError = ingestionConfig != null && ingestionConfig.isContinueOnError();
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
index aa1c02598ca..c493c024928 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/columntransformer/DataTypeColumnTransformerTest.java
@@ -600,4 +600,15 @@ public class DataTypeColumnTransformerTest {
     Object result = transformer.transform("not_a_number");
     assertNull(result, "Invalid conversion should return null when continueOnError=true");
   }
+
+  @Test
+  public void testExplicitDestDataTypeConstructor() {
+    TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+    ColumnReader reader = mockReader(PinotDataType.STRING);
+    // The explicit target type drives the conversion, independent of any field spec.
+    DataTypeColumnTransformer transformer = new DataTypeColumnTransformer(tableConfig, PinotDataType.LONG, reader);
+
+    assertFalse(transformer.isNoOp(), "STRING source, LONG dest should not be a no-op");
+    assertEquals(transformer.transform("12345"), 12345L);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org