You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2026/07/26 20:29:05 UTC
(pinot) branch master updated: [UUID 3/8] UUID result rendering (DataSchema, Arrow/JSON encoders) (#18871)
This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2050fce9bd0 [UUID 3/8] UUID result rendering (DataSchema, Arrow/JSON encoders) (#18871)
2050fce9bd0 is described below
commit 2050fce9bd0dd2c4b2e26df83ecf89dfb3f7f844
Author: Xiang Fu <xi...@apache.org>
AuthorDate: Sun Jul 26 13:28:59 2026 -0700
[UUID 3/8] UUID result rendering (DataSchema, Arrow/JSON encoders) (#18871)
Part 3/8 of splitting apache/pinot#18140 (logical UUID type). Rebased onto
master, which now includes the merged #18869 type foundation and #18870
ingest/storage layer.
Renders UUID result columns as canonical lowercase RFC-4122 strings:
- DataSchema.ColumnDataType.UUID / UUID_ARRAY plus the conversion and format
helpers, backed by the UuidUtils class merged in #18869.
- Arrow and JSON broker response encoders.
UUID null placeholder: ColumnDataType.UUID overrides getNullPlaceholder() to
return the nil UUID, matching the default null sentinel FieldSpec already uses
for UUID columns. Its stored type BYTES supplies a zero-length placeholder,
which is not a valid 16-byte UUID and fails to render, so the three call sites
that build null-aware blocks (DataBlockBuilder, GroupByResultsBlock,
GroupByDataTableReducer) now resolve the placeholder on the logical type rather
than the stored type. For every other logical type the two are identical, an
invariant pinned by a new DataSchemaTest case.
DataSchema.fromBytes now reports an unrecognized ColumnDataType token with a
message naming the mixed-version cause instead of a bare IllegalArgumentException.
Rolling-upgrade note: DataSchema serializes ColumnDataType by name, so an old
broker/server that receives a UUID token from a new node cannot parse it. Treat
as an atomic-upgrade feature - UUID columns should only be queried once the
whole cluster is upgraded. No existing (non-UUID) column is affected.
---
.../response/encoder/ArrowResponseEncoder.java | 6 +
.../response/encoder/JsonResponseEncoder.java | 2 +
.../org/apache/pinot/common/utils/DataSchema.java | 172 ++++++++++++++++++++-
.../response/encoder/ArrowResponseEncoderTest.java | 79 +++++++++-
.../response/encoder/JsonResponseEncoderTest.java | 52 ++++++-
.../apache/pinot/common/utils/DataSchemaTest.java | 88 ++++++++++-
.../core/common/datablock/DataBlockBuilder.java | 40 +++--
.../blocks/results/GroupByResultsBlock.java | 4 +-
.../core/query/reduce/GroupByDataTableReducer.java | 4 +-
.../common/datablock/DataBlockBuilderTest.java | 23 +++
.../core/common/datablock/DataBlockTestUtils.java | 13 ++
.../core/common/datatable/DataTableSerDeTest.java | 26 ++++
.../selection/SelectionOperatorUtilsTest.java | 23 +++
13 files changed, 497 insertions(+), 35 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
index f96027398c5..efd9098991e 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoder.java
@@ -107,6 +107,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case STRING:
case JSON:
case BYTES:
+ case UUID:
case OBJECT:
field = new Field(colName, FieldType.nullable(new ArrowType.Utf8()), null);
vector = new VarCharVector(colName, ALLOCATOR);
@@ -181,6 +182,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
+ case UUID_ARRAY:
// Define the inner field for a string element.
children = List.of(new Field("element", FieldType.nullable(new ArrowType.Utf8()), null));
// Define the field for the list column.
@@ -234,6 +236,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case STRING:
case JSON:
case BYTES:
+ case UUID:
case OBJECT:
byte[] bytes = ((String) value).getBytes(StandardCharsets.UTF_8);
((VarCharVector) vector).setSafe(rowIndex, bytes);
@@ -345,6 +348,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
+ case UUID_ARRAY:
ListVector listVector = (ListVector) vector;
String[] stringArray = (String[]) value;
// Start a new list entry for the current row.
@@ -411,6 +415,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case STRING:
case JSON:
case BYTES:
+ case UUID:
case OBJECT:
row[col] = new String(((VarCharVector) vector).get(i), StandardCharsets.UTF_8);
break;
@@ -469,6 +474,7 @@ public class ArrowResponseEncoder implements ResponseEncoder {
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
+ case UUID_ARRAY:
ListVector listVector = (ListVector) vector;
List<?> arrayValues = listVector.getObject(i);
String[] array = new String[arrayValues.size()];
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
index 0f82daab9e9..aa86400de85 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/response/encoder/JsonResponseEncoder.java
@@ -194,6 +194,7 @@ public class JsonResponseEncoder implements ResponseEncoder {
case TIMESTAMP_ARRAY:
case STRING_ARRAY:
case BYTES_ARRAY:
+ case UUID_ARRAY:
String[] stringArray = new String[jsonValue.size()];
for (int k = 0; k < jsonValue.size(); k++) {
stringArray[k] = jsonValue.get(k).textValue();
@@ -224,6 +225,7 @@ public class JsonResponseEncoder implements ResponseEncoder {
case STRING:
case JSON:
case BYTES:
+ case UUID:
case OBJECT:
return jsonValue.textValue();
case UNKNOWN:
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
index 5149195d68c..978b33aed80 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -40,6 +40,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.type.SqlTypeName;
@@ -51,6 +52,7 @@ import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.CommonConstants.NullValuePlaceHolder;
import org.apache.pinot.spi.utils.EqualityUtils;
import org.apache.pinot.spi.utils.PinotDataType;
+import org.apache.pinot.spi.utils.UuidUtils;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -135,7 +137,7 @@ public class DataSchema {
// Write the column types.
for (ColumnDataType columnDataType : _columnDataTypes) {
// We don't want to use ordinal of the enum since adding a new data type will break things if server and broker
- // use different versions of DataType class.
+ // use different versions of DataType class. See parseColumnDataType() for the mixed-version read side.
byte[] bytes = columnDataType.name().getBytes(UTF_8);
dataOutputStream.writeInt(bytes.length);
dataOutputStream.write(bytes);
@@ -164,7 +166,7 @@ public class DataSchema {
int length = buffer.getInt();
byte[] bytes = new byte[length];
buffer.get(bytes);
- columnDataTypes[i] = ColumnDataType.valueOf(new String(bytes, UTF_8));
+ columnDataTypes[i] = parseColumnDataType(new String(bytes, UTF_8));
}
return new DataSchema(columnNames, columnDataTypes);
}
@@ -187,11 +189,31 @@ public class DataSchema {
int length = buffer.readInt();
byte[] bytes = new byte[length];
buffer.readFully(bytes);
- columnDataTypes[i] = ColumnDataType.valueOf(new String(bytes, UTF_8));
+ columnDataTypes[i] = parseColumnDataType(new String(bytes, UTF_8));
}
return new DataSchema(columnNames, columnDataTypes);
}
+ /// Resolves a [ColumnDataType] token read off the wire, turning the raw [IllegalArgumentException] from
+ /// [ColumnDataType#valueOf] into a message that names the mixed-version cause.
+ ///
+ /// Rolling-upgrade limitation: once a node on this build emits a `UUID` token, an older peer that does not know
+ /// the [ColumnDataType#UUID] constant fails here. There is no version-negotiation shim or fallback to `BYTES`
+ /// today, so brokers and servers must be upgraded atomically (or UUID columns kept out of queries) until the
+ /// whole cluster is on this build. Rolling back to a pre-UUID build is likewise unsafe while UUID-typed query
+ /// results are in flight. No existing (non-UUID) column is affected.
+ private static ColumnDataType parseColumnDataType(String name) {
+ try {
+ return ColumnDataType.valueOf(name);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "Unrecognized ColumnDataType '" + name + "' received from a peer node. This typically means the peer is "
+ + "running a newer build that introduced a data type not yet known to this node. Upgrade all brokers "
+ + "and servers to the same build before querying columns of that type, or keep those columns out of "
+ + "queries until the rolling upgrade is complete.", e);
+ }
+ }
+
@SuppressWarnings("MethodDoesntCallSuperMethod")
@Override
public DataSchema clone() {
@@ -297,6 +319,25 @@ public class DataSchema {
return typeFactory.createSqlType(SqlTypeName.VARBINARY);
}
},
+ // UUID is a logical type backed by BYTES; keep it directly after BYTES. ColumnDataType is serialized by name (not
+ // ordinal) via name()/valueOf(), so enum order does not affect wire compatibility.
+ UUID(BYTES, null) {
+ @Override
+ public RelDataType toType(RelDataTypeFactory typeFactory) {
+ return typeFactory.createSqlType(SqlTypeName.UUID);
+ }
+
+ /// Returns the nil UUID, matching the default null sentinel that [FieldSpec#getDefaultNullValue] uses for
+ /// UUID columns. This is the one type whose placeholder differs from its stored type's: `BYTES` uses a shared
+ /// zero-length [ByteArray], which is not a valid 16-byte UUID and would fail to render.
+ ///
+ /// A fresh instance is returned per call because, unlike every other placeholder (all empty or immutable),
+ /// this one wraps a mutable 16-byte array that callers hand out as a column fill value.
+ @Override
+ public Object getNullPlaceholder() {
+ return new ByteArray(UuidUtils.nullUuidBytes());
+ }
+ },
MAP(NullValuePlaceHolder.MAP) {
@Override
public RelDataType toType(RelDataTypeFactory typeFactory) {
@@ -363,6 +404,12 @@ public class DataSchema {
return typeFactory.createArrayType(BYTES.toType(typeFactory), -1);
}
},
+ UUID_ARRAY(BYTES_ARRAY, NullValuePlaceHolder.INTERNAL_BYTES_ARRAY) {
+ @Override
+ public RelDataType toType(RelDataTypeFactory typeFactory) {
+ return typeFactory.createArrayType(UUID.toType(typeFactory), -1);
+ }
+ },
UNKNOWN(null) {
@Override
public RelDataType toType(RelDataTypeFactory typeFactory) {
@@ -374,7 +421,7 @@ public class DataSchema {
private static final EnumSet<ColumnDataType> INTEGRAL_TYPES = EnumSet.of(INT, LONG);
private static final EnumSet<ColumnDataType> ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, BIG_DECIMAL_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
- STRING_ARRAY, BYTES_ARRAY);
+ STRING_ARRAY, BYTES_ARRAY, UUID_ARRAY);
private static final EnumSet<ColumnDataType> NUMERIC_ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, BIG_DECIMAL_ARRAY);
private static final EnumSet<ColumnDataType> INTEGRAL_ARRAY_TYPES = EnumSet.of(INT_ARRAY, LONG_ARRAY);
@@ -395,6 +442,10 @@ public class DataSchema {
_nullPlaceholder = nullPlaceHolder;
}
+ /// Returns the value used to fill null entries in the serialized column, masked on read by the null bitmap.
+ ///
+ /// Callers must resolve this on the *logical* type, not on [#getStoredType], because [#UUID] overrides it (see
+ /// that constant). For every other logical type the two agree, an invariant pinned by `DataSchemaTest`.
public Object getNullPlaceholder() {
return _nullPlaceholder;
}
@@ -463,6 +514,9 @@ public class DataSchema {
case BYTES:
case BYTES_ARRAY:
return DataType.BYTES;
+ case UUID:
+ case UUID_ARRAY:
+ return DataType.UUID;
case UNKNOWN:
return DataType.UNKNOWN;
default:
@@ -490,6 +544,8 @@ public class DataSchema {
* <li>BYTES: byte[] -> ByteArray</li>
* <li>BOOLEAN_ARRAY: boolean[] -> int[]</li>
* <li>TIMESTAMP_ARRAY: Timestamp[] -> long[]</li>
+ * <li>UUID: UUID/String/byte[]/ByteArray -> ByteArray</li>
+ * <li>UUID_ARRAY: UUID[]/String[]/byte[][]/ByteArray[] -> ByteArray[]</li>
* </ul>
*/
public Object toInternal(Object value) {
@@ -500,10 +556,14 @@ public class DataSchema {
return ((Timestamp) value).getTime();
case BYTES:
return new ByteArray((byte[]) value);
+ case UUID:
+ return new ByteArray(UuidUtils.toBytes(value));
case BOOLEAN_ARRAY:
return fromBooleanArray((boolean[]) value);
case TIMESTAMP_ARRAY:
return fromTimestampArray((Timestamp[]) value);
+ case UUID_ARRAY:
+ return fromUuidArray(value);
case OBJECT:
// For OBJECT type, we need to convert based on the actual type of the value. This can happen when the scalar
// function returns Object type, e.g. cast function.
@@ -516,6 +576,9 @@ public class DataSchema {
if (value instanceof byte[]) {
return new ByteArray((byte[]) value);
}
+ if (value instanceof UUID) {
+ return new ByteArray(UuidUtils.toBytes((UUID) value));
+ }
if (value instanceof boolean[]) {
return fromBooleanArray((boolean[]) value);
}
@@ -539,6 +602,8 @@ public class DataSchema {
* <li>BOOLEAN_ARRAY: int[] -> boolean[]</li>
* <li>TIMESTAMP_ARRAY: long[] -> Timestamp[]</li>
* <li>BYTES_ARRAY: ByteArray[] -> byte[][]</li>
+ * <li>UUID: ByteArray -> UUID</li>
+ * <li>UUID_ARRAY: ByteArray[] -> UUID[]</li>
* </ul>
*/
public Object toExternal(Object value) {
@@ -549,12 +614,16 @@ public class DataSchema {
return new Timestamp((long) value);
case BYTES:
return ((ByteArray) value).getBytes();
+ case UUID:
+ return UuidUtils.toUUID((ByteArray) value);
case BOOLEAN_ARRAY:
return toBooleanArray((int[]) value);
case TIMESTAMP_ARRAY:
return toTimestampArray((long[]) value);
case BYTES_ARRAY:
return toBytesArray(value);
+ case UUID_ARRAY:
+ return toUuidArray(value);
default:
return value;
}
@@ -585,6 +654,8 @@ public class DataSchema {
return value.toString();
case BYTES:
return ((ByteArray) value).getBytes();
+ case UUID:
+ return UuidUtils.toUUID((ByteArray) value);
case INT_ARRAY:
return toIntArray(value);
case LONG_ARRAY:
@@ -603,6 +674,8 @@ public class DataSchema {
return toStringArray(value);
case BYTES_ARRAY:
return toBytesArray(value);
+ case UUID_ARRAY:
+ return toUuidArray(value);
case UNKNOWN: // fall through
case OBJECT:
return (Serializable) value;
@@ -624,12 +697,16 @@ public class DataSchema {
return value.toString();
case BYTES:
return BytesUtils.toHexString((byte[]) value);
+ case UUID:
+ return formatUuid(value);
case BIG_DECIMAL_ARRAY:
return formatBigDecimalArray((BigDecimal[]) value);
case TIMESTAMP_ARRAY:
return formatTimestampArray((Timestamp[]) value);
case BYTES_ARRAY:
return formatBytesArray((byte[][]) value);
+ case UUID_ARRAY:
+ return formatUuidArray(value);
default:
return (Serializable) value;
}
@@ -659,6 +736,8 @@ public class DataSchema {
return value.toString();
case BYTES:
return ((ByteArray) value).toHexString();
+ case UUID:
+ return UuidUtils.toString((ByteArray) value);
case MAP:
return toMap(value);
case INT_ARRAY:
@@ -679,6 +758,8 @@ public class DataSchema {
return (String[]) value;
case BYTES_ARRAY:
return formatBytesArray((ByteArray[]) value);
+ case UUID_ARRAY:
+ return formatUuidArray(value);
default:
throw new IllegalStateException(String.format("Cannot convert and format: '%s' to type: %s", value, this));
}
@@ -869,6 +950,54 @@ public class DataSchema {
throw new IllegalStateException(String.format("Cannot convert: '%s' to byte[][]", value));
}
+ /// Converts any supported UUID array representation to `UUID[]`. Elements may be `UUID`, `byte[]`, [ByteArray]
+ /// or `CharSequence`; per-element dispatch is delegated to [UuidUtils#toUUID(Object)]. Note that `byte[][]`,
+ /// [ByteArray]`[]`, `String[]` and `UUID[]` are all `Object[]`, so one branch covers every array form.
+ private static UUID[] toUuidArray(Object value) {
+ if (value instanceof UUID[]) {
+ return (UUID[]) value;
+ }
+ if (value instanceof ObjectArrayList) {
+ ObjectArrayList<?> list = (ObjectArrayList<?>) value;
+ int size = list.size();
+ UUID[] uuidArray = new UUID[size];
+ for (int i = 0; i < size; i++) {
+ uuidArray[i] = UuidUtils.toUUID(list.get(i));
+ }
+ return uuidArray;
+ }
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ UUID[] uuidArray = new UUID[length];
+ for (int i = 0; i < length; i++) {
+ uuidArray[i] = UuidUtils.toUUID(valueArray[i]);
+ }
+ return uuidArray;
+ }
+
+ /// Inverse of [#toUuidArray]: converts any supported UUID array representation to the internal [ByteArray]`[]`.
+ private static ByteArray[] fromUuidArray(Object value) {
+ if (value instanceof ByteArray[]) {
+ return (ByteArray[]) value;
+ }
+ if (value instanceof ObjectArrayList) {
+ ObjectArrayList<?> list = (ObjectArrayList<?>) value;
+ int size = list.size();
+ ByteArray[] wrapped = new ByteArray[size];
+ for (int i = 0; i < size; i++) {
+ wrapped[i] = new ByteArray(UuidUtils.toBytes(list.get(i)));
+ }
+ return wrapped;
+ }
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ ByteArray[] wrapped = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ wrapped[i] = new ByteArray(UuidUtils.toBytes(valueArray[i]));
+ }
+ return wrapped;
+ }
+
private static String[] formatBytesArray(byte[][] bytesArray) {
int length = bytesArray.length;
String[] formattedBytesArray = new String[length];
@@ -887,6 +1016,28 @@ public class DataSchema {
return formattedBytesArray;
}
+ /// Renders any supported UUID array representation as canonical lowercase RFC 4122 strings. Per-element
+ /// dispatch is delegated to [#formatUuid], so `byte[][]`, [ByteArray]`[]`, `UUID[]` and `String[]` are all
+ /// handled by the single `Object[]` branch.
+ private static String[] formatUuidArray(Object value) {
+ if (value instanceof ObjectArrayList) {
+ ObjectArrayList<?> list = (ObjectArrayList<?>) value;
+ int size = list.size();
+ String[] formattedUuidArray = new String[size];
+ for (int i = 0; i < size; i++) {
+ formattedUuidArray[i] = formatUuid(list.get(i));
+ }
+ return formattedUuidArray;
+ }
+ Object[] valueArray = (Object[]) value;
+ int length = valueArray.length;
+ String[] formattedUuidArray = new String[length];
+ for (int i = 0; i < length; i++) {
+ formattedUuidArray[i] = formatUuid(valueArray[i]);
+ }
+ return formattedUuidArray;
+ }
+
public static ColumnDataType fromDataType(DataType dataType, boolean isSingleValue) {
return isSingleValue ? fromDataTypeSV(dataType) : fromDataTypeMV(dataType);
}
@@ -913,6 +1064,8 @@ public class DataSchema {
return JSON;
case BYTES:
return BYTES;
+ case UUID:
+ return UUID;
case MAP:
return MAP;
case OPEN_STRUCT:
@@ -944,6 +1097,8 @@ public class DataSchema {
return STRING_ARRAY;
case BYTES:
return BYTES_ARRAY;
+ case UUID:
+ return UUID_ARRAY;
default:
throw new IllegalStateException("Unsupported data type: " + dataType);
}
@@ -973,6 +1128,8 @@ public class DataSchema {
return PinotDataType.JSON;
case BYTES:
return PinotDataType.BYTES;
+ case UUID:
+ return PinotDataType.UUID;
case MAP:
return PinotDataType.MAP;
case OBJECT:
@@ -1000,6 +1157,13 @@ public class DataSchema {
}
}
+ /// Renders a single UUID as its canonical lowercase RFC 4122 string. Accepts every representation
+ /// [UuidUtils#toUUID(Object)] does: `UUID`, `byte[]`, [ByteArray] and `CharSequence`. A non-canonical (e.g.
+ /// upper-case) string input is re-canonicalized rather than passed through.
+ private static String formatUuid(Object value) {
+ return UuidUtils.toUUID(value).toString();
+ }
+
public abstract RelDataType toType(RelDataTypeFactory typeFactory);
}
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoderTest.java b/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoderTest.java
index 0d65e3e9f0b..ea141a29398 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoderTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/ArrowResponseEncoderTest.java
@@ -23,15 +23,19 @@ import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
@@ -114,6 +118,72 @@ public class ArrowResponseEncoderTest {
}
}
+ @Test
+ public void testEncodeDecodeUuidColumn()
+ throws IOException {
+ DataSchema schema = new DataSchema(new String[]{"uuidCol"}, new ColumnDataType[]{ColumnDataType.UUID});
+ List<Object[]> rows = Arrays.asList(
+ new Object[]{"550e8400-e29b-41d4-a716-446655440000"},
+ new Object[]{"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"}
+ );
+
+ ResultTable resultTable = new ResultTable(schema, rows);
+ ArrowResponseEncoder encoder = new ArrowResponseEncoder();
+ byte[] encodedBytes = encoder.encodeResultTable(resultTable, 0, rows.size());
+ ResultTable decodedTable = encoder.decodeResultTable(encodedBytes, rows.size(), schema);
+
+ assertEquals(decodedTable.getRows().size(), rows.size(), "Row count should match");
+ for (int i = 0; i < rows.size(); i++) {
+ assertEquals(decodedTable.getRows().get(i)[0], rows.get(i)[0], "UUID row " + i + " should match");
+ }
+ }
+
+ /// Mirrors the real broker path: a UUID column is rendered to its canonical string *before* it reaches the
+ /// encoder -- via `convertAndFormat` in the single-stage engine, and `format(toExternal(..))` in the multi-stage
+ /// engine (see QueryDispatcher#toExternalList). The encoder therefore only ever sees Strings in this group.
+ @Test
+ public void testEncodeDecodeUuidColumnRenderedFromInternalValue()
+ throws IOException {
+ String uuidValue = "550e8400-e29b-41d4-a716-446655440000";
+ DataSchema schema = new DataSchema(new String[]{"uuidCol", "cnt"},
+ new ColumnDataType[]{ColumnDataType.UUID, ColumnDataType.LONG});
+ Object internalValue = ColumnDataType.UUID.toInternal(UUID.fromString(uuidValue));
+ List<Object[]> rows =
+ Collections.singletonList(new Object[]{ColumnDataType.UUID.convertAndFormat(internalValue), 3L});
+
+ ResultTable resultTable = new ResultTable(schema, rows);
+ ArrowResponseEncoder encoder = new ArrowResponseEncoder();
+ byte[] encodedBytes = encoder.encodeResultTable(resultTable, 0, rows.size());
+ ResultTable decodedTable = encoder.decodeResultTable(encodedBytes, rows.size(), schema);
+
+ assertEquals(decodedTable.getRows().size(), 1, "Row count should match");
+ assertEquals(decodedTable.getRows().get(0)[0], uuidValue, "UUID value should round-trip as canonical string");
+ assertEquals(decodedTable.getRows().get(0)[1], 3L, "Non-UUID columns should be preserved");
+ }
+
+ @Test
+ public void testEncodeDecodeUuidColumnWithNulls()
+ throws IOException {
+ String uuidValue = "550e8400-e29b-41d4-a716-446655440000";
+ DataSchema schema = new DataSchema(new String[]{"uuidCol", "uuidArrayCol"},
+ new ColumnDataType[]{ColumnDataType.UUID, ColumnDataType.UUID_ARRAY});
+ List<Object[]> rows = Arrays.asList(
+ new Object[]{uuidValue, new String[]{uuidValue}},
+ new Object[]{null, null}
+ );
+
+ ResultTable resultTable = new ResultTable(schema, rows);
+ ArrowResponseEncoder encoder = new ArrowResponseEncoder();
+ byte[] encodedBytes = encoder.encodeResultTable(resultTable, 0, rows.size());
+ ResultTable decodedTable = encoder.decodeResultTable(encodedBytes, rows.size(), schema);
+
+ assertEquals(decodedTable.getRows().size(), 2, "Row count should match");
+ assertEquals(decodedTable.getRows().get(0)[0], uuidValue, "Non-null UUID should round-trip");
+ assertEquals(decodedTable.getRows().get(0)[1], new String[]{uuidValue}, "Non-null UUID array should round-trip");
+ assertNull(decodedTable.getRows().get(1)[0], "Null UUID should round-trip as null");
+ assertNull(decodedTable.getRows().get(1)[1], "Null UUID array should round-trip as null");
+ }
+
@Test
public void testEncodeDecodeAllDataTypes()
throws IOException {
@@ -122,7 +192,7 @@ public class ArrowResponseEncoderTest {
"intCol", "longCol", "floatCol", "doubleCol", "bigDecimalCol", "booleanCol", "timestampCol",
"stringCol", "jsonCol", "mapCol", "bytesCol", "objectCol", "intArrayCol", "longArrayCol",
"floatArrayCol", "doubleArrayCol", "booleanArrayCol", "timestampArrayCol", "stringArrayCol",
- "bytesArrayCol", "unknownCol"
+ "bytesArrayCol", "uuidArrayCol", "unknownCol"
};
DataSchema.ColumnDataType[] columnTypes = {
@@ -146,6 +216,7 @@ public class ArrowResponseEncoderTest {
ColumnDataType.TIMESTAMP_ARRAY,
ColumnDataType.STRING_ARRAY,
ColumnDataType.BYTES_ARRAY,
+ ColumnDataType.UUID_ARRAY,
ColumnDataType.UNKNOWN
};
@@ -178,6 +249,10 @@ public class ArrowResponseEncoderTest {
byte[][] bytesArrayVal = new byte[][]{
new byte[]{1, 2}, new byte[]{3, 4}
};
+ byte[][] uuidArrayVal = new byte[][]{
+ UuidUtils.toBytes("550e8400-e29b-41d4-a716-446655440000"),
+ UuidUtils.toBytes("550e8400-e29b-41d4-a716-446655440001")
+ };
Object unknownVal = null; // UNKNOWN is represented as null in this example.
// Build a single row that contains all the above values.
@@ -185,7 +260,7 @@ public class ArrowResponseEncoderTest {
Object[] row = new Object[]{
intVal, longVal, floatVal, doubleVal, bigDecimalVal, booleanVal, timestampVal, stringVal,
jsonVal, mapVal, bytesVal, objectVal, intArrayVal, longArrayVal, floatArrayVal, doubleArrayVal,
- booleanArrayVal, timestampArrayVal, stringArrayVal, bytesArrayVal, unknownVal
+ booleanArrayVal, timestampArrayVal, stringArrayVal, bytesArrayVal, uuidArrayVal, unknownVal
};
for (int i = 0; i < row.length; i++) {
row[i] = columnTypes[i].format(row[i]); // Convert to internal representation.
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/JsonResponseEncoderTest.java b/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/JsonResponseEncoderTest.java
index f23088fca01..4625d618be1 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/JsonResponseEncoderTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/response/encoder/JsonResponseEncoderTest.java
@@ -26,12 +26,15 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
import static org.testng.Assert.fail;
public class JsonResponseEncoderTest {
@@ -110,6 +113,46 @@ public class JsonResponseEncoderTest {
}
}
+ @Test
+ public void testEncodeDecodeUuidColumn() throws IOException {
+ DataSchema schema = new DataSchema(
+ new String[] {"uuidCol"},
+ new ColumnDataType[] {ColumnDataType.UUID});
+ String uuidValue = "550e8400-e29b-41d4-a716-446655440000";
+
+ List<Object[]> rows = new ArrayList<>();
+ rows.add(new Object[] {ColumnDataType.UUID.format(UUID.fromString(uuidValue))});
+
+ ResultTable resultTable = new ResultTable(schema, rows);
+ JsonResponseEncoder encoder = new JsonResponseEncoder();
+
+ byte[] encodedBytes = encoder.encodeResultTable(resultTable, 0, rows.size());
+ ResultTable decodedTable = encoder.decodeResultTable(encodedBytes, rows.size(), schema);
+
+ assertEquals(decodedTable.getRows().size(), 1, "Row count should be 1");
+ assertEquals(decodedTable.getRows().get(0)[0], uuidValue, "UUID value should round-trip as canonical string");
+ }
+
+ @Test
+ public void testEncodeDecodeUuidColumnWithNulls() throws IOException {
+ String uuidValue = "550e8400-e29b-41d4-a716-446655440000";
+ DataSchema schema = new DataSchema(new String[] {"uuidCol"}, new ColumnDataType[] {ColumnDataType.UUID});
+
+ List<Object[]> rows = new ArrayList<>();
+ rows.add(new Object[] {uuidValue});
+ rows.add(new Object[] {null});
+
+ ResultTable resultTable = new ResultTable(schema, rows);
+ JsonResponseEncoder encoder = new JsonResponseEncoder();
+
+ byte[] encodedBytes = encoder.encodeResultTable(resultTable, 0, rows.size());
+ ResultTable decodedTable = encoder.decodeResultTable(encodedBytes, rows.size(), schema);
+
+ assertEquals(decodedTable.getRows().size(), 2, "Row count should be 2");
+ assertEquals(decodedTable.getRows().get(0)[0], uuidValue, "Non-null UUID should round-trip");
+ assertNull(decodedTable.getRows().get(1)[0], "Null UUID should round-trip as null");
+ }
+
@Test
public void testEncodeDecodeAllDataTypes() throws IOException {
// Define the column names and corresponding data types.
@@ -117,7 +160,7 @@ public class JsonResponseEncoderTest {
"intCol", "longCol", "floatCol", "doubleCol", "bigDecimalCol", "booleanCol", "timestampCol",
"stringCol", "jsonCol", "mapCol", "bytesCol", "objectCol", "intArrayCol", "longArrayCol",
"floatArrayCol", "doubleArrayCol", "booleanArrayCol", "timestampArrayCol", "stringArrayCol",
- "bytesArrayCol", "unknownCol"
+ "bytesArrayCol", "uuidArrayCol", "unknownCol"
};
DataSchema.ColumnDataType[] columnTypes = {
@@ -141,6 +184,7 @@ public class JsonResponseEncoderTest {
ColumnDataType.TIMESTAMP_ARRAY,
ColumnDataType.STRING_ARRAY,
ColumnDataType.BYTES_ARRAY,
+ ColumnDataType.UUID_ARRAY,
ColumnDataType.UNKNOWN
};
@@ -173,6 +217,10 @@ public class JsonResponseEncoderTest {
byte[][] bytesArrayVal = new byte[][] {
new byte[] {1, 2}, new byte[] {3, 4}
};
+ byte[][] uuidArrayVal = new byte[][] {
+ UuidUtils.toBytes("550e8400-e29b-41d4-a716-446655440000"),
+ UuidUtils.toBytes("550e8400-e29b-41d4-a716-446655440001")
+ };
Object unknownVal = null; // UNKNOWN is represented as null in this example.
// Build a single row that contains all the above values.
@@ -180,7 +228,7 @@ public class JsonResponseEncoderTest {
Object[] row = new Object[] {
intVal, longVal, floatVal, doubleVal, bigDecimalVal, booleanVal, timestampVal, stringVal,
jsonVal, mapVal, bytesVal, objectVal, intArrayVal, longArrayVal, floatArrayVal, doubleArrayVal,
- booleanArrayVal, timestampArrayVal, stringArrayVal, bytesArrayVal, unknownVal
+ booleanArrayVal, timestampArrayVal, stringArrayVal, bytesArrayVal, uuidArrayVal, unknownVal
};
// Convert each value using the schema's formatting (if needed).
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
index 3a22b1d30cc..ba3bdceef4e 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
@@ -21,8 +21,11 @@ package org.apache.pinot.common.utils;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
+import java.util.Locale;
import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -31,14 +34,19 @@ import static org.apache.pinot.common.utils.DataSchema.ColumnDataType.*;
public class DataSchemaTest {
private static final String[] COLUMN_NAMES = {
- "int", "long", "float", "double", "string", "object", "int_array", "long_array", "float_array", "double_array",
- "string_array", "boolean_array", "timestamp_array", "bytes_array"
+ "int", "long", "float", "double", "string", "uuid", "object", "int_array", "long_array", "float_array",
+ "double_array", "string_array", "boolean_array", "timestamp_array", "bytes_array", "uuid_array"
};
private static final int NUM_COLUMNS = COLUMN_NAMES.length;
private static final DataSchema.ColumnDataType[] COLUMN_DATA_TYPES = {
- INT, LONG, FLOAT, DOUBLE, STRING, OBJECT, INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, STRING_ARRAY,
- BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY
+ INT, LONG, FLOAT, DOUBLE, STRING, UUID, OBJECT, INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, STRING_ARRAY,
+ BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY, UUID_ARRAY
};
+ private static final String UUID_VALUE = "550e8400-e29b-41d4-a716-446655440000";
+ private static final String UUID_VALUE_2 = "550e8400-e29b-41d4-a716-446655440001";
+ // Fully qualified: the static ColumnDataType.* import below binds the simple name UUID to the enum constant.
+ private static final java.util.UUID JAVA_UUID = java.util.UUID.fromString(UUID_VALUE);
+ private static final java.util.UUID JAVA_UUID_2 = java.util.UUID.fromString(UUID_VALUE_2);
@Test
public void testGetters() {
@@ -71,9 +79,10 @@ public class DataSchemaTest {
public void testToString() {
DataSchema dataSchema = new DataSchema(COLUMN_NAMES, COLUMN_DATA_TYPES);
Assert.assertEquals(dataSchema.toString(),
- "[int(INT),long(LONG),float(FLOAT),double(DOUBLE),string(STRING),object(OBJECT),int_array(INT_ARRAY),"
- + "long_array(LONG_ARRAY),float_array(FLOAT_ARRAY),double_array(DOUBLE_ARRAY),string_array(STRING_ARRAY),"
- + "boolean_array(BOOLEAN_ARRAY),timestamp_array(TIMESTAMP_ARRAY),bytes_array(BYTES_ARRAY)]");
+ "[int(INT),long(LONG),float(FLOAT),double(DOUBLE),string(STRING),uuid(UUID),object(OBJECT),"
+ + "int_array(INT_ARRAY),long_array(LONG_ARRAY),float_array(FLOAT_ARRAY),double_array(DOUBLE_ARRAY),"
+ + "string_array(STRING_ARRAY),boolean_array(BOOLEAN_ARRAY),timestamp_array(TIMESTAMP_ARRAY),"
+ + "bytes_array(BYTES_ARRAY),uuid_array(UUID_ARRAY)]");
}
@Test
@@ -115,6 +124,16 @@ public class DataSchemaTest {
Assert.assertFalse(STRING.isCompatible(STRING_ARRAY));
Assert.assertFalse(STRING.isCompatible(BYTES_ARRAY));
+ Assert.assertFalse(UUID.isNumber());
+ Assert.assertFalse(UUID.isWholeNumber());
+ Assert.assertFalse(UUID.isArray());
+ Assert.assertFalse(UUID.isNumberArray());
+ Assert.assertFalse(UUID.isWholeNumberArray());
+ Assert.assertFalse(UUID.isCompatible(DOUBLE));
+ Assert.assertTrue(UUID.isCompatible(UUID));
+ Assert.assertFalse(UUID.isCompatible(BYTES));
+ Assert.assertFalse(UUID.isCompatible(STRING));
+
Assert.assertFalse(OBJECT.isNumber());
Assert.assertFalse(OBJECT.isWholeNumber());
Assert.assertFalse(OBJECT.isArray());
@@ -154,7 +173,7 @@ public class DataSchemaTest {
}
for (DataSchema.ColumnDataType columnDataType : new DataSchema.ColumnDataType[]{
- STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY
+ STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY, BYTES_ARRAY, UUID_ARRAY
}) {
Assert.assertFalse(columnDataType.isNumber());
Assert.assertFalse(columnDataType.isWholeNumber());
@@ -178,6 +197,8 @@ public class DataSchemaTest {
Assert.assertEquals(fromDataType(FieldSpec.DataType.DOUBLE, false), DOUBLE_ARRAY);
Assert.assertEquals(fromDataType(FieldSpec.DataType.STRING, true), STRING);
Assert.assertEquals(fromDataType(FieldSpec.DataType.STRING, false), STRING_ARRAY);
+ Assert.assertEquals(fromDataType(FieldSpec.DataType.UUID, true), UUID);
+ Assert.assertEquals(fromDataType(FieldSpec.DataType.UUID, false), UUID_ARRAY);
Assert.assertEquals(fromDataType(FieldSpec.DataType.BOOLEAN, false), BOOLEAN_ARRAY);
Assert.assertEquals(fromDataType(FieldSpec.DataType.TIMESTAMP, false), TIMESTAMP_ARRAY);
Assert.assertEquals(fromDataType(FieldSpec.DataType.BYTES, false), BYTES_ARRAY);
@@ -186,7 +207,58 @@ public class DataSchemaTest {
Assert.assertEquals(BIG_DECIMAL.format(bigDecimalValue), bigDecimalValue.toPlainString());
Timestamp timestampValue = new Timestamp(1234567890123L);
Assert.assertEquals(TIMESTAMP.format(timestampValue), timestampValue.toString());
+ ByteArray uuidValue = new ByteArray(UuidUtils.toBytes(UUID_VALUE));
+ Assert.assertEquals(UUID.convert(uuidValue), JAVA_UUID);
+ Assert.assertEquals(UUID.format(uuidValue), UUID_VALUE);
+ Assert.assertEquals(UUID.convertAndFormat(uuidValue), UUID_VALUE);
+ // format() also accepts the external form and re-canonicalizes non-canonical strings.
+ Assert.assertEquals(UUID.format(JAVA_UUID), UUID_VALUE);
+ Assert.assertEquals(UUID.format(UUID_VALUE.toUpperCase(Locale.ROOT)), UUID_VALUE);
+ byte[][] uuidArrayBytesValue = {UuidUtils.toBytes(UUID_VALUE), UuidUtils.toBytes(UUID_VALUE_2)};
+ ByteArray[] uuidArrayValue = (ByteArray[]) UUID_ARRAY.toInternal(new String[]{UUID_VALUE, UUID_VALUE_2});
+ java.util.UUID[] expectedUuidArray = {JAVA_UUID, JAVA_UUID_2};
+ String[] expectedFormatted = {UUID_VALUE, UUID_VALUE_2};
+ Assert.assertEquals(UUID_ARRAY.toExternal(uuidArrayValue), expectedUuidArray);
+ Assert.assertEquals(UUID_ARRAY.toExternal(uuidArrayBytesValue), expectedUuidArray);
+ Assert.assertEquals(UUID_ARRAY.convert(uuidArrayValue), expectedUuidArray);
+ Assert.assertEquals(UUID_ARRAY.toInternal(expectedUuidArray), uuidArrayValue);
+ Assert.assertEquals(UUID_ARRAY.toInternal(uuidArrayBytesValue), uuidArrayValue);
+ Assert.assertEquals(UUID_ARRAY.format(uuidArrayBytesValue), expectedFormatted);
+ Assert.assertEquals(UUID_ARRAY.format(expectedUuidArray), expectedFormatted);
+ Assert.assertEquals(UUID_ARRAY.format(uuidArrayValue), expectedFormatted);
+ Assert.assertEquals(UUID_ARRAY.convertAndFormat(uuidArrayValue), expectedFormatted);
+ Assert.assertEquals(UUID_ARRAY.convertAndFormat(uuidArrayBytesValue), expectedFormatted);
byte[] bytesValue = {12, 34, 56};
Assert.assertEquals(BYTES.format(bytesValue), BytesUtils.toHexString(bytesValue));
}
+
+ /// The null placeholder must be resolved on the *logical* type. UUID is the only type whose placeholder differs
+ /// from its stored type's: it needs the 16-byte nil UUID, while BYTES supplies a zero-length one. Every other
+ /// logical type must agree with its stored type, otherwise callers that resolve the stored type (DataBlockBuilder,
+ /// GroupByResultsBlock, GroupByDataTableReducer) would silently write the wrong placeholder.
+ @Test
+ public void testNullPlaceholderMatchesStoredTypeExceptUuid() {
+ for (DataSchema.ColumnDataType columnDataType : DataSchema.ColumnDataType.values()) {
+ DataSchema.ColumnDataType storedType = columnDataType.getStoredType();
+ if (columnDataType == UUID) {
+ Assert.assertEquals(storedType, BYTES);
+ Assert.assertEquals(columnDataType.getNullPlaceholder(), new ByteArray(UuidUtils.nullUuidBytes()));
+ Assert.assertNotEquals(columnDataType.getNullPlaceholder(), storedType.getNullPlaceholder());
+ } else {
+ Assert.assertEquals(columnDataType.getNullPlaceholder(), storedType.getNullPlaceholder(),
+ "Null placeholder mismatch between " + columnDataType + " and its stored type " + storedType);
+ }
+ }
+ }
+
+ /// The nil-UUID placeholder wraps a mutable 16-byte array, unlike every other placeholder (all empty or
+ /// immutable), so each call must hand back a fresh instance.
+ @Test
+ public void testUuidNullPlaceholderIsNotShared() {
+ ByteArray first = (ByteArray) UUID.getNullPlaceholder();
+ ByteArray second = (ByteArray) UUID.getNullPlaceholder();
+ Assert.assertNotSame(first, second);
+ first.getBytes()[0] = 1;
+ Assert.assertEquals(second, new ByteArray(UuidUtils.nullUuidBytes()));
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
index 3a15b30184e..6e418e572c4 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/datablock/DataBlockBuilder.java
@@ -81,7 +81,9 @@ public class DataBlockBuilder {
Object[] nullPlaceholders = new Object[numColumns];
for (int colId = 0; colId < numColumns; colId++) {
nullBitmaps[colId] = new RoaringBitmap();
- nullPlaceholders[colId] = storedTypes[colId].getNullPlaceholder();
+ // Resolved on the logical type, not the stored type: UUID overrides getNullPlaceholder() to return the nil
+ // UUID, whereas its stored type BYTES would yield a zero-length placeholder that is not a valid UUID.
+ nullPlaceholders[colId] = dataSchema.getColumnDataType(colId).getNullPlaceholder();
}
int nullFixedBytes = numColumns * Integer.BYTES * 2;
int rowSizeInBytes = calculateBytesPerRow(dataSchema);
@@ -254,7 +256,11 @@ public class DataBlockBuilder {
ByteBuffer fixedSize, PagedPinotOutputStream varSize, RoaringBitmap nullBitmap,
Object2IntOpenHashMap<String> dictionary, @Nullable AggregationFunction aggFunction)
throws IOException {
- ColumnDataType storedType = dataSchema.getColumnDataType(colId).getStoredType();
+ // Dispatch on the stored type, but read null placeholders off the logical type: UUID overrides
+ // getNullPlaceholder() to return the nil UUID, whereas its stored type BYTES would yield a zero-length
+ // placeholder that is not a valid UUID. The two agree for every other type (see DataSchemaTest).
+ ColumnDataType columnDataType = dataSchema.getColumnDataType(colId);
+ ColumnDataType storedType = columnDataType.getStoredType();
int numRows = columns.get(colId).length;
Object[] column = columns.get(colId);
@@ -266,7 +272,7 @@ public class DataBlockBuilder {
switch (storedType) {
// Single-value column
case INT: {
- int nullPlaceholder = (int) storedType.getNullPlaceholder();
+ int nullPlaceholder = (int) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -281,7 +287,7 @@ public class DataBlockBuilder {
break;
}
case LONG: {
- long nullPlaceholder = (long) storedType.getNullPlaceholder();
+ long nullPlaceholder = (long) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -296,7 +302,7 @@ public class DataBlockBuilder {
break;
}
case FLOAT: {
- float nullPlaceholder = (float) storedType.getNullPlaceholder();
+ float nullPlaceholder = (float) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -311,7 +317,7 @@ public class DataBlockBuilder {
break;
}
case DOUBLE: {
- double nullPlaceholder = (double) storedType.getNullPlaceholder();
+ double nullPlaceholder = (double) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -326,7 +332,7 @@ public class DataBlockBuilder {
break;
}
case BIG_DECIMAL: {
- BigDecimal nullPlaceholder = (BigDecimal) storedType.getNullPlaceholder();
+ BigDecimal nullPlaceholder = (BigDecimal) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -342,7 +348,7 @@ public class DataBlockBuilder {
}
case STRING: {
ToIntFunction<String> didSupplier = k -> dictionary.size();
- int nullPlaceHolder = dictionary.computeIfAbsent((String) storedType.getNullPlaceholder(), didSupplier);
+ int nullPlaceHolder = dictionary.computeIfAbsent((String) columnDataType.getNullPlaceholder(), didSupplier);
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -358,7 +364,7 @@ public class DataBlockBuilder {
break;
}
case BYTES: {
- ByteArray nullPlaceholder = (ByteArray) storedType.getNullPlaceholder();
+ ByteArray nullPlaceholder = (ByteArray) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -373,7 +379,7 @@ public class DataBlockBuilder {
break;
}
case MAP: {
- Map nullPlaceholder = (Map) storedType.getNullPlaceholder();
+ Map nullPlaceholder = (Map) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -389,7 +395,7 @@ public class DataBlockBuilder {
}
// Multi-value column
case INT_ARRAY: {
- int[] nullPlaceholder = (int[]) storedType.getNullPlaceholder();
+ int[] nullPlaceholder = (int[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -404,7 +410,7 @@ public class DataBlockBuilder {
break;
}
case LONG_ARRAY: {
- long[] nullPlaceholder = (long[]) storedType.getNullPlaceholder();
+ long[] nullPlaceholder = (long[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -419,7 +425,7 @@ public class DataBlockBuilder {
break;
}
case FLOAT_ARRAY: {
- float[] nullPlaceholder = (float[]) storedType.getNullPlaceholder();
+ float[] nullPlaceholder = (float[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -434,7 +440,7 @@ public class DataBlockBuilder {
break;
}
case DOUBLE_ARRAY: {
- double[] nullPlaceholder = (double[]) storedType.getNullPlaceholder();
+ double[] nullPlaceholder = (double[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -449,7 +455,7 @@ public class DataBlockBuilder {
break;
}
case BIG_DECIMAL_ARRAY: {
- BigDecimal[] nullPlaceholder = (BigDecimal[]) storedType.getNullPlaceholder();
+ BigDecimal[] nullPlaceholder = (BigDecimal[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -464,7 +470,7 @@ public class DataBlockBuilder {
break;
}
case STRING_ARRAY: {
- String[] nullPlaceholder = (String[]) storedType.getNullPlaceholder();
+ String[] nullPlaceholder = (String[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
@@ -479,7 +485,7 @@ public class DataBlockBuilder {
break;
}
case BYTES_ARRAY: {
- ByteArray[] nullPlaceholder = (ByteArray[]) storedType.getNullPlaceholder();
+ ByteArray[] nullPlaceholder = (ByteArray[]) columnDataType.getNullPlaceholder();
interruptableLoop(0, numRows, interruptableLoopStep, (start, end) -> {
for (int rowId = start; rowId < end; rowId++) {
Object value = column[rowId];
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
index 6cfc8511818..0a5565be7b9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
@@ -220,7 +220,9 @@ public class GroupByResultsBlock extends BaseResultsBlock {
Object[] nullPlaceholders = new Object[numColumns];
for (int colId = 0; colId < numColumns; colId++) {
nullBitmaps[colId] = new RoaringBitmap();
- nullPlaceholders[colId] = storedColumnDataTypes[colId].getNullPlaceholder();
+ // Resolved on the logical type, not the stored type: UUID overrides getNullPlaceholder() to return the nil
+ // UUID, whereas its stored type BYTES would yield a zero-length placeholder that is not a valid UUID.
+ nullPlaceholders[colId] = _dataSchema.getColumnDataType(colId).getNullPlaceholder();
}
int rowId = 0;
while (iterator.hasNext()) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
index bfbd8bb8486..85b8f614f3d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/GroupByDataTableReducer.java
@@ -564,7 +564,9 @@ public class GroupByDataTableReducer implements DataTableReducer {
Object[] nullPlaceholders = new Object[_numColumns];
for (int colId = 0; colId < _numColumns; colId++) {
nullBitmaps[colId] = new RoaringBitmap();
- nullPlaceholders[colId] = storedColumnDataTypes[colId].getNullPlaceholder();
+ // Resolved on the logical type, not the stored type: UUID overrides getNullPlaceholder() to return the nil
+ // UUID, whereas its stored type BYTES would yield a zero-length placeholder that is not a valid UUID.
+ nullPlaceholders[colId] = dataSchema.getColumnDataType(colId).getNullPlaceholder();
}
int rowId = 0;
while (iterator.hasNext()) {
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
index f7206f59189..24e4c03b11c 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockBuilderTest.java
@@ -32,6 +32,7 @@ import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.roaringbitmap.RoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
@@ -194,6 +195,28 @@ public class DataBlockBuilderTest {
runColumnBlockTest(type, 25_000);
}
+ /// A null in a UUID column must serialize as the nil UUID, not as the zero-length placeholder its stored type
+ /// (BYTES) supplies. The value is normally masked by the null bitmap, but it must still decode as a valid 16-byte
+ /// UUID for any consumer that renders the raw column, and [UuidUtils#toString] rejects any other width.
+ @Test
+ void testUuidNullPlaceholderIsNilUuid()
+ throws IOException {
+ ByteArray uuid = new ByteArray(UuidUtils.toBytes("550e8400-e29b-41d4-a716-446655440000"));
+ DataSchema dataSchema = new DataSchema(new String[]{"uuidCol"}, new ColumnDataType[]{ColumnDataType.UUID});
+ Object[] column = {uuid, null};
+ List<Object[]> rows = List.of(new Object[]{uuid}, new Object[]{null});
+
+ List<DataBlock> blocks = List.of(DataBlockBuilder.buildFromRows(rows, dataSchema),
+ DataBlockBuilder.buildFromColumns(List.<Object[]>of(column), dataSchema));
+ for (DataBlock block : blocks) {
+ assertEquals(block.getNumberOfRows(), 2);
+ assertEquals(new ByteArray(block.getBytes(0, 0).getBytes()), uuid);
+ // Row 1 is null: the bitmap flags it, and the placeholder underneath still renders as the nil UUID.
+ assertEquals(block.getNullRowIds(0), RoaringBitmap.bitmapOf(1));
+ assertEquals(UuidUtils.toString(block.getBytes(1, 0).getBytes()), "00000000-0000-0000-0000-000000000000");
+ }
+ }
+
private void runColumnBlockTest(ColumnDataType type, int numRows)
throws IOException {
Object[] column = generateColumns(type, numRows);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
index c96a21b5e86..a9fd19f0900 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datablock/DataBlockTestUtils.java
@@ -24,11 +24,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.UUID;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.pinot.common.datablock.DataBlock;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.roaringbitmap.RoaringBitmap;
@@ -73,6 +75,9 @@ public class DataBlockTestUtils {
case BYTES:
row[colId] = new ByteArray(RandomStringUtils.secure().next(RANDOM.nextInt(20)).getBytes());
break;
+ case UUID:
+ row[colId] = new ByteArray(UuidUtils.toBytes(new UUID(RANDOM.nextLong(), RANDOM.nextLong())));
+ break;
case INT_ARRAY:
int length = RANDOM.nextInt(ARRAY_SIZE);
int[] intArray = new int[length];
@@ -147,6 +152,14 @@ public class DataBlockTestUtils {
}
row[colId] = bytesArray;
break;
+ case UUID_ARRAY:
+ length = RANDOM.nextInt(ARRAY_SIZE);
+ ByteArray[] uuidArray = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ uuidArray[i] = new ByteArray(UuidUtils.toBytes(new UUID(RANDOM.nextLong(), RANDOM.nextLong())));
+ }
+ row[colId] = uuidArray;
+ break;
case MAP:
length = RANDOM.nextInt(ARRAY_SIZE);
Map<String, Object> map = new HashMap<>();
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
index 7f416c9f414..f2ac8811035 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableSerDeTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import java.util.UUID;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.datatable.DataTable;
@@ -34,6 +35,7 @@ import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.spi.accounting.ThreadResourceUsageProvider;
import org.apache.pinot.spi.exception.QueryErrorCode;
import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.roaringbitmap.RoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
@@ -60,6 +62,7 @@ public class DataTableSerDeTest {
private static final String[] STRINGS = new String[NUM_ROWS];
private static final String[] JSONS = new String[NUM_ROWS];
private static final byte[][] BYTES = new byte[NUM_ROWS][];
+ private static final byte[][] UUIDS = new byte[NUM_ROWS][];
private static final Object[] OBJECTS = new Object[NUM_ROWS];
private static final int[][] INT_ARRAYS = new int[NUM_ROWS][];
private static final long[][] LONG_ARRAYS = new long[NUM_ROWS][];
@@ -69,6 +72,7 @@ public class DataTableSerDeTest {
private static final long[][] TIMESTAMP_ARRAYS = new long[NUM_ROWS][];
private static final String[][] STRING_ARRAYS = new String[NUM_ROWS][];
private static final ByteArray[][] BYTES_ARRAYS = new ByteArray[NUM_ROWS][];
+ private static final ByteArray[][] UUID_ARRAYS = new ByteArray[NUM_ROWS][];
private static final BigDecimal[][] BIG_DECIMAL_ARRAYS = new BigDecimal[NUM_ROWS][];
private static final Map<String, Object>[] MAPS = new Map[NUM_ROWS];
@@ -367,6 +371,11 @@ public class DataTableSerDeTest {
BYTES[rowId] = isNull ? new byte[0] : RandomStringUtils.secure().next(RANDOM.nextInt(20)).getBytes();
dataTableBuilder.setColumn(colId, new ByteArray(BYTES[rowId]));
break;
+ case UUID:
+ UUIDS[rowId] = isNull ? UuidUtils.nullUuidBytes()
+ : UuidUtils.toBytes(new UUID(RANDOM.nextLong(), RANDOM.nextLong()));
+ dataTableBuilder.setColumn(colId, new ByteArray(UUIDS[rowId]));
+ break;
case INT_ARRAY:
int length = RANDOM.nextInt(20);
int[] intArray = new int[length];
@@ -440,6 +449,15 @@ public class DataTableSerDeTest {
BYTES_ARRAYS[rowId] = bytesArray;
dataTableBuilder.setColumn(colId, bytesArray);
break;
+ case UUID_ARRAY:
+ length = RANDOM.nextInt(20);
+ ByteArray[] uuidArray = new ByteArray[length];
+ for (int i = 0; i < length; i++) {
+ uuidArray[i] = new ByteArray(UuidUtils.toBytes(new UUID(RANDOM.nextLong(), RANDOM.nextLong())));
+ }
+ UUID_ARRAYS[rowId] = uuidArray;
+ dataTableBuilder.setColumn(colId, uuidArray);
+ break;
case STRING_ARRAY:
length = RANDOM.nextInt(20);
String[] stringArray = new String[length];
@@ -520,6 +538,10 @@ public class DataTableSerDeTest {
Assert.assertEquals(newDataTable.getBytes(rowId, colId).getBytes(), isNull ? new byte[0] : BYTES[rowId],
ERROR_MESSAGE);
break;
+ case UUID:
+ Assert.assertEquals(newDataTable.getBytes(rowId, colId).getBytes(),
+ isNull ? UuidUtils.nullUuidBytes() : UUIDS[rowId], ERROR_MESSAGE);
+ break;
case INT_ARRAY:
Assert.assertTrue(Arrays.equals(newDataTable.getIntArray(rowId, colId), INT_ARRAYS[rowId]), ERROR_MESSAGE);
break;
@@ -551,6 +573,10 @@ public class DataTableSerDeTest {
Assert.assertTrue(Arrays.equals(newDataTable.getBytesArray(rowId, colId), BYTES_ARRAYS[rowId]),
ERROR_MESSAGE);
break;
+ case UUID_ARRAY:
+ Assert.assertTrue(Arrays.equals(newDataTable.getBytesArray(rowId, colId), UUID_ARRAYS[rowId]),
+ ERROR_MESSAGE);
+ break;
case STRING_ARRAY:
Assert.assertTrue(Arrays.equals(newDataTable.getStringArray(rowId, colId), STRING_ARRAYS[rowId]),
ERROR_MESSAGE);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/query/selection/SelectionOperatorUtilsTest.java b/pinot-core/src/test/java/org/apache/pinot/core/query/selection/SelectionOperatorUtilsTest.java
index 196f4dd168e..1df11e8cf86 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/query/selection/SelectionOperatorUtilsTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/query/selection/SelectionOperatorUtilsTest.java
@@ -18,17 +18,23 @@
*/
package org.apache.pinot.core.query.selection;
+import java.util.Collections;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pinot.common.response.broker.ResultTable;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
+import org.apache.pinot.spi.utils.ByteArray;
+import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.UuidUtils;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class SelectionOperatorUtilsTest {
+ private static final String UUID_VALUE = "550e8400-e29b-41d4-a716-446655440000";
@Test
public void testGetResultTableColumnIndices() {
@@ -207,4 +213,21 @@ public class SelectionOperatorUtilsTest {
ColumnDataType.STRING, ColumnDataType.STRING, ColumnDataType.STRING
}));
}
+
+ @Test
+ public void testRenderResultTableWithoutOrderingFormatsUUIDAndBytes() {
+ byte[] bytesValue = new byte[]{0x01, 0x23, 0x45};
+ DataSchema dataSchema = new DataSchema(new String[]{"uuidCol", "bytesCol"},
+ new ColumnDataType[]{ColumnDataType.UUID, ColumnDataType.BYTES});
+
+ ResultTable resultTable = SelectionOperatorUtils.renderResultTableWithoutOrdering(
+ Collections.singletonList(
+ new Object[]{new ByteArray(UuidUtils.toBytes(UUID_VALUE)), new ByteArray(bytesValue)}),
+ dataSchema,
+ new int[]{0, 1});
+
+ Object[] row = resultTable.getRows().get(0);
+ assertEquals(row[0], UUID_VALUE);
+ assertEquals(row[1], BytesUtils.toHexString(bytesValue));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org